training_code/terminal_dataset.py

"""
Terminal Commands Dataset Generator
Generates synthetic and realistic terminal command sequences, bash scripts, command Q&A pairs, and interactive CLI sessions.
"""
import random
import os

COMMAND_TEMPLATES = [
    # Navigation
    ("How do I navigate up one directory?", "cd .."),
    ("How do I go to the home directory?", "cd ~"),
    ("How do I check my current directory path?", "pwd"),
    ("How do I list all files including hidden files?", "ls -la"),
    ("How do I list files with human readable file sizes?", "ls -lh"),
    ("How do I create a nested directory structure?", "mkdir -p path/to/nested/directory"),
    ("How do I print directory tree structure?", "tree -L 2"),
    
    # File Manipulation
    ("How do I copy a directory recursively?", "cp -r source_dir/ target_dir/"),
    ("How do I move or rename a file?", "mv old_filename.txt new_filename.txt"),
    ("How do I force remove a folder and all contents?", "rm -rf target_folder/"),
    ("How do I create an empty file?", "touch index.js"),
    ("How do I inspect the first 20 lines of a file?", "head -n 20 logfile.log"),
    ("How do I monitor a log file in real-time?", "tail -f /var/log/syslog"),
    ("How do I count lines in a text file?", "wc -l dataset.txt"),
    
    # Search and Filter
    ("How do I recursively search for text in files?", "grep -rn \"search_term\" ."),
    ("How do I find all python files in the current folder?", "find . -type f -name \"*.py\""),
    ("How do I find files larger than 100MB?", "find / -size +100M 2>/dev/null"),
    ("How do I sort lines and remove duplicates?", "sort input.txt | uniq -c"),
    ("How do I replace text in a file inline?", "sed -i 's/old_text/new_text/g' config.yaml"),
    
    # Permissions and Processes
    ("How do I make a shell script executable?", "chmod +x script.sh"),
    ("How do I give full read/write/execute permissions to owner?", "chmod 755 binary_file"),
    ("How do I change owner of a folder recursively?", "chown -R www-data:www-data /var/www/html"),
    ("How do I list running processes matching python?", "ps aux | grep python"),
    ("How do I kill a process by process ID?", "kill -9 12345"),
    ("How do I check system RAM usage?", "free -h"),
    ("How do I check disk space usage in human readable format?", "df -h"),
    ("How do I check disk usage of current directories?", "du -sh * | sort -hr"),

    # Networking & Web
    ("How do I test network connectivity to a host?", "ping -c 4 google.com"),
    ("How do I download a file silently with curl?", "curl -sSL https://example.com/file.tar.gz -o file.tar.gz"),
    ("How do I download a file using wget?", "wget -q https://example.com/data.json"),
    ("How do I SSH into a remote server with custom port?", "ssh -p 2222 user@remote-host.com"),
    ("How do I copy a local file to a remote server using scp?", "scp -P 2222 local_file.txt user@remote-host:/tmp/"),
    ("How do I view open listening network ports?", "netstat -tulpn"),

    # Git Version Control
    ("How do I check repository status in git?", "git status"),
    ("How do I stage all changed files in git?", "git add ."),
    ("How do I commit staged changes with a message?", "git commit -m \"feat: implement new terminal parser\""),
    ("How do I push commits to remote main branch?", "git push origin main"),
    ("How do I create and switch to a new git branch?", "git checkout -b feature/new-architecture"),
    ("How do I view compact git log history?", "git log --oneline -n 10"),
    ("How do I discard all unstaged local changes in git?", "git checkout -- ."),
    ("How do I stash working directory changes?", "git stash pop"),

    # Package Managers & Containers
    ("How do I install python packages from requirements?", "pip install -r requirements.txt"),
    ("How do I install npm dependencies?", "npm install"),
    ("How do I update package lists on Ubuntu?", "sudo apt update && sudo apt upgrade -y"),
    ("How do I build a Docker image with tag?", "docker build -t myapp:latest ."),
    ("How do I run an interactive container with volume mount?", "docker run -it -v $(pwd):/app -p 8080:8080 myapp:latest /bin/bash"),
    ("How do I view running docker containers?", "docker ps -a"),

    # Compression
    ("How do I compress a folder into a tar.gz archive?", "tar -czvf archive.tar.gz target_directory/"),
    ("How do I extract a tar.gz archive?", "tar -xzvf archive.tar.gz"),
    ("How do I unzip a zip file to a specific destination?", "unzip file.zip -d /path/to/destination/")
]

SHELL_INTERACTIONS = [
    "$ cd ..\n$ pwd\n/home/user\n$ ls -la\ntotal 32\ndrwxr-xr-x 4 user user 4096 Aug 2 00:00 .\ndrwxr-xr-x 8 user user 4096 Aug 2 00:00 ..\n-rw-r--r-- 1 user user  220 Aug 2 00:00 .bashrc",
    "$ mkdir project && cd project\n$ git init\nInitialized empty Git repository in /home/user/project/.git/\n$ touch main.py README.md\n$ git status\nOn branch main\nUntracked files:\n  (use \"git add <file>...\" to include in what will be committed)\n\tREADME.md\n\tmain.py",
    "$ grep -rn \"import torch\" src/\nsrc/model.py:1:import torch\nsrc/train.py:2:import torch\nsrc/utils.py:1:import torch",
    "$ chmod +x build.sh\n$ ./build.sh\n[INFO] Building release binary...\n[SUCCESS] Build completed in 2.4s.",
    "$ curl -I https://api.github.com\nHTTP/2 200\nserver: GitHub.com\ndate: Sun, 02 Aug 2026 00:00:00 GMT\ncontent-type: application/json; charset=utf-8",
    "$ ps aux | grep python\nuser      12345 98.2  4.1 452104 338102 ?        Rsl  00:00  12:30 python train.py\nuser      12390  0.0  0.0   6200   892 pts/0    S+   00:15   0:00 grep python",
    "$ df -h\nFilesystem      Size  Used Avail Use% Mounted on\n/dev/sda1        99G   32G   63G  34% /\ntmpfs           7.8G     0  7.8G   0% /dev/shm",
    "$ git commit -m \"fix: resolve permission issue\"\n[main a1b2c3d] fix: resolve permission issue\n 2 files changed, 14 insertions(+), 3 deletions(-)\n$ git push origin main\nTo github.com:user/repo.git\n   e4f5g6h..a1b2c3d  main -> main"
]

def generate_terminal_dataset(num_samples=15000):
    lines = []
    
    # 1. Q&A pairs
    for _ in range(num_samples // 3):
        qa = random.choice(COMMAND_TEMPLATES)
        fmt = random.choice([
            f"User: {qa[0]}\nAssistant: Run `{qa[1]}`\n",
            f"Question: {qa[0]}\nAnswer: You can use the following command:\n```bash\n{qa[1]}\n```\n",
            f"$ {qa[1]}\n# Executed command for: {qa[0]}\n"
        ])
        lines.append(fmt)

    # 2. Shell session logs
    for _ in range(num_samples // 3):
        interaction = random.choice(SHELL_INTERACTIONS)
        lines.append(f"```session\n{interaction}\n```\n")

    # 3. Command variations & chained commands
    cmds = [qa[1] for qa in COMMAND_TEMPLATES]
    for _ in range(num_samples // 3):
        c1, c2 = random.sample(cmds, 2)
        chain = f"$ {c1} && {c2}\n"
        lines.append(chain)

    random.shuffle(lines)
    return "\n".join(lines)

if __name__ == "__main__":
    text = generate_terminal_dataset(30000)
    print(f"Generated Terminal dataset preview:\n{text[:400]}")
    print(f"Total dataset characters: {len(text):,}")