download_github_activity.py

#!/usr/bin/env python3
"""
GitHub Activity Downloader for Julien Serbanescu RAG System

Downloads issues and pull requests authored by the user across all repos,
saving them as markdown documents for indexing.

Usage:
    python download_github_activity.py --username Julien-ser
    python download_github_activity.py --username Julien-ser --token YOUR_GITHUB_TOKEN
"""

import os
import re
import time
import argparse
import requests
from pathlib import Path


def clean_filename(text):
    cleaned = re.sub(r'[^\w\-_.]', '_', text)
    cleaned = re.sub(r'_+', '_', cleaned)
    return cleaned.strip('_')[:120]


def fetch_paginated(url, headers, params=None, max_pages=10):
    """Fetch all pages from a GitHub API endpoint."""
    results = []
    params = params or {}
    params.setdefault("per_page", 100)

    for page in range(1, max_pages + 1):
        params["page"] = page
        try:
            resp = requests.get(url, headers=headers, params=params, timeout=15)
            if resp.status_code == 401:
                print(f"  Token rejected (401). Retrying without authentication...")
                headers.pop("Authorization", None)
                resp = requests.get(url, headers=headers, params=params, timeout=15)
            if resp.status_code == 403:
                print(f"  Rate limit hit, waiting 60s...")
                time.sleep(60)
                resp = requests.get(url, headers=headers, params=params, timeout=15)
            resp.raise_for_status()
            data = resp.json()

            items = data.get("items", data) if isinstance(data, dict) else data
            if not items:
                break
            results.extend(items)
            if len(items) < params["per_page"]:
                break
            time.sleep(0.5)
        except Exception as e:
            print(f"  Error fetching {url} page {page}: {e}")
            break

    return results


def download_issues_and_prs(username, token=None, output_dir="docs/github_activity"):
    """Download issues and PRs authored by the user."""
    output_path = Path(output_dir)
    output_path.mkdir(parents=True, exist_ok=True)

    headers = {"Accept": "application/vnd.github.v3+json"}
    if token:
        headers["Authorization"] = f"token {token}"

    saved = 0

    for item_type, qualifier in [("PR", "is:pr"), ("Issue", "is:issue")]:
        print(f"\nFetching {item_type}s authored by {username}...")
        query = f"author:{username} {qualifier} is:public"
        items = fetch_paginated(
            "https://api.github.com/search/issues",
            headers,
            params={"q": query, "sort": "updated", "order": "desc"},
        )
        print(f"  Found {len(items)} {item_type}(s)")

        for item in items:
            repo_name = item.get("repository_url", "").split("/")[-2:]
            repo_full = "/".join(repo_name) if len(repo_name) == 2 else "unknown"
            number = item.get("number", 0)
            title = item.get("title", "Untitled")
            body = item.get("body") or ""
            state = item.get("state", "unknown")
            labels = [l["name"] for l in item.get("labels", [])]
            created = item.get("created_at", "")
            updated = item.get("updated_at", "")
            html_url = item.get("html_url", "")

            content = f"""# {item_type}: {title}

**Repository:** {repo_full}
**Number:** #{number}
**State:** {state}
**Labels:** {', '.join(labels) if labels else 'None'}
**Created:** {created}
**Updated:** {updated}
**URL:** {html_url}

---

{body}
"""
            filename = clean_filename(f"{repo_full.replace('/', '_')}_{item_type}_{number}_{title}")
            filepath = output_path / f"{filename}.md"
            filepath.write_text(content, encoding="utf-8")
            saved += 1

        time.sleep(1)

    print(f"\nSaved {saved} issues/PRs to {output_dir}/")
    return saved


def main():
    parser = argparse.ArgumentParser(description="Download GitHub issues & PRs for RAG indexing")
    parser.add_argument("--username", required=True, help="GitHub username")
    parser.add_argument("--token", help="GitHub personal access token (recommended)")
    parser.add_argument("--output-dir", default="docs/github_activity", help="Output directory")
    args = parser.parse_args()

    print(f"=== Downloading GitHub Activity for {args.username} ===")
    download_issues_and_prs(args.username, args.token, args.output_dir)


if __name__ == "__main__":
    main()