import os import base64 import requests API_BASE = "https://api.github.com" DEPENDENCY_FILENAMES = [ "requirements.txt", "environment.yml", "environment.yaml", "setup.py", "setup.cfg", "pyproject.toml", "Pipfile", "Pipfile.lock", "Dockerfile", "Makefile", "CMakeLists.txt", ] def _get_headers(): token = os.getenv("GITHUB_TOKEN", "") headers = {"Accept": "application/vnd.github.v3+json"} if token: headers["Authorization"] = f"Bearer {token}" return headers def _github_get(path: str) -> dict | None: """统一的 GitHub API GET 请求,失败返回 None""" headers = _get_headers() url = f"{API_BASE}{path}" try: resp = requests.get(url, headers=headers, timeout=15) if resp.status_code == 404: return None if resp.status_code in (403, 429): print(f"[警告] GitHub API 限速: {path}") return None resp.raise_for_status() return resp.json() except Exception as e: print(f"[警告] GitHub API 请求失败 ({path}): {e}") return None def fetch_readme(owner: str, name: str) -> str | None: """获取仓库 README 文件内容(带缓存,1 小时 TTL)""" from cache import github_readme_cache cache_key = f"{owner}/{name}" cached = github_readme_cache.get(cache_key) if cached is not None: return cached try: data = _github_get(f"/repos/{owner}/{name}/readme") if not data: return None content = data.get("content", "") result = base64.b64decode(content).decode("utf-8", errors="replace") github_readme_cache.set(cache_key, result) return result except Exception: return None def fetch_dependencies(owner: str, name: str) -> dict[str, str]: """获取仓库的依赖文件(带缓存,1 小时 TTL) Returns: dict: {文件名: 文件内容},找不到任何依赖文件时返回 {} """ from cache import github_deps_cache cache_key = f"{owner}/{name}" cached = github_deps_cache.get(cache_key) if cached is not None: return cached result = {} try: contents = _github_get(f"/repos/{owner}/{name}/contents") or [] for item in contents: fname = item.get("name", "") if fname in DEPENDENCY_FILENAMES: file_data = _github_get(f"/repos/{owner}/{name}/contents/{fname}") if file_data and file_data.get("content"): try: decoded = base64.b64decode(file_data["content"]).decode("utf-8", errors="replace") result[fname] = decoded except Exception: pass except Exception: pass github_deps_cache.set(cache_key, result) return result if __name__ == "__main__": # 自测:对 anomalib 仓库调用 owner, name = "openvinotoolkit", "anomalib" print("=== repo_fetcher 自测 ===") readme = fetch_readme(owner, name) print(f"README 长度: {len(readme) if readme else 0} 字符") if readme: print(f" 前 200 字符: {readme[:200]}...") deps = fetch_dependencies(owner, name) print(f"依赖文件: {len(deps)} 个 — {list(deps.keys())}") for fname, content in deps.items(): print(f" {fname}: {len(content)} 字符")