import os import json import gradio as gr import git import shutil from datetime import datetime from typing import List, Dict, Optional # =============================== # Persistent paths (HF Spaces) # =============================== PERSISTENT_REPO_DIR = "/data/transformers_repo" CACHE_DIR = "/data/cache" CACHE_RESULT = f"{CACHE_DIR}/translation_status.json" CACHE_HEAD = f"{CACHE_DIR}/head_sha.txt" class TranslationTracker: def __init__(self): self.repo_url = "https://github.com/huggingface/transformers.git" self.repo_path = None self.en_docs_path = None self.ko_docs_path = None self.repo = None # =============================== # Repo management (cached) # =============================== def clone_repo(self, progress=gr.Progress()): try: if os.path.exists(PERSISTENT_REPO_DIR): progress(0.2, desc="기존 레포 재사용 중 (fetch)...") self.repo = git.Repo(PERSISTENT_REPO_DIR) self.repo.git.fetch() self.repo.git.reset("--hard", "origin/main") else: progress(0.1, desc="레포지토리 최초 클론 중...") self.repo = git.Repo.clone_from( self.repo_url, PERSISTENT_REPO_DIR, ) self.repo_path = PERSISTENT_REPO_DIR self.en_docs_path = os.path.join(self.repo_path, "docs", "source", "en") self.ko_docs_path = os.path.join(self.repo_path, "docs", "source", "ko") return "레포지토리 준비 완료 (캐시 사용)" except Exception as e: return f"레포지토리 준비 실패: {str(e)}" # =============================== # Cache helpers # =============================== def get_repo_head_sha(self) -> str: return self.repo.head.commit.hexsha def load_cached_result(self): if not os.path.exists(CACHE_RESULT) or not os.path.exists(CACHE_HEAD): return None with open(CACHE_HEAD) as f: cached_head = f.read().strip() if cached_head != self.get_repo_head_sha(): return None with open(CACHE_RESULT) as f: return json.load(f) def save_cache(self, results: List[Dict]): os.makedirs(CACHE_DIR, exist_ok=True) with open(CACHE_HEAD, "w") as f: f.write(self.get_repo_head_sha()) with open(CACHE_RESULT, "w") as f: json.dump(results, f, ensure_ascii=False, indent=2) # =============================== # Git helpers # =============================== def _get_last_commit(self, file_path: str): if not os.path.exists(file_path): return None rel_path = os.path.relpath(file_path, self.repo_path) for commit in self.repo.iter_commits(paths=rel_path, max_count=1): return commit return None def get_last_commit_date(self, file_path: str) -> Optional[datetime]: commit = self._get_last_commit(file_path) return commit.committed_datetime if commit else None def get_last_content_change_commit( self, file_path: str, max_commits: int = 200, ): if not os.path.exists(file_path): return None rel_path = os.path.relpath(file_path, self.repo_path) for commit in self.repo.iter_commits(paths=rel_path, max_count=max_commits): stats = commit.stats.files.get(rel_path) if stats and (stats.get("insertions", 0) > 0 or stats.get("deletions", 0) > 0): return commit return None def is_outdated_by_diff(self, en_file: str, ko_file: str) -> bool: en_change_commit = self.get_last_content_change_commit(en_file) if not en_change_commit: return False ko_commit = self._get_last_commit(ko_file) if not ko_commit: return True return ko_commit.committed_datetime < en_change_commit.committed_datetime # =============================== # Main logic (cached) # =============================== def get_translation_status(self, progress=gr.Progress()) -> List[Dict]: if not self.repo: return [{"error": "레포지토리가 준비되지 않았습니다."}] cached = self.load_cached_result() if cached: return cached results = [] en_md_files = [] progress(0.1, desc="영어 문서 스캔 중...") for root, _, files in os.walk(self.en_docs_path): for file in files: if file.endswith(".md"): en_md_files.append(os.path.join(root, file)) total = len(en_md_files) for i, en_file in enumerate(en_md_files): progress((i + 1) / total, desc=f"번역 상태 계산 중 ({i+1}/{total})") rel_path = os.path.relpath(en_file, self.en_docs_path) ko_file = os.path.join(self.ko_docs_path, rel_path) file_name = os.path.basename(en_file) en_change_commit = self.get_last_content_change_commit(en_file) ko_commit = self._get_last_commit(ko_file) if not ko_commit: status = "미번역" outdate = False ko_date = None else: outdate = self.is_outdated_by_diff(en_file, ko_file) status = "번역됨 (업데이트 필요)" if outdate else "번역됨 (최신)" ko_date = ko_commit.committed_datetime results.append({ "file_name": file_name, "en_path": rel_path, "ko_path": rel_path if os.path.exists(ko_file) else "없음", "en_latest": ( en_change_commit.committed_datetime.strftime("%Y-%m-%d %H:%M:%S") if en_change_commit else "변경 없음" ), "ko_base": ( ko_date.strftime("%Y-%m-%d %H:%M:%S") if ko_date else "없음" ), "status": status, "outdate": outdate, }) results.sort(key=lambda x: x["file_name"]) self.save_cache(results) return results # =============================== # UI # =============================== def create_ui(): tracker = TranslationTracker() with gr.Blocks(title="Transformers 문서 번역 추적기") as app: gr.Markdown("# Transformers 문서 번역 추적기") gr.Markdown("영어 문서의 **실제 내용 변경(diff)** 기준 + 캐시 기반 추적기") with gr.Row(): clone_btn = gr.Button("레포지토리 준비", variant="primary") status_btn = gr.Button("번역 상태 확인", variant="secondary") status_output = gr.Textbox(label="상태") with gr.Tabs(): with gr.TabItem("모든 문서"): all_table = gr.DataFrame( headers=[ "파일명", "영어 경로", "한글 경로", "영어 실제 변경", "한글 번역 기준", "상태", ], datatype=["str"] * 6, ) with gr.TabItem("번역 필요"): untranslated_table = gr.DataFrame( headers=["파일명", "영어 경로", "상태"], datatype=["str"] * 3, ) with gr.TabItem("업데이트 필요"): outdated_table = gr.DataFrame( headers=[ "파일명", "영어 경로", "영어 실제 변경", "한글 번역 기준", ], datatype=["str"] * 4, ) clone_btn.click(tracker.clone_repo, outputs=status_output) def process(): results = tracker.get_translation_status() if results and "error" in results[0]: return results[0]["error"], None, None, None all_data = [ [ r["file_name"], r["en_path"], r["ko_path"], r["en_latest"], r["ko_base"], r["status"], ] for r in results ] untranslated = [ [r["file_name"], r["en_path"], r["status"]] for r in results if r["status"] == "미번역" ] outdated = [ [r["file_name"], r["en_path"], r["en_latest"], r["ko_base"]] for r in results if r["outdate"] ] return "번역 상태 조회 완료 (캐시 적용)", all_data, untranslated, outdated status_btn.click( process, outputs=[status_output, all_table, untranslated_table, outdated_table], ) return app def main(): app = create_ui() app.launch() if __name__ == "__main__": main()