fix: eval dùng 8b-instant + checkpoint resume + cắt context để tiết kiệm token

evaluate.py

@@ -21,6 +21,22 @@ rag_module.LLM_MODEL = JUDGE_MODEL
 from src.rag import answer as rag_answer
 print("rag ok")
 
+CHECKPOINT_PATH = "eval_checkpoint.json"
+
+
+def _save_checkpoint(results: list):
+    with open(CHECKPOINT_PATH, "w", encoding="utf-8") as f:
+        json.dump(results, f, ensure_ascii=False, indent=2)
+
+
+def _load_checkpoint() -> list:
+    if os.path.exists(CHECKPOINT_PATH):
+        with open(CHECKPOINT_PATH, encoding="utf-8") as f:
+            data = json.load(f)
+        print(f"Tiếp tục từ checkpoint: {len(data)} câu đã xong.")
+        return data
+    return []
+
 
 def judge(prompt: str) -> float:
     """Gọi LLM judge, trả về score 0.0-1.0."""
@@ -39,7 +55,7 @@ def judge(prompt: str) -> float:
 
 
 def eval_faithfulness(question: str, answer: str, contexts: list[str]) -> float:
-    context_text = "\n".join(contexts)
+    context_text = "\n".join(c[:600] for c in contexts[:3])  # top-3, max 600 chars mỗi chunk
     prompt = f"""Đánh giá xem câu trả lời có trung thực với ngữ cảnh không.
 Chỉ trả về một số từ 0.0 đến 1.0 (1.0 = hoàn toàn trung thực, 0.0 = bịa đặt).
 
@@ -62,7 +78,7 @@ Score:"""
 
 
 def eval_context_recall(question: str, ground_truth: str, contexts: list[str]) -> float:
-    context_text = "\n".join(contexts)
+    context_text = "\n".join(c[:600] for c in contexts[:3])
     prompt = f"""Đánh giá xem ngữ cảnh có chứa đủ thông tin để trả lời câu hỏi theo đáp án chuẩn không.
 Chỉ trả về một số từ 0.0 đến 1.0 (1.0 = đủ thông tin, 0.0 = thiếu hoàn toàn).
 
@@ -74,17 +90,36 @@ Score:"""
     return judge(prompt)
 
 
+def _print_summary(results: list):
+    avg_faith = sum(r["faithfulness"] for r in results) / len(results)
+    avg_rel = sum(r["answer_relevancy"] for r in results) / len(results)
+    avg_recall = sum(r["context_recall"] for r in results) / len(results)
+
+    print(f"\n{'='*50}")
+    print(f"KẾT QUẢ ĐÁNH GIÁ ({len(results)} câu hỏi)")
+    print(f"Judge model: {JUDGE_MODEL}")
+    print(f"{'='*50}")
+    print(f"Faithfulness:     {avg_faith:.3f}")
+    print(f"Answer Relevancy: {avg_rel:.3f}")
+    print(f"Context Recall:   {avg_recall:.3f}")
+    print(f"Average:          {(avg_faith + avg_rel + avg_recall) / 3:.3f}")
+    return avg_faith, avg_rel, avg_recall
+
+
 def evaluate(golden_set_path: str = "data/golden_set.json", sample: int = 20):
     with open(golden_set_path, encoding="utf-8") as f:
         golden = json.load(f)
 
-    # Lấy sample câu hỏi có ground_truth thực (bỏ out_of_scope)
     items = [q for q in golden if q.get("type") != "out_of_scope"][:sample]
 
-    results = []
-    print(f"Đánh giá {len(items)} câu hỏi...\n")
+    # Resume từ checkpoint nếu có
+    results = _load_checkpoint()
+    done_questions = {r["question"] for r in results}
+    remaining = [it for it in items if it["question"] not in done_questions]
 
-    for i, item in enumerate(items, 1):
+    print(f"Đánh giá {len(items)} câu hỏi ({len(remaining)} còn lại)...\n")
+
+    for i, item in enumerate(remaining, len(results) + 1):
         question = item["question"]
         ground_truth = item["answer"]
 
@@ -108,24 +143,15 @@ def evaluate(golden_set_path: str = "data/golden_set.json", sample: int = 20):
             })
 
             print(f"  Faithfulness: {faith:.2f} | Relevancy: {relevancy:.2f} | Recall: {recall:.2f}")
-            time.sleep(2)  # tránh rate limit
+            _save_checkpoint(results)
+            time.sleep(2)
 
         except Exception as e:
             print(f"  Lỗi: {e}")
             time.sleep(5)
 
     if results:
-        avg_faith = sum(r["faithfulness"] for r in results) / len(results)
-        avg_rel = sum(r["answer_relevancy"] for r in results) / len(results)
-        avg_recall = sum(r["context_recall"] for r in results) / len(results)
-
-        print(f"\n{'='*50}")
-        print(f"KẾT QUẢ ĐÁNH GIÁ ({len(results)} câu hỏi)")
-        print(f"{'='*50}")
-        print(f"Faithfulness:     {avg_faith:.3f}")
-        print(f"Answer Relevancy: {avg_rel:.3f}")
-        print(f"Context Recall:   {avg_recall:.3f}")
-        print(f"Average:          {(avg_faith + avg_rel + avg_recall) / 3:.3f}")
+        avg_faith, avg_rel, avg_recall = _print_summary(results)
 
         with open("eval_results.json", "w", encoding="utf-8") as f:
             json.dump({
@@ -133,11 +159,17 @@ def evaluate(golden_set_path: str = "data/golden_set.json", sample: int = 20):
                     "faithfulness": avg_faith,
                     "answer_relevancy": avg_rel,
                     "context_recall": avg_recall,
+                    "judge_model": JUDGE_MODEL,
                 },
                 "details": results,
             }, f, ensure_ascii=False, indent=2)
         print("\nĐã lưu chi tiết vào eval_results.json")
 
+        # Xóa checkpoint khi hoàn thành
+        if len(results) == len(items) and os.path.exists(CHECKPOINT_PATH):
+            os.remove(CHECKPOINT_PATH)
+            print("Đã xóa checkpoint.")
+
 
 if __name__ == "__main__":
     import sys