Critical QA Fixes: Bug 1-4 (Data Isolation, Hotfix Sanity, History Save, Visuals Stability)
Browse files- orchestrator.py +21 -5
- problem_understanding.py +7 -4
- prompts.py +6 -0
- visuals.py +14 -2
orchestrator.py
CHANGED
|
@@ -16,6 +16,7 @@ from domain.validator import ConsistencyGate
|
|
| 16 |
from smart_solver import sign_step, resolve_ast_target, execute_action
|
| 17 |
import domain.telemetry as telemetry
|
| 18 |
from domain.schemas import BuddyEvent, BuddyState # V8.5: Streaming contract
|
|
|
|
| 19 |
|
| 20 |
# V8.6.9: Global Guardrails (Increased for High-Complexity 5-Unit Problems)
|
| 21 |
GLOBAL_TOKEN_LIMIT = 50000
|
|
@@ -1597,7 +1598,13 @@ ctx.finish("$$ 4 $$", "מעולה! הגענו לתוצאה.")
|
|
| 1597 |
solve_prompt = f"FIX ERROR: {last_error_context}\n\nORIGINAL QUESTION: {solve_prompt}"
|
| 1598 |
|
| 1599 |
# V8.5: Reset context on retry to reduce token pressure
|
| 1600 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1601 |
|
| 1602 |
result = await safe_llm_call(
|
| 1603 |
lambda: self.strategy_manager.solve_with_strategy(
|
|
@@ -1656,10 +1663,19 @@ ctx.finish("$$ 4 $$", "מעולה! הגענו לתוצאה.")
|
|
| 1656 |
|
| 1657 |
# מעקף: אם השגיאה היא רק בעיית קריאה של סימנים (אי-שוויונים/חיצים), סומכים על ה-LLM ויוצאים
|
| 1658 |
if "SYMPY_PARSE_ERROR" in str(poly_reason):
|
| 1659 |
-
|
| 1660 |
-
|
| 1661 |
-
|
| 1662 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1663 |
elif attempts == max_attempts:
|
| 1664 |
print(f"⚠️ [HOTFIX] Max attempts reached. Forcing LLM response despite Polygraph failure.")
|
| 1665 |
is_degraded = True
|
|
|
|
| 16 |
from smart_solver import sign_step, resolve_ast_target, execute_action
|
| 17 |
import domain.telemetry as telemetry
|
| 18 |
from domain.schemas import BuddyEvent, BuddyState # V8.5: Streaming contract
|
| 19 |
+
from firebase_manager import firebase_manager
|
| 20 |
|
| 21 |
# V8.6.9: Global Guardrails (Increased for High-Complexity 5-Unit Problems)
|
| 22 |
GLOBAL_TOKEN_LIMIT = 50000
|
|
|
|
| 1598 |
solve_prompt = f"FIX ERROR: {last_error_context}\n\nORIGINAL QUESTION: {solve_prompt}"
|
| 1599 |
|
| 1600 |
# V8.5: Reset context on retry to reduce token pressure
|
| 1601 |
+
# V8.6.9: Inject sub-question specific data to prevent leakage (Data Isolation)
|
| 1602 |
+
local_anchor = {**data_anchor}
|
| 1603 |
+
if sub_q.get("specific_values"):
|
| 1604 |
+
print(f"📦 [V8.6.9] Injecting local data for {sub_q['id']}: {sub_q['specific_values']}")
|
| 1605 |
+
local_anchor["specific_values"] = list(set(local_anchor.get("specific_values", []) + sub_q["specific_values"]))
|
| 1606 |
+
|
| 1607 |
+
effective_context = {**local_anchor, **context} if attempts == 1 else {**local_anchor}
|
| 1608 |
|
| 1609 |
result = await safe_llm_call(
|
| 1610 |
lambda: self.strategy_manager.solve_with_strategy(
|
|
|
|
| 1663 |
|
| 1664 |
# מעקף: אם השגיאה היא רק בעיית קריאה של סימנים (אי-שוויונים/חיצים), סומכים על ה-LLM ויוצאים
|
| 1665 |
if "SYMPY_PARSE_ERROR" in str(poly_reason):
|
| 1666 |
+
# V8.6.9: Sanity Check for forbidden words in response to prevent blindly trusting "contradiction" answers
|
| 1667 |
+
forbidden_words = ["סתירה", "לא ייתכן", "שגיאה", "לא הגיוני", "אין פתרון"]
|
| 1668 |
+
response_text = str(llm_resp) # Search in full JSON representation
|
| 1669 |
+
if any(word in response_text for word in forbidden_words):
|
| 1670 |
+
print(f"🛑 [HOTFIX BLOCKED] Forbidden word detected in SYMPY_PARSE_ERROR response. Not Trusting LLM.")
|
| 1671 |
+
is_degraded = True
|
| 1672 |
+
degraded_reason = "polygraph_fail_forbidden_words"
|
| 1673 |
+
# Continue to second attempt instead of breaking
|
| 1674 |
+
else:
|
| 1675 |
+
print(f"🛡️ [HOTFIX] Bypassing SymPy Parse Error. Trusting LLM output for sub-q {sub_q['id']}.")
|
| 1676 |
+
is_degraded = True
|
| 1677 |
+
degraded_reason = "sympy_parse_bypass"
|
| 1678 |
+
break # יוצאים מהלופ מיד! חוסך ניסיון שני ומונע בזבוז טוקנים
|
| 1679 |
elif attempts == max_attempts:
|
| 1680 |
print(f"⚠️ [HOTFIX] Max attempts reached. Forcing LLM response despite Polygraph failure.")
|
| 1681 |
is_degraded = True
|
problem_understanding.py
CHANGED
|
@@ -41,6 +41,7 @@ Return JSON:
|
|
| 41 |
"id": "א",
|
| 42 |
"question": "Full text of sub-question א",
|
| 43 |
"requires": ["center", "radius"],
|
|
|
|
| 44 |
"expected_output": "equation | number | point | etc.",
|
| 45 |
"topic": "CIRCLE_EQUATION"
|
| 46 |
}},
|
|
@@ -48,6 +49,7 @@ Return JSON:
|
|
| 48 |
"id": "ב",
|
| 49 |
"question": "Full text of sub-question ב",
|
| 50 |
"requires": ["equation_from_א", "point"],
|
|
|
|
| 51 |
"expected_output": "line_equation",
|
| 52 |
"topic": "LINE_TANGENT"
|
| 53 |
}}
|
|
@@ -61,13 +63,14 @@ Return JSON:
|
|
| 61 |
|
| 62 |
CRITICAL RULES:
|
| 63 |
1. Include ALL sub-questions (א, ב, ג, ד, etc.)
|
| 64 |
-
2. **
|
|
|
|
| 65 |
- This is a SINGLE QUESTION (even if it looks long).
|
| 66 |
- Set `problem_type` = "GEOMETRIC_LOCUS".
|
| 67 |
- Create ONLY ONE sub-question (id="א") containing the entire text.
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
|
| 72 |
Return ONLY valid JSON.
|
| 73 |
"""
|
|
|
|
| 41 |
"id": "א",
|
| 42 |
"question": "Full text of sub-question א",
|
| 43 |
"requires": ["center", "radius"],
|
| 44 |
+
"specific_values": ["m=2"],
|
| 45 |
"expected_output": "equation | number | point | etc.",
|
| 46 |
"topic": "CIRCLE_EQUATION"
|
| 47 |
}},
|
|
|
|
| 49 |
"id": "ב",
|
| 50 |
"question": "Full text of sub-question ב",
|
| 51 |
"requires": ["equation_from_א", "point"],
|
| 52 |
+
"specific_values": ["a=1"],
|
| 53 |
"expected_output": "line_equation",
|
| 54 |
"topic": "LINE_TANGENT"
|
| 55 |
}}
|
|
|
|
| 63 |
|
| 64 |
CRITICAL RULES:
|
| 65 |
1. Include ALL sub-questions (א, ב, ג, ד, etc.)
|
| 66 |
+
2. **CHRONOLOGICAL DATA ISOLATION (V8.6.9):** If a specific value (e.g., a=1, m=2) is mentioned ONLY in a specific sub-question, you MUST include it in the `specific_values` array for THAT sub-question only. NEVER put it in the top-level anchor if it's not global.
|
| 67 |
+
3. **EXCEPTION:** If the problem asks for a **Geometric Locus (מקום גיאומטרי)**:
|
| 68 |
- This is a SINGLE QUESTION (even if it looks long).
|
| 69 |
- Set `problem_type` = "GEOMETRIC_LOCUS".
|
| 70 |
- Create ONLY ONE sub-question (id="א") containing the entire text.
|
| 71 |
+
4. Identify dependencies (ב needs א's result)
|
| 72 |
+
5. Determine topic for EACH sub-question
|
| 73 |
+
6. DO NOT solve - only analyze structure
|
| 74 |
|
| 75 |
Return ONLY valid JSON.
|
| 76 |
"""
|
prompts.py
CHANGED
|
@@ -281,6 +281,7 @@ CRITICAL INSTRUCTIONS:
|
|
| 281 |
5. **Mathematical Logic over OCR (V8.6.6):** In case of OCR contradictions (e.g. $e^x$ vs $e^{{-x}}$ in the same problem), use mathematical logic to choose the correct formula based on the problem's context (e.g., if a question asks for a limit at $\infty$ that only exists for $e^{{-x}}$, prefer that).
|
| 282 |
6. Example: Do not assume `f(x) = ax - x^2` just because it looks like a standard problem. Only extract what is literally there.
|
| 283 |
7. **SUB-QUESTION MAPPING (CRITICAL - V231.14):** You MUST map ALL sub-questions present in the OCR text (e.g., א, ב, ג, ד). Do NOT group them into one question and do NOT stop at the first one. Each sub-question letter requires its own entry in the `sub_questions` array.
|
|
|
|
| 284 |
r"""
|
| 285 |
|
| 286 |
def get_specialist_prompt(category, problem_text, solver_hint, grade, student_name, student_gender="M", data_anchor=None):
|
|
@@ -623,6 +624,11 @@ def get_master_prompt_v860():
|
|
| 623 |
- You MUST find certain mathematical anchors: intersection points with axes ($x=0$ or $y=0$), extrema, or asymptotes.
|
| 624 |
- Cross-reference your algebraic findings with the provided graphs.
|
| 625 |
- If the image is ambiguous, state clearly: "בגלל שהשרטוט אינו חד-משמעי, עלינו לוודא את נכונות הגרף באמצעות הצבת נקודות חיתוך".
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 626 |
|
| 627 |
|
| 628 |
═══════════════════════════════════════════
|
|
|
|
| 281 |
5. **Mathematical Logic over OCR (V8.6.6):** In case of OCR contradictions (e.g. $e^x$ vs $e^{{-x}}$ in the same problem), use mathematical logic to choose the correct formula based on the problem's context (e.g., if a question asks for a limit at $\infty$ that only exists for $e^{{-x}}$, prefer that).
|
| 282 |
6. Example: Do not assume `f(x) = ax - x^2` just because it looks like a standard problem. Only extract what is literally there.
|
| 283 |
7. **SUB-QUESTION MAPPING (CRITICAL - V231.14):** You MUST map ALL sub-questions present in the OCR text (e.g., א, ב, ג, ד). Do NOT group them into one question and do NOT stop at the first one. Each sub-question letter requires its own entry in the `sub_questions` array.
|
| 284 |
+
8. **DATA SCOPING (V8.6.9 - CRITICAL):** Only extract data that is truly GLOBAL (applies to all sections). If a value or constraint is explicitly tied to a specific section (e.g., "בסעיף ב' נתון כי a=1"), do NOT put it in the `specific_values` of the main anchor. The problem understanding phase will handle the local data.
|
| 285 |
r"""
|
| 286 |
|
| 287 |
def get_specialist_prompt(category, problem_text, solver_hint, grade, student_name, student_gender="M", data_anchor=None):
|
|
|
|
| 624 |
- You MUST find certain mathematical anchors: intersection points with axes ($x=0$ or $y=0$), extrema, or asymptotes.
|
| 625 |
- Cross-reference your algebraic findings with the provided graphs.
|
| 626 |
- If the image is ambiguous, state clearly: "בגלל שהשרטוט אינו חד-משמעי, עלינו לוודא את נכונות הגרף באמצעות הצבת נקודות חיתוך".
|
| 627 |
+
16. **CHRONOLOGICAL LOGIC & DATA ISOLATION (V8.6.9 - CRITICAL):**
|
| 628 |
+
- You MUST solve sub-questions strictly in order.
|
| 629 |
+
- NEVER use data, parameters (like $a=1$), or specific values that explicitly belong to a LATER sub-question (e.g., Section ב') to solve an EARLIER sub-question (e.g., Section א').
|
| 630 |
+
- Solve earlier sections algebraically using general variables unless the data is part of the global question anchor.
|
| 631 |
+
- If a student's finding in Section א' is required for Section ב', you may use it, but NEVER the other way around.
|
| 632 |
|
| 633 |
|
| 634 |
═══════════════════════════════════════════
|
visuals.py
CHANGED
|
@@ -140,7 +140,13 @@ def _plot_func(expr_str):
|
|
| 140 |
|
| 141 |
for idx, single_expr in enumerate(expressions):
|
| 142 |
try:
|
| 143 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
if not expr: continue
|
| 145 |
|
| 146 |
# טיפול בפרמטרים חופשיים
|
|
@@ -270,7 +276,13 @@ def _plot_geo(solution_expr, context_text, geometric_entities=None):
|
|
| 270 |
try:
|
| 271 |
x, y = symbols('x y')
|
| 272 |
# V290.0: Watchdog for Geo Sympify & Contour
|
| 273 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
if expr:
|
| 275 |
f_np = _run_with_timeout(lambdify, ((x, y), expr, 'numpy'), timeout_duration=1.0)
|
| 276 |
if f_np:
|
|
|
|
| 140 |
|
| 141 |
for idx, single_expr in enumerate(expressions):
|
| 142 |
try:
|
| 143 |
+
# V8.6.9: Explicit try-except for SYMPY_PARSE_ERROR prevention
|
| 144 |
+
try:
|
| 145 |
+
expr = _run_with_timeout(sympify, (single_expr, None, local_dict), timeout_duration=2.0)
|
| 146 |
+
except Exception as e:
|
| 147 |
+
print(f"⚠️ [VISUALS] SymPy Parse Error on '{single_expr}': {e}")
|
| 148 |
+
expr = None
|
| 149 |
+
|
| 150 |
if not expr: continue
|
| 151 |
|
| 152 |
# טיפול בפרמטרים חופשיים
|
|
|
|
| 276 |
try:
|
| 277 |
x, y = symbols('x y')
|
| 278 |
# V290.0: Watchdog for Geo Sympify & Contour
|
| 279 |
+
# V8.6.9: Local try-except for crash prevention
|
| 280 |
+
try:
|
| 281 |
+
expr = _run_with_timeout(sympify, (eq_str, None, {"sin": sin, "cos": cos, "tan": tan, "sqrt": sqrt, "Abs": Abs, "pi": np.pi, "ln": ln, "log": log, "exp": exp, "e": exp(1)}), timeout_duration=1.5)
|
| 282 |
+
except Exception as e:
|
| 283 |
+
print(f"⚠️ [VISUALS] Geo SymPy Parse Error on '{eq_str}': {e}")
|
| 284 |
+
expr = None
|
| 285 |
+
|
| 286 |
if expr:
|
| 287 |
f_np = _run_with_timeout(lambdify, ((x, y), expr, 'numpy'), timeout_duration=1.0)
|
| 288 |
if f_np:
|