refactor: let Controller reason from chat history

agents/orchestrator.py

@@ -603,19 +603,13 @@ Return JSON:
 
         # Autonomous reasoning features
         self._reasoning_trace: Optional[ReasoningTrace] = None
-        self._pending_follow_up: Optional[CompletenessAnalysis] = None
-
-        # Accumulated context from incomplete input gathering (for multi-turn)
-        self._accumulated_context: Dict[str, Any] = {}
-        # Track the last question we asked (for context-aware classification)
-        self._last_question: Optional[str] = None
 
         # Location data captured from tool results for map updates
         self._captured_location: Optional[Dict[str, Any]] = None
 
-        # Multi-turn conversation history (per-session, cross-user isolated via gr.State)
+        # Completed reports history (for session awareness)
         self._conversation_history: List[Dict[str, Any]] = []
-        self._max_history_turns: int = 5  # Keep last 5 reports for context
+        self._max_history_turns: int = 5
 
         # Eager initialization to avoid delay on first request
         if eager_init and self.api_key:
@@ -993,7 +987,7 @@ Return JSON:
         self,
         user_message: str,
         image_analysis: Optional[str] = None,
-        log_catcher: Optional[Any] = None
+        chat_history: Optional[List[Dict[str, str]]] = None,
     ) -> Generator[Tuple[str, dict], None, None]:
         """
         Process infrastructure report through TRUE AUTONOMOUS multi-agent system.
@@ -1008,7 +1002,7 @@ Return JSON:
         Args:
             user_message: User's description of the issue
             image_analysis: Optional image analysis from Claude Vision
-            log_catcher: Optional LogCatcher (kept for backward compatibility)
+            chat_history: Full conversation history for Controller to reason about
 
         Yields:
             Tuples of (event_type, event_data) for UI updates
@@ -1047,15 +1041,19 @@ Return JSON:
             except Empty:
                 break
 
-        # Build context from previous turns
-        context_parts = []
-        if self._accumulated_context:
-            context_parts.append(f"Previous context: {json.dumps(self._accumulated_context)}")
-        if self._last_question:
-            context_parts.append(f"Last question asked: \"{self._last_question}\"")
-        if self._conversation_history:
-            context_parts.append(f"Previous reports in session: {len(self._conversation_history)}")
-        context = "\n".join(context_parts) if context_parts else "No previous context."
+        # Format conversation history for Controller (TRUE AUTONOMOUS - Controller reasons about context)
+        conversation_context = ""
+        if chat_history and len(chat_history) > 0:
+            conv_lines = ["CONVERSATION HISTORY (you have full context - reason about what user needs):"]
+            for msg in chat_history:
+                role = msg.get("role", "unknown")
+                content = msg.get("content", "")
+                if role == "user":
+                    conv_lines.append(f"User: {content}")
+                elif role == "assistant":
+                    # Truncate long assistant responses
+                    conv_lines.append(f"Agent: {content[:500]}{'...' if len(content) > 500 else ''}")
+            conversation_context = "\n".join(conv_lines)
 
         trace.think("Delegating full control to Controller agent", "autonomy")
         yield ("planning", {"status": "started", "message": "Controller is planning..."})
@@ -1064,11 +1062,10 @@ Return JSON:
         # TRUE AUTONOMOUS TASK - Controller has full decision-making power
         task = f"""You are the autonomous Controller for FixMyNeighborhood, an NYC infrastructure reporting system.
 
-USER INPUT: "{user_message}"
+CURRENT USER INPUT: "{user_message}"
 {f'IMAGE ANALYSIS: {image_analysis}' if image_analysis else 'No image provided.'}
 
-CONTEXT:
-{context}
+{conversation_context if conversation_context else 'No previous conversation.'}
 
 YOUR AGENTS:
 - triage_agent: Validates NYC addresses (call validate_address tool), classifies images as infrastructure or not
@@ -1239,6 +1236,7 @@ NOW: Analyze the input and decide what to do. You have full autonomy."""
         print(f"[Controller] Raw result: {raw_result[:500]}...")
 
         # Detect response type (Controller decided what to do)
+        # No Python context extraction - Controller reasons from chat history
         if "FOLLOW_UP:" in raw_result:
             # Controller decided more info is needed
             follow_up_match = re.search(r'FOLLOW_UP:\s*(.+?)(?:$|REJECTION:|REPORT:)', raw_result, re.DOTALL)
@@ -1246,10 +1244,6 @@ NOW: Analyze the input and decide what to do. You have full autonomy."""
                 follow_up_text = follow_up_match.group(1).strip()
                 trace.decide("Controller decided: need more information", confidence=0.9)
 
-                # Extract what Controller understood for context
-                self._accumulated_context = self._extract_context_from_result(raw_result)
-                self._last_question = follow_up_text
-
                 yield ("reasoning_update", {"trace": trace.to_display()})
                 yield ("needs_info", {
                     "message": follow_up_text,
@@ -1264,10 +1258,6 @@ NOW: Analyze the input and decide what to do. You have full autonomy."""
                 rejection_text = rejection_match.group(1).strip()
                 trace.decide("Controller decided: reject request", confidence=0.9)
 
-                # Clear context on rejection
-                self._accumulated_context = {}
-                self._last_question = None
-
                 yield ("reasoning_update", {"trace": trace.to_display()})
                 yield ("needs_info", {
                     "message": rejection_text,
@@ -1310,8 +1300,6 @@ NOW: Analyze the input and decide what to do. You have full autonomy."""
             "response_days": self._extract_field(raw_result, r'(?:Expected Response|response)[:\s]+~?(\d+)', "7"),
         }
         self._add_to_history(user_message, report_summary)
-        self._accumulated_context = {}
-        self._last_question = None
         trace.think(f"Report stored (total: {len(self._conversation_history)})", "multi-turn")
 
         yield ("reasoning_update", {"trace": trace.to_display()})
@@ -1325,19 +1313,6 @@ NOW: Analyze the input and decide what to do. You have full autonomy."""
             "reasoning_trace": trace.to_display(),
         })
 
-    def _extract_context_from_result(self, result: str) -> Dict[str, Any]:
-        """Extract understood context from Controller's response."""
-        context = {}
-        # Try to extract what Controller understood
-        if "pothole" in result.lower():
-            context["issue_type"] = "pothole"
-        elif "streetlight" in result.lower():
-            context["issue_type"] = "streetlight"
-        elif "drain" in result.lower():
-            context["issue_type"] = "drain"
-        # Add more as needed
-        return context
-
     def _format_result(self, result: str, user_message: str) -> str:
         """Format the controller result into a user-friendly message."""
         # More flexible patterns to handle various agent output formats

app.py

@@ -330,6 +330,7 @@ def process_report(message: str, image, history: list, session_logs: list, sessi
         """Build all output values for yield."""
         return (
             history,
+            None,  # Clear image_input after submit
             get_processing_status(),
             get_status_text(agent_states["research"]["status"], agent_states["research"].get("duration_formatted")),
             get_tools_markdown(agent_states["research"]["tools"]),
@@ -428,8 +429,8 @@ def process_report(message: str, image, history: list, session_logs: list, sessi
         progress_msg_idx += 1
 
     try:
-        # Process through orchestrator with log_catcher for real-time events
-        for event_type, data in orchestrator.process(message, image_analysis, log_catcher):
+        # Process through orchestrator with chat history (Controller reasons about context)
+        for event_type, data in orchestrator.process(message, image_analysis, chat_history=history):
 
             # Handle reasoning trace updates (Autonomous Feature #1)
             if event_type == "reasoning_update":
@@ -816,7 +817,7 @@ with gr.Blocks(title="FixMyNeighborhood - Multi-Agent Reporter") as demo:
         process_report,
         inputs=[message_input, image_input, chatbot, session_logs, session_orchestrator, reasoning_display],
         outputs=[
-            chatbot, processing_status, research_status, research_tools,
+            chatbot, image_input, processing_status, research_status, research_tools,
             report_status, report_tools, map_display, logs_display,
             session_logs, session_orchestrator, reasoning_display
         ]
@@ -826,7 +827,7 @@ with gr.Blocks(title="FixMyNeighborhood - Multi-Agent Reporter") as demo:
         process_report,
         inputs=[message_input, image_input, chatbot, session_logs, session_orchestrator, reasoning_display],
         outputs=[
-            chatbot, processing_status, research_status, research_tools,
+            chatbot, image_input, processing_status, research_status, research_tools,
             report_status, report_tools, map_display, logs_display,
             session_logs, session_orchestrator, reasoning_display
         ]