Update app.py

app.py

@@ -1093,12 +1093,73 @@ async def websocket_chat(ws: WebSocket):
         # Internal endpoint (avoids Hugging Face proxy redirect)
         internal_url = "http://127.0.0.1:7860/gen/chat/completions"
 
-        # Persistent HTTP client for streaming
-        async with httpx.AsyncClient(
-            timeout=None,
-            follow_redirects=False
-        ) as client:
+        # Queue to ensure sequential request processing
+        import asyncio
+        request_queue = asyncio.Queue()
+        
+        async def process_requests():
+            # Persistent HTTP client for streaming
+            async with httpx.AsyncClient(
+                timeout=None,
+                follow_redirects=False
+            ) as client:
+                while True:
+                    try:
+                        request_id, body, headers = await request_queue.get()
+                    except asyncio.CancelledError:
+                        break
+
+                    try:
+                        async with client.stream(
+                            "POST",
+                            internal_url,
+                            json=body,
+                            headers=headers
+                        ) as response:
+
+                            # Handle upstream errors
+                            if response.status_code >= 400:
+                                error_text = ""
+                                try:
+                                    error_text = (await response.aread()).decode(
+                                        "utf-8", errors="replace"
+                                    )[:500]
+                                except Exception:
+                                    pass
+
+                                await ws.send_json({
+                                    "id": request_id,
+                                    "error": "Upstream request failed",
+                                    "status": response.status_code,
+                                    "detail": error_text
+                                })
+                                continue
+
+                            # Stream tokens/lines to the websocket
+                            async for line in response.aiter_lines():
+                                if not line:
+                                    continue
+
+                                try:
+                                    await ws.send_text(f"{request_id}:{line}")
+                                except RuntimeError:
+                                    return
+
+                    except Exception as stream_error:
+                        try:
+                            await ws.send_json({
+                                "id": request_id,
+                                "error": "Streaming error",
+                                "detail": str(stream_error)
+                            })
+                        except Exception:
+                            pass
 
+        # Start the request processor task
+        processor_task = asyncio.create_task(process_requests())
+
+        try:
+            request_counter = 0
             while True:
                 try:
                     msg = await ws.receive_text()
@@ -1118,46 +1179,15 @@ async def websocket_chat(ws: WebSocket):
                     await ws.send_json({"error": "Missing body"})
                     continue
 
-                try:
-                    async with client.stream(
-                        "POST",
-                        internal_url,
-                        json=body,
-                        headers=headers
-                    ) as response:
+                request_counter += 1
+                await request_queue.put((request_counter, body, headers))
 
-                        # Handle upstream errors
-                        if response.status_code >= 400:
-                            error_text = ""
-                            try:
-                                error_text = (await response.aread()).decode(
-                                    "utf-8", errors="replace"
-                                )[:500]
-                            except Exception:
-                                pass
-
-                            await ws.send_json({
-                                "error": "Upstream request failed",
-                                "status": response.status_code,
-                                "detail": error_text
-                            })
-                            continue
-
-                        # Stream tokens/lines to the websocket
-                        async for line in response.aiter_lines():
-                            if not line:
-                                continue
-
-                            try:
-                                await ws.send_text(line)
-                            except RuntimeError:
-                                break
-
-                except Exception as stream_error:
-                    await ws.send_json({
-                        "error": "Streaming error",
-                        "detail": str(stream_error)
-                    })
+        finally:
+            processor_task.cancel()
+            try:
+                await processor_task
+            except asyncio.CancelledError:
+                pass
 
     except WebSocketDisconnect:
         return