File size: 1,028 Bytes
b05542b 4ff45b3 b05542b 4ff45b3 b05542b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | import os
import httpx
MODAL_ENDPOINT = os.environ.get("MODAL_ENDPOINT_QWEN", "")
MODAL_AUTH_TOKEN = os.environ.get("MODAL_AUTH_TOKEN", "")
TRUNCATION_LIMIT = 20_000
def chat(messages: list[dict]) -> str | None:
if not MODAL_ENDPOINT:
raise RuntimeError("MODAL_ENDPOINT_QWEN not set")
_trim(messages)
resp = httpx.post(
MODAL_ENDPOINT,
json={"messages": messages, "token": MODAL_AUTH_TOKEN},
timeout=600,
)
resp.raise_for_status()
return resp.json().get("text")
def _trim(messages: list[dict]):
if len(messages) < 4:
return
total_chars = sum(len(m.get("content", "")) for m in messages)
if total_chars < TRUNCATION_LIMIT * 3.5:
return
system = [m for m in messages if m.get("role") == "system"]
rest = [m for m in messages if m.get("role") != "system"]
while rest and total_chars >= TRUNCATION_LIMIT * 3.5:
dropped = rest.pop(0)
total_chars -= len(dropped.get("content", ""))
messages[:] = system + rest
|