mengxaingshuo commited on
Commit
dcea8ab
·
1 Parent(s): 8312ded

fix: sync v4 runtime and fast inventory

Browse files
DEPLOY_20260908.md CHANGED
@@ -55,11 +55,22 @@ directory, and extract it there. In an administrator SSH session, run:
55
  ```bash
56
  cd /data0/zqyan/<new-app-directory>
57
  chmod +x start.sh deploy/restart-school.sh
 
58
  ./deploy/restart-school.sh --confirm-restart
59
  curl -fsS http://127.0.0.1:7861/api/status
60
  curl -fsS http://127.0.0.1:7861/marine/health
61
  ```
62
 
 
 
 
 
 
 
 
 
 
 
63
  The restart script automatically uses the known `squid_agent` Python 3.11 path
64
  instead of Ubuntu's legacy `python3`, and checks the existing runtime
65
  configuration from the currently running service *before* it stops that
 
55
  ```bash
56
  cd /data0/zqyan/<new-app-directory>
57
  chmod +x start.sh deploy/restart-school.sh
58
+ export PUBLIC_BASE_URL="https://subway-unbiased-barcode.ngrok-free.dev/marine"
59
  ./deploy/restart-school.sh --confirm-restart
60
  curl -fsS http://127.0.0.1:7861/api/status
61
  curl -fsS http://127.0.0.1:7861/marine/health
62
  ```
63
 
64
+ `PUBLIC_BASE_URL` is required for browser-facing Ocean export links. Without
65
+ it, the model can only see the private `127.0.0.1:8000` URL. The fixed ngrok
66
+ endpoint proxies `/marine/*` to that private API, so the correct public link
67
+ has the form `https://subway-unbiased-barcode.ngrok-free.dev/marine/download/<token>`.
68
+
69
+ The school inventory is local-first. It returns the local mirror immediately
70
+ when Hugging Face egress is blocked, and merges remote-only paths when HF is
71
+ reachable; a blocked HF connection no longer holds a metadata-only chat until
72
+ the request times out.
73
+
74
  The restart script automatically uses the known `squid_agent` Python 3.11 path
75
  instead of Ubuntu's legacy `python3`, and checks the existing runtime
76
  configuration from the currently running service *before* it stops that
DEPLOY_UNIFIED.md CHANGED
@@ -14,6 +14,15 @@ caches: a same-path local file is preferred, while every missing file is read
14
  from the configured Hugging Face Dataset. Do not set different repository
15
  names on the two deployments.
16
 
 
 
 
 
 
 
 
 
 
17
  The public ngrok URL exposes the UI on `/` and the Marine API through
18
  `/marine/*`. Therefore its health endpoint is `/marine/health`, not `/health`.
19
 
 
14
  from the configured Hugging Face Dataset. Do not set different repository
15
  names on the two deployments.
16
 
17
+ Set the browser-facing export base before restarting the school UI:
18
+
19
+ ```bash
20
+ export PUBLIC_BASE_URL="https://subway-unbiased-barcode.ngrok-free.dev/marine"
21
+ ```
22
+
23
+ This prevents localhost download links. Public exports then use
24
+ `/marine/download/<token>` through the fixed ngrok UI endpoint.
25
+
26
  The public ngrok URL exposes the UI on `/` and the Marine API through
27
  `/marine/*`. Therefore its health endpoint is `/marine/health`, not `/health`.
28
 
deploy/restart-school.sh CHANGED
@@ -52,7 +52,7 @@ allowed = {
52
  "CODEX_HOME", "PORT", "PYTHONDONTWRITEBYTECODE", "ZAI_API_KEY",
53
  "GLM_API_KEY", "ZHIPU_API_KEY", "HF_TOKEN", "HF_SQUID_DATASET_REPO",
54
  "HF_TUNA_DATASET_REPO", "FISHERIES_EXPORT_ROOT", "HF_FISHERIES_CACHE_ROOT",
55
- "MARINE_EXPORT_RANGE_MAX_DAYS",
56
  }
57
  with open(f"/proc/{pid}/environ", "rb") as stream:
58
  entries = stream.read().split(b"\0")
 
52
  "CODEX_HOME", "PORT", "PYTHONDONTWRITEBYTECODE", "ZAI_API_KEY",
53
  "GLM_API_KEY", "ZHIPU_API_KEY", "HF_TOKEN", "HF_SQUID_DATASET_REPO",
54
  "HF_TUNA_DATASET_REPO", "FISHERIES_EXPORT_ROOT", "HF_FISHERIES_CACHE_ROOT",
55
+ "MARINE_EXPORT_RANGE_MAX_DAYS", "PUBLIC_BASE_URL", "MARINE_PUBLIC_BASE_URL",
56
  }
57
  with open(f"/proc/{pid}/environ", "rb") as stream:
58
  entries = stream.read().split(b"\0")
marine_mcp.py CHANGED
@@ -15,6 +15,22 @@ from fisheries_hf import (
15
  API_URL = os.environ.get("MARINE_API_URL", "").strip().rstrip("/")
16
  if not API_URL:
17
  raise RuntimeError("MARINE_API_URL is not configured")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
 
19
  mcp = MCPServer(
20
  "Marine Data",
@@ -150,7 +166,7 @@ def marine_subset(
150
  result = _post("/data/export", payload)
151
  path = result.get("download_path")
152
  if isinstance(path, str) and path.startswith("/download/"):
153
- result["download_url"] = f"{API_URL}{path}"
154
  return result
155
 
156
  @mcp.tool()
@@ -159,7 +175,7 @@ def marine_download(token: str) -> dict[str, Any]:
159
  token = token.strip()
160
  if not re.fullmatch(r"[A-Za-z0-9_-]{20,160}", token):
161
  raise ValueError("invalid download token")
162
- return {"download_url": f"{API_URL}/download/{token}"}
163
 
164
 
165
 
@@ -199,7 +215,7 @@ def marine_export(
199
  result = _post("/data/export", payload)
200
  path = result.get("download_path")
201
  if isinstance(path, str) and path.startswith("/download/"):
202
- result["download_url"] = f"{API_URL}{path}"
203
  return result
204
 
205
 
 
15
  API_URL = os.environ.get("MARINE_API_URL", "").strip().rstrip("/")
16
  if not API_URL:
17
  raise RuntimeError("MARINE_API_URL is not configured")
18
+ # Browser-facing download URLs must not expose the private Marine API address.
19
+ # On the school deployment this is set to the fixed UI proxy prefix, e.g.
20
+ # https://subway-unbiased-barcode.ngrok-free.dev/marine. If it is unset we
21
+ # retain the old local URL for private/operator-only deployments.
22
+ PUBLIC_BASE_URL = (
23
+ os.environ.get("MARINE_PUBLIC_BASE_URL")
24
+ or os.environ.get("PUBLIC_BASE_URL")
25
+ or ""
26
+ ).strip().rstrip("/")
27
+
28
+
29
+ def _download_url(path: str) -> str:
30
+ path = str(path or "").strip()
31
+ if not path.startswith("/download/"):
32
+ return ""
33
+ return f"{PUBLIC_BASE_URL or API_URL}{path}"
34
 
35
  mcp = MCPServer(
36
  "Marine Data",
 
166
  result = _post("/data/export", payload)
167
  path = result.get("download_path")
168
  if isinstance(path, str) and path.startswith("/download/"):
169
+ result["download_url"] = _download_url(path)
170
  return result
171
 
172
  @mcp.tool()
 
175
  token = token.strip()
176
  if not re.fullmatch(r"[A-Za-z0-9_-]{20,160}", token):
177
  raise ValueError("invalid download token")
178
+ return {"download_url": _download_url(f"/download/{token}")}
179
 
180
 
181
 
 
215
  result = _post("/data/export", payload)
216
  path = result.get("download_path")
217
  if isinstance(path, str) and path.startswith("/download/"):
218
+ result["download_url"] = _download_url(path)
219
  return result
220
 
221
 
services/chat_runtime.py CHANGED
@@ -672,13 +672,20 @@ def _is_fisheries_inventory_prompt(prompt: str) -> bool:
672
  reads, analysis and exports on their normal guarded paths.
673
  """
674
  text = str(prompt or "").strip().lower()
675
- if not _is_fisheries_prompt(text):
 
 
 
 
 
 
 
676
  return False
677
  inventory_terms = (
678
  "可用数据集", "当前可用", "文件来源", "文件清单", "列出",
679
  "有哪些文件", "入库", "目录", "inventory", "catalog",
680
  )
681
- if not any(term in text for term in inventory_terms):
682
  return False
683
  # “不读取文件内容” is explicitly a metadata-only request, rather than a
684
  # request to open a file. Remove these negated phrases before testing for
@@ -885,7 +892,20 @@ async def dispatch_chat_stream(tid,prompt):
885
  return
886
 
887
  if _is_fisheries_inventory_prompt(prompt):
888
- answer = await _direct_fisheries_inventory_answer(prompt)
 
 
 
 
 
 
 
 
 
 
 
 
 
889
  yield out("token", {"text": answer})
890
  yield out(
891
  "done",
 
672
  reads, analysis and exports on their normal guarded paths.
673
  """
674
  text = str(prompt or "").strip().lower()
675
+ # Be explicit about the wording used by the school/HF comparison tests.
676
+ # These requests mention storage provenance rather than a named source
677
+ # such as GFW or FAO, but they are still metadata-only Squid/Tuna queries.
678
+ explicit_inventory = (
679
+ any(term in text for term in ("完整文件数", "总大小", "本地镜像", "回退读取"))
680
+ and any(term in text for term in ("squid", "鱿鱼", "柔鱼", "tuna", "金枪鱼"))
681
+ )
682
+ if not _is_fisheries_prompt(text) and not explicit_inventory:
683
  return False
684
  inventory_terms = (
685
  "可用数据集", "当前可用", "文件来源", "文件清单", "列出",
686
  "有哪些文件", "入库", "目录", "inventory", "catalog",
687
  )
688
+ if not any(term in text for term in inventory_terms) and not explicit_inventory:
689
  return False
690
  # “不读取文件内容” is explicitly a metadata-only request, rather than a
691
  # request to open a file. Remove these negated phrases before testing for
 
892
  return
893
 
894
  if _is_fisheries_inventory_prompt(prompt):
895
+ # Never let a blocked HF connection or a malformed inventory response
896
+ # fall through to the 900-second Codex Harness timeout. Metadata-only
897
+ # inventory is deterministic and must always terminate promptly.
898
+ log.info("direct fisheries inventory fast path: prompt_chars=%s", len(str(prompt or "")))
899
+ try:
900
+ answer = await asyncio.wait_for(
901
+ _direct_fisheries_inventory_answer(prompt),
902
+ timeout=float(os.environ.get("FISHERIES_INVENTORY_TIMEOUT", "45")),
903
+ )
904
+ except asyncio.TimeoutError:
905
+ answer = (
906
+ "学校服务器的 Squid 清单查询超过 45 秒,已停止等待模型调用。"
907
+ "请稍后重试;本次没有读取文件内容,也没有生成文件。"
908
+ )
909
  yield out("token", {"text": answer})
910
  yield out(
911
  "done",
services/ocean_batch.py CHANGED
@@ -3,6 +3,7 @@ from __future__ import annotations
3
  import asyncio
4
  import calendar
5
  import json
 
6
  import re
7
  import secrets
8
  import time
@@ -138,6 +139,11 @@ class OceanBatchManager:
138
  def __init__(self,root:Path,api_url:str,concurrency:int=6):
139
  self.root=Path(root);self.root.mkdir(parents=True,exist_ok=True)
140
  self.api_url=str(api_url).rstrip("/")
 
 
 
 
 
141
  self.concurrency=max(1,min(int(concurrency),12))
142
  self.running:dict[str,asyncio.Task]={}
143
  # Jobs interrupted by process restart are resumable, not falsely running.
@@ -152,6 +158,12 @@ class OceanBatchManager:
152
  pass
153
 
154
  def _path(self,jid):return self.root/f"{jid}.json"
 
 
 
 
 
 
155
  def _read(self,jid):
156
  p=self._path(jid)
157
  if not p.exists():return None
@@ -249,7 +261,7 @@ class OceanBatchManager:
249
  sub=next((x for x in job["subtasks"] if x["subtask_id"]==subtask_id),None)
250
  if not sub:return
251
  if isinstance(result,dict) and result.get("download_path") and not result.get("download_url"):
252
- result["download_url"]=self.api_url+str(result["download_path"])
253
  if isinstance(result,dict) and result.get("download_url"):
254
  sub["status"]="completed"
255
  sub["download_url"]=str(result["download_url"])
@@ -386,7 +398,7 @@ class OceanBatchManager:
386
  detail=str(exc)[:500]
387
  if attempt<2:await asyncio.sleep(1.0*(2**attempt))
388
  if isinstance(result,dict) and result.get("download_path") and not result.get("download_url"):
389
- result["download_url"]=self.api_url+str(result["download_path"])
390
  if not isinstance(result,dict) or not result.get("download_url"):
391
  raise RuntimeError(detail or "无法刷新 Ocean 下载链接")
392
  job=self._read(jid)
 
3
  import asyncio
4
  import calendar
5
  import json
6
+ import os
7
  import re
8
  import secrets
9
  import time
 
139
  def __init__(self,root:Path,api_url:str,concurrency:int=6):
140
  self.root=Path(root);self.root.mkdir(parents=True,exist_ok=True)
141
  self.api_url=str(api_url).rstrip("/")
142
+ self.public_base_url=(
143
+ os.environ.get("MARINE_PUBLIC_BASE_URL")
144
+ or os.environ.get("PUBLIC_BASE_URL")
145
+ or ""
146
+ ).strip().rstrip("/")
147
  self.concurrency=max(1,min(int(concurrency),12))
148
  self.running:dict[str,asyncio.Task]={}
149
  # Jobs interrupted by process restart are resumable, not falsely running.
 
158
  pass
159
 
160
  def _path(self,jid):return self.root/f"{jid}.json"
161
+
162
+ def _download_url(self,path):
163
+ path=str(path or "").strip()
164
+ if not path.startswith("/download/"):
165
+ return ""
166
+ return f"{self.public_base_url or self.api_url}{path}"
167
  def _read(self,jid):
168
  p=self._path(jid)
169
  if not p.exists():return None
 
261
  sub=next((x for x in job["subtasks"] if x["subtask_id"]==subtask_id),None)
262
  if not sub:return
263
  if isinstance(result,dict) and result.get("download_path") and not result.get("download_url"):
264
+ result["download_url"]=self._download_url(result["download_path"])
265
  if isinstance(result,dict) and result.get("download_url"):
266
  sub["status"]="completed"
267
  sub["download_url"]=str(result["download_url"])
 
398
  detail=str(exc)[:500]
399
  if attempt<2:await asyncio.sleep(1.0*(2**attempt))
400
  if isinstance(result,dict) and result.get("download_path") and not result.get("download_url"):
401
+ result["download_url"]=self._download_url(result["download_path"])
402
  if not isinstance(result,dict) or not result.get("download_url"):
403
  raise RuntimeError(detail or "无法刷新 Ocean 下载链接")
404
  job=self._read(jid)
start.sh CHANGED
@@ -121,6 +121,7 @@ env = {
121
  ),
122
  "HF_TUNA_DATASET_REVISION": os.environ.get("HF_TUNA_DATASET_REVISION", ""),
123
  "PUBLIC_BASE_URL": os.environ.get("PUBLIC_BASE_URL", ""),
 
124
  "MARINE_EXPORT_RANGE_MAX_DAYS": os.environ.get("MARINE_EXPORT_RANGE_MAX_DAYS", "31"),
125
  }
126
 
@@ -179,6 +180,7 @@ marine_env = {
179
  "FISHERIES_EXPORT_ROOT": os.environ.get("FISHERIES_EXPORT_ROOT", "/tmp/squid_fisheries_exports"),
180
  "HF_FISHERIES_CACHE_ROOT": os.environ.get("HF_FISHERIES_CACHE_ROOT", "/tmp/squid_hf_fisheries_cache"),
181
  "PUBLIC_BASE_URL": os.environ.get("PUBLIC_BASE_URL", ""),
 
182
  "MARINE_EXPORT_RANGE_MAX_DAYS": os.environ.get("MARINE_EXPORT_RANGE_MAX_DAYS", "31"),
183
  }
184
  for name in ("HF_TOKEN", "HF_SQUID_DATASET_REVISION", "HF_TUNA_DATASET_REVISION"):
 
121
  ),
122
  "HF_TUNA_DATASET_REVISION": os.environ.get("HF_TUNA_DATASET_REVISION", ""),
123
  "PUBLIC_BASE_URL": os.environ.get("PUBLIC_BASE_URL", ""),
124
+ "MARINE_PUBLIC_BASE_URL": os.environ.get("MARINE_PUBLIC_BASE_URL", ""),
125
  "MARINE_EXPORT_RANGE_MAX_DAYS": os.environ.get("MARINE_EXPORT_RANGE_MAX_DAYS", "31"),
126
  }
127
 
 
180
  "FISHERIES_EXPORT_ROOT": os.environ.get("FISHERIES_EXPORT_ROOT", "/tmp/squid_fisheries_exports"),
181
  "HF_FISHERIES_CACHE_ROOT": os.environ.get("HF_FISHERIES_CACHE_ROOT", "/tmp/squid_hf_fisheries_cache"),
182
  "PUBLIC_BASE_URL": os.environ.get("PUBLIC_BASE_URL", ""),
183
+ "MARINE_PUBLIC_BASE_URL": os.environ.get("MARINE_PUBLIC_BASE_URL", ""),
184
  "MARINE_EXPORT_RANGE_MAX_DAYS": os.environ.get("MARINE_EXPORT_RANGE_MAX_DAYS", "31"),
185
  }
186
  for name in ("HF_TOKEN", "HF_SQUID_DATASET_REVISION", "HF_TUNA_DATASET_REVISION"):
ui_server.py CHANGED
@@ -1984,8 +1984,18 @@ async def hf_live_tree(repo: str | None = None, force: bool = False) -> list[dic
1984
  items: list[dict] = []
1985
  tree_error = ""
1986
 
 
 
 
 
 
 
 
 
 
 
1987
  async with httpx.AsyncClient(
1988
- timeout=httpx.Timeout(connect=10, read=30, write=20, pool=20),
1989
  follow_redirects=True,
1990
  ) as c:
1991
  next_url = tree_url
@@ -2017,7 +2027,15 @@ async def hf_live_tree(repo: str | None = None, force: bool = False) -> list[dic
2017
 
2018
  # Reliable fallback: dataset metadata contains repository siblings.
2019
  if not _hf_live_files(items):
2020
- r = await c.get(info_url, headers=headers)
 
 
 
 
 
 
 
 
2021
  if r.status_code in {401, 403}:
2022
  if local_items:
2023
  return merge_local_with_remote_tree([], repo)
 
1984
  items: list[dict] = []
1985
  tree_error = ""
1986
 
1987
+ # A local mirror is the school server's first data plane. Use a short
1988
+ # remote probe there: a blocked Hugging Face egress must fall back to the
1989
+ # local metadata instead of holding the chat open for 40+ seconds.
1990
+ local_probe = bool(local_items)
1991
+ remote_timeout = httpx.Timeout(
1992
+ connect=3 if local_probe else 10,
1993
+ read=5 if local_probe else 30,
1994
+ write=10 if local_probe else 20,
1995
+ pool=10 if local_probe else 20,
1996
+ )
1997
  async with httpx.AsyncClient(
1998
+ timeout=remote_timeout,
1999
  follow_redirects=True,
2000
  ) as c:
2001
  next_url = tree_url
 
2027
 
2028
  # Reliable fallback: dataset metadata contains repository siblings.
2029
  if not _hf_live_files(items):
2030
+ try:
2031
+ r = await c.get(info_url, headers=headers)
2032
+ except Exception as exc:
2033
+ if local_items:
2034
+ return merge_local_with_remote_tree([], repo)
2035
+ detail = tree_error or str(exc)[:300]
2036
+ raise RuntimeError(
2037
+ f"Hugging Face Dataset 暂时不可达 ({repo}):{detail}"
2038
+ ) from exc
2039
  if r.status_code in {401, 403}:
2040
  if local_items:
2041
  return merge_local_with_remote_tree([], repo)