Clean deploy v3.3.0
Browse files- #U53cc#U4ed3#U5e93#U4fee#U590d#U8bf4#U660e.md +37 -0
- .codewhale/mcp.json +26 -0
- .dockerignore +11 -0
- .gitattributes +0 -35
- .gitignore +6 -207
- AGENTS.md +135 -0
- DEPLOY.md +55 -0
- Dockerfile +32 -0
- HuggingFace_Spaces_使用指南.md +0 -1380
- README.md +176 -206
- VERSION +1 -0
- app.html +0 -0
- app.py +0 -966
- data/.gitkeep +0 -1
- data/logbook_availability.csv +0 -246
- fisheries_hf.py +575 -0
- harness_safe.cordis.yml +31 -0
- marine_mcp.py +755 -0
- query_tools/__init__.py +0 -21
- query_tools/literature_cpue_query.py +0 -384
- query_tools/logbook_availability_query.py +0 -401
- query_tools/query_gfw.py +0 -310
- query_tools/query_sprfmo.py +0 -311
- requirements.txt +6 -6
- scripts/preflight.py +82 -0
- sidebar_catalog.py +116 -0
- skills/marine-data/SKILL.md +60 -0
- start.sh +182 -0
- tests/test_data_routing_and_export_guard.py +86 -0
- tests/test_fisheries_hf.py +128 -0
- tests/test_followup_routing.py +60 -0
- tests/test_marine_inventory_aliases.py +85 -0
- tests/test_output_sanitizer.py +42 -0
- tests/test_release_stability.py +30 -0
- ui_server.py +0 -0
#U53cc#U4ed3#U5e93#U4fee#U590d#U8bf4#U660e.md
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# 数据服务修复说明(V2.2.4)
|
| 2 |
+
|
| 3 |
+
本版本以当前 `main` 为基础恢复提交 `e1879a6` 的双仓库能力,没有回退后续侧栏、设置、元数据审计和真实 CSV 分析功能。
|
| 4 |
+
|
| 5 |
+
## 已修复链路
|
| 6 |
+
|
| 7 |
+
- 实时文件树同时读取:
|
| 8 |
+
- `globalsquiddatabase/squid_dataset`
|
| 9 |
+
- `globalsquiddatabase/Tuna-Fisheries-Dataset`
|
| 10 |
+
- `fisheries_catalog`、`fisheries_inventory` 和 `fisheries_search` 返回每个文件的 `repository`。
|
| 11 |
+
- `fisheries_analyze_export` 接受 `repository`,并从同一仓库完成路径核验、下载、分析和导出。
|
| 12 |
+
- UI 数据集、详情、质量检查、元数据审计和服务状态均统计两个仓库。
|
| 13 |
+
- 两个仓库分别缓存文件树、提交 SHA 和下载文件,避免同路径串库。
|
| 14 |
+
- ICCAT `YearC` 可作为年份字段,并分别导出筛选明细、去重明细和年度汇总。
|
| 15 |
+
- 清单工具兼容 `query/source` 参数,并过滤未执行的内部工具调用标记。
|
| 16 |
+
- “确认/继续”等短回复会续接15分钟内的上一项数据请求;Ocean导出未指定格式时默认NetCDF并直接执行。
|
| 17 |
+
|
| 18 |
+
## 部署后测试
|
| 19 |
+
|
| 20 |
+
先执行:
|
| 21 |
+
|
| 22 |
+
```text
|
| 23 |
+
只查询 Tuna-Fisheries-Dataset,查找 ICCAT/EFFDIS_LL2000-2024.csv,并返回真实repository和path。
|
| 24 |
+
```
|
| 25 |
+
|
| 26 |
+
再执行:
|
| 27 |
+
|
| 28 |
+
```text
|
| 29 |
+
使用刚才返回的repository和path实际读取文件,报告字段、记录数、时间范围、缺失和完全重复记录,并导出CSV。
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
结果中的 `repository` 必须为 `globalsquiddatabase/Tuna-Fisheries-Dataset`。
|
| 33 |
+
# V2.2.5 数据路由与真实执行校验
|
| 34 |
+
|
| 35 |
+
- 补齐 IATTC、ICCAT、IOTC、CCSBT、EFFDIS 等金枪鱼数据源关键词,避免误入无工具的 Harness。
|
| 36 |
+
- 补齐 v10、u10、t2m、msl 等 Ocean 变量关键词,并在来源缺省时按变量映射到对应数据源。
|
| 37 |
+
- Ocean 导出必须有真实 export 工具完成事件,并返回下载链接或明确服务端错误;不再把“正在提交”当作完成。
|
.codewhale/mcp.json
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"timeouts": {
|
| 3 |
+
"connect_timeout": 15,
|
| 4 |
+
"execute_timeout": 900,
|
| 5 |
+
"read_timeout": 900
|
| 6 |
+
},
|
| 7 |
+
"servers": {
|
| 8 |
+
"marine": {
|
| 9 |
+
"command": "python3",
|
| 10 |
+
"args": [
|
| 11 |
+
"/home/user/app/marine_mcp.py"
|
| 12 |
+
],
|
| 13 |
+
"disabled": false,
|
| 14 |
+
"required": false,
|
| 15 |
+
"env": {
|
| 16 |
+
"MARINE_API_URL": "${MARINE_API_URL}",
|
| 17 |
+
"HF_TOKEN": "${HF_TOKEN}",
|
| 18 |
+
"HF_DATASET_REPO": "${HF_DATASET_REPO}",
|
| 19 |
+
"HF_DATASET_REVISION": "${HF_DATASET_REVISION}",
|
| 20 |
+
"FISHERIES_EXPORT_ROOT": "${FISHERIES_EXPORT_ROOT}",
|
| 21 |
+
"HF_FISHERIES_CACHE_ROOT": "${HF_FISHERIES_CACHE_ROOT}",
|
| 22 |
+
"PUBLIC_BASE_URL": "${PUBLIC_BASE_URL}"
|
| 23 |
+
}
|
| 24 |
+
}
|
| 25 |
+
}
|
| 26 |
+
}
|
.dockerignore
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
.git/**
|
| 3 |
+
.pytest_cache
|
| 4 |
+
.pytest_cache/**
|
| 5 |
+
__pycache__
|
| 6 |
+
**/__pycache__
|
| 7 |
+
*.pyc
|
| 8 |
+
*.pyo
|
| 9 |
+
.env
|
| 10 |
+
*.log
|
| 11 |
+
.DS_Store
|
.gitattributes
DELETED
|
@@ -1,35 +0,0 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
.gitignore
CHANGED
|
@@ -1,209 +1,8 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
| 2 |
__pycache__/
|
| 3 |
-
*.py[cod]
|
| 4 |
-
*$py.class
|
| 5 |
-
*.pem
|
| 6 |
-
|
| 7 |
-
# C extensions
|
| 8 |
-
*.so
|
| 9 |
-
|
| 10 |
-
# Distribution / packaging
|
| 11 |
-
.Python
|
| 12 |
-
build/
|
| 13 |
-
develop-eggs/
|
| 14 |
-
dist/
|
| 15 |
-
downloads/
|
| 16 |
-
eggs/
|
| 17 |
-
.eggs/
|
| 18 |
-
lib/
|
| 19 |
-
lib64/
|
| 20 |
-
parts/
|
| 21 |
-
sdist/
|
| 22 |
-
var/
|
| 23 |
-
wheels/
|
| 24 |
-
share/python-wheels/
|
| 25 |
-
*.egg-info/
|
| 26 |
-
.installed.cfg
|
| 27 |
-
*.egg
|
| 28 |
-
MANIFEST
|
| 29 |
-
|
| 30 |
-
# PyInstaller
|
| 31 |
-
# Usually these files are written by a python script from a template
|
| 32 |
-
# before PyInstaller builds the exe, so as to inject date/other info into it.
|
| 33 |
-
*.manifest
|
| 34 |
-
*.spec
|
| 35 |
-
|
| 36 |
-
# Installer logs
|
| 37 |
-
pip-log.txt
|
| 38 |
-
pip-delete-this-directory.txt
|
| 39 |
-
|
| 40 |
-
# Unit test / coverage reports
|
| 41 |
-
htmlcov/
|
| 42 |
-
.tox/
|
| 43 |
-
.nox/
|
| 44 |
-
.coverage
|
| 45 |
-
.coverage.*
|
| 46 |
-
.cache
|
| 47 |
-
nosetests.xml
|
| 48 |
-
coverage.xml
|
| 49 |
-
*.cover
|
| 50 |
-
*.py,cover
|
| 51 |
-
.hypothesis/
|
| 52 |
.pytest_cache/
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
*.mo
|
| 57 |
-
*.pot
|
| 58 |
-
|
| 59 |
-
# Django stuff:
|
| 60 |
-
*.log
|
| 61 |
-
local_settings.py
|
| 62 |
-
db.sqlite3
|
| 63 |
-
db.sqlite3-journal
|
| 64 |
-
|
| 65 |
-
# Flask stuff:
|
| 66 |
-
instance/
|
| 67 |
-
.webassets-cache
|
| 68 |
-
|
| 69 |
-
# Scrapy stuff:
|
| 70 |
-
.scrapy
|
| 71 |
-
|
| 72 |
-
# Sphinx documentation
|
| 73 |
-
docs/_build/
|
| 74 |
-
|
| 75 |
-
# PyBuilder
|
| 76 |
-
.pybuilder/
|
| 77 |
-
target/
|
| 78 |
-
|
| 79 |
-
# Jupyter Notebook
|
| 80 |
-
.ipynb_checkpoints
|
| 81 |
-
|
| 82 |
-
# IPython
|
| 83 |
-
profile_default/
|
| 84 |
-
ipython_config.py
|
| 85 |
-
|
| 86 |
-
# pyenv
|
| 87 |
-
# For a library or package, you might want to ignore these files since the code is
|
| 88 |
-
# having to support multiple versions of Python.
|
| 89 |
-
.python-version
|
| 90 |
-
|
| 91 |
-
# pdm
|
| 92 |
-
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
| 93 |
-
#pdm.lock
|
| 94 |
-
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
| 95 |
-
# in version control.
|
| 96 |
-
# https://pdm.fming.dev/#use-with-ide
|
| 97 |
-
.pdm.toml
|
| 98 |
-
|
| 99 |
-
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
| 100 |
-
__pypackages__/
|
| 101 |
-
|
| 102 |
-
# Celery stuff
|
| 103 |
-
celerybeat-schedule
|
| 104 |
-
celerybeat.pid
|
| 105 |
-
|
| 106 |
-
# SageMath parsed files
|
| 107 |
-
*.sage.py
|
| 108 |
-
|
| 109 |
-
# Environments
|
| 110 |
-
.env
|
| 111 |
-
.venv
|
| 112 |
-
env/
|
| 113 |
-
venv/
|
| 114 |
-
ENV/
|
| 115 |
-
env.bak/
|
| 116 |
-
venv.bak/
|
| 117 |
-
|
| 118 |
-
# Spyder project settings
|
| 119 |
-
.spyderproject
|
| 120 |
-
.spyproject
|
| 121 |
-
|
| 122 |
-
# Rope project settings
|
| 123 |
-
.ropeproject
|
| 124 |
-
|
| 125 |
-
# mkdocs documentation
|
| 126 |
-
/site
|
| 127 |
-
|
| 128 |
-
# mypy
|
| 129 |
-
.mypy_cache/
|
| 130 |
-
.dmypy.json
|
| 131 |
-
dmypy.json
|
| 132 |
-
|
| 133 |
-
# Pyre type checker
|
| 134 |
-
.pyre/
|
| 135 |
-
|
| 136 |
-
# pytype static type analyzer
|
| 137 |
-
.pytype/
|
| 138 |
-
|
| 139 |
-
# Cython debug symbols
|
| 140 |
-
cython_debug/
|
| 141 |
-
|
| 142 |
-
# IDE - VS Code
|
| 143 |
-
.vscode/
|
| 144 |
-
*.code-workspace
|
| 145 |
-
|
| 146 |
-
# IDE - PyCharm
|
| 147 |
-
.idea/
|
| 148 |
-
*.iml
|
| 149 |
-
|
| 150 |
-
# IDE - Sublime Text
|
| 151 |
-
*.sublime-workspace
|
| 152 |
-
|
| 153 |
-
# IDE - Vim
|
| 154 |
-
*.swp
|
| 155 |
-
*.swo
|
| 156 |
-
*~
|
| 157 |
-
|
| 158 |
-
# IDE - Emacs
|
| 159 |
-
*~
|
| 160 |
-
\#*\#
|
| 161 |
-
/.emacs.desktop
|
| 162 |
-
/.emacs.desktop.lock
|
| 163 |
-
*.elc
|
| 164 |
-
auto-save-list
|
| 165 |
-
tramp
|
| 166 |
-
|
| 167 |
-
# macOS
|
| 168 |
-
.DS_Store
|
| 169 |
-
.AppleDouble
|
| 170 |
-
.LSOverride
|
| 171 |
-
Icon
|
| 172 |
-
._*
|
| 173 |
-
.DocumentRevisions-V100
|
| 174 |
-
.fseventsd
|
| 175 |
-
.Spotlight-V100
|
| 176 |
-
.TemporaryItems
|
| 177 |
-
.Trashes
|
| 178 |
-
.VolumeIcon.icns
|
| 179 |
-
.com.apple.timemachine.donotpresent
|
| 180 |
-
.AppleDB
|
| 181 |
-
.AppleDesktop
|
| 182 |
-
Network Trash Folder
|
| 183 |
-
Temporary Items
|
| 184 |
-
.apdisk
|
| 185 |
-
|
| 186 |
-
# Windows
|
| 187 |
-
Thumbs.db
|
| 188 |
-
Thumbs.db:encryptable
|
| 189 |
-
ehthumbs.db
|
| 190 |
-
ehthumbs_vista.db
|
| 191 |
-
*.stackdump
|
| 192 |
-
[Dd]esktop.ini
|
| 193 |
-
$RECYCLE.BIN/
|
| 194 |
-
*.cab
|
| 195 |
-
*.msi
|
| 196 |
-
*.msix
|
| 197 |
-
*.msm
|
| 198 |
-
*.msp
|
| 199 |
-
*.lnk
|
| 200 |
-
|
| 201 |
-
# Linux
|
| 202 |
-
*~
|
| 203 |
-
.fuse_hidden*
|
| 204 |
-
.directory
|
| 205 |
-
.Trash-*
|
| 206 |
-
.nfs*
|
| 207 |
-
|
| 208 |
-
# Project specific
|
| 209 |
-
# Add any project-specific files or directories to ignore below
|
|
|
|
| 1 |
+
.DS_Store
|
| 2 |
+
.env
|
| 3 |
+
*.log
|
| 4 |
__pycache__/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
.pytest_cache/
|
| 6 |
+
*.pyc
|
| 7 |
+
*.pyo
|
| 8 |
+
project-data-package-*.zip
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
AGENTS.md
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Marine Foundation Data Agent
|
| 2 |
+
|
| 3 |
+
Use the `marine` MCP tools for all school-server requests.
|
| 4 |
+
Never search `/tmp`, the Hugging Face container, or the workspace as a substitute for the school server.
|
| 5 |
+
|
| 6 |
+
## Status tools
|
| 7 |
+
- `mcp_marine_marine_health`
|
| 8 |
+
- `mcp_marine_marine_domains`
|
| 9 |
+
- `mcp_marine_marine_status`
|
| 10 |
+
|
| 11 |
+
## Data retrieval tools
|
| 12 |
+
- `mcp_marine_marine_query`: check dated data availability
|
| 13 |
+
- `mcp_marine_marine_subset`: create a geographic NetCDF subset on the school server
|
| 14 |
+
- `mcp_marine_marine_download`: convert an export token to an HTTPS URL
|
| 15 |
+
|
| 16 |
+
Current retrieval MVP supports `ocean` + `oisst`, variables `sst`, `anom`, `err`, `ice`.
|
| 17 |
+
For a requested file, use query/subset and return `download_url` directly.
|
| 18 |
+
Do not claim retrieval is unavailable before trying these tools.
|
| 19 |
+
Never present `/tmp/...` or school-server filesystem paths as download links.
|
| 20 |
+
`done` means completed tasks, not necessarily physical file count.
|
| 21 |
+
Default to concise Chinese.
|
| 22 |
+
|
| 23 |
+
## Multi-format export (MVP)
|
| 24 |
+
Use `mcp_marine_marine_export` when the user requests a specific output format.
|
| 25 |
+
Supported formats: `netcdf`, `csv`, `xlsx`, `json`, `geotiff`, `png`.
|
| 26 |
+
If export/download is requested without a format, default to `netcdf` and call
|
| 27 |
+
`mcp_marine_marine_export` immediately; do not ask for confirmation. A short
|
| 28 |
+
confirmation continues the most recent data request in the same thread.
|
| 29 |
+
Return the absolute `download_url` directly to the user.
|
| 30 |
+
Do not claim a format is unavailable before attempting this tool.
|
| 31 |
+
|
| 32 |
+
## Ocean multi-source retrieval V3
|
| 33 |
+
|
| 34 |
+
Validated sources:
|
| 35 |
+
- cmems_physics: so, thetao, uo, vo
|
| 36 |
+
- cmems_surface: mlotst, zos
|
| 37 |
+
- cmems_bgc: chl, no3, nppv, o2, po4, si
|
| 38 |
+
- cmems_carbonate: ph, spco2
|
| 39 |
+
- era5: msl, t2m, u10, v10
|
| 40 |
+
- era5_accum: e, slhf, sshf, ssrd, strd, tp
|
| 41 |
+
- occci: chlor_a, chlor_a_log10_bias, chlor_a_log10_rmsd, total_nobs_sum
|
| 42 |
+
- oisst: sst, anom, err, ice
|
| 43 |
+
|
| 44 |
+
Tools:
|
| 45 |
+
- mcp_marine_marine_catalog
|
| 46 |
+
- mcp_marine_marine_query
|
| 47 |
+
- mcp_marine_marine_subset
|
| 48 |
+
- mcp_marine_marine_export
|
| 49 |
+
|
| 50 |
+
Formats: netcdf, csv, xlsx, json, geotiff, png.
|
| 51 |
+
For 3-D CMEMS variables, GeoTIFF/PNG require depth.
|
| 52 |
+
Always return the absolute HTTPS download_url.
|
| 53 |
+
Variable-only requests are still data requests: `v10`, `u10`, `t2m` and `msl`
|
| 54 |
+
map to `era5`; do not send them to a tool-free chat path.
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
## Fast retrieval routing V4
|
| 58 |
+
|
| 59 |
+
When the user already specifies the requested date, geographic area,
|
| 60 |
+
variable/source and output format, call `mcp_marine_marine_export`
|
| 61 |
+
directly.
|
| 62 |
+
|
| 63 |
+
Do not call catalog or query before export unless they are actually needed.
|
| 64 |
+
|
| 65 |
+
Routing:
|
| 66 |
+
- ask what sources/variables exist -> catalog
|
| 67 |
+
- ask only whether a date/variable exists -> query
|
| 68 |
+
- ask for an actual file -> export directly
|
| 69 |
+
|
| 70 |
+
If export returns `status=error`, report its `detail` directly.
|
| 71 |
+
|
| 72 |
+
Never silently:
|
| 73 |
+
- change the requested date
|
| 74 |
+
- change the requested source
|
| 75 |
+
- change the requested variable
|
| 76 |
+
- clamp an out-of-range depth
|
| 77 |
+
- invent another depth
|
| 78 |
+
- retry using nearby data
|
| 79 |
+
|
| 80 |
+
Do not guess data values or unavailable model levels.
|
| 81 |
+
|
| 82 |
+
## Hugging Face fisheries data plane V1
|
| 83 |
+
|
| 84 |
+
The project has TWO different data planes:
|
| 85 |
+
|
| 86 |
+
1. School Marine Server:
|
| 87 |
+
- Ocean environmental/physical/BGC/atmospheric data
|
| 88 |
+
- use `mcp_marine_marine_*`
|
| 89 |
+
|
| 90 |
+
2. Hugging Face Dataset:
|
| 91 |
+
- Squid: `globalsquiddatabase/squid_dataset`
|
| 92 |
+
- Tuna: `globalsquiddatabase/Tuna-Fisheries-Dataset`
|
| 93 |
+
- use:
|
| 94 |
+
- `mcp_marine_fisheries_catalog`
|
| 95 |
+
- `mcp_marine_fisheries_inventory`
|
| 96 |
+
- `mcp_marine_fisheries_search`
|
| 97 |
+
- `mcp_marine_fisheries_data_rules`
|
| 98 |
+
- `mcp_marine_fisheries_analyze_export`
|
| 99 |
+
|
| 100 |
+
Never conclude that fisheries data is absent merely because the school-server
|
| 101 |
+
`tuna_data` or `squid_data` task database is empty.
|
| 102 |
+
|
| 103 |
+
Aggregation:
|
| 104 |
+
- catch -> SUM
|
| 105 |
+
- effort -> SUM only within compatible units
|
| 106 |
+
- CPUE -> recompute total catch / total effort
|
| 107 |
+
- never average monthly CPUE
|
| 108 |
+
- never fabricate daily fisheries data from monthly/annual products
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
## Fisheries live inventory fast path
|
| 112 |
+
|
| 113 |
+
For squid/tuna fisheries inventory, catalog, source availability, and "what files
|
| 114 |
+
are already present" questions, the UI backend injects a
|
| 115 |
+
`[HF_FISHERIES_LIVE_CONTEXT]` block from both live Hugging Face Dataset main trees.
|
| 116 |
+
|
| 117 |
+
When that context is present:
|
| 118 |
+
- for inventory/presence questions, answer directly from it;
|
| 119 |
+
- do not call `start_mcp_server`;
|
| 120 |
+
- do not infer availability from planning spreadsheets;
|
| 121 |
+
- for actual fields, row counts, missing/duplicate checks, time/space filters,
|
| 122 |
+
aggregation or CSV export, call `mcp_marine_fisheries_analyze_export` and pass
|
| 123 |
+
the repository returned by search/inventory;
|
| 124 |
+
- when `exports` contains filtered_raw, deduplicated and annual_summary, return
|
| 125 |
+
each distinct filename and download_url exactly once;
|
| 126 |
+
- inventory example: `fisheries_inventory(domain="tuna", keyword="IATTC")`;
|
| 127 |
+
IATTC, ICCAT, IOTC and CCSBT names always identify Fisheries requests;
|
| 128 |
+
never print `<function_calls>`, `<invoke>` or `<parameter>` markup as text;
|
| 129 |
+
- never substitute Web, Shell, Run, JS or a subagent for the restricted
|
| 130 |
+
fisheries content tool;
|
| 131 |
+
- return its `download_url` directly and never invent theoretical rows as
|
| 132 |
+
actual observations.
|
| 133 |
+
|
| 134 |
+
Marine MCP is reserved for school-server Ocean data and is connected lazily only
|
| 135 |
+
when an Ocean request needs it.
|
DEPLOY.md
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Deployment checklist
|
| 2 |
+
|
| 3 |
+
Hugging Face Space type: Docker, preferably Private while testing.
|
| 4 |
+
|
| 5 |
+
Required Space Secrets:
|
| 6 |
+
- `OPENCODE_GO_API_KEY` (or the same key stored as `OPENAI_API_KEY`)
|
| 7 |
+
- `CODEWHALE_RUNTIME_TOKEN`
|
| 8 |
+
|
| 9 |
+
Recommended Space Secret:
|
| 10 |
+
- `HF_TOKEN` with read access to any private/restricted fisheries Dataset used
|
| 11 |
+
by the Space. The two default public repositories can be read without a token.
|
| 12 |
+
|
| 13 |
+
Required Space Variable:
|
| 14 |
+
- `MARINE_API_URL`
|
| 15 |
+
|
| 16 |
+
Optional Space Variables:
|
| 17 |
+
- `HF_SQUID_DATASET_REPO` (default `globalsquiddatabase/squid_dataset`)
|
| 18 |
+
- `HF_TUNA_DATASET_REPO` (default `globalsquiddatabase/Tuna-Fisheries-Dataset`)
|
| 19 |
+
- `HF_SQUID_DATASET_REVISION` and `HF_TUNA_DATASET_REVISION` (optional approved
|
| 20 |
+
commit SHAs; otherwise each repository's current SHA is resolved and cached)
|
| 21 |
+
- `PUBLIC_BASE_URL` (normally inferred from `SPACE_HOST`)
|
| 22 |
+
- `HF_FISHERIES_MAX_CSV_BYTES` and `HF_FISHERIES_MAX_ZIP_BYTES`
|
| 23 |
+
|
| 24 |
+
The current temporary Marine API URL can be used for the first smoke test.
|
| 25 |
+
Later replace the Quick Tunnel with a fixed Named Tunnel URL.
|
| 26 |
+
|
| 27 |
+
After the Space is Running:
|
| 28 |
+
1. Open `/mobile` if the root does not redirect there.
|
| 29 |
+
2. Test Ocean status: `检查我的海洋数据服务器,并告诉我三个数据中心状态。`
|
| 30 |
+
3. Test a real fisheries CSV read/export:
|
| 31 |
+
`实际读取 current/sprfmo_effort.csv,报告字段、记录数、缺失、重复并导出 CSV。`
|
| 32 |
+
4. Test Tuna routing and export:
|
| 33 |
+
`只查询 Tuna-Fisheries-Dataset,查找并读取 ICCAT/EFFDIS_LL2000-2024.csv,报告真实字段、记录数、缺失和重复并导出 CSV。`
|
| 34 |
+
Expected exports: filtered raw detail, exact-deduplicated detail and annual
|
| 35 |
+
summary. ICCAT `YearC` must be reported as the year field.
|
| 36 |
+
5. Expected model-visible tools include
|
| 37 |
+
`mcp_marine_fisheries_analyze_export`.
|
| 38 |
+
6. Open the returned `/api/fisheries/download/{token}` URL and confirm the CSV
|
| 39 |
+
downloads successfully.
|
| 40 |
+
|
| 41 |
+
Next server phase:
|
| 42 |
+
- add `/query`
|
| 43 |
+
- add `/subset`
|
| 44 |
+
- add `/download/{token}`
|
| 45 |
+
- add job/update APIs
|
| 46 |
+
Then extend `marine_mcp.py` with corresponding tools.
|
| 47 |
+
|
| 48 |
+
## v3.3.0 deployment stability
|
| 49 |
+
|
| 50 |
+
- `VERSION` is the single application version source.
|
| 51 |
+
- Only root `app.html` is served; do not re-create `templates/app.html`.
|
| 52 |
+
- `scripts/preflight.py` runs automatically from `start.sh` and fails fast if Space metadata or the frontend entry is inconsistent.
|
| 53 |
+
- For durable user favorites and project-package downloads, attach Hugging Face Persistent Storage and set:
|
| 54 |
+
- `USER_STATE_ROOT=/data/squid_user_state`
|
| 55 |
+
- `PROJECT_PACKAGE_ROOT=/data/squid_project_packages`
|
Dockerfile
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM node:22-bookworm-slim
|
| 2 |
+
|
| 3 |
+
USER root
|
| 4 |
+
|
| 5 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 6 |
+
python3 python3-pip python3-venv ca-certificates curl git bash procps \
|
| 7 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 8 |
+
|
| 9 |
+
# CodeWhale is the current continuation of DeepSeek-TUI.
|
| 10 |
+
RUN npm install -g codewhale@0.9.10
|
| 11 |
+
|
| 12 |
+
RUN usermod -l user -d /home/user -m node && groupmod -n user node
|
| 13 |
+
|
| 14 |
+
ENV HOME=/home/user \
|
| 15 |
+
PATH=/home/user/.local/bin:/usr/local/bin:$PATH \
|
| 16 |
+
PYTHONUNBUFFERED=1 \
|
| 17 |
+
PORT=7860 \
|
| 18 |
+
DEEPSEEK_MCP_CONFIG=/home/user/app/.codewhale/mcp.json
|
| 19 |
+
|
| 20 |
+
WORKDIR /home/user/app
|
| 21 |
+
|
| 22 |
+
COPY --chown=user:user requirements.txt /home/user/app/requirements.txt
|
| 23 |
+
|
| 24 |
+
RUN python3 -m pip install --break-system-packages --no-cache-dir -r /home/user/app/requirements.txt
|
| 25 |
+
|
| 26 |
+
COPY --chown=user:user . /home/user/app
|
| 27 |
+
|
| 28 |
+
RUN chmod +x /home/user/app/start.sh
|
| 29 |
+
|
| 30 |
+
USER user
|
| 31 |
+
|
| 32 |
+
CMD ["/home/user/app/start.sh"]
|
HuggingFace_Spaces_使用指南.md
DELETED
|
@@ -1,1380 +0,0 @@
|
|
| 1 |
-
# Hugging Face Spaces 使用指南
|
| 2 |
-
|
| 3 |
-
## 目录
|
| 4 |
-
|
| 5 |
-
- [简介](#简介)
|
| 6 |
-
- [第一部分:创建个人空间](#第一部分创建个人空间)
|
| 7 |
-
- [1.1 账号注册](#11-账号注册)
|
| 8 |
-
- [1.2 创建新空间](#12-创建新空间)
|
| 9 |
-
- [1.3 空间基础配置](#13-空间基础配置)
|
| 10 |
-
- [1.4 初始化设置](#14-初始化设置)
|
| 11 |
-
- [第二部分:克隆他人空间](#第二部分克隆他人空间)
|
| 12 |
-
- [2.1 网页控制台克隆(推荐)](#21-网页控制台克隆推荐)
|
| 13 |
-
- [2.2 Git命令克隆](#22-git命令克隆)
|
| 14 |
-
- [2.3 克隆私有空间](#23-克隆私有空间)
|
| 15 |
-
- [2.4 依赖环境配置](#24-依赖环境配置)
|
| 16 |
-
- [第三部分:使用Git推送代码](#第三部分使用git推送代码)
|
| 17 |
-
- [3.1 本地仓库初始化](#31-本地仓库初始化)
|
| 18 |
-
- [3.2 Git认证配置](#32-git认证配置)
|
| 19 |
-
- [3.3 分支管理](#33-分支管理)
|
| 20 |
-
- [3.4 提交与推送](#34-提交与推送)
|
| 21 |
-
- [常见问题与解决方案](#常见问题与解决方案)
|
| 22 |
-
- [最佳实践建议](#最佳实践建议)
|
| 23 |
-
|
| 24 |
-
---
|
| 25 |
-
|
| 26 |
-
## 简介
|
| 27 |
-
|
| 28 |
-
Hugging Face Spaces 是一个免费托管机器学习应用的平台,支持 Gradio、Streamlit、Docker 等多种应用框架。Spaces 的底层是一个 Git 仓库,因此可以使用标准的 Git 工具进行代码管理和推送。本指南将详细介绍如何创建、克隆和管理 Hugging Face Spaces。
|
| 29 |
-
|
| 30 |
-
---
|
| 31 |
-
|
| 32 |
-
## 第一部分:创建个人空间
|
| 33 |
-
|
| 34 |
-
### 1.1 被号注册
|
| 35 |
-
|
| 36 |
-
**步骤说明:**
|
| 37 |
-
|
| 38 |
-
1. 访问 Hugging Face 官网:https://huggingface.co
|
| 39 |
-
2. 点击右上角"Sign Up"按钮
|
| 40 |
-
3. 填写注册信息:
|
| 41 |
-
- 用户名(username)
|
| 42 |
-
- 电子邮箱
|
| 43 |
-
- 密码
|
| 44 |
-
4. 验证邮箱地址
|
| 45 |
-
5. 登录账号
|
| 46 |
-
|
| 47 |
-
**注意事项:**
|
| 48 |
-
- 用户名一旦注册不可更改
|
| 49 |
-
- 建议使用常用的邮箱地址,以便接收重要通知
|
| 50 |
-
|
| 51 |
-
### 1.2 创建新空间
|
| 52 |
-
|
| 53 |
-
**详细步骤:**
|
| 54 |
-
|
| 55 |
-
1. **访问创建页面**
|
| 56 |
-
- 登录后,访问 https://huggingface.co/new
|
| 57 |
-
- 或点击顶部导航栏的"Spaces" → "Create new Space"
|
| 58 |
-
|
| 59 |
-
2. **填写基本信息**
|
| 60 |
-
|
| 61 |
-
| 配置项 | 说明 | 示例 |
|
| 62 |
-
|--------|------|------|
|
| 63 |
-
| Owner | 仓库所有者(个人或组织) | your-username |
|
| 64 |
-
| Space name | 空间名称 | my-demo-space |
|
| 65 |
-
| License | 许可证(可选) | mit, apache-2.0 |
|
| 66 |
-
| SDK | 应用框架选择 | gradio, streamlit, docker, static |
|
| 67 |
-
|
| 68 |
-
3. **选择SDK类型**
|
| 69 |
-
|
| 70 |
-
- **Gradio**: 最常用的选择,适合创建交互式 ML 演示
|
| 71 |
-
- **Streamlit**: 适合数据科学应用和可视化
|
| 72 |
-
- **Docker**: 完全自定义环境,适合复杂应用
|
| 73 |
-
- **Static**: 纯静态 HTML 网页
|
| 74 |
-
|
| 75 |
-
4. **设置可见性**
|
| 76 |
-
|
| 77 |
-
- **Public**: 公开,所有人可见(免费额度推荐)
|
| 78 |
-
- **Protected**: 需登录访问(PRO 计划)
|
| 79 |
-
- **Private**: 私有,仅自己和团队成员可见(Enterprise 计划)
|
| 80 |
-
|
| 81 |
-
5. **点击"Create Space"完成创建**
|
| 82 |
-
|
| 83 |
-
**创建后的初始页面:**
|
| 84 |
-
|
| 85 |
-
创建成功后,你会看到一个空的仓库页面,包含以下预设文件:
|
| 86 |
-
- `README.md`: 包含空间元数据的配置文件
|
| 87 |
-
|
| 88 |
-
### 1.3 空间基础配置
|
| 89 |
-
|
| 90 |
-
**README.md 元数据配置:**
|
| 91 |
-
|
| 92 |
-
在 `README.md` 文件顶部有一个 YAML 配置块,定义了空间的运行参数:
|
| 93 |
-
|
| 94 |
-
```yaml
|
| 95 |
-
---
|
| 96 |
-
title: Demo Space # 空间标题
|
| 97 |
-
emoji: 🚀 # 空间图标
|
| 98 |
-
colorFrom: yellow # 主题渐变起始颜色
|
| 99 |
-
colorTo: orange # 主题渐变结束颜色
|
| 100 |
-
sdk: gradio # SDK类型
|
| 101 |
-
sdk_version: 4.44.0 # SDK版本号
|
| 102 |
-
app_file: app.py # 应用入口文件
|
| 103 |
-
pinned: false # 是否固定在首页
|
| 104 |
-
license: mit # 许可证类型
|
| 105 |
-
---
|
| 106 |
-
```
|
| 107 |
-
|
| 108 |
-
**关键配置项详解:**
|
| 109 |
-
|
| 110 |
-
| 配置项 | 必填 | 说明 | 推荐值 |
|
| 111 |
-
|--------|------|------|--------|
|
| 112 |
-
| `sdk` | ✓ | 应用框架 | gradio, streamlit, docker |
|
| 113 |
-
| `app_file` | ✓ | 入口文件名 | app.py 或 gradio_app.py |
|
| 114 |
-
| `sdk_version` | ✓ | SDK版本 | 使用最新稳定版 |
|
| 115 |
-
| `python_version` | ○ | Python版本 | 3.10.13(兼容性好) |
|
| 116 |
-
| `hardware` | ○ | 硬件配置 | cpu-basic(免费), l4sx1(GPU) |
|
| 117 |
-
| `startup_duration_timeout` | ○ | 启动超时 | 默认10分钟,大模型建议1h |
|
| 118 |
-
|
| 119 |
-
**硬件配置选项:**
|
| 120 |
-
|
| 121 |
-
```yaml
|
| 122 |
-
# CPU 空间(免费)
|
| 123 |
-
hardware: cpu-basic
|
| 124 |
-
|
| 125 |
-
# GPU 空间(付费)
|
| 126 |
-
hardware: t4-small # T4 GPU,适合轻量模型
|
| 127 |
-
hardware: a10g-small # A10G GPU
|
| 128 |
-
hardware: l4sx1 # L4S GPU,适合大模型
|
| 129 |
-
```
|
| 130 |
-
|
| 131 |
-
### 1.4 初始化设置
|
| 132 |
-
|
| 133 |
-
**Web UI 方式(适合新手):**
|
| 134 |
-
|
| 135 |
-
1. **添加应用文件**
|
| 136 |
-
- 点击"Files"标签页
|
| 137 |
-
- 点击"Add file" → "Upload file"
|
| 138 |
-
- 上传 `app.py`(应用代码)
|
| 139 |
-
- 上传 `requirements.txt`(依赖文件)
|
| 140 |
-
|
| 141 |
-
2. **配置依赖**
|
| 142 |
-
|
| 143 |
-
创建 `requirements.txt` 文件,列出所有 Python 依赖:
|
| 144 |
-
|
| 145 |
-
```txt
|
| 146 |
-
# 基础依赖示例
|
| 147 |
-
gradio>=4.44.0
|
| 148 |
-
transformers>=4.40.0
|
| 149 |
-
torch>=2.3.0
|
| 150 |
-
|
| 151 |
-
# 可选:指定 PyTorch CUDA 版本
|
| 152 |
-
--extra-index-url https://download.pytorch.org/whl/cu121
|
| 153 |
-
torch==2.3.0+cu121
|
| 154 |
-
```
|
| 155 |
-
|
| 156 |
-
3. **添加系统依赖**
|
| 157 |
-
|
| 158 |
-
如需安装 Debian 系统包,创建 `packages.txt`:
|
| 159 |
-
|
| 160 |
-
```txt
|
| 161 |
-
ffmpeg
|
| 162 |
-
libsm6
|
| 163 |
-
libxext6
|
| 164 |
-
```
|
| 165 |
-
|
| 166 |
-
**终端方式(推荐):**
|
| 167 |
-
|
| 168 |
-
详见第三部分的 Git 推送流程。
|
| 169 |
-
|
| 170 |
-
---
|
| 171 |
-
|
| 172 |
-
## 第二部分:克隆他人空间
|
| 173 |
-
|
| 174 |
-
### 2.1 网页控制台克隆(推荐)
|
| 175 |
-
|
| 176 |
-
Hugging Face 提供便捷的网页控制台克隆功能,可以快速将他人的 Space 复制到自己的账号下,无需使用 Git 命令。
|
| 177 |
-
|
| 178 |
-
**操作步骤:**
|
| 179 |
-
|
| 180 |
-
1. **访问目标 Space**
|
| 181 |
-
- 登录 Hugging Face
|
| 182 |
-
- 打开想要克隆的 Space 页面
|
| 183 |
-
- 例如:https://huggingface.co/spaces/username/space-name
|
| 184 |
-
|
| 185 |
-
2. **点击克隆按钮**
|
| 186 |
-
- 在 Space 页面右上角找到三个点菜单(⋮)或链接/附件按钮
|
| 187 |
-
- 点击 **"Duplicate this Space"** 选项
|
| 188 |
-
|
| 189 |
-
3. **配置克隆参数**
|
| 190 |
-
|
| 191 |
-
在弹出的配置页面,可以修改以下属性:
|
| 192 |
-
|
| 193 |
-
| 配置项 | 说明 | 修改建议 |
|
| 194 |
-
|--------|------|----------|
|
| 195 |
-
| **Owner** | 新 Space 的所有者 | 选择你的用户名或组织 |
|
| 196 |
-
| **Space name** | 新 Space 的名称 | 可自定义新名称 |
|
| 197 |
-
| **Visibility** | 可见性设置 | 选择 Public 或 Private |
|
| 198 |
-
| **Hardware** | 硬件配置 | 可根据需求选择 CPU 或 GPU |
|
| 199 |
-
|
| 200 |
-
4. **完成克隆**
|
| 201 |
-
- 点击"Duplicate Space"按钮
|
| 202 |
-
- 系统自动创建完整副本,包括所有文件、配置和历史记录
|
| 203 |
-
- 克隆完成后自动跳转到新 Space 页面
|
| 204 |
-
|
| 205 |
-
**克隆特点:**
|
| 206 |
-
|
| 207 |
-
- ✓ **完整复制**: 包含所有代码文件(app.py, requirements.txt 等)
|
| 208 |
-
- ✓ **保留配置**: README.md 元数据、SDK 设置自动保留
|
| 209 |
-
- ✓ **独立运行**: 克隆后的 Space 完全独立,不影响原 Space
|
| 210 |
-
- ✓ **可自定义**: 可修改硬件配置、可见性等参数
|
| 211 |
-
|
| 212 |
-
**重要提示:**
|
| 213 |
-
|
| 214 |
-
⚠️ **Secrets不会自动复制**: 原 Space 中的环境变量、API密钥等 Secrets 需要手动重新添加。
|
| 215 |
-
|
| 216 |
-
添加 Secrets 的步骤:
|
| 217 |
-
1. 进入克隆后的 Space 页面
|
| 218 |
-
2. 点击"Settings"标签页
|
| 219 |
-
3. 找到"Repository secrets"部分
|
| 220 |
-
4. 点击"Add secret"
|
| 221 |
-
5. 添加所需的环境变量(如 HF_TOKEN, API_KEY 等)
|
| 222 |
-
|
| 223 |
-
**使用场景:**
|
| 224 |
-
|
| 225 |
-
网页控制台克隆特别适合以下情况:
|
| 226 |
-
- 快速基于他人模板创建自己的应用
|
| 227 |
-
- 测试和修改公开的演示 Space
|
| 228 |
-
- 学习他人的实现方式并进行定制
|
| 229 |
-
- 为团队创建私有副本用于内部开发
|
| 230 |
-
|
| 231 |
-
### 2.2 Git命令克隆
|
| 232 |
-
|
| 233 |
-
如果你需要将 Space 克隆到本地进行修改,可以使用 Git 命令。
|
| 234 |
-
|
| 235 |
-
**准备工作:**
|
| 236 |
-
|
| 237 |
-
1. **安装 Git**
|
| 238 |
-
|
| 239 |
-
```bash
|
| 240 |
-
# macOS
|
| 241 |
-
brew install git
|
| 242 |
-
|
| 243 |
-
# Linux (Ubuntu/Debian)
|
| 244 |
-
sudo apt-get install git
|
| 245 |
-
|
| 246 |
-
# Windows
|
| 247 |
-
# 从 https://git-scm.com/download/win 下载安装包
|
| 248 |
-
```
|
| 249 |
-
|
| 250 |
-
2. **验证安装**
|
| 251 |
-
|
| 252 |
-
```bash
|
| 253 |
-
git --version # 检查 Git 版本
|
| 254 |
-
```
|
| 255 |
-
|
| 256 |
-
**标准克隆流程:**
|
| 257 |
-
|
| 258 |
-
```bash
|
| 259 |
-
# 1. 克隆 Space 仓库
|
| 260 |
-
git clone https://huggingface.co/spaces/username/space-name
|
| 261 |
-
|
| 262 |
-
# 2. 进入克隆的目录
|
| 263 |
-
cd space-name
|
| 264 |
-
|
| 265 |
-
# 3. 查看文件结构
|
| 266 |
-
ls -la
|
| 267 |
-
```
|
| 268 |
-
|
| 269 |
-
**克隆不同类型的仓库:**
|
| 270 |
-
|
| 271 |
-
```bash
|
| 272 |
-
# 克隆模型仓库
|
| 273 |
-
git clone https://huggingface.co/username/model-name
|
| 274 |
-
|
| 275 |
-
# 克隆数据集仓库
|
| 276 |
-
git clone https://huggingface.co/datasets/username/dataset-name
|
| 277 |
-
|
| 278 |
-
# 克隆 Space
|
| 279 |
-
git clone https://huggingface.co/spaces/username/space-name
|
| 280 |
-
|
| 281 |
-
# 使用 SSH 克隆(需配置 SSH 密钥)
|
| 282 |
-
git clone git@hf.co:username/space-name
|
| 283 |
-
```
|
| 284 |
-
|
| 285 |
-
**克隆后的操作:**
|
| 286 |
-
|
| 287 |
-
克隆到本地后,你可以:
|
| 288 |
-
- 查看代码结构和实现逻辑
|
| 289 |
-
- 在本地修改并测试
|
| 290 |
-
- 推送到你自己的 Space(需先创建新的 Space)
|
| 291 |
-
- 学习他人的最佳实践
|
| 292 |
-
|
| 293 |
-
### 2.3 克隆私有空间
|
| 294 |
-
|
| 295 |
-
克隆私有 Space 需要认证配置。
|
| 296 |
-
|
| 297 |
-
**步骤1:生成访问令牌(Access Token)**
|
| 298 |
-
|
| 299 |
-
1. 登录 Hugging Face
|
| 300 |
-
2. 访问 https://huggingface.co/settings/tokens
|
| 301 |
-
3. 点击"Create new token"
|
| 302 |
-
4. 填写令牌信息:
|
| 303 |
-
- **Name**: 令牌名称(如"dev-machine")
|
| 304 |
-
- **Role**: 权限类型
|
| 305 |
-
- `read`: 仅下载(克隆私有仓库)
|
| 306 |
-
- `write`: 可上传(推送代码)
|
| 307 |
-
- **Expiration**: 过期时间(建议30天)
|
| 308 |
-
5. 点击"Generate token"
|
| 309 |
-
6. **立即复制并保存令牌**(令牌格式:`hf_xxxxxxxxxxxx`,只显示一次)
|
| 310 |
-
|
| 311 |
-
**步骤2:配置认证**
|
| 312 |
-
|
| 313 |
-
**方法1:使用 Hugging Face CLI(推荐)**
|
| 314 |
-
|
| 315 |
-
```bash
|
| 316 |
-
# 安装 huggingface_hub CLI
|
| 317 |
-
pip install huggingface_hub[cli]
|
| 318 |
-
|
| 319 |
-
# 交互式登录
|
| 320 |
-
huggingface-cli login
|
| 321 |
-
# 输入粘贴令牌: hf_xxxxxxxxxxxxxxxxxxxxxxxx
|
| 322 |
-
# 添加 Git 凭证? [Y/n] 输入 Y
|
| 323 |
-
|
| 324 |
-
# 验证登录状态
|
| 325 |
-
huggingface-cli whoami
|
| 326 |
-
# 输出:
|
| 327 |
-
# username: your-username
|
| 328 |
-
# auth_token: hf_xxxxxxxxxxxxxxxxxxxx
|
| 329 |
-
```
|
| 330 |
-
|
| 331 |
-
**方法2:使用环境变量(适合 CI/CD)**
|
| 332 |
-
|
| 333 |
-
```bash
|
| 334 |
-
# 设置环境变量
|
| 335 |
-
export HF_TOKEN="hf_xxxxxxxxxxxxxxxxxxxxxxxx"
|
| 336 |
-
|
| 337 |
-
# 克隆私有仓库(自动使用 HF_TOKEN)
|
| 338 |
-
git clone https://huggingface.co/spaces/username/private-space
|
| 339 |
-
```
|
| 340 |
-
|
| 341 |
-
**方法3:在克隆 URL 中嵌入令牌(不推荐)**
|
| 342 |
-
|
| 343 |
-
```bash
|
| 344 |
-
# 直接在 URL 中使用令牌
|
| 345 |
-
git clone https://oauth2:hf_xxxxxxxxxxxxxxxxxxxx@huggingface.co/spaces/username/private-space
|
| 346 |
-
|
| 347 |
-
# 注意:这种方式会将令牌暴露在命令历史中,不安全
|
| 348 |
-
```
|
| 349 |
-
|
| 350 |
-
**步骤3:克隆私有空间**
|
| 351 |
-
|
| 352 |
-
```bash
|
| 353 |
-
# 克隆私有空间(已配置认证后)
|
| 354 |
-
git clone https://huggingface.co/spaces/username/private-space
|
| 355 |
-
|
| 356 |
-
# 进入目录并查看
|
| 357 |
-
cd private-space
|
| 358 |
-
ls -la
|
| 359 |
-
```
|
| 360 |
-
|
| 361 |
-
**Windows 用户特别注意:**
|
| 362 |
-
|
| 363 |
-
Windows 系统中,令牌存储位置为 `%USERPROFILE%\\.huggingface\token`,而非 Linux/Mac 的 `~/.huggingface/token`。如果 Git 推送失败,需配置 Windows 凭证管理器:
|
| 364 |
-
|
| 365 |
-
```bash
|
| 366 |
-
# Windows PowerShell
|
| 367 |
-
git config --global credential.helper wincred
|
| 368 |
-
huggingface-cli login
|
| 369 |
-
```
|
| 370 |
-
|
| 371 |
-
### 2.4 依赖环境配置
|
| 372 |
-
|
| 373 |
-
**Python 依赖管理:**
|
| 374 |
-
|
| 375 |
-
克隆后的空间通常包含 `requirements.txt`,需要在本地安装依赖:
|
| 376 |
-
|
| 377 |
-
```bash
|
| 378 |
-
# 进入克隆的 Space 目录
|
| 379 |
-
cd space-name
|
| 380 |
-
|
| 381 |
-
# 创建虚拟环境(推荐)
|
| 382 |
-
python -m venv venv
|
| 383 |
-
source venv/bin/activate # Linux/Mac
|
| 384 |
-
# 或 venv\Scripts\activate # Windows
|
| 385 |
-
|
| 386 |
-
# 安装依赖
|
| 387 |
-
pip install -r requirements.txt
|
| 388 |
-
|
| 389 |
-
# 如果包含自定义 PyTorch CUDA 版本,确保匹配本地环境
|
| 390 |
-
cat requirements.txt | grep extra-index-url
|
| 391 |
-
```
|
| 392 |
-
|
| 393 |
-
**处理特殊依赖:**
|
| 394 |
-
|
| 395 |
-
某些 Space 可能需要额外的配置:
|
| 396 |
-
|
| 397 |
-
```bash
|
| 398 |
-
# 1. pre-requirements.txt(优先安装的依赖)
|
| 399 |
-
# 用于更新 pip 或安装前置依赖
|
| 400 |
-
pip install -r pre-requirements.txt
|
| 401 |
-
|
| 402 |
-
# 2. packages.txt(系统级依赖)
|
| 403 |
-
# Debian 系统包,需要在 Linux 环境安装
|
| 404 |
-
# macOS/Windows 无法直接安装,需手动替代方案
|
| 405 |
-
|
| 406 |
-
# 3. apt-packages.txt(另一种系统依赖格式)
|
| 407 |
-
sudo apt-get update && sudo apt-get install -y $(cat apt-packages.txt)
|
| 408 |
-
```
|
| 409 |
-
|
| 410 |
-
**检查依赖完整性:**
|
| 411 |
-
|
| 412 |
-
```bash
|
| 413 |
-
# 验证关键库是否安装
|
| 414 |
-
pip list | grep -E "gradio|streamlit|torch|transformers"
|
| 415 |
-
|
| 416 |
-
# 检查版本匹配
|
| 417 |
-
pip show gradio
|
| 418 |
-
pip show torch
|
| 419 |
-
```
|
| 420 |
-
|
| 421 |
-
---
|
| 422 |
-
|
| 423 |
-
## 第三部分:使用Git推送代码
|
| 424 |
-
|
| 425 |
-
### 3.1 本地仓库初始化
|
| 426 |
-
|
| 427 |
-
**场景1:克隆已有空间并修改**
|
| 428 |
-
|
| 429 |
-
```bash
|
| 430 |
-
# 1. 克隆你的 Space
|
| 431 |
-
git clone https://huggingface.co/spaces/your-username/space-name
|
| 432 |
-
cd space-name
|
| 433 |
-
|
| 434 |
-
# 2. 查看当前状态
|
| 435 |
-
git status
|
| 436 |
-
git log --oneline -5 # 查看最近5次提交
|
| 437 |
-
```
|
| 438 |
-
|
| 439 |
-
**场景2:本地项目推送到新建的空 Space**
|
| 440 |
-
|
| 441 |
-
```bash
|
| 442 |
-
# 1. 本地项目目录
|
| 443 |
-
cd my-local-project
|
| 444 |
-
|
| 445 |
-
# 2. 初始化 Git 仓库
|
| 446 |
-
git init
|
| 447 |
-
|
| 448 |
-
# 3. 添加远程仓库
|
| 449 |
-
git remote add origin https://huggingface.co/spaces/your-username/space-name
|
| 450 |
-
|
| 451 |
-
# 4. 查看远程仓库配置
|
| 452 |
-
git remote -v
|
| 453 |
-
# 输出:
|
| 454 |
-
# origin https://huggingface.co/spaces/your-username/space-name (fetch)
|
| 455 |
-
# origin https://huggingface.co/spaces/your-username/space-name (push)
|
| 456 |
-
```
|
| 457 |
-
|
| 458 |
-
**必要文件准备:**
|
| 459 |
-
|
| 460 |
-
推送前确保包含以下文件:
|
| 461 |
-
|
| 462 |
-
```bash
|
| 463 |
-
# 检查必要文件
|
| 464 |
-
ls -la
|
| 465 |
-
|
| 466 |
-
# 必须包含:
|
| 467 |
-
# - README.md (包含 YAML 元数据)
|
| 468 |
-
# - app.py (应用入口)
|
| 469 |
-
# - requirements.txt (Python 依赖)
|
| 470 |
-
|
| 471 |
-
# 可选文件:
|
| 472 |
-
# - packages.txt (系统依赖)
|
| 473 |
-
# - LICENSE (许可证)
|
| 474 |
-
```
|
| 475 |
-
|
| 476 |
-
### 3.2 Git认证配置
|
| 477 |
-
|
| 478 |
-
**认证方法对比:**
|
| 479 |
-
|
| 480 |
-
| 方法 | 适用场景 | 安全性 | 配置位置 |
|
| 481 |
-
|------|----------|--------|----------|
|
| 482 |
-
| huggingface-cli login | 本地开发 | ✓ 安全 | ~/.huggingface/token |
|
| 483 |
-
| HF_TOKEN 环境变量 | CI/CD | ✓ 安全(需 masking) | 环境变量 |
|
| 484 |
-
| SSH 密钥 | 频繁推送 | ✓✓ 最安全 | ~/.ssh/id_rsa |
|
| 485 |
-
| URL 嵌入令牌 | 临时使用 | ✗ 不安全 | 命令历史 |
|
| 486 |
-
|
| 487 |
-
**推荐配置流程:**
|
| 488 |
-
|
| 489 |
-
```bash
|
| 490 |
-
# 1. 生成访问令牌
|
| 491 |
-
# 访问 https://huggingface.co/settings/tokens
|
| 492 |
-
# 创建 "write" 类型令牌
|
| 493 |
-
|
| 494 |
-
# 2. 配置 Git 凭证
|
| 495 |
-
huggingface-cli login --token hf_xxxxxxxxxxxxxxxxxxxx
|
| 496 |
-
|
| 497 |
-
# 3. 配置 Git 凭证持久化
|
| 498 |
-
git config --global credential.helper store # Linux/Mac
|
| 499 |
-
# 或 git config --global credential.helper wincred # Windows
|
| 500 |
-
|
| 501 |
-
# 4. 验证认证
|
| 502 |
-
huggingface-cli whoami
|
| 503 |
-
|
| 504 |
-
# 5. 测试推送权限
|
| 505 |
-
git push --dry-run # 模拟推送,不实际提交
|
| 506 |
-
```
|
| 507 |
-
|
| 508 |
-
**SSH 认证配置(高级):**
|
| 509 |
-
|
| 510 |
-
```bash
|
| 511 |
-
# 1. 生成 SSH 密钥
|
| 512 |
-
ssh-keygen -t ed25519 -C "your-email@example.com"
|
| 513 |
-
# 保存到 ~/.ssh/id_ed25519_hf
|
| 514 |
-
|
| 515 |
-
# 2. 添加公钥到 Hugging Face
|
| 516 |
-
cat ~/.ssh/id_ed25519_hf.pub
|
| 517 |
-
# 复制内容,访问 https://huggingface.co/settings/keys
|
| 518 |
-
# 添加新的 SSH 密钥
|
| 519 |
-
|
| 520 |
-
# 3. 配置 SSH 别名
|
| 521 |
-
cat >> ~/.ssh/config << EOF
|
| 522 |
-
Host hf.co
|
| 523 |
-
HostName huggingface.co
|
| 524 |
-
User git
|
| 525 |
-
IdentityFile ~/.ssh/id_ed25519_hf
|
| 526 |
-
EOF
|
| 527 |
-
|
| 528 |
-
# 4. 使用 SSH 推送
|
| 529 |
-
git remote set-url origin git@hf.co:spaces/your-username/space-name
|
| 530 |
-
git push
|
| 531 |
-
```
|
| 532 |
-
|
| 533 |
-
### 3.3 分支管理
|
| 534 |
-
|
| 535 |
-
**Hugging Face Spaces 的分支特点:**
|
| 536 |
-
|
| 537 |
-
- Spaces 仅使用 `main` 分支作为运行环境
|
| 538 |
-
- 其他分支可用于开发,但不会触发 Space 构建
|
| 539 |
-
- 推送到 `main` 分支会自动重建并重启 Space
|
| 540 |
-
|
| 541 |
-
**分支操作示例:**
|
| 542 |
-
|
| 543 |
-
```bash
|
| 544 |
-
# 1. 查看当前分支
|
| 545 |
-
git branch
|
| 546 |
-
# * main
|
| 547 |
-
|
| 548 |
-
# 2. 创建开发分支
|
| 549 |
-
git checkout -b dev-feature
|
| 550 |
-
|
| 551 |
-
# 3. 在开发分支提交修改
|
| 552 |
-
git add app.py
|
| 553 |
-
git commit -m "Add new feature"
|
| 554 |
-
|
| 555 |
-
# 4. 切换回 main 分支
|
| 556 |
-
git checkout main
|
| 557 |
-
|
| 558 |
-
# 5. 合并开发分支
|
| 559 |
-
git merge dev-feature
|
| 560 |
-
|
| 561 |
-
# 6. 推送 main 分支(触发 Space 重建)
|
| 562 |
-
git push origin main
|
| 563 |
-
```
|
| 564 |
-
|
| 565 |
-
**使用 Pull Request 流程:**
|
| 566 |
-
|
| 567 |
-
```bash
|
| 568 |
-
# 1. 创建功能分支
|
| 569 |
-
git checkout -b feature-new-model
|
| 570 |
-
|
| 571 |
-
# 2. 提交修改
|
| 572 |
-
git add .
|
| 573 |
-
git commit -m "Add new model integration"
|
| 574 |
-
|
| 575 |
-
# 3. 推送功能分支
|
| 576 |
-
git push origin feature-new-model
|
| 577 |
-
|
| 578 |
-
# 4. 在 Hugging Face Web UI 创建 Pull Request
|
| 579 |
-
# 访问 Space 页面 → "Pull requests" → "New pull request"
|
| 580 |
-
|
| 581 |
-
# 5. 合并 PR 后,Space 自动重建
|
| 582 |
-
```
|
| 583 |
-
|
| 584 |
-
### 3.4 提交与推送
|
| 585 |
-
|
| 586 |
-
**标准提交流程:**
|
| 587 |
-
|
| 588 |
-
```bash
|
| 589 |
-
# 1. 查看待提交的文件
|
| 590 |
-
git status
|
| 591 |
-
|
| 592 |
-
# 2. 添加文件到暂存区
|
| 593 |
-
# 方式A:添加所有修改
|
| 594 |
-
git add .
|
| 595 |
-
|
| 596 |
-
# 方式B:添加特定文件(推荐)
|
| 597 |
-
git add app.py requirements.txt README.md
|
| 598 |
-
|
| 599 |
-
# ⚠️ 不要添加敏感文件
|
| 600 |
-
# 检查是否有 .env, credentials.json 等
|
| 601 |
-
git status | grep -E ".env|credentials|secrets"
|
| 602 |
-
|
| 603 |
-
# 3. 创建提交
|
| 604 |
-
git commit -m "Update app with new model"
|
| 605 |
-
|
| 606 |
-
# 提交信息最佳实践:
|
| 607 |
-
# - 使用简洁描述(不超过50字符)
|
| 608 |
-
# - 说明修改内容,而非修改方式
|
| 609 |
-
# - 示例:
|
| 610 |
-
# ✓ "Add sentiment analysis feature"
|
| 611 |
-
# ✓ "Fix memory leak in model loading"
|
| 612 |
-
# ✓ "Update transformers to 4.40.0"
|
| 613 |
-
# ✗ "Fixed stuff"
|
| 614 |
-
# ✗ "Update"
|
| 615 |
-
|
| 616 |
-
# 4. 推送到远程仓库
|
| 617 |
-
git push origin main
|
| 618 |
-
# 或
|
| 619 |
-
git push # 如果默认分支已设置
|
| 620 |
-
|
| 621 |
-
# 5. 查看推送结果
|
| 622 |
-
git log --oneline -3
|
| 623 |
-
```
|
| 624 |
-
|
| 625 |
-
**使用 Hugging Face CLI 上传(简化流程):**
|
| 626 |
-
|
| 627 |
-
```bash
|
| 628 |
-
# 安装 CLI
|
| 629 |
-
pip install huggingface_hub[cli]
|
| 630 |
-
|
| 631 |
-
# 直接上传文件或文件夹
|
| 632 |
-
huggingface-cli upload your-username/space-name ./app.py
|
| 633 |
-
|
| 634 |
-
# 上传整个目录
|
| 635 |
-
huggingface-cli upload your-username/space-name ./my-project
|
| 636 |
-
|
| 637 |
-
# 指定提交信息
|
| 638 |
-
huggingface-cli upload your-username/space-name ./model.safetensors \
|
| 639 |
-
--commit-message "Add trained model weights"
|
| 640 |
-
|
| 641 |
-
# 查看上传结果
|
| 642 |
-
huggingface-cli repo-files your-username/space-name
|
| 643 |
-
```
|
| 644 |
-
|
| 645 |
-
**自动化推送(使用 Python):**
|
| 646 |
-
|
| 647 |
-
```python
|
| 648 |
-
from huggingface_hub import HfApi
|
| 649 |
-
import os
|
| 650 |
-
|
| 651 |
-
# 初始化 API 客户端
|
| 652 |
-
api = HfApi(token=os.getenv('HF_TOKEN'))
|
| 653 |
-
|
| 654 |
-
# 上传单个文件
|
| 655 |
-
api.upload_file(
|
| 656 |
-
path_or_fileobj="./app.py",
|
| 657 |
-
path_in_repo="app.py",
|
| 658 |
-
repo_id="your-username/space-name",
|
| 659 |
-
repo_type="space",
|
| 660 |
-
commit_message="Update application"
|
| 661 |
-
)
|
| 662 |
-
|
| 663 |
-
# 上传整个文件夹
|
| 664 |
-
api.upload_folder(
|
| 665 |
-
folder_path="./my-project",
|
| 666 |
-
repo_id="your-username/space-name",
|
| 667 |
-
repo_type="space",
|
| 668 |
-
commit_message="Deploy new version"
|
| 669 |
-
)
|
| 670 |
-
```
|
| 671 |
-
|
| 672 |
-
---
|
| 673 |
-
|
| 674 |
-
## 常见问题与解决方案
|
| 675 |
-
|
| 676 |
-
### 问题1:推送时认证失败(401 Unauthorized)
|
| 677 |
-
|
| 678 |
-
**原因**: Token 未配置或权限不足
|
| 679 |
-
|
| 680 |
-
**解决方案**:
|
| 681 |
-
|
| 682 |
-
```bash
|
| 683 |
-
# 1. 检查当前认证状态
|
| 684 |
-
huggingface-cli whoami
|
| 685 |
-
|
| 686 |
-
# 2. 如果未认证,重新登录
|
| 687 |
-
huggingface-cli login --token hf_xxxxxxxxxxxxxxxx
|
| 688 |
-
|
| 689 |
-
# 3. 确认 Token 类型为 "write"
|
| 690 |
-
# 访问 https://huggingface.co/settings/tokens 检查
|
| 691 |
-
|
| 692 |
-
# 4. 配置 Git 凭证持久化
|
| 693 |
-
git config --global credential.helper store
|
| 694 |
-
|
| 695 |
-
# 5. 测试推送
|
| 696 |
-
git push --dry-run
|
| 697 |
-
```
|
| 698 |
-
|
| 699 |
-
### 问题2:推送后 Space 启动失败
|
| 700 |
-
|
| 701 |
-
**原因**: 配置文件缺失或格式错误
|
| 702 |
-
|
| 703 |
-
**解决方案**:
|
| 704 |
-
|
| 705 |
-
```bash
|
| 706 |
-
# 1. 检查 README.md 的 YAML 配置
|
| 707 |
-
head -n 20 README.md
|
| 708 |
-
|
| 709 |
-
# 确保 YAML 块格式正确:
|
| 710 |
-
# ---
|
| 711 |
-
# title: ...
|
| 712 |
-
# sdk: gradio
|
| 713 |
-
# app_file: app.py
|
| 714 |
-
# ---
|
| 715 |
-
# (注意 "---" 分隔符)
|
| 716 |
-
|
| 717 |
-
# 2. 检查 requirements.txt 格式
|
| 718 |
-
cat requirements.txt
|
| 719 |
-
|
| 720 |
-
# 避免常见错误:
|
| 721 |
-
# ✗ 使用 Windows 换行符(CRLF)
|
| 722 |
-
# ✓ 使用 Unix 换行符(LF)
|
| 723 |
-
|
| 724 |
-
# 转换换行符:
|
| 725 |
-
dos2unix requirements.txt # Linux/Mac
|
| 726 |
-
# 或在文本编辑器设置 LF 换行
|
| 727 |
-
|
| 728 |
-
# 3. 检查 app.py 是否存在且语法正确
|
| 729 |
-
python -m py_compile app.py
|
| 730 |
-
|
| 731 |
-
# 4. 查看 Space 构建日志
|
| 732 |
-
# 访问 Space 页面 → "Logs" 标签页
|
| 733 |
-
```
|
| 734 |
-
|
| 735 |
-
### 问题3:依赖安装失败
|
| 736 |
-
|
| 737 |
-
**原因**: 版本冲突或平台不兼容
|
| 738 |
-
|
| 739 |
-
**解决方案**:
|
| 740 |
-
|
| 741 |
-
```txt
|
| 742 |
-
# 修改 requirements.txt
|
| 743 |
-
|
| 744 |
-
# 1. 使用灵活的版本约束
|
| 745 |
-
# ✗ transformers==4.40.0 (严格版本)
|
| 746 |
-
# ✓ transformers>=4.40.0 (最低版本)
|
| 747 |
-
|
| 748 |
-
# 2. 处理 PyTorch CUDA 版本
|
| 749 |
-
# 如果本地无 GPU,安装 CPU 版本
|
| 750 |
-
--extra-index-url https://download.pytorch.org/whl/cpu
|
| 751 |
-
torch>=2.3.0
|
| 752 |
-
|
| 753 |
-
# 3. 分离系统依赖和 Python 依赖
|
| 754 |
-
# packages.txt 内容:
|
| 755 |
-
ffmpeg
|
| 756 |
-
libgl1-mesa-glx
|
| 757 |
-
|
| 758 |
-
# 4. 检查依赖冲突
|
| 759 |
-
pip check
|
| 760 |
-
```
|
| 761 |
-
|
| 762 |
-
### 问题4:Space 冷启动超时
|
| 763 |
-
|
| 764 |
-
**原因**: 模型加载时间过长
|
| 765 |
-
|
| 766 |
-
**解决方案**:
|
| 767 |
-
|
| 768 |
-
```yaml
|
| 769 |
-
# 在 README.md 添加超时配置
|
| 770 |
-
---
|
| 771 |
-
title: Large Model Demo
|
| 772 |
-
sdk: gradio
|
| 773 |
-
startup_duration_timeout: 1h # 增加到 1 小时
|
| 774 |
-
---
|
| 775 |
-
```
|
| 776 |
-
|
| 777 |
-
```python
|
| 778 |
-
# 在 app.py 中优化模型加载
|
| 779 |
-
import gradio as gr
|
| 780 |
-
from transformers import pipeline
|
| 781 |
-
|
| 782 |
-
# 使用缓存
|
| 783 |
-
@gr.cache_resource
|
| 784 |
-
def load_model():
|
| 785 |
-
return pipeline("text-generation", model="large-model")
|
| 786 |
-
|
| 787 |
-
# 分步加载(如果模型组件多)
|
| 788 |
-
def load_model_lazy():
|
| 789 |
-
# 首次请求时加载
|
| 790 |
-
pass
|
| 791 |
-
```
|
| 792 |
-
|
| 793 |
-
### 问题5:克隆私有空间权限被拒绝
|
| 794 |
-
|
| 795 |
-
**原因**: Token 权限不足或已过期
|
| 796 |
-
|
| 797 |
-
**解决方案**:
|
| 798 |
-
|
| 799 |
-
```bash
|
| 800 |
-
# 1. 检查 Token 状态
|
| 801 |
-
# 访问 https://huggingface.co/settings/tokens
|
| 802 |
-
# 检查 Token 是否被撤销或过期
|
| 803 |
-
|
| 804 |
-
# 2. 重新生成 Token
|
| 805 |
-
# 创建新的 "read" 类型 Token
|
| 806 |
-
|
| 807 |
-
# 3. 清除旧 Token
|
| 808 |
-
huggingface-cli logout
|
| 809 |
-
|
| 810 |
-
# 4. 使用新 Token 登录
|
| 811 |
-
huggingface-cli login --token hf_new_token_xxxxxx
|
| 812 |
-
|
| 813 |
-
# 5. 配置 Git 凭证
|
| 814 |
-
git config --global credential.helper store
|
| 815 |
-
|
| 816 |
-
# 6. 再次克隆
|
| 817 |
-
git clone https://huggingface.co/spaces/username/private-space
|
| 818 |
-
```
|
| 819 |
-
|
| 820 |
-
### 问题6:推送包含敏感文件
|
| 821 |
-
|
| 822 |
-
**原因**: 误提交了 .env 或 credentials 文件
|
| 823 |
-
|
| 824 |
-
**解决方案**:
|
| 825 |
-
|
| 826 |
-
```bash
|
| 827 |
-
# 1. 立即撤销 Token
|
| 828 |
-
# 访问 https://huggingface.co/settings/tokens
|
| 829 |
-
# 撤销相关 Token
|
| 830 |
-
|
| 831 |
-
# 2. 从 Git 历史中删除敏感文件
|
| 832 |
-
git filter-branch --force --index-filter \
|
| 833 |
-
'git rm --cached --ignore-unmatch .env credentials.json' \
|
| 834 |
-
--prune-empty --tag-name-filter cat -- --all
|
| 835 |
-
|
| 836 |
-
# 3. 强制推送清理后的历史
|
| 837 |
-
git push origin main --force
|
| 838 |
-
|
| 839 |
-
# ⚠️ 注意:强制推送会重写历史,谨慎操作
|
| 840 |
-
|
| 841 |
-
# 4. 配置 .gitignore
|
| 842 |
-
cat >> .gitignore << EOF
|
| 843 |
-
.env
|
| 844 |
-
*.env
|
| 845 |
-
credentials.json
|
| 846 |
-
secrets.yaml
|
| 847 |
-
*.pem
|
| 848 |
-
EOF
|
| 849 |
-
|
| 850 |
-
# 5. 生成新 Token
|
| 851 |
-
huggingface-cli login
|
| 852 |
-
```
|
| 853 |
-
|
| 854 |
-
### 问题7:网页克隆后 Secrets 未复制
|
| 855 |
-
|
| 856 |
-
**原因**: Hugging Face 安全机制,Secrets 需手动添加
|
| 857 |
-
|
| 858 |
-
**解决方案**:
|
| 859 |
-
|
| 860 |
-
```bash
|
| 861 |
-
# 1. 进入克隆后的 Space
|
| 862 |
-
# 访问 https://huggingface.co/spaces/your-username/cloned-space
|
| 863 |
-
|
| 864 |
-
# 2. 点击 Settings 标签页
|
| 865 |
-
|
| 866 |
-
# 3. 找到 Repository secrets 部分
|
| 867 |
-
|
| 868 |
-
# 4. 手动添加所需 Secrets
|
| 869 |
-
# 示例:
|
| 870 |
-
# HF_TOKEN = hf_xxxxxxxxxxxxxxxx
|
| 871 |
-
# API_KEY = your_api_key
|
| 872 |
-
# DATABASE_URL = your_database_url
|
| 873 |
-
|
| 874 |
-
# 5. 在代码中使用 Secrets
|
| 875 |
-
import os
|
| 876 |
-
hf_token = os.getenv('HF_TOKEN')
|
| 877 |
-
api_key = os.getenv('API_KEY')
|
| 878 |
-
```
|
| 879 |
-
|
| 880 |
-
### 问题8:克隆后本地运行失败
|
| 881 |
-
|
| 882 |
-
**原因**: 系统依赖缺失或环境不匹配
|
| 883 |
-
|
| 884 |
-
**解决方案**:
|
| 885 |
-
|
| 886 |
-
```bash
|
| 887 |
-
# 1. 检查 Python 版本匹配
|
| 888 |
-
python --version
|
| 889 |
-
cat README.md | grep python_version
|
| 890 |
-
|
| 891 |
-
# 2. 安装系统依赖
|
| 892 |
-
# 查看 packages.txt
|
| 893 |
-
cat packages.txt
|
| 894 |
-
|
| 895 |
-
# Linux: 直接安装
|
| 896 |
-
sudo apt-get install -y $(cat packages.txt)
|
| 897 |
-
|
| 898 |
-
# macOS: 使用替代方案
|
| 899 |
-
brew install ffmpeg libsm libxext
|
| 900 |
-
|
| 901 |
-
# Windows: 手动下载安装包
|
| 902 |
-
|
| 903 |
-
# 3. 检查 CUDA 版本(如使用 GPU)
|
| 904 |
-
nvidia-smi
|
| 905 |
-
cat requirements.txt | grep extra-index-url
|
| 906 |
-
|
| 907 |
-
# 4. 创建虚拟环境并重新安装依赖
|
| 908 |
-
python -m venv venv
|
| 909 |
-
source venv/bin/activate
|
| 910 |
-
pip install -r requirements.txt
|
| 911 |
-
```
|
| 912 |
-
|
| 913 |
-
---
|
| 914 |
-
|
| 915 |
-
## 最佳实践建议
|
| 916 |
-
|
| 917 |
-
### 1. 文件组织结构
|
| 918 |
-
|
| 919 |
-
**推荐的 Space 目录结构:**
|
| 920 |
-
|
| 921 |
-
```
|
| 922 |
-
my-space/
|
| 923 |
-
├── README.md # 元数据配置(必须)
|
| 924 |
-
├── app.py # 应用入口(必须)
|
| 925 |
-
├── requirements.txt # Python 依赖(必须)
|
| 926 |
-
├── packages.txt # 系统依赖(可选)
|
| 927 |
-
├── .gitignore # Git 忽略规则(推荐)
|
| 928 |
-
├── assets/ # 静态资源
|
| 929 |
-
│ ├── logo.png
|
| 930 |
-
│ └── styles.css
|
| 931 |
-
├── utils/ # 工具函数
|
| 932 |
-
│ ├── __init__.py
|
| 933 |
-
│ └── helpers.py
|
| 934 |
-
└── tests/ # 测试文件(可选)
|
| 935 |
-
└── test_app.py
|
| 936 |
-
```
|
| 937 |
-
|
| 938 |
-
### 2. README.md 配置规范
|
| 939 |
-
|
| 940 |
-
**完整配置示例:**
|
| 941 |
-
|
| 942 |
-
```yaml
|
| 943 |
-
---
|
| 944 |
-
# 基本信息
|
| 945 |
-
title: My ML Demo
|
| 946 |
-
emoji: 🤖
|
| 947 |
-
colorFrom: blue
|
| 948 |
-
colorTo: green
|
| 949 |
-
|
| 950 |
-
# SDK 配置
|
| 951 |
-
sdk: gradio
|
| 952 |
-
sdk_version: 4.44.0
|
| 953 |
-
python_version: "3.10.13"
|
| 954 |
-
app_file: app.py
|
| 955 |
-
|
| 956 |
-
# 硬件配置
|
| 957 |
-
hardware: cpu-basic # 免费 CPU
|
| 958 |
-
# 或 hardware: l4sx1 # GPU(付费)
|
| 959 |
-
|
| 960 |
-
# 性能配置
|
| 961 |
-
startup_duration_timeout: 15m # 启动超时(默认)
|
| 962 |
-
pinned: false # 是否固定在首页
|
| 963 |
-
|
| 964 |
-
# 许可证
|
| 965 |
-
license: apache-2.0
|
| 966 |
-
|
| 967 |
-
# 描述
|
| 968 |
-
short_description: A machine learning demo application
|
| 969 |
-
tags:
|
| 970 |
-
- machine-learning
|
| 971 |
-
- transformers
|
| 972 |
-
- gradio
|
| 973 |
-
---
|
| 974 |
-
```
|
| 975 |
-
|
| 976 |
-
### 3. requirements.txt 编写规范
|
| 977 |
-
|
| 978 |
-
**示例:**
|
| 979 |
-
|
| 980 |
-
```txt
|
| 981 |
-
# 优先设置 PyTorch 源(如需 CUDA)
|
| 982 |
-
--extra-index-url https://download.pytorch.org/whl/cu121
|
| 983 |
-
|
| 984 |
-
# 核心框架
|
| 985 |
-
gradio>=4.44.0,<5.0.0 # 主框架(限制版本范围)
|
| 986 |
-
streamlit>=1.40.0 # Streamlit 版本(如使用)
|
| 987 |
-
|
| 988 |
-
# ML 库
|
| 989 |
-
transformers>=4.40.0 # 灵活版本约束
|
| 990 |
-
torch>=2.3.0 # PyTorch
|
| 991 |
-
accelerate>=0.25.0 # 加速库
|
| 992 |
-
|
| 993 |
-
# 数据处理
|
| 994 |
-
datasets>=2.19.0
|
| 995 |
-
numpy>=1.24.0,<2.0.0
|
| 996 |
-
pandas>=2.0.0
|
| 997 |
-
|
| 998 |
-
# 可选:精确版本(确保兼容)
|
| 999 |
-
# transformers==4.40.0
|
| 1000 |
-
# torch==2.3.0+cu121
|
| 1001 |
-
```
|
| 1002 |
-
|
| 1003 |
-
**版本约束最佳实践:**
|
| 1004 |
-
|
| 1005 |
-
- 使用 `>=` 确保最低版本
|
| 1006 |
-
- 使用 `<` 限制上限版本,避免破坏性更新
|
| 1007 |
-
- 关键依赖使用精确版本 `==`,确保一致性
|
| 1008 |
-
- 测试兼容性后再推送到 Space
|
| 1009 |
-
|
| 1010 |
-
### 4. Git 工作流程
|
| 1011 |
-
|
| 1012 |
-
**推荐的开发流程:**
|
| 1013 |
-
|
| 1014 |
-
```bash
|
| 1015 |
-
# 1. 本地开发(在功能分支)
|
| 1016 |
-
git checkout -b feature-improvement
|
| 1017 |
-
|
| 1018 |
-
# 2. 本地测试
|
| 1019 |
-
python app.py # 确保应用可运行
|
| 1020 |
-
|
| 1021 |
-
# 3. 提交修改
|
| 1022 |
-
git add app.py
|
| 1023 |
-
git commit -m "Improve model loading performance"
|
| 1024 |
-
|
| 1025 |
-
# 4. 推送功能分支
|
| 1026 |
-
git push origin feature-improvement
|
| 1027 |
-
|
| 1028 |
-
# 5. 创建 Pull Request(在 Web UI)
|
| 1029 |
-
# 审核修改,确保无误
|
| 1030 |
-
|
| 1031 |
-
# 6. 合并 PR 到 main
|
| 1032 |
-
# Space 自动重建
|
| 1033 |
-
|
| 1034 |
-
# 7. 监控构建日志
|
| 1035 |
-
# 访问 Space → "Logs" 标签页
|
| 1036 |
-
|
| 1037 |
-
# 8. 测试线上应用
|
| 1038 |
-
# 如失败,快速回滚:
|
| 1039 |
-
git revert HEAD
|
| 1040 |
-
git push origin main
|
| 1041 |
-
```
|
| 1042 |
-
|
| 1043 |
-
### 5. 安全配置
|
| 1044 |
-
|
| 1045 |
-
**敏感信息管理:**
|
| 1046 |
-
|
| 1047 |
-
```bash
|
| 1048 |
-
# 1. 配置 .gitignore
|
| 1049 |
-
cat > .gitignore << EOF
|
| 1050 |
-
# 环境变量
|
| 1051 |
-
.env
|
| 1052 |
-
*.env.local
|
| 1053 |
-
|
| 1054 |
-
# 凭证文件
|
| 1055 |
-
credentials.json
|
| 1056 |
-
secrets.yaml
|
| 1057 |
-
*.pem
|
| 1058 |
-
*.key
|
| 1059 |
-
|
| 1060 |
-
# 缓存和临时文件
|
| 1061 |
-
__pycache__/
|
| 1062 |
-
*.pyc
|
| 1063 |
-
.cache/
|
| 1064 |
-
*.tmp
|
| 1065 |
-
|
| 1066 |
-
# IDE 配置
|
| 1067 |
-
.vscode/
|
| 1068 |
-
.idea/
|
| 1069 |
-
*.swp
|
| 1070 |
-
EOF
|
| 1071 |
-
|
| 1072 |
-
# 2. 使用 Space Secrets(推荐)
|
| 1073 |
-
# 在 Space Settings → "Repository secrets" 添加:
|
| 1074 |
-
# HF_TOKEN = hf_xxxxxxxx
|
| 1075 |
-
# API_KEY = your_api_key
|
| 1076 |
-
|
| 1077 |
-
# 在代码中读取:
|
| 1078 |
-
import os
|
| 1079 |
-
api_key = os.getenv('API_KEY')
|
| 1080 |
-
```
|
| 1081 |
-
|
| 1082 |
-
**Python 代码示例:**
|
| 1083 |
-
|
| 1084 |
-
```python
|
| 1085 |
-
import os
|
| 1086 |
-
import gradio as gr
|
| 1087 |
-
|
| 1088 |
-
# 从环境变量读取 Secrets(推荐)
|
| 1089 |
-
hf_token = os.getenv('HF_TOKEN')
|
| 1090 |
-
api_key = os.getenv('API_KEY')
|
| 1091 |
-
|
| 1092 |
-
# ✗ 不要硬编码
|
| 1093 |
-
# hf_token = "hf_xxxxxxxxxxxxxx" # 危险!
|
| 1094 |
-
|
| 1095 |
-
# ✓ 使用环境变量
|
| 1096 |
-
if not hf_token:
|
| 1097 |
-
raise ValueError("HF_TOKEN environment variable not set")
|
| 1098 |
-
|
| 1099 |
-
# 使用 huggingface_hub 的内置认证
|
| 1100 |
-
from huggingface_hub import HfApi
|
| 1101 |
-
api = HfApi() # 自动读取 HF_TOKEN 或 ~/.huggingface/token
|
| 1102 |
-
```
|
| 1103 |
-
|
| 1104 |
-
### 6. 性能优化
|
| 1105 |
-
|
| 1106 |
-
**模型加载优化:**
|
| 1107 |
-
|
| 1108 |
-
```python
|
| 1109 |
-
# 1. 使用缓存
|
| 1110 |
-
import gradio as gr
|
| 1111 |
-
from transformers import pipeline
|
| 1112 |
-
|
| 1113 |
-
# Gradio 缓存装饰器
|
| 1114 |
-
@gr.cache_resource
|
| 1115 |
-
def load_model():
|
| 1116 |
-
return pipeline("text-generation", model="gpt2")
|
| 1117 |
-
|
| 1118 |
-
# 2. 指定缓存目录
|
| 1119 |
-
from transformers import AutoModel
|
| 1120 |
-
import os
|
| 1121 |
-
|
| 1122 |
-
cache_dir = "/tmp/model_cache" # Spaces 推荐使用 /tmp
|
| 1123 |
-
model = AutoModel.from_pretrained(
|
| 1124 |
-
"bert-base-uncased",
|
| 1125 |
-
cache_dir=cache_dir
|
| 1126 |
-
)
|
| 1127 |
-
|
| 1128 |
-
# 3. 懒加载(首次请求时加载)
|
| 1129 |
-
class ModelLoader:
|
| 1130 |
-
_model = None
|
| 1131 |
-
|
| 1132 |
-
@classmethod
|
| 1133 |
-
def get_model(cls):
|
| 1134 |
-
if cls._model is None:
|
| 1135 |
-
cls._model = pipeline("sentiment-analysis")
|
| 1136 |
-
return cls._model
|
| 1137 |
-
|
| 1138 |
-
# 使用
|
| 1139 |
-
model = ModelLoader.get_model()
|
| 1140 |
-
```
|
| 1141 |
-
|
| 1142 |
-
**requirements.txt 优化:**
|
| 1143 |
-
|
| 1144 |
-
```txt
|
| 1145 |
-
# 减少依赖大小
|
| 1146 |
-
# ✗ 安装完整的 transformers[torch]
|
| 1147 |
-
# ✓ 仅安装必要组件
|
| 1148 |
-
transformers
|
| 1149 |
-
torch --index-url https://download.pytorch.org/whl/cpu # CPU 版本更小
|
| 1150 |
-
|
| 1151 |
-
# 使用轻量级替代
|
| 1152 |
-
# ✗ tensorflow>=2.0 (体积大)
|
| 1153 |
-
# ✓ torch>=2.0 (相对更小)
|
| 1154 |
-
|
| 1155 |
-
# 禁用不必要的功能
|
| 1156 |
-
# 在 torch 安装时添加:
|
| 1157 |
-
--no-deps torch # 跳过可选依赖
|
| 1158 |
-
pip install torch --no-deps
|
| 1159 |
-
```
|
| 1160 |
-
|
| 1161 |
-
### 7. 文档与注释
|
| 1162 |
-
|
| 1163 |
-
**README.md 内容建议:**
|
| 1164 |
-
|
| 1165 |
-
```markdown
|
| 1166 |
-
---
|
| 1167 |
-
title: My ML Demo
|
| 1168 |
-
sdk: gradio
|
| 1169 |
-
app_file: app.py
|
| 1170 |
-
---
|
| 1171 |
-
|
| 1172 |
-
# My ML Demo
|
| 1173 |
-
|
| 1174 |
-
## 简介
|
| 1175 |
-
这是一个使用 Transformers 库的情感分析演示应用。
|
| 1176 |
-
|
| 1177 |
-
## 功能
|
| 1178 |
-
- 文本情感分析
|
| 1179 |
-
- 多语言支持(英语、中文)
|
| 1180 |
-
- 实时结果展示
|
| 1181 |
-
|
| 1182 |
-
## 使用方法
|
| 1183 |
-
1. 输入文本
|
| 1184 |
-
2. 点击"分析"按钮
|
| 1185 |
-
3. 查看情感分类结果
|
| 1186 |
-
|
| 1187 |
-
## 技术栈
|
| 1188 |
-
- Gradio 4.44.0
|
| 1189 |
-
- Transformers 4.40.0
|
| 1190 |
-
- PyTorch 2.3.0
|
| 1191 |
-
|
| 1192 |
-
## 作者
|
| 1193 |
-
Your Name (@your-username)
|
| 1194 |
-
|
| 1195 |
-
## 许可证
|
| 1196 |
-
Apache 2.0
|
| 1197 |
-
```
|
| 1198 |
-
|
| 1199 |
-
**代码注释规范:**
|
| 1200 |
-
|
| 1201 |
-
```python
|
| 1202 |
-
"""
|
| 1203 |
-
应用主文件:app.py
|
| 1204 |
-
功能:Hugging Face Spaces 情感分析演示
|
| 1205 |
-
作者:Your Name
|
| 1206 |
-
日期:2026-07-13
|
| 1207 |
-
"""
|
| 1208 |
-
|
| 1209 |
-
import gradio as gr
|
| 1210 |
-
from transformers import pipeline
|
| 1211 |
-
import os
|
| 1212 |
-
|
| 1213 |
-
# 模型加载函数(使用缓存优化性能)
|
| 1214 |
-
@gr.cache_resource
|
| 1215 |
-
def load_sentiment_model():
|
| 1216 |
-
"""
|
| 1217 |
-
加载情感分析模型
|
| 1218 |
-
返回:Transformers pipeline 对象
|
| 1219 |
-
"""
|
| 1220 |
-
model_name = "distilbert-base-uncased-finetuned-sst-2-english"
|
| 1221 |
-
|
| 1222 |
-
# 从环境变量读取 Token(如有私有模型)
|
| 1223 |
-
hf_token = os.getenv('HF_TOKEN')
|
| 1224 |
-
|
| 1225 |
-
# 创建 pipeline(自动处理模型下载和缓存)
|
| 1226 |
-
pipe = pipeline(
|
| 1227 |
-
"sentiment-analysis",
|
| 1228 |
-
model=model_name,
|
| 1229 |
-
token=hf_token # 可选:访问私有模型
|
| 1230 |
-
)
|
| 1231 |
-
|
| 1232 |
-
return pipe
|
| 1233 |
-
|
| 1234 |
-
# Gradio 接口定义
|
| 1235 |
-
def analyze_text(text):
|
| 1236 |
-
"""
|
| 1237 |
-
分析输入文本的情感
|
| 1238 |
-
参数:text(str) - 用户输入文本
|
| 1239 |
-
返回:dict - 情感分析结果(标签和分数)
|
| 1240 |
-
"""
|
| 1241 |
-
if not text:
|
| 1242 |
-
return {"label": "无输入", "score": 0.0}
|
| 1243 |
-
|
| 1244 |
-
# 获取模型(缓存的)
|
| 1245 |
-
pipe = load_sentiment_model()
|
| 1246 |
-
|
| 1247 |
-
# 执行分析
|
| 1248 |
-
result = pipe(text)[0]
|
| 1249 |
-
|
| 1250 |
-
return result
|
| 1251 |
-
|
| 1252 |
-
# 创建 Gradio 应用
|
| 1253 |
-
demo = gr.Interface(
|
| 1254 |
-
fn=analyze_text,
|
| 1255 |
-
inputs=gr.Textbox(label="输入文本", placeholder="请输入要分析的文本..."),
|
| 1256 |
-
outputs=gr.JSON(label="分析结果"),
|
| 1257 |
-
title="情感分析演示",
|
| 1258 |
-
description="使用 DistilBERT 模型进行文本情感分析"
|
| 1259 |
-
)
|
| 1260 |
-
|
| 1261 |
-
# 启动应用(Spaces 自动调用)
|
| 1262 |
-
if __name__ == "__main__":
|
| 1263 |
-
demo.launch()
|
| 1264 |
-
```
|
| 1265 |
-
|
| 1266 |
-
### 8. 版本控制最佳实践
|
| 1267 |
-
|
| 1268 |
-
**提交信息规范:**
|
| 1269 |
-
|
| 1270 |
-
```bash
|
| 1271 |
-
# 提交信息格式
|
| 1272 |
-
git commit -m "<类型>: <简短描述>"
|
| 1273 |
-
|
| 1274 |
-
# 类型标签:
|
| 1275 |
-
# feat: 新功能
|
| 1276 |
-
# fix: Bug 修复
|
| 1277 |
-
# docs: 文档更新
|
| 1278 |
-
# style: 格式调整
|
| 1279 |
-
# refactor: 重构代码
|
| 1280 |
-
# test: 测试相关
|
| 1281 |
-
# chore: 构建/工具更新
|
| 1282 |
-
|
| 1283 |
-
# 示例:
|
| 1284 |
-
git commit -m "feat: Add multi-language support"
|
| 1285 |
-
git commit -m "fix: Resolve model loading timeout issue"
|
| 1286 |
-
git commit -m "docs: Update README with usage instructions"
|
| 1287 |
-
git commit -m "chore: Upgrade transformers to 4.40.0"
|
| 1288 |
-
```
|
| 1289 |
-
|
| 1290 |
-
**分支命名规范:**
|
| 1291 |
-
|
| 1292 |
-
```bash
|
| 1293 |
-
# 功能分支
|
| 1294 |
-
feature-add-model
|
| 1295 |
-
feature-optimize-performance
|
| 1296 |
-
|
| 1297 |
-
# 修复分支
|
| 1298 |
-
fix-authentication-error
|
| 1299 |
-
fix-memory-leak
|
| 1300 |
-
|
| 1301 |
-
# 文档分支
|
| 1302 |
-
docs-readme-update
|
| 1303 |
-
docs-api-guide
|
| 1304 |
-
|
| 1305 |
-
# 发布分支
|
| 1306 |
-
release-v1.0.0
|
| 1307 |
-
release-v2.0.0
|
| 1308 |
-
```
|
| 1309 |
-
|
| 1310 |
-
---
|
| 1311 |
-
|
| 1312 |
-
## 附录
|
| 1313 |
-
|
| 1314 |
-
### A.常用命令速查表
|
| 1315 |
-
|
| 1316 |
-
| 操作 | 命令 |
|
| 1317 |
-
|------|------|
|
| 1318 |
-
| 安装 Git | `brew install git` (macOS) |
|
| 1319 |
-
| 克隆 Space | `git clone https://huggingface.co/spaces/user/space` |
|
| 1320 |
-
| 配置认证 | `huggingface-cli login --token hf_xxxx` |
|
| 1321 |
-
| 查看认证状态 | `huggingface-cli whoami` |
|
| 1322 |
-
| 添加文件 | `git add app.py requirements.txt` |
|
| 1323 |
-
| 提交修改 | `git commit -m "Update message"` |
|
| 1324 |
-
| 推送到远程 | `git push origin main` |
|
| 1325 |
-
| 上传文件(CLI) | `huggingface-cli upload user/space ./file` |
|
| 1326 |
-
| 查看远程仓库 | `git remote -v` |
|
| 1327 |
-
| 查看提交历史 | `git log --oneline -10` |
|
| 1328 |
-
|
| 1329 |
-
### B.Hugging Face CLI 常用命令
|
| 1330 |
-
|
| 1331 |
-
```bash
|
| 1332 |
-
# 认证管理
|
| 1333 |
-
huggingface-cli login # 交互式登录
|
| 1334 |
-
huggingface-cli login --token hf_xxx # Token 登录
|
| 1335 |
-
huggingface-cli logout # 退出登录
|
| 1336 |
-
huggingface-cli whoami # 查看当前用户
|
| 1337 |
-
|
| 1338 |
-
# 仓库管理
|
| 1339 |
-
huggingface-cli repo create my-space --type space --space-sdk gradio
|
| 1340 |
-
huggingface-cli repo delete user/space --type space
|
| 1341 |
-
|
| 1342 |
-
# 文件上传
|
| 1343 |
-
huggingface-cli upload user/space ./app.py
|
| 1344 |
-
huggingface-cli upload user/space ./folder --commit-message "Add files"
|
| 1345 |
-
|
| 1346 |
-
# 文件下载
|
| 1347 |
-
huggingface-cli download user/model ./model.safetensors
|
| 1348 |
-
|
| 1349 |
-
# 仓库信息
|
| 1350 |
-
huggingface-cli repo-info user/space --repo-type space
|
| 1351 |
-
```
|
| 1352 |
-
|
| 1353 |
-
### C.环境变量说明
|
| 1354 |
-
|
| 1355 |
-
| 变量名 | 说明 | 使用场景 |
|
| 1356 |
-
|--------|------|----------|
|
| 1357 |
-
| `HF_TOKEN` | Hugging Face 访问令牌 | CI/CD,私有仓库访问 |
|
| 1358 |
-
| `HF_HOME` | Hugging Face 缓存目录 | 自定义缓存位置 |
|
| 1359 |
-
| `HUGGINGFACE_HUB_CACHE` | Hub 缓存目录 | 模型缓存路径 |
|
| 1360 |
-
| `TRANSFORMERS_CACHE` | Transformers 缓存 | 模型文件缓存 |
|
| 1361 |
-
| `HF_DATASETS_CACHE` | 数据集缓存 | 数据集文件缓存 |
|
| 1362 |
-
|
| 1363 |
-
### D.相关资源链接
|
| 1364 |
-
|
| 1365 |
-
- **官方文档**: https://huggingface.co/docs
|
| 1366 |
-
- **Spaces 概览**: https://huggingface.co/docs/hub/spaces-overview
|
| 1367 |
-
- **配置参考**: https://huggingface.co/docs/hub/spaces-config-reference
|
| 1368 |
-
- **依赖管理**: https://huggingface.co/docs/hub/spaces-dependencies
|
| 1369 |
-
- **Git 操作指南**: https://huggingface.co/docs/hub/repositories-getting-started
|
| 1370 |
-
- **Token 设置**: https://huggingface.co/settings/tokens
|
| 1371 |
-
- **SSH 密钥配置**: https://huggingface.co/settings/keys
|
| 1372 |
-
- **Python SDK 文档**: https://huggingface.co/docs/huggingface_hub
|
| 1373 |
-
- **CLI 工具指南**: https://huggingface.co/docs/huggingface_hub/guides/cli
|
| 1374 |
-
|
| 1375 |
-
---
|
| 1376 |
-
|
| 1377 |
-
**文档版本**: 2.0
|
| 1378 |
-
**更新日期**: 2026-07-13
|
| 1379 |
-
**基于**: Hugging Face 官方文档(2026版本)及社区最佳实践
|
| 1380 |
-
**更新说明**: 移除 Git LFS 相关内容,新增网页控制台克隆功能说明
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
README.md
CHANGED
|
@@ -1,257 +1,227 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
-
sdk:
|
| 7 |
-
|
| 8 |
-
python_version: '3.13'
|
| 9 |
-
app_file: app.py
|
| 10 |
-
pinned: false
|
| 11 |
-
license: mit
|
| 12 |
-
short_description: 测试空间
|
| 13 |
---
|
| 14 |
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
-
|
| 24 |
-
-
|
| 25 |
-
-
|
| 26 |
-
-
|
| 27 |
-
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
```
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
-
|
|
|
|
|
|
|
| 70 |
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
-
|
| 74 |
-
-
|
| 75 |
-
- Logbook:本地需有 `../data/logbook_availability.csv` 数据文件
|
| 76 |
-
- GFW:本地需有 GFW ZIP 数据包,或自动从 `~/Desktop/资料下载/柔鱼数据/` 搜索
|
| 77 |
|
| 78 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
|
| 80 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
|
|
|
|
|
|
| 87 |
|
| 88 |
-
|
| 89 |
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
```
|
| 94 |
|
| 95 |
-
##
|
| 96 |
|
| 97 |
-
|
| 98 |
-
pip install -r requirements.txt
|
| 99 |
-
```
|
| 100 |
|
| 101 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 102 |
|
| 103 |
-
|
| 104 |
-
|------|---------|------|
|
| 105 |
-
| `smolagents[gradio]` | >=1.0.0 | Agent 框架 + Gradio 集成 |
|
| 106 |
-
| `requests` | >=2.31.0 | HTTP 请求 |
|
| 107 |
-
| `pandas` | >=2.0.0 | 数据处理 |
|
| 108 |
-
| `huggingface_hub` | >=0.23.0 | HF 数据集 API |
|
| 109 |
-
| `openpyxl` | >=3.1.0 | Excel 文件解析 |
|
| 110 |
-
| `openai` | >=2.0.0 | OpenAI 兼容 LLM 接入 |
|
| 111 |
|
| 112 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
|
| 114 |
-
|
|
|
|
| 115 |
|
| 116 |
-
```python
|
| 117 |
-
# Hugging Face 数据集访问 Token
|
| 118 |
-
HF_TOKEN = "hf_你的Token"
|
| 119 |
-
|
| 120 |
-
# LLM API 配置
|
| 121 |
-
OPENAI_API_KEY = "sk-你的APIKey"
|
| 122 |
-
OPENAI_API_BASE = "https://api.deepseek.com/v1" # API 地址
|
| 123 |
-
MODEL_ID = "deepseek-chat" # 模型 ID
|
| 124 |
-
|
| 125 |
-
# DeepSeek 思考模式
|
| 126 |
-
THINKING_MODE_ENABLED = True # True 启用,False 禁用
|
| 127 |
-
THINKING_EFFORT = "high" # low/medium/high/max
|
| 128 |
-
```
|
| 129 |
|
| 130 |
-
|
|
|
|
|
|
|
|
|
|
| 131 |
|
| 132 |
-
| 服务 | `OPENAI_API_BASE` | `MODEL_ID` |
|
| 133 |
-
|------|-------------------|------------|
|
| 134 |
-
| DeepSeek | `https://api.deepseek.com/v1` | `deepseek-chat` |
|
| 135 |
-
| OpenAI | `https://api.openai.com/v1` | `gpt-4o-mini` |
|
| 136 |
-
| 本地 Ollama | `http://localhost:11434/v1` | 模型名称 |
|
| 137 |
-
| 智谱 GLM | `https://open.bigmodel.cn/api/paas/v4` | `glm-4-flash` |
|
| 138 |
|
| 139 |
-
|
| 140 |
|
| 141 |
-
|
|
|
|
|
|
|
|
|
|
| 142 |
|
| 143 |
-
```bash
|
| 144 |
-
cd /your/workspace/myspace
|
| 145 |
-
python3 app.py
|
| 146 |
-
```
|
| 147 |
|
| 148 |
-
|
| 149 |
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
|
| 156 |
-
浏览器打开上述地址即可使用 Chat 界面进行查询。
|
| 157 |
|
| 158 |
-
##
|
| 159 |
|
| 160 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 161 |
|
| 162 |
-
```bash
|
| 163 |
-
# 前台运行
|
| 164 |
-
python3 app.py
|
| 165 |
|
| 166 |
-
#
|
| 167 |
-
nohup python3 app.py > agent.log 2>&1 &
|
| 168 |
-
```
|
| 169 |
|
| 170 |
-
|
|
|
|
|
|
|
|
|
|
| 171 |
|
| 172 |
-
```dockerfile
|
| 173 |
-
FROM python:3.12-slim
|
| 174 |
|
| 175 |
-
|
| 176 |
-
COPY requirements.txt .
|
| 177 |
-
RUN pip install --no-cache-dir -r requirements.txt
|
| 178 |
|
| 179 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 180 |
|
| 181 |
-
#
|
| 182 |
-
EXPOSE 7860
|
| 183 |
|
| 184 |
-
|
| 185 |
-
```
|
|
|
|
|
|
|
|
|
|
| 186 |
|
| 187 |
-
```bash
|
| 188 |
-
docker build -t fishery-agent .
|
| 189 |
-
docker run -d -p 7860:7860 \
|
| 190 |
-
-e HF_TOKEN=hf_你的Token \
|
| 191 |
-
-e OPENAI_API_KEY=sk_你的Key \
|
| 192 |
-
fishery-agent
|
| 193 |
-
```
|
| 194 |
|
| 195 |
-
##
|
| 196 |
|
| 197 |
-
```
|
| 198 |
-
[Unit]
|
| 199 |
-
Description=Fishery Data Query Agent
|
| 200 |
-
After=network.target
|
| 201 |
|
| 202 |
-
|
| 203 |
-
Type=simple
|
| 204 |
-
User=your_user
|
| 205 |
-
WorkingDirectory=/your/workspace/myspace
|
| 206 |
-
ExecStart=/your/venv/bin/python3 app.py
|
| 207 |
-
Restart=on-failure
|
| 208 |
-
RestartSec=5
|
| 209 |
|
| 210 |
-
|
| 211 |
-
WantedBy=multi-user.target
|
| 212 |
-
```
|
| 213 |
|
| 214 |
-
## 使用示例
|
| 215 |
|
| 216 |
-
##
|
|
|
|
|
|
|
|
|
|
| 217 |
|
| 218 |
-
```
|
| 219 |
-
用户:2010到2020年东南太平洋有哪些鱿鱼捕捞日志数据?
|
| 220 |
-
```
|
| 221 |
|
| 222 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
|
| 224 |
-
##
|
| 225 |
|
| 226 |
-
```
|
| 227 |
-
用户:查询2012年1月太平洋区域中国渔船的作业努力量
|
| 228 |
-
```
|
| 229 |
|
| 230 |
-
|
| 231 |
|
| 232 |
-
|
| 233 |
|
| 234 |
-
### 示例 3:无明确工具指向的查询
|
| 235 |
|
| 236 |
-
|
| 237 |
-
用户:帮我查一下有什么数据可以用
|
| 238 |
-
```
|
| 239 |
|
| 240 |
-
|
| 241 |
-
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
| 工具名 | 类名 | 功能 | 必填参数 |
|
| 245 |
-
|--------|------|------|---------|
|
| 246 |
-
| `parameter_extractor` | `ParameterExtractionTool` | 从自然语言提取数据集查询参数 | `user_input` |
|
| 247 |
-
| `logbook_query` | `LogbookQueryTool` | 查询 Logbook 捕捞日志可用性 | 无(均为可选) |
|
| 248 |
-
| `gfw_query` | `GfwQueryTool` | 查询 GFW 渔船作业努力量(2012-2024) | `year`, `month`, `lat_min`, `lat_max`, `lon_min`, `lon_max` |
|
| 249 |
-
| `literature_cpue_query` | `LiteratureCpueQueryTool` | 查询 CPUE 相关文献数据 | 无(均为可选) |
|
| 250 |
-
| `sprfmo_query` | `SprfmoQueryTool` | 查询 SPRFMO 南太平洋数据 | `data_type` |
|
| 251 |
-
|
| 252 |
-
**注意:**
|
| 253 |
-
- `logbook_query` 支持可选参数:`region`(海区)、`year_start`(起始年份)、`year_end`(结束年份)、`species`(物种)、`data_type`(数据类型)
|
| 254 |
-
- `gfw_query` 仅支持查询 **2012-2024 年**的数据,超出范围会返回错误
|
| 255 |
-
- `gfw_query` 支持可选参数:`flag`(船旗国)、`geartype`(渔具类型)、`min_fishing_hours`(最小捕捞小时数)
|
| 256 |
-
- `literature_cpue_query` 支持可选参数:`region`(海区)、`year_start`、`year_end`、`species`、`response_variable`(响应变量)、`paper_type`(论文类型)
|
| 257 |
-
- `sprfmo_query` 必须指定 `data_type`('catch' 或 'effort'),支持可选参数:`country`(国家)、`year_start`、`year_end`、`species`(仅捕捞量)、`gear_type`(仅努力量)、`group_by`(聚合维度)
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Global Marine Foundation Data Agent
|
| 3 |
+
emoji: 🌊
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
+
# Global Marine Foundation Data Agent
|
| 11 |
+
|
| 12 |
+
Current UI release: **v3.3.0**.
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
## v3.3.0 稳定性重构(第一阶段)
|
| 16 |
+
|
| 17 |
+
- 新增根目录 `VERSION`,后端与前端统一从同一版本源渲染,避免版本号漂移。
|
| 18 |
+
- 运行时仅服务根目录 `app.html`;发布包不再携带第二份 `templates/app.html`。
|
| 19 |
+
- 新增 `scripts/preflight.py`,启动前检查 Hugging Face README 元数据、Docker 入口、单一模板、版本占位符与缓存污染。
|
| 20 |
+
- 新增 `.dockerignore`,发布镜像排除 `.pytest_cache`、`__pycache__`、`.pyc` 等开发缓存。
|
| 21 |
+
- 项目数据包下载 token 改为磁盘索引;配置 Persistent Storage 时可在 Space 重启后继续恢复未过期 ZIP。
|
| 22 |
+
- `/api/ui/info` 增加版本来源、单模板状态与收藏/项目数据包存储模式,方便部署诊断。
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
V2.3.0 makes the home Ocean / Tuna / Squid cards functional: they now load live catalog/inventory state, open a data-center panel, jump to filtered dataset views, and start domain-aware queries. The sidebar now adds live badges, Ocean/Tuna/Squid shortcuts, recent-conversation search, and synced favorites/task counters.
|
| 26 |
+
|
| 27 |
+
V2.2.5 routes IATTC/ICCAT/IOTC/CCSBT and Ocean variable-only prompts such as
|
| 28 |
+
`v10` directly to the Fisheries/Ocean data runtime. Ocean export requests are
|
| 29 |
+
now accepted as complete only after the real export tool returns either an
|
| 30 |
+
actual download URL or an explicit server error; progress-only claims are
|
| 31 |
+
rejected instead of being shown as successful submissions.
|
| 32 |
+
|
| 33 |
+
V2.2.4 keeps a 15-minute per-thread pending data request so short follow-ups
|
| 34 |
+
such as “确认” or “继续” are routed back to the Ocean/Fisheries tool runtime with
|
| 35 |
+
the original parameters instead of being sent to the tool-free Harness path.
|
| 36 |
+
Ocean export requests without an explicit format now default to NetCDF and run
|
| 37 |
+
immediately without asking for a second confirmation.
|
| 38 |
+
|
| 39 |
+
V2.2.3 accepts `query` and `source` as compatibility aliases for Fisheries
|
| 40 |
+
inventory calls, while retaining `domain` and `keyword` as the preferred
|
| 41 |
+
schema. Literal `<function_calls>`, `<invoke>` and `<parameter>` blocks are now
|
| 42 |
+
removed before browser output; a response containing only leaked tool-planning
|
| 43 |
+
text fails closed instead of exposing internal markup.
|
| 44 |
+
|
| 45 |
+
V2.2.2 recognizes ICCAT `YearC` as a year field and returns separately named
|
| 46 |
+
filtered-raw, exact-deduplicated and annual-summary CSV exports. Repeated copies
|
| 47 |
+
of the same download URL are shown only once in the chat UI.
|
| 48 |
+
|
| 49 |
+
V2.2.1 restores the dual Hugging Face fisheries repository route after the
|
| 50 |
+
V2.2 content-analysis refactor. Inventory, search, live UI context, file
|
| 51 |
+
validation, download, analysis and export now preserve repository provenance
|
| 52 |
+
for both `globalsquiddatabase/squid_dataset` and
|
| 53 |
+
`globalsquiddatabase/Tuna-Fisheries-Dataset`. Repository selection is restricted
|
| 54 |
+
to these two configured datasets.
|
| 55 |
+
|
| 56 |
+
V2.2 adds a restricted Hugging Face fisheries content plane. The Agent can
|
| 57 |
+
now validate an exact live-tree path, pin the current Dataset commit SHA,
|
| 58 |
+
download CSV/TSV or CSV/TSV members inside ZIP files within configured size
|
| 59 |
+
limits, report actual fields/rows/missing values/duplicates, apply year and
|
| 60 |
+
bounding-box filters, calculate monthly counts and annual sums, and return a
|
| 61 |
+
tokenized CSV download URL. It does not enable arbitrary Shell, URL, repository
|
| 62 |
+
or filesystem access. Large ZIP processing uses a streamed local cache and a
|
| 63 |
+
longer MCP timeout; production-scale repeated GFW queries should still be moved
|
| 64 |
+
to query-ready partitions on the school server.
|
| 65 |
+
|
| 66 |
+
V1.6.1 fixes Hugging Face inventory counting: directory entries carrying
|
| 67 |
+
`size=0` are no longer counted as files. Sources that are absent from the live
|
| 68 |
+
main branch are labeled as not ingested rather than as read failures.
|
| 69 |
|
| 70 |
+
V1.7 adds Fisheries source-category filtering, ingested/missing source totals,
|
| 71 |
+
classification evidence in dataset details, and a UTF-8 CSV export of the
|
| 72 |
+
current live inventory.
|
| 73 |
|
| 74 |
+
V1.8 adds a live Hugging Face repository hygiene report covering zero-byte
|
| 75 |
+
files, duplicate basenames, unmapped paths, large files, extension distribution
|
| 76 |
+
and repository-residue files, with a downloadable JSON report. These checks do
|
| 77 |
+
not claim that same-name files have identical content.
|
|
|
|
|
|
|
| 78 |
|
| 79 |
+
V1.9 translates common service fields and states into Chinese, adds actionable
|
| 80 |
+
chat error recovery controls, task progress indicators and JSON result export,
|
| 81 |
+
and introduces functional longitude-range and local/UTC display preferences.
|
| 82 |
+
Conversation history and favorites remain browser-local until the Memory API
|
| 83 |
+
offers stable record update/delete operations; the UI does not claim that they
|
| 84 |
+
are synchronized across devices.
|
| 85 |
|
| 86 |
+
V2.0 adds explicit dataset metadata-completeness evidence, including returned
|
| 87 |
+
and missing core fields in details and comparisons. Settings now include a
|
| 88 |
+
default export format, answer detail level, 10/30/50-item local history limits,
|
| 89 |
+
and optional download confirmation. Static dataset catalogs are maintained in
|
| 90 |
+
`sidebar_catalog.py`, while the main workspace page lives in
|
| 91 |
+
`templates/app.html`; this reduces the risk of UI edits changing data-routing
|
| 92 |
+
rules. Cross-device conversation and favorite synchronization remains disabled
|
| 93 |
+
until stable server-side update/delete endpoints exist.
|
| 94 |
|
| 95 |
+
V2.1 adds a consolidated metadata audit for all configured Ocean and Fisheries
|
| 96 |
+
sources. It reports evidence-based completeness, missing core fields and a
|
| 97 |
+
recommended next action, and exports the audit as UTF-8 CSV. Settings also add
|
| 98 |
+
a validated JSON workspace backup/restore flow for browser-local conversations,
|
| 99 |
+
favorites and preferences. Backups exclude credentials, tokens and uploaded
|
| 100 |
+
file contents, providing a manual cross-device migration path without claiming
|
| 101 |
+
server-side synchronization.
|
| 102 |
|
| 103 |
+
Custom chat UI backed by CodeWhale/DeepSeek and a restricted Marine MCP bridge.
|
| 104 |
|
| 105 |
+
Public UI: port 7860.
|
| 106 |
+
Private CodeWhale Runtime: 127.0.0.1:7878.
|
| 107 |
+
The Runtime token never enters the browser.
|
|
|
|
| 108 |
|
| 109 |
+
## Functional sidebar
|
| 110 |
|
| 111 |
+
The main UI provides six real views:
|
|
|
|
|
|
|
| 112 |
|
| 113 |
+
- 对话助手:chat, uploads, recent conversations and generated downloads.
|
| 114 |
+
- 数据集:live Ocean catalog plus the Hugging Face fisheries main-tree inventory, with Chinese labels, filters, favorites, on-demand details, live Ocean date/variable availability checks, searchable Fisheries file inventory, comparison for up to three datasets and CSV comparison export.
|
| 115 |
+
- 数据服务:runtime, Marine API, Ocean/Tuna/Squid domains and Hugging Face health.
|
| 116 |
+
- 我的任务:user-isolated processing tasks and generated assets, with Chinese operation/status labels, search/status filters and structured quality-check summaries.
|
| 117 |
+
- 收藏:browser-persistent dataset and assistant-answer bookmarks scoped by user id.
|
| 118 |
+
- 设置:font size, Enter behavior, status refresh, accent color, default view, compact/reduced-motion modes, local-history controls and deployment version information.
|
| 119 |
|
| 120 |
+
Read-only sidebar endpoints:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
|
| 122 |
+
- `GET /api/sidebar/datasets`
|
| 123 |
+
- `GET /api/sidebar/datasets/{group}/{source_key}`
|
| 124 |
+
- `POST /api/sidebar/datasets/ocean/{source_key}/availability`
|
| 125 |
+
- `GET /api/sidebar/datasets/fisheries/{source_key}/files`
|
| 126 |
+
- `GET /api/sidebar/datasets/quality`
|
| 127 |
+
- `GET /api/sidebar/datasets/metadata-audit`
|
| 128 |
+
- `GET /api/sidebar/services`
|
| 129 |
+
- `GET /api/sidebar/tasks?user_id=...`
|
| 130 |
|
| 131 |
+
Authenticated deployments validate the Supabase bearer token on every endpoint.
|
| 132 |
+
The task endpoint filters server-side by the resolved authenticated user id.
|
| 133 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
|
| 135 |
+
## v2.3.1 card click reliability fix
|
| 136 |
+
- Ocean/Tuna/Squid homepage cards now have direct click and keyboard handlers plus JS fallbacks.
|
| 137 |
+
- Added visible v2.3.1 marker in header to verify the deployed frontend is the new build.
|
| 138 |
+
- If the modal container is unavailable, card click falls back to the filtered datasets view instead of doing nothing.
|
| 139 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
+
## v2.3.3 fisheries service summary fix
|
| 142 |
|
| 143 |
+
- Fixed `Hugging Face Fisheries` showing `undefined` in the Data Services view.
|
| 144 |
+
- The services API now exposes a stable combined repository label and repository availability counts.
|
| 145 |
+
- Added frontend fallbacks so missing optional API fields never render as `undefined`.
|
| 146 |
+
- Updated the visible build marker and default app version to v2.3.3.
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
|
| 149 |
+
## v2.4.0 fisheries file workflow
|
| 150 |
|
| 151 |
+
- Fisheries file rows are now clickable instead of display-only.
|
| 152 |
+
- Added per-file detail card with repository, file size and extension.
|
| 153 |
+
- Added one-click handoff from a concrete Tuna/Squid file into the chat for analysis.
|
| 154 |
+
- Added file-level favorites and copy-path actions.
|
| 155 |
+
- Kept live Hugging Face inventory and existing v2.3.3 service summary behavior unchanged.
|
| 156 |
|
|
|
|
| 157 |
|
| 158 |
+
## v2.4.1 metadata audit only
|
| 159 |
|
| 160 |
+
This build is based directly on v2.4.0. It intentionally does **not** include the
|
| 161 |
+
experimental dataset-card click-handler changes. The only functional change is the
|
| 162 |
+
metadata audit: Ocean and Fisheries now use separate evidence rules, Fisheries
|
| 163 |
+
file-content fields are shown as pending file-level verification rather than false
|
| 164 |
+
0% missing metadata, and Ocean audit evidence is merged from the Marine API catalog,
|
| 165 |
+
status and domains endpoints.
|
| 166 |
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
+
## v2.5.0 favorites workflow
|
|
|
|
|
|
|
| 169 |
|
| 170 |
+
- Preserves the v2.4.1 metadata-audit-only behavior and all existing dataset buttons/data connections.
|
| 171 |
+
- Fisheries file favorites are now a first-class `file` type instead of being mixed with dataset favorites.
|
| 172 |
+
- Favorites page adds dataset/file/answer counts, type filter, search, file repository/path display, copy-path action, and one-click return to chat analysis.
|
| 173 |
+
- Workspace backup/restore preserves the new file favorite type.
|
| 174 |
|
|
|
|
|
|
|
| 175 |
|
| 176 |
+
## v2.6.0 task workflow
|
|
|
|
|
|
|
| 177 |
|
| 178 |
+
- “我的任务”升级为任务工作台:区分处理中、已完成、失败和数据资产。
|
| 179 |
+
- 任务支持详情抽屉、复制任务编号、导出 JSON、携带结构化结果继续对话分析。
|
| 180 |
+
- 后端任务摘要增加关联会话、上传文件和数据来源字段,便于追溯任务上下文。
|
| 181 |
+
- 数据资产支持详情、来源/文件类型展示、下载以及一键进入对话继续分析。
|
| 182 |
+
- 任务和资产均增加搜索筛选,保留 v2.5.0 收藏工作流和 v2.4.1 元数据审计逻辑。
|
| 183 |
|
| 184 |
+
## v2.7.0 service diagnostics
|
|
|
|
| 185 |
|
| 186 |
+
- 数据服务页面新增综合健康状态、健康检查计数、平均响应时延和慢响应统计。
|
| 187 |
+
- Marine API `/health`、`/domains`、`/status/ocean` 与 Hugging Face Fisheries 读取增加毫秒级响应耗时。
|
| 188 |
+
- 新增“服务诊断详情”抽屉,集中展示逐项状态、诊断建议与失败/慢响应项。
|
| 189 |
+
- 支持复制诊断摘要和导出诊断 JSON,便于部署排障和路演现场快速反馈。
|
| 190 |
+
- 保留 v2.6.0 任务工作台、v2.5.0 收藏工作流、v2.4.1 元数据审计与既有数据集按钮行为。
|
| 191 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 192 |
|
| 193 |
+
## v2.8.1 项目数据包
|
| 194 |
|
| 195 |
+
侧边栏新增“项目数据包”。输入研究/项目描述后,系统会基于当前已接入的 Ocean、Tuna 和 Squid 数据目录推荐所需数据,并可生成 ZIP。ZIP 按数据库分类为 `Ocean/`、`Tuna-Fisheries-Dataset/`、`squid_dataset/`;同时包含 `manifest.json` 和 `README.md`。Hugging Face 渔业数据会在配置的包大小上限内复制真实仓库文件;Ocean 在描述中能够识别具体日期和海域时会调用学校 Marine API 导出真实 NetCDF,否则在 `Ocean/data_requests.json` 中保留待补充的请求清单,避免伪造数据。
|
|
|
|
|
|
|
|
|
|
| 196 |
|
| 197 |
+
## v2.8.1 项目数据包
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
|
| 199 |
+
侧边栏新增“项目数据包”。输入研究/项目描述后,系统会基于当前已接入的 Ocean、Tuna 和 Squid 数据目录推荐所需数据,并可生成 ZIP。ZIP 按数据库分类为 `Ocean/`、`Tuna-Fisheries-Dataset/`、`squid_dataset/`;同时包含 `manifest.json` 和 `README.md`。Hugging Face 渔业数据会在配置的包大小上限内复制真实仓库文件;Ocean 在描述中能够识别具体日期和海域时会调用学校 Marine API 导出真实 NetCDF,否则在 `Ocean/data_requests.json` 中保留待补充的请求清单,避免伪造数据。
|
|
|
|
|
|
|
| 200 |
|
|
|
|
| 201 |
|
| 202 |
+
## v2.8.1
|
| 203 |
+
- 项目数据包侧边栏入口增加静态节点 + JavaScript 自愈注入双保险。
|
| 204 |
+
- 新增 `/api/ui/info` 用于确认实际运行模板、版本和项目数据包功能是否加载。
|
| 205 |
+
- 运行时统一以 `templates/app.html` 为唯一服务模板,根目录 `app.html` 仅保持镜像一致。
|
| 206 |
|
|
|
|
|
|
|
|
|
|
| 207 |
|
| 208 |
+
## v2.8.2 单一前端模板修复
|
| 209 |
+
- 运行时只读取仓库根目录 `app.html`,不再依赖 `templates/app.html`。
|
| 210 |
+
- 修复 Hugging Face 网页上传时根目录文件已更新、嵌套模板仍停留旧版导致“版本/功能不一致”的问题。
|
| 211 |
+
- 应用版本改为代码内固定 `2.8.2`,不再被 Space 中遗留的 `APP_VERSION=2.8.0` 变量覆盖。
|
| 212 |
+
- 保留项目数据包侧栏入口及其 ZIP 按数据库分类打包功能。
|
| 213 |
|
| 214 |
+
## v2.9.0 项目数据方案确认
|
| 215 |
|
| 216 |
+
“项目数据包”升级为两阶段工作流:先分析项目并生成推荐数据方案,再由用户勾选/取消具体 Ocean 与 Fisheries 数据源,最后只打包所选来源。界面会提示 Ocean 自动导出所缺的日期/海域条件;未补充时仍可生成 ZIP,并在 Ocean 目录保留可复现的数据请求清单。后端新增 `selected_ocean_keys` 与 `selected_fisheries_names`,确保 ZIP 内容与前端确认清单一致。
|
|
|
|
|
|
|
| 217 |
|
| 218 |
+
## v3.1.0 数据包预估与清单预览
|
| 219 |
|
| 220 |
+
“项目数据包”新增生成前预估:根据 Hugging Face 实时文件树计算所选 Fisheries 候选文件数量与已知大小,并按 Ocean / Tuna-Fisheries-Dataset / squid_dataset 汇总。Ocean NetCDF 因需实际导出,预估阶段明确标记为“大小未知”,避免伪造估算值。ZIP 生成完成后可直接查看已打包文件清单,再决定是否下载;完整来源与跳过原因仍写入 `manifest.json`。
|
| 221 |
|
|
|
|
| 222 |
|
| 223 |
+
## v3.2.0 用户消息快捷操作触发方式
|
|
|
|
|
|
|
| 224 |
|
| 225 |
+
- 设置中新增“用户问题快捷操作”,可选择左键单击或右键单击打开复制、重试、编辑文本。
|
| 226 |
+
- 默认保持左键单击,升级后不改变原有操作习惯。
|
| 227 |
+
- 右键模式只拦截用户消息气泡上的浏览器右键菜单,页面其他区域右键行为不变。
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
VERSION
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3.3.0
|
app.html
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
app.py
DELETED
|
@@ -1,966 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
基于 smolagents 框架的 Hugging Face 数据集自然语言查询智能代理
|
| 3 |
-
|
| 4 |
-
功能:
|
| 5 |
-
1. 启动时自动获取数据集文件列表,作为系统提示词告知用户可用数据
|
| 6 |
-
2. 接收用户自然语言输入,从中提取数据集查询所需的参数信息
|
| 7 |
-
3. 使用提取的参数从 Hugging Face 数据集获取数据(支持 ZIP/CSV/Excel 等格式)
|
| 8 |
-
4. 返回结构化的查询结果
|
| 9 |
-
|
| 10 |
-
依赖安装:
|
| 11 |
-
pip install 'smolagents[gradio]' requests pandas huggingface_hub openpyxl
|
| 12 |
-
|
| 13 |
-
使用前:
|
| 14 |
-
1. 将文件顶部 HF_TOKEN 常量的值替换为个人 Hugging Face Settings 下申请的 Read Token(用于数据集访问)
|
| 15 |
-
2. 将文件顶部 OPENAI_API_KEY 常量的值替换为你的 OpenAI API Key(用于 LLM 接入)
|
| 16 |
-
3. 如使用第三方兼容服务,修改 OPENAI_API_BASE 常量的地址
|
| 17 |
-
"""
|
| 18 |
-
|
| 19 |
-
import os
|
| 20 |
-
import json
|
| 21 |
-
import logging
|
| 22 |
-
import re
|
| 23 |
-
from typing import Optional, List
|
| 24 |
-
|
| 25 |
-
from huggingface_hub import HfApi
|
| 26 |
-
from smolagents import Tool, ToolCallingAgent, OpenAIServerModel, GradioUI
|
| 27 |
-
|
| 28 |
-
# 导入专用查询函数(从 query_tools 子模块导入)
|
| 29 |
-
from query_tools.logbook_availability_query import query_logbook_availability
|
| 30 |
-
from query_tools.query_gfw import query_gfw
|
| 31 |
-
|
| 32 |
-
# ---------------------------------------------------------------------------
|
| 33 |
-
# 日志配置(提前配置,以便在后续代码中使用)
|
| 34 |
-
# ---------------------------------------------------------------------------
|
| 35 |
-
logging.basicConfig(
|
| 36 |
-
level=logging.INFO,
|
| 37 |
-
format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
|
| 38 |
-
)
|
| 39 |
-
logger = logging.getLogger("hf_data_agent")
|
| 40 |
-
|
| 41 |
-
# ===========================================================================
|
| 42 |
-
# Hugging Face 数据集访问 Token(从环境变量加载)
|
| 43 |
-
# ===========================================================================
|
| 44 |
-
HF_TOKEN = os.getenv('HF_TOKEN') # 从环境变量 HF_TOKEN 读取
|
| 45 |
-
|
| 46 |
-
# 启动时校验 HF_TOKEN 是否存在,并给出提示
|
| 47 |
-
if not HF_TOKEN:
|
| 48 |
-
logger.warning(
|
| 49 |
-
"未找到环境变量 HF_TOKEN,将无法访问 Hugging Face 数据集。"
|
| 50 |
-
"请设置环境变量: export HF_TOKEN=hf_你的Token"
|
| 51 |
-
)
|
| 52 |
-
|
| 53 |
-
# ===========================================================================
|
| 54 |
-
# 数据集仓库 ID
|
| 55 |
-
# ===========================================================================
|
| 56 |
-
HF_DATASET_REPO = "squid-lab/squid_dataset" # Hugging Face 数据集仓库 ID
|
| 57 |
-
|
| 58 |
-
# ===========================================================================
|
| 59 |
-
# OpenAI 模型配置常量(从环境变量加载)
|
| 60 |
-
# ===========================================================================
|
| 61 |
-
OPENAI_API_KEY = os.getenv('OPENAI_API_KEY') # 从环境变量 OPENAI_API_KEY 读取
|
| 62 |
-
OPENAI_API_BASE = os.getenv('OPENAI_API_BASE', 'https://api.deepseek.com/v1') # 默认 DeepSeek API 地址
|
| 63 |
-
MODEL_ID = os.getenv('MODEL_ID', 'deepseek-chat')
|
| 64 |
-
|
| 65 |
-
# 启动时校验 OPENAI_API_KEY 是否存在,并给出提示
|
| 66 |
-
if not OPENAI_API_KEY:
|
| 67 |
-
logger.warning(
|
| 68 |
-
"未找到环境变量 OPENAI_API_KEY,Agent 将无法调用 LLM。"
|
| 69 |
-
"请设置环境变量: export OPENAI_API_KEY=sk_你的Key"
|
| 70 |
-
)
|
| 71 |
-
|
| 72 |
-
# 注意:deepseek-chat 和 deepseek-reasoner 将于 2026/07/24 弃用
|
| 73 |
-
# 推荐使用新模型:deepseek-v4-flash(快速)或 deepseek-v4-pro(专业)
|
| 74 |
-
|
| 75 |
-
# ===========================================================================
|
| 76 |
-
# DeepSeek 思考模式配置(已停用)
|
| 77 |
-
# ===========================================================================
|
| 78 |
-
# 注意:DeepSeek 思考模式已停用,原因如下:
|
| 79 |
-
# 1. deepseek-chat 和 deepseek-reasoner 将于 2026/07/24 弃用
|
| 80 |
-
# 2. 新模型 deepseek-v4-flash/v4-pro 不需要特殊的思考模式适配
|
| 81 |
-
# 3. 思考模式不支持 tool_choice 参数,限制 Agent 工具调用能力
|
| 82 |
-
# 4. 使用标准 OpenAIServerModel 更稳定,兼容性更好
|
| 83 |
-
THINKING_MODE_ENABLED = False # 已停用思考模式
|
| 84 |
-
THINKING_EFFORT = "high" # 保留参数但不再使用
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
# ===========================================================================
|
| 88 |
-
# 数据集文件列表获取函数
|
| 89 |
-
# ===========================================================================
|
| 90 |
-
def get_dataset_file_list() -> List[str]:
|
| 91 |
-
"""
|
| 92 |
-
从 Hugging Face 数据集仓库获取一级文件和文件夹名称列表。
|
| 93 |
-
|
| 94 |
-
使用 HfApi.dataset_info() 获取数据集元信息,提取 siblings 中的文件路径,
|
| 95 |
-
只保留一级目录/文件(不包含深层嵌套的文件)。
|
| 96 |
-
|
| 97 |
-
Returns:
|
| 98 |
-
一级文件和文件夹名称列表,如 ['中西太平洋WCPFC/', 'README.md', 'data.csv']
|
| 99 |
-
"""
|
| 100 |
-
logger.info("正在获取数据集文件列表: %s", HF_DATASET_REPO)
|
| 101 |
-
|
| 102 |
-
try:
|
| 103 |
-
api = HfApi(token=HF_TOKEN)
|
| 104 |
-
dataset_info = api.dataset_info(repo_id=HF_DATASET_REPO)
|
| 105 |
-
|
| 106 |
-
# 提取所有文件路径
|
| 107 |
-
all_files = [sibling.rfilename for sibling in dataset_info.siblings]
|
| 108 |
-
|
| 109 |
-
# 只保留一级目录/文件(路径中不包含 '/' 的项,或者一级文件夹)
|
| 110 |
-
first_level_items = set()
|
| 111 |
-
for file_path in all_files:
|
| 112 |
-
parts = file_path.split("/")
|
| 113 |
-
# 一级文件(无子目录)
|
| 114 |
-
if len(parts) == 1:
|
| 115 |
-
first_level_items.add(parts[0])
|
| 116 |
-
# 一级文件夹(取第一部分,添加 '/' 后缀标识为文件夹)
|
| 117 |
-
else:
|
| 118 |
-
first_level_items.add(parts[0] + "/")
|
| 119 |
-
|
| 120 |
-
file_list = sorted(list(first_level_items))
|
| 121 |
-
logger.info("获取到 %d 个一级文件/文件夹: %s", len(file_list), file_list)
|
| 122 |
-
return file_list
|
| 123 |
-
|
| 124 |
-
except Exception as e:
|
| 125 |
-
logger.error("获取数据集文件列表失败: %s", e, exc_info=True)
|
| 126 |
-
return []
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
def build_system_prompt_with_file_list() -> str:
|
| 130 |
-
"""
|
| 131 |
-
构建包含数据集文件列表的系统提示词。
|
| 132 |
-
|
| 133 |
-
在对话启动时告知用户当前可用的数据集文件,帮助用户了解可查询的内容。
|
| 134 |
-
|
| 135 |
-
Returns:
|
| 136 |
-
格式化的系统提示词字符串
|
| 137 |
-
"""
|
| 138 |
-
file_list = get_dataset_file_list()
|
| 139 |
-
|
| 140 |
-
if not file_list:
|
| 141 |
-
return (
|
| 142 |
-
"你是一个 Hugging Face 数据集查询助手。"
|
| 143 |
-
"用户可以询问数据集内容,你将帮助用户查询和获取数据。"
|
| 144 |
-
"当前无法获取数据集文件列表,请检查 HF_TOKEN 配置是否正确。"
|
| 145 |
-
)
|
| 146 |
-
|
| 147 |
-
# 格式化文件列表为提示词
|
| 148 |
-
files_str = "\n".join([f" - {item}" for item in file_list])
|
| 149 |
-
system_prompt = (
|
| 150 |
-
"你是一个渔业数据查询助手,拥有以下工具:\n\n"
|
| 151 |
-
"1. **logbook_query** — 查询 logbook(捕捞日志)数据可用性,支持按海区、年份、物种、数据类型筛选\n"
|
| 152 |
-
"2. **gfw_query** — 查询 GFW(Global Fishing Watch)渔船作业努力量,需要指定年份(2012-2024)、月份和空间范围\n"
|
| 153 |
-
"3. **literature_cpue_query** — 查询 CPUE(单位捕捞努力量渔获量)相关文献数据,支持按海区、年份、物种、响应变量筛选\n"
|
| 154 |
-
"4. **sprfmo_query** — 查询 SPRFMO(南太平洋区域渔业管理组织)数据,支持捕捞量或努力量查询,可按国家、年份、物种等筛选\n\n"
|
| 155 |
-
"当前可查询的 Hugging Face 数据集仓库: " + HF_DATASET_REPO + "\n\n"
|
| 156 |
-
"数据集一级文件和文件夹列表:\n" + files_str + "\n\n"
|
| 157 |
-
"用户查询示例:\n"
|
| 158 |
-
" - '2010-2020年东南太平洋有哪些鱿鱼捕捞日志数据?' → 使用 logbook_query\n"
|
| 159 |
-
" - '查询2012年1月太平洋区域的渔船作业努力量' → 使用 gfw_query\n"
|
| 160 |
-
" - '有哪些关于鱿鱼CPUE的研究文献?' → 使用 literature_cpue_query\n"
|
| 161 |
-
" - '查询中国2015-2020年在南太平洋的渔获量' → 使用 sprfmo_query\n\n"
|
| 162 |
-
"请根据用户需求选择合适的工具,不要混用。\n\n"
|
| 163 |
-
"【重要】最终回答格式要求:\n"
|
| 164 |
-
"在展示查询数据后,必须附加以下信息:\n"
|
| 165 |
-
"---\n"
|
| 166 |
-
"**查询执行详情:**\n"
|
| 167 |
-
"- 使用工具:[本次调用的Tool名称]\n"
|
| 168 |
-
"- 查询参数:\n"
|
| 169 |
-
" - 参数1名称: 参数1值\n"
|
| 170 |
-
" - 参数2名称: 参数2值\n"
|
| 171 |
-
" - ...(列出所有实际传入的参数)\n"
|
| 172 |
-
"---\n"
|
| 173 |
-
"请严格遵守此格式,确保用户清楚了解每次查询的具体执行过程。"
|
| 174 |
-
)
|
| 175 |
-
|
| 176 |
-
logger.info("系统提示词已构建,包含 %d 个文件/文件夹", len(file_list))
|
| 177 |
-
return system_prompt
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
# ===========================================================================
|
| 181 |
-
# DeepSeek 思考模式自定义模型类(已移除)
|
| 182 |
-
# ===========================================================================
|
| 183 |
-
# DeepSeekThinkingModel 类已被移除,原因如下:
|
| 184 |
-
# 1. deepseek-chat 和 deepseek-reasoner 将于 2026/07/24 弃用
|
| 185 |
-
# 2. 新模型 deepseek-v4-flash/v4-pro 使用标准 OpenAI API,无需特殊适配
|
| 186 |
-
# 3. 思考模式不支持 tool_choice,限制了 Agent 的工具调用能力
|
| 187 |
-
# 4. 使用标准 OpenAIServerModel 更稳定,兼容性更好
|
| 188 |
-
# 现在使用标准 OpenAIServerModel,参见 create_hf_data_agent 函数
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
# ===========================================================================
|
| 192 |
-
# Tool 1: 参数提取 —— 从自然语言中解析出数据集查询参数
|
| 193 |
-
# ===========================================================================
|
| 194 |
-
class ParameterExtractionTool(Tool):
|
| 195 |
-
"""从用户的自然语言描述中提取 Hugging Face 数据集查询所需的结构化参数。"""
|
| 196 |
-
|
| 197 |
-
name = "parameter_extractor"
|
| 198 |
-
description = (
|
| 199 |
-
"从用户的自然语言查询请求中提取数据集查询参数,包括:"
|
| 200 |
-
"region(区域,如中西太平洋)、time_range(时间范围,如1967-2024)、"
|
| 201 |
-
"time_scale(时间尺度,如月尺度)、spatial_resolution(空间分辨率,如1x1)、"
|
| 202 |
-
"file_name(文件名)。返回 JSON 格式的参数字典。"
|
| 203 |
-
)
|
| 204 |
-
inputs = {
|
| 205 |
-
"user_query": {
|
| 206 |
-
"type": "string",
|
| 207 |
-
"description": (
|
| 208 |
-
"用户的自然语言查询请求,例如'查询中西太平洋1967-2024年月尺度1x1分辨率的渔获数据'"
|
| 209 |
-
),
|
| 210 |
-
}
|
| 211 |
-
}
|
| 212 |
-
output_type = "string"
|
| 213 |
-
|
| 214 |
-
# --- 区域关键字映射表 ---
|
| 215 |
-
REGION_KEYWORDS = {
|
| 216 |
-
"中西太平洋": "中西太平洋WCPFC",
|
| 217 |
-
"wcpfc": "中西太平洋WCPFC",
|
| 218 |
-
"东太平洋": "东太平洋",
|
| 219 |
-
"太平洋": "中西太平洋WCPFC",
|
| 220 |
-
"印度洋": "印度洋",
|
| 221 |
-
"大西洋": "大西洋",
|
| 222 |
-
"南海": "南海",
|
| 223 |
-
}
|
| 224 |
-
|
| 225 |
-
# --- 时间尺度映射表 ---
|
| 226 |
-
TIME_SCALE_KEYWORDS = {
|
| 227 |
-
"月尺度": "月尺度",
|
| 228 |
-
"月度": "月尺度",
|
| 229 |
-
"月": "月尺度",
|
| 230 |
-
"年尺度": "年尺度",
|
| 231 |
-
"年度": "年尺度",
|
| 232 |
-
"年": "年尺度",
|
| 233 |
-
"日尺度": "日尺度",
|
| 234 |
-
"日度": "日尺度",
|
| 235 |
-
"日": "日尺度",
|
| 236 |
-
}
|
| 237 |
-
|
| 238 |
-
# --- 空间分辨率映射表 ---
|
| 239 |
-
SPATIAL_RESOLUTION_KEYWORDS = {
|
| 240 |
-
"1x1": "1x1",
|
| 241 |
-
"1度": "1x1",
|
| 242 |
-
"0.5x0.5": "0.5x0.5",
|
| 243 |
-
"0.5度": "0.5x0.5",
|
| 244 |
-
"0.1x0.1": "0.1x0.1",
|
| 245 |
-
"0.1度": "0.1x0.1",
|
| 246 |
-
}
|
| 247 |
-
|
| 248 |
-
def forward(self, user_query: str) -> str:
|
| 249 |
-
"""
|
| 250 |
-
解析自然语言,提取数据集查询参数。
|
| 251 |
-
|
| 252 |
-
参数提取流程:
|
| 253 |
-
1. 识别目标区域(优先匹配关键字映射表)
|
| 254 |
-
2. 识别时间范围(起始年份-结束年份)
|
| 255 |
-
3. 识别时间尺度(月尺度/年尺度/日尺度)
|
| 256 |
-
4. 识别空间分辨率(1x1/0.5x0.5等)
|
| 257 |
-
5. 识别文件名(用户指定的具体文件名)
|
| 258 |
-
"""
|
| 259 |
-
logger.info("开始提取参数,输入: %s", user_query)
|
| 260 |
-
|
| 261 |
-
try:
|
| 262 |
-
params = {
|
| 263 |
-
"region": self._extract_region(user_query),
|
| 264 |
-
"time_range": self._extract_time_range(user_query),
|
| 265 |
-
"time_scale": self._extract_time_scale(user_query),
|
| 266 |
-
"spatial_resolution": self._extract_spatial_resolution(user_query),
|
| 267 |
-
"file_name": self._extract_file_name(user_query),
|
| 268 |
-
}
|
| 269 |
-
|
| 270 |
-
# 校验必要参数:区域不能为空
|
| 271 |
-
if not params["region"]:
|
| 272 |
-
error_msg = (
|
| 273 |
-
"无法从输入中识别目标区域,请在查询中明确指定区域。"
|
| 274 |
-
f"支持的区域关键字: {list(self.REGION_KEYWORDS.keys())}"
|
| 275 |
-
)
|
| 276 |
-
logger.warning(error_msg)
|
| 277 |
-
return json.dumps({"error": error_msg}, ensure_ascii=False)
|
| 278 |
-
|
| 279 |
-
logger.info("参数提取成功: %s", params)
|
| 280 |
-
return json.dumps(params, ensure_ascii=False)
|
| 281 |
-
|
| 282 |
-
except Exception as e:
|
| 283 |
-
logger.error("参数提取失败: %s", e, exc_info=True)
|
| 284 |
-
return json.dumps(
|
| 285 |
-
{"error": f"参数提取过程发生错误: {str(e)}"},
|
| 286 |
-
ensure_ascii=False,
|
| 287 |
-
)
|
| 288 |
-
|
| 289 |
-
# ----- 以下为私有辅助方法,按提取维度拆分 -----
|
| 290 |
-
|
| 291 |
-
def _extract_region(self, query: str) -> Optional[str]:
|
| 292 |
-
"""
|
| 293 |
-
从自然语言中识别目标区域。
|
| 294 |
-
优先级:关键字映射 → 直接匹配区域名称
|
| 295 |
-
"""
|
| 296 |
-
# 优先通过关键字映射识别
|
| 297 |
-
for cn_keyword, region_name in self.REGION_KEYWORDS.items():
|
| 298 |
-
if cn_keyword.lower() in query.lower():
|
| 299 |
-
return region_name
|
| 300 |
-
|
| 301 |
-
return None
|
| 302 |
-
|
| 303 |
-
def _extract_time_range(self, query: str) -> Optional[str]:
|
| 304 |
-
"""
|
| 305 |
-
从自然语言中识别时间范围。
|
| 306 |
-
匹配"XXXX-XXXX"格式或"XXXX年到XXXX年"格式。
|
| 307 |
-
"""
|
| 308 |
-
# 匹配"1967-2024"格式
|
| 309 |
-
match = re.search(r"(\d{4})-(\d{4})", query)
|
| 310 |
-
if match:
|
| 311 |
-
return f"{match.group(1)}-{match.group(2)}"
|
| 312 |
-
|
| 313 |
-
# 匹配"1967年到2024年"或"1967至2024"格式
|
| 314 |
-
match = re.search(r"(\d{4})\s*(?:年到|至|-)\s*(\d{4})", query)
|
| 315 |
-
if match:
|
| 316 |
-
return f"{match.group(1)}-{match.group(2)}"
|
| 317 |
-
|
| 318 |
-
# 匹配单一年份"2024年"
|
| 319 |
-
match = re.search(r"(\d{4})\s*年", query)
|
| 320 |
-
if match:
|
| 321 |
-
year = match.group(1)
|
| 322 |
-
return f"{year}-{year}"
|
| 323 |
-
|
| 324 |
-
return None
|
| 325 |
-
|
| 326 |
-
def _extract_time_scale(self, query: str) -> Optional[str]:
|
| 327 |
-
"""
|
| 328 |
-
从自然语言中识别时间尺度。
|
| 329 |
-
优先级:关键字映射 → 默认月尺度
|
| 330 |
-
"""
|
| 331 |
-
for cn_keyword, scale_name in self.TIME_SCALE_KEYWORDS.items():
|
| 332 |
-
if cn_keyword in query:
|
| 333 |
-
return scale_name
|
| 334 |
-
|
| 335 |
-
return "月尺度" # 默认月尺度
|
| 336 |
-
|
| 337 |
-
def _extract_spatial_resolution(self, query: str) -> Optional[str]:
|
| 338 |
-
"""
|
| 339 |
-
从自然语言中识别空间分辨率。
|
| 340 |
-
匹配"XxX"格式或"X度"格式。
|
| 341 |
-
"""
|
| 342 |
-
# 优先通过关键字映射识别
|
| 343 |
-
for cn_keyword, resolution in self.SPATIAL_RESOLUTION_KEYWORDS.items():
|
| 344 |
-
if cn_keyword in query:
|
| 345 |
-
return resolution
|
| 346 |
-
|
| 347 |
-
# 匹配"1x1"格式
|
| 348 |
-
match = re.search(r"(\d+(?:\.\d+)?)x(\d+(?:\.\d+)?)", query)
|
| 349 |
-
if match:
|
| 350 |
-
return f"{match.group(1)}x{match.group(2)}"
|
| 351 |
-
|
| 352 |
-
return "1x1" # 默认1x1分辨率
|
| 353 |
-
|
| 354 |
-
def _extract_file_name(self, query: str) -> Optional[str]:
|
| 355 |
-
"""
|
| 356 |
-
从自然语言中识别具体文件名。
|
| 357 |
-
匹配"xxx.zip"或"xxx数据"模式。
|
| 358 |
-
"""
|
| 359 |
-
# 匹配".zip"结尾的文件名
|
| 360 |
-
match = re.search(r"(\S+)\.zip", query)
|
| 361 |
-
if match:
|
| 362 |
-
return f"{match.group(1)}.zip"
|
| 363 |
-
|
| 364 |
-
# 匹配"xxx文件"或"xxx数据"模式
|
| 365 |
-
match = re.search(r"(\w+)\s*(?:文件|数据)", query)
|
| 366 |
-
if match:
|
| 367 |
-
return f"{match.group(1)}.zip"
|
| 368 |
-
|
| 369 |
-
return None
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
# ===========================================================================
|
| 373 |
-
# Tool 2: Hugging Face 数据集获取 —— 从云端获取并解析数据
|
| 374 |
-
# ===========================================================================
|
| 375 |
-
# ===========================================================================
|
| 376 |
-
# HFDataQueryTool 工具类(已移除)
|
| 377 |
-
# ===========================================================================
|
| 378 |
-
# HFDataQueryTool 已被移除,原因:
|
| 379 |
-
# - 该工具的功能已整合到其他工具中,不再需要单独的数据获取工具
|
| 380 |
-
# - 移除了约 350 行代码,包括文件解析、ZIP 处理、多种格式支持等
|
| 381 |
-
# - Agent 现在主要使用 logbook_query 和 gfw_query 两个专用查询工具
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
# ===========================================================================
|
| 385 |
-
# Tool 3: Logbook 数据可用性查询
|
| 386 |
-
# ===========================================================================
|
| 387 |
-
class LogbookQueryTool(Tool):
|
| 388 |
-
"""查询 logbook 数据可用性,支持按海区、年份、物种、数据类型筛选。"""
|
| 389 |
-
|
| 390 |
-
name = "logbook_query"
|
| 391 |
-
description = (
|
| 392 |
-
"查询 logbook(捕捞日志)数据的可用性。"
|
| 393 |
-
"支持按海区(region)、年份范围(year_start/year_end)、"
|
| 394 |
-
"物种(species)、数据类型(data_type)筛选。"
|
| 395 |
-
"返回统计摘要和预览数据。"
|
| 396 |
-
)
|
| 397 |
-
inputs = {
|
| 398 |
-
"region": {
|
| 399 |
-
"type": "string",
|
| 400 |
-
"description": "海区名称(模糊匹配),如 '东南太平洋'、'中西太平洋'。不指定则不筛选。",
|
| 401 |
-
"nullable": True,
|
| 402 |
-
},
|
| 403 |
-
"year_start": {
|
| 404 |
-
"type": "integer",
|
| 405 |
-
"description": "起始年份(包含),如 2010。不指定则不筛选。",
|
| 406 |
-
"nullable": True,
|
| 407 |
-
},
|
| 408 |
-
"year_end": {
|
| 409 |
-
"type": "integer",
|
| 410 |
-
"description": "结束年份(包含),如 2020。不指定则不筛选。",
|
| 411 |
-
"nullable": True,
|
| 412 |
-
},
|
| 413 |
-
"species": {
|
| 414 |
-
"type": "string",
|
| 415 |
-
"description": "物种名称(模糊匹配),如 '鱿鱼'。不指定则不筛选。",
|
| 416 |
-
"nullable": True,
|
| 417 |
-
},
|
| 418 |
-
"data_type": {
|
| 419 |
-
"type": "string",
|
| 420 |
-
"description": "数据类型(模糊匹配),如 '捕捞日志'。不指定则不筛选。",
|
| 421 |
-
"nullable": True,
|
| 422 |
-
},
|
| 423 |
-
}
|
| 424 |
-
output_type = "string"
|
| 425 |
-
|
| 426 |
-
def forward(
|
| 427 |
-
self,
|
| 428 |
-
region: Optional[str] = None,
|
| 429 |
-
year_start: Optional[int] = None,
|
| 430 |
-
year_end: Optional[int] = None,
|
| 431 |
-
species: Optional[str] = None,
|
| 432 |
-
data_type: Optional[str] = None,
|
| 433 |
-
) -> str:
|
| 434 |
-
"""
|
| 435 |
-
调用 query_logbook_availability 查询 logbook 数据可用性。
|
| 436 |
-
|
| 437 |
-
处理流程:
|
| 438 |
-
1. 将参数传递给 query_logbook_availability 函数
|
| 439 |
-
2. 捕获异常并返回错误信息
|
| 440 |
-
3. 将结果转为 JSON 字符串返回
|
| 441 |
-
"""
|
| 442 |
-
logger.info(
|
| 443 |
-
"Logbook 查询: region=%s, year_start=%s, year_end=%s, species=%s, data_type=%s",
|
| 444 |
-
region, year_start, year_end, species, data_type,
|
| 445 |
-
)
|
| 446 |
-
|
| 447 |
-
try:
|
| 448 |
-
result = query_logbook_availability(
|
| 449 |
-
region=region if region else None,
|
| 450 |
-
year_start=year_start,
|
| 451 |
-
year_end=year_end,
|
| 452 |
-
species=species if species else None,
|
| 453 |
-
data_type=data_type if data_type else None,
|
| 454 |
-
output_format="markdown", # 默认使用 markdown 输出,不写文件
|
| 455 |
-
)
|
| 456 |
-
|
| 457 |
-
# 格式化 summary 为可读字符串(兼容新旧版本)
|
| 458 |
-
summary_data = result.get("summary", {})
|
| 459 |
-
if isinstance(summary_data, dict):
|
| 460 |
-
# 新版本:summary 是字典,格式化为友好字符串
|
| 461 |
-
summary_str = (
|
| 462 |
-
f"找到 {summary_data.get('records_count', 0)} 条 logbook 数据记录\n"
|
| 463 |
-
f"覆盖年份: {summary_data.get('year_range', ['未知', '未知'])[0]}-{summary_data.get('year_range', ['未知', '未知'])[1]}\n"
|
| 464 |
-
f"海区: {', '.join(summary_data.get('regions', ['未知']))}\n"
|
| 465 |
-
f"物种: {', '.join(summary_data.get('species', ['未知']))}"
|
| 466 |
-
)
|
| 467 |
-
else:
|
| 468 |
-
# 旧版本:summary 已经是字符串
|
| 469 |
-
summary_str = summary_data
|
| 470 |
-
|
| 471 |
-
# 确保返回结构完整
|
| 472 |
-
complete_result = {
|
| 473 |
-
"summary": summary_str,
|
| 474 |
-
"records": result.get("records", []),
|
| 475 |
-
"preview_markdown": result.get("preview_markdown", ""),
|
| 476 |
-
"source_files": result.get("source_files", []),
|
| 477 |
-
}
|
| 478 |
-
|
| 479 |
-
return json.dumps(complete_result, ensure_ascii=False, default=str)
|
| 480 |
-
|
| 481 |
-
except Exception as e:
|
| 482 |
-
logger.error("Logbook 查询失败: %s", e, exc_info=True)
|
| 483 |
-
return json.dumps(
|
| 484 |
-
{"error": f"Logbook 查询失败: {str(e)}"},
|
| 485 |
-
ensure_ascii=False,
|
| 486 |
-
)
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
# ===========================================================================
|
| 490 |
-
# Tool 4: GFW 渔船作业努力量查询
|
| 491 |
-
# ===========================================================================
|
| 492 |
-
class GfwQueryTool(Tool):
|
| 493 |
-
"""查询 GFW(Global Fishing Watch)渔船作业努力量数据。"""
|
| 494 |
-
|
| 495 |
-
name = "gfw_query"
|
| 496 |
-
description = (
|
| 497 |
-
"查询 GFW 渔船作业努力量数据。"
|
| 498 |
-
"需要指定年份(仅支持 2012-2024)、月份和空间范围(经纬度)。"
|
| 499 |
-
"可选按船旗国(flag)和渔具类型(geartype)筛选。"
|
| 500 |
-
"注意:lon_min > lon_max 表示查询范围跨越 180° 经线。"
|
| 501 |
-
)
|
| 502 |
-
inputs = {
|
| 503 |
-
"year": {
|
| 504 |
-
"type": "integer",
|
| 505 |
-
"description": "查询年份(仅支持 2012-2024),如 2012。",
|
| 506 |
-
},
|
| 507 |
-
"month": {
|
| 508 |
-
"type": "integer",
|
| 509 |
-
"description": "查询月份(1-12)。",
|
| 510 |
-
},
|
| 511 |
-
"lat_min": {
|
| 512 |
-
"type": "number",
|
| 513 |
-
"description": "最小纬度,如 -10。",
|
| 514 |
-
},
|
| 515 |
-
"lat_max": {
|
| 516 |
-
"type": "number",
|
| 517 |
-
"description": "最大纬度,如 10。",
|
| 518 |
-
},
|
| 519 |
-
"lon_min": {
|
| 520 |
-
"type": "number",
|
| 521 |
-
"description": "最小经度,如 145。大于 lon_max 时表示跨越 180° 经线。",
|
| 522 |
-
},
|
| 523 |
-
"lon_max": {
|
| 524 |
-
"type": "number",
|
| 525 |
-
"description": "最大经度,如 -175。",
|
| 526 |
-
},
|
| 527 |
-
"flag": {
|
| 528 |
-
"type": "string",
|
| 529 |
-
"description": "船旗国代码(精确匹配,不区分大小写),如 'CN'。不指定则不筛选。",
|
| 530 |
-
"nullable": True,
|
| 531 |
-
},
|
| 532 |
-
"geartype": {
|
| 533 |
-
"type": "string",
|
| 534 |
-
"description": "渔具类型(精确匹配,不区分大小写),如 'drifting_longlines'。不指定则不筛选。",
|
| 535 |
-
"nullable": True,
|
| 536 |
-
},
|
| 537 |
-
"min_fishing_hours": {
|
| 538 |
-
"type": "number",
|
| 539 |
-
"description": "最小捕捞小时数阈值,如 10。不指定则不筛选。",
|
| 540 |
-
"nullable": True,
|
| 541 |
-
},
|
| 542 |
-
}
|
| 543 |
-
output_type = "string"
|
| 544 |
-
|
| 545 |
-
def forward(
|
| 546 |
-
self,
|
| 547 |
-
year: int,
|
| 548 |
-
month: int,
|
| 549 |
-
lat_min: float,
|
| 550 |
-
lat_max: float,
|
| 551 |
-
lon_min: float,
|
| 552 |
-
lon_max: float,
|
| 553 |
-
flag: Optional[str] = None,
|
| 554 |
-
geartype: Optional[str] = None,
|
| 555 |
-
min_fishing_hours: Optional[float] = None,
|
| 556 |
-
) -> str:
|
| 557 |
-
"""
|
| 558 |
-
调用 query_gfw 查询 GFW 渔船作业努力量数据。
|
| 559 |
-
|
| 560 |
-
处理流程:
|
| 561 |
-
1. 将参数传递给 query_gfw 函数
|
| 562 |
-
2. 捕获异常并返回错误信息
|
| 563 |
-
3. 将 DataFrame 结果和 summary 转为 JSON 字符串返回
|
| 564 |
-
"""
|
| 565 |
-
logger.info(
|
| 566 |
-
"GFW 查询: year=%s, month=%s, lat=[%s,%s], lon=[%s,%s], flag=%s, geartype=%s, min_fishing_hours=%s",
|
| 567 |
-
year, month, lat_min, lat_max, lon_min, lon_max, flag, geartype, min_fishing_hours,
|
| 568 |
-
)
|
| 569 |
-
|
| 570 |
-
try:
|
| 571 |
-
result_df, summary = query_gfw(
|
| 572 |
-
year=year,
|
| 573 |
-
month=month,
|
| 574 |
-
lat_min=lat_min,
|
| 575 |
-
lat_max=lat_max,
|
| 576 |
-
lon_min=lon_min,
|
| 577 |
-
lon_max=lon_max,
|
| 578 |
-
flag=flag if flag else None,
|
| 579 |
-
geartype=geartype if geartype else None,
|
| 580 |
-
min_fishing_hours=min_fishing_hours,
|
| 581 |
-
)
|
| 582 |
-
|
| 583 |
-
# 将 DataFrame 转为记录列表(限制预览条数)
|
| 584 |
-
display_limit = 100
|
| 585 |
-
records = result_df.head(display_limit).to_dict(orient="records")
|
| 586 |
-
|
| 587 |
-
output = {
|
| 588 |
-
"summary": summary,
|
| 589 |
-
"record_count": len(result_df),
|
| 590 |
-
"display_records": min(len(result_df), display_limit),
|
| 591 |
-
"data_preview": records,
|
| 592 |
-
"columns": list(result_df.columns) if not result_df.empty else [],
|
| 593 |
-
}
|
| 594 |
-
|
| 595 |
-
return json.dumps(output, ensure_ascii=False, default=str)
|
| 596 |
-
|
| 597 |
-
except FileNotFoundError as e:
|
| 598 |
-
# GFW 数据文件未找到
|
| 599 |
-
logger.error("GFW 数据文件未找到: %s", e)
|
| 600 |
-
return json.dumps(
|
| 601 |
-
{"error": f"GFW 数据文件未找到: {str(e)}"},
|
| 602 |
-
ensure_ascii=False,
|
| 603 |
-
)
|
| 604 |
-
except ValueError as e:
|
| 605 |
-
# 参数校验失败(如月份越界、纬度范围错误)
|
| 606 |
-
logger.error("GFW 查询参数错误: %s", e)
|
| 607 |
-
return json.dumps(
|
| 608 |
-
{"error": f"查询参数错误: {str(e)}"},
|
| 609 |
-
ensure_ascii=False,
|
| 610 |
-
)
|
| 611 |
-
except Exception as e:
|
| 612 |
-
logger.error("GFW 查询失败: %s", e, exc_info=True)
|
| 613 |
-
return json.dumps(
|
| 614 |
-
{"error": f"GFW 查询失败: {str(e)}"},
|
| 615 |
-
ensure_ascii=False,
|
| 616 |
-
)
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
# ===========================================================================
|
| 620 |
-
# Tool 5: 文献 CPUE 查询
|
| 621 |
-
# ===========================================================================
|
| 622 |
-
class LiteratureCpueQueryTool(Tool):
|
| 623 |
-
"""查询 CPUE(单位捕捞努力量渔获量)相关文献数据。"""
|
| 624 |
-
|
| 625 |
-
name = "literature_cpue_query"
|
| 626 |
-
description = (
|
| 627 |
-
"查询 CPUE(单位捕捞努力量渔获量)相关文献数据。"
|
| 628 |
-
"支持按海区、年份范围、物种、响应变量、论文类型筛选。"
|
| 629 |
-
"返回文献统计摘要和预览数据。"
|
| 630 |
-
)
|
| 631 |
-
inputs = {
|
| 632 |
-
"region": {
|
| 633 |
-
"type": "string",
|
| 634 |
-
"description": "海区名称(模糊匹配),如 '东南太平洋'、'中西太平洋'。不指定则不筛选。",
|
| 635 |
-
"nullable": True,
|
| 636 |
-
},
|
| 637 |
-
"year_start": {
|
| 638 |
-
"type": "integer",
|
| 639 |
-
"description": "起始年份(包含),如 2010。不指定则不筛选。",
|
| 640 |
-
"nullable": True,
|
| 641 |
-
},
|
| 642 |
-
"year_end": {
|
| 643 |
-
"type": "integer",
|
| 644 |
-
"description": "结束年份(包含),如 2020。不指定则不筛选。",
|
| 645 |
-
"nullable": True,
|
| 646 |
-
},
|
| 647 |
-
"species": {
|
| 648 |
-
"type": "string",
|
| 649 |
-
"description": "物种名称(模糊匹配),如 '鱿鱼'。不指定则不筛选。",
|
| 650 |
-
"nullable": True,
|
| 651 |
-
},
|
| 652 |
-
"response_variable": {
|
| 653 |
-
"type": "string",
|
| 654 |
-
"description": "响应变量(模糊匹配),如 'CPUE'、'biomass'。不指定则不筛选。",
|
| 655 |
-
"nullable": True,
|
| 656 |
-
},
|
| 657 |
-
"paper_type": {
|
| 658 |
-
"type": "string",
|
| 659 |
-
"description": "论文类型(模糊匹配)。不指定则不筛选。",
|
| 660 |
-
"nullable": True,
|
| 661 |
-
},
|
| 662 |
-
}
|
| 663 |
-
output_type = "string"
|
| 664 |
-
|
| 665 |
-
def forward(
|
| 666 |
-
self,
|
| 667 |
-
region: Optional[str] = None,
|
| 668 |
-
year_start: Optional[int] = None,
|
| 669 |
-
year_end: Optional[int] = None,
|
| 670 |
-
species: Optional[str] = None,
|
| 671 |
-
response_variable: Optional[str] = None,
|
| 672 |
-
paper_type: Optional[str] = None,
|
| 673 |
-
) -> str:
|
| 674 |
-
"""
|
| 675 |
-
调用 query_literature_cpue 查询 CPUE 文献数据。
|
| 676 |
-
|
| 677 |
-
处理流程:
|
| 678 |
-
1. 将参数传递给 query_literature_cpue 函数
|
| 679 |
-
2. 捕获异常并返回错误信息
|
| 680 |
-
3. 将结果转为 JSON 字符串返回
|
| 681 |
-
"""
|
| 682 |
-
logger.info(
|
| 683 |
-
"CPUE 文献查询: region=%s, year_start=%s, year_end=%s, species=%s, response_variable=%s, paper_type=%s",
|
| 684 |
-
region, year_start, year_end, species, response_variable, paper_type,
|
| 685 |
-
)
|
| 686 |
-
|
| 687 |
-
try:
|
| 688 |
-
# 动态导入以避免循环依赖
|
| 689 |
-
from query_tools.literature_cpue_query import query_literature_cpue
|
| 690 |
-
|
| 691 |
-
result = query_literature_cpue(
|
| 692 |
-
region=region if region else None,
|
| 693 |
-
year_start=year_start,
|
| 694 |
-
year_end=year_end,
|
| 695 |
-
species=species if species else None,
|
| 696 |
-
response_variable=response_variable if response_variable else None,
|
| 697 |
-
paper_type=paper_type if paper_type else None,
|
| 698 |
-
output_format="markdown", # 默认使用 markdown 输出,不写文件
|
| 699 |
-
)
|
| 700 |
-
|
| 701 |
-
# 格式化 summary 为可读字符串
|
| 702 |
-
summary_data = result.get("summary", {})
|
| 703 |
-
if isinstance(summary_data, dict):
|
| 704 |
-
summary_str = (
|
| 705 |
-
f"找到 {summary_data.get('paper_count', 0)} 篇 CPUE 相关文献\n"
|
| 706 |
-
f"覆盖年份: {summary_data.get('year_range', ['未知', '未知'])[0]}-{summary_data.get('year_range', ['未知', '未知'])[1]}\n"
|
| 707 |
-
f"涉及海区: {', '.join(summary_data.get('regions', ['未知']))}\n"
|
| 708 |
-
f"涉及物种: {', '.join(summary_data.get('species', ['未知']))}\n"
|
| 709 |
-
f"使用模型: {', '.join(summary_data.get('models', ['未知']))}"
|
| 710 |
-
)
|
| 711 |
-
else:
|
| 712 |
-
summary_str = summary_data
|
| 713 |
-
|
| 714 |
-
# 确保返回结构完整
|
| 715 |
-
complete_result = {
|
| 716 |
-
"summary": summary_str,
|
| 717 |
-
"records": result.get("records", []),
|
| 718 |
-
"preview_markdown": result.get("preview_markdown", ""),
|
| 719 |
-
"source_files": result.get("source_files", []),
|
| 720 |
-
}
|
| 721 |
-
|
| 722 |
-
return json.dumps(complete_result, ensure_ascii=False, default=str)
|
| 723 |
-
|
| 724 |
-
except Exception as e:
|
| 725 |
-
logger.error("CPUE 文献查询失败: %s", e, exc_info=True)
|
| 726 |
-
return json.dumps(
|
| 727 |
-
{"error": f"CPUE 文献查询失败: {str(e)}"},
|
| 728 |
-
ensure_ascii=False,
|
| 729 |
-
)
|
| 730 |
-
|
| 731 |
-
|
| 732 |
-
# ===========================================================================
|
| 733 |
-
# Tool 6: SPRFMO 南太平洋数据查询
|
| 734 |
-
# ===========================================================================
|
| 735 |
-
class SprfmoQueryTool(Tool):
|
| 736 |
-
"""查询 SPRFMO(南太平洋区域渔业管理组织)数据。"""
|
| 737 |
-
|
| 738 |
-
name = "sprfmo_query"
|
| 739 |
-
description = (
|
| 740 |
-
"查询 SPRFMO(南太平洋区域渔业管理组织)数据。"
|
| 741 |
-
"支持查询捕捞量(catch)或努力量(effort)数据。"
|
| 742 |
-
"可按国家、年份范围、物种等筛选,支持按维度聚合统计。"
|
| 743 |
-
"空间分辨率:5x5 度;时间分辨率:年度。"
|
| 744 |
-
)
|
| 745 |
-
inputs = {
|
| 746 |
-
"data_type": {
|
| 747 |
-
"type": "string",
|
| 748 |
-
"description": "数据类型,'catch'(捕捞量)或 'effort'(努力量)。必填。",
|
| 749 |
-
},
|
| 750 |
-
"country": {
|
| 751 |
-
"type": "string",
|
| 752 |
-
"description": "国家代码或名称(如 'CHN'、'中国'、'JPN')。不指定则不筛选。",
|
| 753 |
-
"nullable": True,
|
| 754 |
-
},
|
| 755 |
-
"year_start": {
|
| 756 |
-
"type": "integer",
|
| 757 |
-
"description": "起始年份(包含)。不指定则不筛选。",
|
| 758 |
-
"nullable": True,
|
| 759 |
-
},
|
| 760 |
-
"year_end": {
|
| 761 |
-
"type": "integer",
|
| 762 |
-
"description": "结束年份(包含)。不指定则不筛选。",
|
| 763 |
-
"nullable": True,
|
| 764 |
-
},
|
| 765 |
-
"species": {
|
| 766 |
-
"type": "string",
|
| 767 |
-
"description": "物种名称(仅捕捞量数据)。不指定则不筛选。",
|
| 768 |
-
"nullable": True,
|
| 769 |
-
},
|
| 770 |
-
"gear_type": {
|
| 771 |
-
"type": "string",
|
| 772 |
-
"description": "渔具类型(仅努力量数据)。不指定则不筛选。",
|
| 773 |
-
"nullable": True,
|
| 774 |
-
},
|
| 775 |
-
"group_by": {
|
| 776 |
-
"type": "string",
|
| 777 |
-
"description": "聚合维度,如 'year'、'country'、'species'。多个维度用逗号分隔。不指定则不聚合。",
|
| 778 |
-
"nullable": True,
|
| 779 |
-
},
|
| 780 |
-
}
|
| 781 |
-
output_type = "string"
|
| 782 |
-
|
| 783 |
-
def forward(
|
| 784 |
-
self,
|
| 785 |
-
data_type: str,
|
| 786 |
-
country: Optional[str] = None,
|
| 787 |
-
year_start: Optional[int] = None,
|
| 788 |
-
year_end: Optional[int] = None,
|
| 789 |
-
species: Optional[str] = None,
|
| 790 |
-
gear_type: Optional[str] = None,
|
| 791 |
-
group_by: Optional[str] = None,
|
| 792 |
-
) -> str:
|
| 793 |
-
"""
|
| 794 |
-
调用 query_sprfmo 查询 SPRFMO 数据。
|
| 795 |
-
|
| 796 |
-
处理流程:
|
| 797 |
-
1. 构建 filters 字典
|
| 798 |
-
2. 解析 group_by 参数
|
| 799 |
-
3. 调用 query_sprfmo 函数
|
| 800 |
-
4. 捕获异常并返回错误信息
|
| 801 |
-
5. 将结果转为 JSON 字符串返回
|
| 802 |
-
"""
|
| 803 |
-
logger.info(
|
| 804 |
-
"SPRFMO 查询: data_type=%s, country=%s, year=[%s,%s], species=%s, gear_type=%s, group_by=%s",
|
| 805 |
-
data_type, country, year_start, year_end, species, gear_type, group_by,
|
| 806 |
-
)
|
| 807 |
-
|
| 808 |
-
try:
|
| 809 |
-
# 动态导入以避免循环依赖
|
| 810 |
-
from query_tools.query_sprfmo import query_sprfmo
|
| 811 |
-
|
| 812 |
-
# 构建 filters 字典
|
| 813 |
-
filters = {}
|
| 814 |
-
if country:
|
| 815 |
-
filters["country"] = country
|
| 816 |
-
if year_start is not None:
|
| 817 |
-
filters["year_start"] = year_start
|
| 818 |
-
if year_end is not None:
|
| 819 |
-
filters["year_end"] = year_end
|
| 820 |
-
if species and data_type == "catch":
|
| 821 |
-
filters["species"] = species
|
| 822 |
-
if gear_type and data_type == "effort":
|
| 823 |
-
filters["gear_type"] = gear_type
|
| 824 |
-
|
| 825 |
-
# 解析 group_by 参数
|
| 826 |
-
group_by_list = None
|
| 827 |
-
if group_by:
|
| 828 |
-
group_by_list = [g.strip() for g in group_by.split(",")]
|
| 829 |
-
|
| 830 |
-
# 调用查询函数
|
| 831 |
-
result = query_sprfmo(
|
| 832 |
-
filters=filters,
|
| 833 |
-
group_by=group_by_list,
|
| 834 |
-
metrics=data_type, # metrics 参数直接使用 data_type
|
| 835 |
-
data_type=data_type,
|
| 836 |
-
output_format="markdown",
|
| 837 |
-
)
|
| 838 |
-
|
| 839 |
-
# 格式化 summary
|
| 840 |
-
summary_data = result.get("summary", {})
|
| 841 |
-
summary_str = (
|
| 842 |
-
f"数据类型: {summary_data.get('data_type', '未知')}\n"
|
| 843 |
-
f"总记录数: {summary_data.get('total_records', 0)}\n"
|
| 844 |
-
f"数据来源: {summary_data.get('data_source', 'SPRFMO')}\n"
|
| 845 |
-
)
|
| 846 |
-
if "year_range" in summary_data:
|
| 847 |
-
summary_str += f"年份范围: {summary_data['year_range']}\n"
|
| 848 |
-
if "total_catch" in summary_data:
|
| 849 |
-
summary_str += f"总捕捞量: {summary_data['total_catch']:.2f} kg\n"
|
| 850 |
-
if "total_effort" in summary_data:
|
| 851 |
-
summary_str += f"总努力量: {summary_data['total_effort']:.2f} 天\n"
|
| 852 |
-
|
| 853 |
-
# 确保返回结构完整
|
| 854 |
-
complete_result = {
|
| 855 |
-
"summary": summary_str,
|
| 856 |
-
"records": result.get("records", []),
|
| 857 |
-
"preview_markdown": result.get("preview_markdown", ""),
|
| 858 |
-
"source_files": result.get("source_files", []),
|
| 859 |
-
"warnings": result.get("warnings", []),
|
| 860 |
-
"metadata": result.get("metadata", {}),
|
| 861 |
-
}
|
| 862 |
-
|
| 863 |
-
return json.dumps(complete_result, ensure_ascii=False, default=str)
|
| 864 |
-
|
| 865 |
-
except FileNotFoundError as e:
|
| 866 |
-
# SPRFMO 数据文件未找到
|
| 867 |
-
logger.error("SPRFMO 数据文件未找到: %s", e)
|
| 868 |
-
return json.dumps(
|
| 869 |
-
{"error": f"SPRFMO 数据文件未找到: {str(e)}"},
|
| 870 |
-
ensure_ascii=False,
|
| 871 |
-
)
|
| 872 |
-
except Exception as e:
|
| 873 |
-
logger.error("SPRFMO 查询失败: %s", e, exc_info=True)
|
| 874 |
-
return json.dumps(
|
| 875 |
-
{"error": f"SPRFMO 查询失败: {str(e)}"},
|
| 876 |
-
ensure_ascii=False,
|
| 877 |
-
)
|
| 878 |
-
|
| 879 |
-
|
| 880 |
-
# ===========================================================================
|
| 881 |
-
# Agent 工厂函数:创建配置好的 ToolCallingAgent
|
| 882 |
-
# ===========================================================================
|
| 883 |
-
def create_hf_data_agent(
|
| 884 |
-
model_id: str = MODEL_ID,
|
| 885 |
-
api_key: Optional[str] = None,
|
| 886 |
-
api_base: Optional[str] = None,
|
| 887 |
-
max_steps: int = 5,
|
| 888 |
-
) -> ToolCallingAgent:
|
| 889 |
-
"""
|
| 890 |
-
创建并返回一个配置好的 Hugging Face 数据集查询智能代理。
|
| 891 |
-
|
| 892 |
-
在 Agent 启动时会自动获取数据集文件列表并作为系统提示词告知用户。
|
| 893 |
-
|
| 894 |
-
注意:DeepSeek 思考模式已停用,现在使用标准 OpenAIServerModel。
|
| 895 |
-
原因:deepseek-chat/deepseek-reasoner 将于 2026/07/24 弃用,
|
| 896 |
-
新模型 deepseek-v4-flash/v4-pro 使用标准 OpenAI API。
|
| 897 |
-
|
| 898 |
-
Args:
|
| 899 |
-
model_id: 模型 ID,默认使用文件顶部 MODEL_ID 常量
|
| 900 |
-
推荐: deepseek-v4-flash 或 deepseek-v4-pro
|
| 901 |
-
api_key: API Key,默认使用文件顶部 OPENAI_API_KEY 常量
|
| 902 |
-
api_base: API 地址,默认使用文件顶部 OPENAI_API_BASE 常量
|
| 903 |
-
max_steps: Agent 最大推理步骤数
|
| 904 |
-
|
| 905 |
-
Returns:
|
| 906 |
-
配置好的 ToolCallingAgent 实例
|
| 907 |
-
"""
|
| 908 |
-
key = api_key or OPENAI_API_KEY
|
| 909 |
-
base = api_base or OPENAI_API_BASE
|
| 910 |
-
|
| 911 |
-
# 初始化 LLM 模型(使用标准 OpenAIServerModel,已停用思考模式)
|
| 912 |
-
model = OpenAIServerModel(
|
| 913 |
-
model_id=model_id,
|
| 914 |
-
api_key=key,
|
| 915 |
-
api_base=base,
|
| 916 |
-
)
|
| 917 |
-
|
| 918 |
-
# 实例化工具(包含新增的 CPUE 文献和 SPRFMO 查询工具)
|
| 919 |
-
param_tool = ParameterExtractionTool()
|
| 920 |
-
logbook_tool = LogbookQueryTool()
|
| 921 |
-
gfw_tool = GfwQueryTool()
|
| 922 |
-
literature_cpue_tool = LiteratureCpueQueryTool()
|
| 923 |
-
sprfmo_tool = SprfmoQueryTool()
|
| 924 |
-
|
| 925 |
-
# 构建包含数据集文件列表的指令(会被插入到系统提示词中)
|
| 926 |
-
instructions = build_system_prompt_with_file_list()
|
| 927 |
-
logger.info("Agent 自定义指令: %s", instructions[:200] + "...") # 只打印前200字符
|
| 928 |
-
|
| 929 |
-
# 创建 ToolCallingAgent,使用 instructions 参数设置自定义指令
|
| 930 |
-
# 工具列表包含5个查询工具(已移除 hf_data_tool)
|
| 931 |
-
agent = ToolCallingAgent(
|
| 932 |
-
tools=[param_tool, logbook_tool, gfw_tool, literature_cpue_tool, sprfmo_tool],
|
| 933 |
-
model=model,
|
| 934 |
-
max_steps=max_steps,
|
| 935 |
-
instructions=instructions,
|
| 936 |
-
)
|
| 937 |
-
|
| 938 |
-
logger.info("Hugging Face 数据集查询代理创建成功,模型: %s", model_id)
|
| 939 |
-
return agent
|
| 940 |
-
|
| 941 |
-
|
| 942 |
-
# ===========================================================================
|
| 943 |
-
# 主入口:GradioUI 交互式 Web 界面
|
| 944 |
-
# ===========================================================================
|
| 945 |
-
def main():
|
| 946 |
-
"""
|
| 947 |
-
使用 GradioUI 启动交互式 Web 界面。
|
| 948 |
-
用户在浏览器中输入自然语言查询,代理实时展示思考过程和数据获取结果。
|
| 949 |
-
界面基于 gr.ChatInterface,支持流式输出和步骤可视化。
|
| 950 |
-
"""
|
| 951 |
-
agent = create_hf_data_agent()
|
| 952 |
-
|
| 953 |
-
# GradioUI 将 agent 包装为 gr.ChatInterface Web 应用
|
| 954 |
-
# - share=True: 生成公网可访问的临时链接(72小时有效)
|
| 955 |
-
# - reset_agent_memory=False: 保留对话上下文,允许多轮交互
|
| 956 |
-
demo = GradioUI(
|
| 957 |
-
agent,
|
| 958 |
-
reset_agent_memory=False,
|
| 959 |
-
)
|
| 960 |
-
|
| 961 |
-
logger.info("正在启动 GradioUI Web 界面...")
|
| 962 |
-
demo.launch(share=True)
|
| 963 |
-
|
| 964 |
-
|
| 965 |
-
if __name__ == "__main__":
|
| 966 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
data/.gitkeep
DELETED
|
@@ -1 +0,0 @@
|
|
| 1 |
-
placeholder
|
|
|
|
|
|
data/logbook_availability.csv
DELETED
|
@@ -1,246 +0,0 @@
|
|
| 1 |
-
region,year,species,data_type,resolution,source_file,notes
|
| 2 |
-
中西太平洋 WCPFC,1967,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 3 |
-
中西太平洋 WCPFC,1968,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 4 |
-
中西太平洋 WCPFC,1969,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 5 |
-
中西太平洋 WCPFC,1970,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 6 |
-
中西太平洋 WCPFC,1971,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 7 |
-
中西太平洋 WCPFC,1972,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 8 |
-
中西太平洋 WCPFC,1973,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 9 |
-
中西太平洋 WCPFC,1974,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 10 |
-
中西太平洋 WCPFC,1975,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 11 |
-
中西太平洋 WCPFC,1976,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 12 |
-
中西太平洋 WCPFC,1977,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 13 |
-
中西太平洋 WCPFC,1978,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 14 |
-
中西太平洋 WCPFC,1979,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 15 |
-
中西太平洋 WCPFC,1980,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 16 |
-
中西太平洋 WCPFC,1981,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 17 |
-
中西太平洋 WCPFC,1982,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 18 |
-
中西太平洋 WCPFC,1983,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 19 |
-
中西太平洋 WCPFC,1984,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 20 |
-
中西太平洋 WCPFC,1985,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 21 |
-
中西太平洋 WCPFC,1986,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 22 |
-
中西太平洋 WCPFC,1987,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 23 |
-
中西太平洋 WCPFC,1988,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 24 |
-
中西太平洋 WCPFC,1989,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 25 |
-
中西太平洋 WCPFC,1990,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 26 |
-
中西太平洋 WCPFC,1991,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 27 |
-
中西太平洋 WCPFC,1992,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 28 |
-
中西太平洋 WCPFC,1993,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 29 |
-
中西太平洋 WCPFC,1994,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 30 |
-
中西太平洋 WCPFC,1995,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 31 |
-
中西太平洋 WCPFC,1996,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 32 |
-
中西太平洋 WCPFC,1997,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 33 |
-
中西太平洋 WCPFC,1998,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 34 |
-
中西太平洋 WCPFC,1999,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 35 |
-
中西太平洋 WCPFC,2000,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 36 |
-
中西太平洋 WCPFC,2001,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 37 |
-
中西太平洋 WCPFC,2002,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 38 |
-
中西太平洋 WCPFC,2003,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 39 |
-
中西太平洋 WCPFC,2004,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 40 |
-
中西太平洋 WCPFC,2005,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 41 |
-
中西太平洋 WCPFC,2006,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 42 |
-
中西太平洋 WCPFC,2007,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 43 |
-
中西太平洋 WCPFC,2008,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 44 |
-
中西太平洋 WCPFC,2009,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 45 |
-
中西太平洋 WCPFC,2010,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 46 |
-
中西太平洋 WCPFC,2011,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 47 |
-
中西太平洋 WCPFC,2012,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 48 |
-
中西太平洋 WCPFC,2013,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 49 |
-
中西太平洋 WCPFC,2014,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 50 |
-
中西太平洋 WCPFC,2015,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 51 |
-
中西太平洋 WCPFC,2016,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 52 |
-
中西太平洋 WCPFC,2017,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 53 |
-
中西太平洋 WCPFC,2018,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 54 |
-
中西太平洋 WCPFC,2019,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 55 |
-
中西太平洋 WCPFC,2020,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 56 |
-
中西太平洋 WCPFC,2021,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 57 |
-
中西太平洋 WCPFC,2022,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 58 |
-
中西太平洋 WCPFC,2023,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 59 |
-
中西太平洋 WCPFC,2024,柔鱼类,捕捞量(月尺度),1°×1°,中西太平洋WCPFC 1967-2024月尺1x1/,根据 Hugging Face 数据集目录名称整理
|
| 60 |
-
东南太平洋(南太平洋 SPRFMO),2007,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 61 |
-
东南太平洋(南太平洋 SPRFMO),2008,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 62 |
-
东南太平洋(南太平洋 SPRFMO),2009,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 63 |
-
东南太平洋(南太平洋 SPRFMO),2010,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 64 |
-
东南太平洋(南太平洋 SPRFMO),2011,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 65 |
-
东南太平洋(南太平洋 SPRFMO),2012,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 66 |
-
东南太平洋(南太平洋 SPRFMO),2013,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 67 |
-
东南太平洋(南太平洋 SPRFMO),2014,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 68 |
-
东南太平洋(南太平洋 SPRFMO),2015,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 69 |
-
东南太平洋(南太平洋 SPRFMO),2016,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 70 |
-
东南太平洋(南太平洋 SPRFMO),2017,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 71 |
-
东南太平洋(南太平洋 SPRFMO),2018,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 72 |
-
东南太平洋(南太平洋 SPRFMO),2019,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 73 |
-
东南太平洋(南太平洋 SPRFMO),2020,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 74 |
-
东南太平洋(南太平洋 SPRFMO),2021,柔鱼类,捕捞量+捕捞努力度(年尺度),5°×5°,SPRFMO 南太平洋 2007-2021 捕捞量+努力度 5x5 年尺/,根据 Hugging Face 数据集目录名称整理
|
| 75 |
-
全球(太平洋、印度洋、大西洋、南海),1998,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 76 |
-
全球(太平洋、印度洋、大西洋、南海),1999,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 77 |
-
全球(太平洋、印度洋、大西洋、南海),2000,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 78 |
-
全球(太平洋、印度洋、大西洋、南海),2001,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 79 |
-
全球(太平洋、印度洋、大西洋、南海),2002,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 80 |
-
全球(太平洋、印度洋、大西洋、南海),2003,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 81 |
-
全球(太平洋、印度洋、大西洋、南海),2004,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 82 |
-
全球(太平洋、印度洋、大西洋、南海),2005,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 83 |
-
全球(太平洋、印度洋、大西洋、南海),2006,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 84 |
-
全球(太平洋、印度洋、大西洋、南海),2007,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 85 |
-
全球(太平洋、印度洋、大西洋、南海),2008,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 86 |
-
全球(太平洋、印度洋、大西洋、南海),2009,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 87 |
-
全球(太平洋、印度洋、大西洋、南海),2010,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 88 |
-
全球(太平洋、印度洋、大西洋、南海),2011,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 89 |
-
全球(太平洋、印度洋、大西洋、南海),2012,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 90 |
-
全球(太平洋、印度洋、大西洋、南海),2013,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 91 |
-
全球(太平洋、印度洋、大西洋、南海),2014,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 92 |
-
全球(太平洋、印度洋、大西洋、南海),2015,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 93 |
-
全球(太平洋、印度洋、大西洋、南海),2016,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 94 |
-
全球(太平洋、印度洋、大西洋、南海),2017,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 95 |
-
全球(太平洋、印度洋、大西洋、南海),2018,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 96 |
-
全球(太平洋、印度洋、大西洋、南海),2019,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 97 |
-
全球(太平洋、印度洋、大西洋、南海),2020,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 98 |
-
全球(太平洋、印度洋、大西洋、南海),2021,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 99 |
-
全球(太平洋、印度洋、大西洋、南海),2022,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 100 |
-
全球(太平洋、印度洋、大西洋、南海),2023,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 101 |
-
全球(太平洋、印度洋、大西洋、南海),2024,柔鱼科,捕捞量(年尺度),无空间分辨率,FAO全球柔鱼科捕捞量1998-2024年无测量分辨率.xlsx,根据 Hugging Face 数据集文件名称整理
|
| 102 |
-
全球(太平洋、印度洋、大西洋、南海),1950,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 103 |
-
全球(太平洋、印度洋、大西洋、南海),1951,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 104 |
-
全球(太平洋、印度洋、大西洋、南海),1952,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 105 |
-
全球(太平洋、印度洋、大西洋、南海),1953,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 106 |
-
全球(太平洋、印度洋、大西洋、南海),1954,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 107 |
-
全球(太平洋、印度洋、大西洋、南海),1955,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 108 |
-
全球(太平洋、印度洋、大西洋、南海),1956,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 109 |
-
全球(太平洋、印度洋、大西洋、南海),1957,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 ��球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 110 |
-
全球(太平洋、印度洋、大西洋、南海),1958,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 111 |
-
全球(太平洋、印度洋、大西洋、南海),1959,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 112 |
-
全球(太平洋、印度洋、大西洋、南海),1960,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 113 |
-
全球(太平洋、印度洋、大西洋、南海),1961,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 114 |
-
全球(太平洋、印度洋、大西洋、南海),1962,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 115 |
-
全球(太平洋、印度洋、大西洋、南海),1963,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 116 |
-
全球(太平洋、印度洋、大西洋、南海),1964,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 117 |
-
全球(太平洋、印度洋、大西洋、南海),1965,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 118 |
-
全球(太平洋、印度洋、大西洋、南海),1966,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 119 |
-
全球(太平洋、印度洋、大西洋、南海),1967,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 120 |
-
全球(太平洋、印度洋、大西洋、南海),1968,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 121 |
-
全球(太平洋、印度洋、大西洋、南海),1969,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 122 |
-
全球(太平洋、印度洋、大西洋、南海),1970,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 123 |
-
全球(太平洋、印度洋、大西洋、南海),1971,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 124 |
-
全球(太平洋、印度洋、大西洋、南海),1972,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 125 |
-
全球(太平洋、印度洋、大西洋、南海),1973,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 126 |
-
全球(太平洋、印度洋、大西洋、南海),1974,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 127 |
-
全球(太平洋、印度洋、大西洋、南海),1975,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 128 |
-
全球(太平洋、印度洋、大西洋、南海),1976,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 129 |
-
全球(太平洋、印度洋、大西洋、南海),1977,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 130 |
-
全球(太平洋、印度洋、大西洋、南海),1978,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 131 |
-
全球(太平洋、印度洋、大西洋、南海),1979,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 132 |
-
全球(太平洋、印度洋、大西洋、南海),1980,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 133 |
-
全球(太平洋、印度洋、大西洋、南海),1981,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 134 |
-
全球(太平洋、印度洋、大西洋、南海),1982,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 135 |
-
全球(太平洋、印度洋、大西洋、南海),1983,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 136 |
-
全球(太平洋、印度洋、大西洋、南海),1984,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 137 |
-
全球(太平洋、印度洋、大西洋、南海),1985,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 138 |
-
全球(太平洋、印度洋、大西洋、南海),1986,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 139 |
-
全球(太平洋、印度洋、大西洋、南海),1987,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 140 |
-
全球(太平洋、印度洋、大西洋、南海),1988,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 141 |
-
全球(太平洋、印度洋、大西洋、南海),1989,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 142 |
-
全球(太平洋、印度洋、大西洋、南海),1990,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 143 |
-
全球(太平洋、印度洋、大西洋、南海),1991,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 144 |
-
全球(太平洋、印度洋、大西洋、南海),1992,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 145 |
-
全球(太平洋、印度洋、大西洋、南海),1993,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 146 |
-
全球(太平洋、印度洋、大西洋、南海),1994,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 147 |
-
全球(太平洋、印度洋、大西洋、南海),1995,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 148 |
-
全球(太平洋、印度洋、大西洋、南海),1996,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 149 |
-
全球(太平洋、印度洋、大西洋、南海),1997,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 150 |
-
全球(太平洋、印度洋、大西洋、南海),1998,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 151 |
-
全球(太平洋、印度洋、大西洋、南海),1999,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 152 |
-
全球(太平洋、印度洋、大西洋、南海),2000,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 153 |
-
全球(太平洋、印度洋、大西洋、南海),2001,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 154 |
-
全球(太平洋、印度洋、大西洋、南海),2002,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 155 |
-
全球(太平洋、印度洋、大西洋、南海),2003,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 156 |
-
全球(太平洋、印度洋、大西洋、南海),2004,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 157 |
-
全球(太平洋、印度洋、大西洋、南海),2005,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 158 |
-
全球(太平洋、印度洋、大西洋、南海),2006,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录���称整理
|
| 159 |
-
全球(太平洋、印度洋、大西洋、南海),2007,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 160 |
-
全球(太平洋、印度洋、大西洋、南海),2008,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 161 |
-
全球(太平洋、印度洋、大西洋、南海),2009,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 162 |
-
全球(太平洋、印度洋、大西洋、南海),2010,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 163 |
-
全球(太平洋、印度洋、大西洋、南海),2011,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 164 |
-
全球(太平洋、印度洋、大西洋、南海),2012,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 165 |
-
全球(太平洋、印度洋、大西洋、南海),2013,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 166 |
-
全球(太平洋、印度洋、大西洋、南海),2014,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 167 |
-
全球(太平洋、印度洋、大西洋、南海),2015,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 168 |
-
全球(太平洋、印度洋、大西洋、南海),2016,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 169 |
-
全球(太平洋、印度洋、大西洋、南海),2017,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 170 |
-
全球(太平洋、印度洋、大西洋、南海),2018,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 171 |
-
全球(太平洋、印度洋、大西洋、南海),2019,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 172 |
-
全球(太平洋、印度洋、大西洋、南海),2020,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 173 |
-
全球(太平洋、印度洋、大西洋、南海),2021,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 174 |
-
全球(太平洋、印度洋、大西洋、南海),2022,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 175 |
-
全球(太平洋、印度洋、大西洋、南海),2023,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 176 |
-
全球(太平洋、印度洋、大西洋、南海),2024,茎柔鱼,捕捞量(年尺度),全球年尺度,RAM Legacy 茎柔鱼 1950 - 2024 全球年尺/,根据 Hugging Face 数据集目录名称整理
|
| 177 |
-
全球(太平洋、印度洋、大西洋、南海),1950,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 178 |
-
全球(太平洋、印度洋、大西洋、南海),1951,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 179 |
-
全球(太平洋、印度洋、大西洋、南海),1952,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 180 |
-
全球(太平洋、印度洋、大西洋、南海),1953,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 181 |
-
全球(太平洋、印度洋、大西洋、南海),1954,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 182 |
-
全球(太平洋、印度洋、大西洋、南海),1955,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 183 |
-
全球(太平洋、印度洋、大西洋、南海),1956,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 184 |
-
全球(太平洋、印度洋、大西洋、南海),1957,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 185 |
-
全球(太平洋、印度洋、大西洋、南海),1958,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 186 |
-
全球(太平洋、印度洋、大西洋、南海),1959,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 187 |
-
全球(太平洋、印度洋、大西洋、南海),1960,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 188 |
-
全球(太平洋、印度洋、大西洋、南海),1961,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 189 |
-
全球(太平洋、印度洋、大西洋、南海),1962,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 190 |
-
全球(太平洋、印度洋、大西洋、南海),1963,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 191 |
-
全球(太平洋、印度洋、大西洋、南海),1964,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 192 |
-
全球(太平洋、印度洋、大西洋、南海),1965,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 193 |
-
全球(太平洋、印度洋、大西洋、南海),1966,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 194 |
-
全球(太平洋、印度洋、大西洋、南海),1967,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 195 |
-
全球(太平洋、印度洋、大西洋、南海),1968,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 196 |
-
全球(太平洋、印度洋、大西洋、南海),1969,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 197 |
-
全球(太平洋、印度洋、大西洋、南海),1970,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 198 |
-
全球(太平洋、印度洋、大西洋、南海),1971,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 199 |
-
全球(太平洋、印度洋、大西洋、南海),1972,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 200 |
-
全球(太平洋、印度洋、大西洋、南海),1973,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 201 |
-
全球(太平洋、印度洋、大西洋、南海),1974,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 202 |
-
全球(太平洋、印度洋、大西洋、南海),1975,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 203 |
-
全球(太平洋、印度洋、大西洋、南海),1976,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 204 |
-
全球(太平洋、印度洋、大西洋、南海),1977,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 205 |
-
全球(太平洋、印度洋、大西洋、南海),1978,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 206 |
-
全球(太平洋、印度洋、大西洋、南海),1979,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 207 |
-
全球(太平洋、印度洋、大西洋、南海),1980,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 208 |
-
全球(太平洋、印度洋、大西洋、南海),1981,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 209 |
-
全球(太平洋、印度洋、大西洋、南海),1982,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 210 |
-
全球(太平洋、印度洋、大西洋、南海),1983,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 211 |
-
全球(太平洋、印度洋、大西洋、南海),1984,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 212 |
-
全球(太平洋、印度洋、大西洋、南海),1985,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 213 |
-
全球(太平洋、印度洋、大西洋、南海),1986,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 214 |
-
全球(太平洋、印度洋、大西洋、南海),1987,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 215 |
-
全球(太平洋、印度洋、大西洋、南海),1988,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 216 |
-
全球(太平洋、印度洋、大西洋、南海),1989,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 217 |
-
全球(太平洋、印度洋、大西洋、南海),1990,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 218 |
-
全球(太平洋、印度洋、大西洋、南海),1991,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 219 |
-
全球(太平洋、印度洋、大西洋、南海),1992,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 220 |
-
全球(太平洋、印度洋、大西洋、南海),1993,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 221 |
-
全球(太平洋、印度洋、大西洋、南海),1994,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 222 |
-
全球(太平洋、印度洋、大西洋、南海),1995,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 223 |
-
全球(太平洋、印度洋、大西洋、南海),1996,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 224 |
-
全球(太平洋、印度洋、大西洋、南海),1997,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 225 |
-
全球(太平洋、印度洋、大西洋、南海),1998,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 226 |
-
全球(太平洋、印度洋、大西洋、南海),1999,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 227 |
-
全球(太平洋、印度洋、大西洋、南海),2000,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 228 |
-
全球(太平洋、印���洋、大西洋、南海),2001,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 229 |
-
全球(太平洋、印度洋、大西洋、南海),2002,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 230 |
-
全球(太平洋、印度洋、大西洋、南海),2003,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 231 |
-
全球(太平洋、印度洋、大西洋、南海),2004,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 232 |
-
全球(太平洋、印度洋、大西洋、南海),2005,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 233 |
-
全球(太平洋、印度洋、大西洋、南海),2006,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 234 |
-
全球(太平洋、印度洋、大西洋、南海),2007,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 235 |
-
全球(太平洋、印度洋、大西洋、南海),2008,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 236 |
-
全球(太平洋、印度洋、大西洋、南海),2009,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 237 |
-
全球(太平洋、印度洋、大西洋、南海),2010,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 238 |
-
全球(太平洋、印度洋、大西洋、南海),2011,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 239 |
-
全球(太平洋、印度洋、大西洋、南海),2012,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 240 |
-
全球(太平洋、印度洋、大西洋、南海),2013,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 241 |
-
全球(太平洋、印度洋、大西洋、南海),2014,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 242 |
-
全球(太平洋、印度洋、大西洋、南海),2015,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 243 |
-
全球(太平洋、印度洋、大西洋、南海),2016,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 244 |
-
全球(太平洋、印度洋、大西洋、南海),2017,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 245 |
-
全球(太平洋、印度洋、大西洋、南海),2018,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
| 246 |
-
全球(太平洋、印度洋、大西洋、南海),2019,柔鱼科,捕捞量(年尺度),0.5°×0.5°,我们周围的海域全球柔鱼科捕量年尺 1950-2019 0.5×0.5/,根据 Hugging Face 数据集目录名称整理
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fisheries_hf.py
ADDED
|
@@ -0,0 +1,575 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Restricted Hugging Face fisheries file reader and CSV export helpers.
|
| 2 |
+
|
| 3 |
+
The module deliberately exposes no arbitrary URL, shell, or filesystem access.
|
| 4 |
+
Callers must first validate a path against the configured Dataset live tree and
|
| 5 |
+
pass the expected file size to :func:`download_dataset_file`.
|
| 6 |
+
"""
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import csv
|
| 10 |
+
import hashlib
|
| 11 |
+
import io
|
| 12 |
+
import itertools
|
| 13 |
+
import json
|
| 14 |
+
import os
|
| 15 |
+
import re
|
| 16 |
+
import secrets
|
| 17 |
+
import shutil
|
| 18 |
+
import time
|
| 19 |
+
import zipfile
|
| 20 |
+
from collections import defaultdict
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
from typing import Any, Iterable
|
| 23 |
+
from urllib.parse import quote
|
| 24 |
+
|
| 25 |
+
import httpx
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
HF_SQUID_DATASET_REPO = (
|
| 29 |
+
os.environ.get("HF_SQUID_DATASET_REPO")
|
| 30 |
+
or os.environ.get("HF_DATASET_REPO")
|
| 31 |
+
or "globalsquiddatabase/squid_dataset"
|
| 32 |
+
).strip()
|
| 33 |
+
HF_TUNA_DATASET_REPO = (
|
| 34 |
+
os.environ.get("HF_TUNA_DATASET_REPO")
|
| 35 |
+
or "globalsquiddatabase/Tuna-Fisheries-Dataset"
|
| 36 |
+
).strip()
|
| 37 |
+
HF_DATASET_REPOS = {
|
| 38 |
+
"squid": HF_SQUID_DATASET_REPO,
|
| 39 |
+
"tuna": HF_TUNA_DATASET_REPO,
|
| 40 |
+
}
|
| 41 |
+
# Backwards-compatible default used when old callers omit a repository.
|
| 42 |
+
HF_DATASET_REPO = HF_SQUID_DATASET_REPO
|
| 43 |
+
HF_SQUID_DATASET_REVISION = (
|
| 44 |
+
os.environ.get("HF_SQUID_DATASET_REVISION")
|
| 45 |
+
or os.environ.get("HF_DATASET_REVISION")
|
| 46 |
+
or ""
|
| 47 |
+
).strip()
|
| 48 |
+
HF_TUNA_DATASET_REVISION = os.environ.get("HF_TUNA_DATASET_REVISION", "").strip()
|
| 49 |
+
HF_TOKEN = os.environ.get("HF_TOKEN", "").strip()
|
| 50 |
+
PUBLIC_BASE_URL = os.environ.get("PUBLIC_BASE_URL", "").strip().rstrip("/")
|
| 51 |
+
|
| 52 |
+
CACHE_ROOT = Path(
|
| 53 |
+
os.environ.get("HF_FISHERIES_CACHE_ROOT", "/tmp/squid_hf_fisheries_cache")
|
| 54 |
+
)
|
| 55 |
+
EXPORT_ROOT = Path(
|
| 56 |
+
os.environ.get("FISHERIES_EXPORT_ROOT", "/tmp/squid_fisheries_exports")
|
| 57 |
+
)
|
| 58 |
+
CACHE_ROOT.mkdir(parents=True, exist_ok=True)
|
| 59 |
+
EXPORT_ROOT.mkdir(parents=True, exist_ok=True)
|
| 60 |
+
|
| 61 |
+
MAX_CSV_BYTES = int(os.environ.get("HF_FISHERIES_MAX_CSV_BYTES", 100 * 1024 * 1024))
|
| 62 |
+
MAX_ZIP_BYTES = int(os.environ.get("HF_FISHERIES_MAX_ZIP_BYTES", 1536 * 1024 * 1024))
|
| 63 |
+
EXPORT_TTL_SECONDS = int(os.environ.get("FISHERIES_EXPORT_TTL_SECONDS", "86400"))
|
| 64 |
+
|
| 65 |
+
_REVISION_CACHE: dict[str, dict[str, Any]] = {}
|
| 66 |
+
|
| 67 |
+
YEAR_ALIASES = ("year", "yearc", "年份", "yr")
|
| 68 |
+
MONTH_ALIASES = ("month", "月份", "mon")
|
| 69 |
+
DATE_ALIASES = ("date", "time", "datetime", "日期", "时间", "year_month")
|
| 70 |
+
LON_ALIASES = ("lon", "longitude", "decimal_longitude", "经度", "x")
|
| 71 |
+
LAT_ALIASES = ("lat", "latitude", "decimal_latitude", "纬度", "y")
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _headers() -> dict[str, str]:
|
| 75 |
+
return {"Authorization": f"Bearer {HF_TOKEN}"} if HF_TOKEN else {}
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def normalize_repository(repository: str | None = None) -> str:
|
| 79 |
+
"""Resolve a safe repository selector to one of the two configured repos."""
|
| 80 |
+
value = str(repository or "squid").strip()
|
| 81 |
+
lowered = value.lower()
|
| 82 |
+
if lowered in HF_DATASET_REPOS:
|
| 83 |
+
return HF_DATASET_REPOS[lowered]
|
| 84 |
+
for repo in HF_DATASET_REPOS.values():
|
| 85 |
+
if value == repo:
|
| 86 |
+
return repo
|
| 87 |
+
raise ValueError("repository must be squid, tuna, or an exact configured repository id")
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _revision(repository: str) -> str:
|
| 91 |
+
pinned = (
|
| 92 |
+
HF_TUNA_DATASET_REVISION
|
| 93 |
+
if repository == HF_TUNA_DATASET_REPO
|
| 94 |
+
else HF_SQUID_DATASET_REVISION
|
| 95 |
+
)
|
| 96 |
+
if pinned:
|
| 97 |
+
return pinned
|
| 98 |
+
now = time.time()
|
| 99 |
+
cache = _REVISION_CACHE.get(repository) or {}
|
| 100 |
+
cached = str(cache.get("sha") or "")
|
| 101 |
+
if cached and now - float(cache.get("ts") or 0) < 300:
|
| 102 |
+
return cached
|
| 103 |
+
url = f"https://huggingface.co/api/datasets/{repository}"
|
| 104 |
+
with httpx.Client(timeout=30.0, follow_redirects=True) as client:
|
| 105 |
+
response = client.get(url, headers=_headers())
|
| 106 |
+
if response.status_code in {401, 403}:
|
| 107 |
+
raise RuntimeError("Hugging Face Dataset 无读取权限,请检查 HF_TOKEN。")
|
| 108 |
+
response.raise_for_status()
|
| 109 |
+
body = response.json()
|
| 110 |
+
sha = str(body.get("sha") or "").strip()
|
| 111 |
+
if not re.fullmatch(r"[0-9a-fA-F]{40}", sha):
|
| 112 |
+
raise RuntimeError("无法获得 Hugging Face Dataset 的固定提交 SHA。")
|
| 113 |
+
_REVISION_CACHE[repository] = {"ts": now, "sha": sha}
|
| 114 |
+
return sha
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _safe_cache_path(repository: str, path: str, revision: str) -> Path:
|
| 118 |
+
suffix = Path(path).suffix.lower()
|
| 119 |
+
digest = hashlib.sha256(f"{repository}:{revision}:{path}".encode()).hexdigest()
|
| 120 |
+
return CACHE_ROOT / f"{digest}{suffix}"
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def download_dataset_file(
|
| 124 |
+
path: str,
|
| 125 |
+
expected_size: int,
|
| 126 |
+
repository: str | None = None,
|
| 127 |
+
) -> tuple[Path, str]:
|
| 128 |
+
"""Download one validated Dataset file to a revision-keyed local cache."""
|
| 129 |
+
clean = str(path or "").strip().lstrip("/")
|
| 130 |
+
if not clean or "\x00" in clean or any(part in {"", ".", ".."} for part in clean.split("/")):
|
| 131 |
+
raise ValueError("invalid dataset path")
|
| 132 |
+
suffix = Path(clean).suffix.lower()
|
| 133 |
+
limit = MAX_ZIP_BYTES if suffix == ".zip" else MAX_CSV_BYTES
|
| 134 |
+
if suffix not in {".csv", ".tsv", ".zip"}:
|
| 135 |
+
raise ValueError("当前内容查询仅支持 CSV、TSV 和包含 CSV/TSV 的 ZIP。")
|
| 136 |
+
if expected_size <= 0:
|
| 137 |
+
raise ValueError("live tree did not provide a positive file size")
|
| 138 |
+
if expected_size > limit:
|
| 139 |
+
raise ValueError(
|
| 140 |
+
f"文件大小 {expected_size} bytes 超过在线处理上限 {limit} bytes;"
|
| 141 |
+
"请先在学校服务器生成查询就绪的分区文件。"
|
| 142 |
+
)
|
| 143 |
+
|
| 144 |
+
repository = normalize_repository(repository)
|
| 145 |
+
revision = _revision(repository)
|
| 146 |
+
target = _safe_cache_path(repository, clean, revision)
|
| 147 |
+
if target.exists() and target.stat().st_size == expected_size:
|
| 148 |
+
return target, revision
|
| 149 |
+
|
| 150 |
+
partial = target.with_suffix(target.suffix + ".part")
|
| 151 |
+
partial.unlink(missing_ok=True)
|
| 152 |
+
encoded_path = quote(clean, safe="/")
|
| 153 |
+
url = (
|
| 154 |
+
f"https://huggingface.co/datasets/{repository}/resolve/"
|
| 155 |
+
f"{revision}/{encoded_path}"
|
| 156 |
+
)
|
| 157 |
+
total = 0
|
| 158 |
+
try:
|
| 159 |
+
with httpx.stream(
|
| 160 |
+
"GET",
|
| 161 |
+
url,
|
| 162 |
+
headers=_headers(),
|
| 163 |
+
follow_redirects=True,
|
| 164 |
+
timeout=httpx.Timeout(connect=20, read=300, write=30, pool=30),
|
| 165 |
+
) as response:
|
| 166 |
+
if response.status_code in {401, 403}:
|
| 167 |
+
raise RuntimeError("Hugging Face 文件无读取权限,请检查 HF_TOKEN。")
|
| 168 |
+
response.raise_for_status()
|
| 169 |
+
with partial.open("wb") as stream:
|
| 170 |
+
for chunk in response.iter_bytes(1024 * 1024):
|
| 171 |
+
if not chunk:
|
| 172 |
+
continue
|
| 173 |
+
total += len(chunk)
|
| 174 |
+
if total > limit:
|
| 175 |
+
raise ValueError("download exceeded configured size limit")
|
| 176 |
+
stream.write(chunk)
|
| 177 |
+
if total != expected_size:
|
| 178 |
+
raise RuntimeError(
|
| 179 |
+
f"文件下载不完整:expected={expected_size}, received={total}"
|
| 180 |
+
)
|
| 181 |
+
partial.replace(target)
|
| 182 |
+
except Exception:
|
| 183 |
+
partial.unlink(missing_ok=True)
|
| 184 |
+
raise
|
| 185 |
+
return target, revision
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def _decode_text(path: Path) -> tuple[str, str]:
|
| 189 |
+
raw = path.read_bytes()
|
| 190 |
+
for encoding in ("utf-8-sig", "utf-8", "gb18030"):
|
| 191 |
+
try:
|
| 192 |
+
return raw.decode(encoding), encoding
|
| 193 |
+
except UnicodeDecodeError:
|
| 194 |
+
pass
|
| 195 |
+
return raw.decode("utf-8", errors="replace"), "utf-8-replace"
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _dialect(text: str, suffix: str = ".csv") -> str:
|
| 199 |
+
sample = text[:20000]
|
| 200 |
+
try:
|
| 201 |
+
return csv.Sniffer().sniff(sample, delimiters=",\t;|").delimiter
|
| 202 |
+
except Exception:
|
| 203 |
+
return "\t" if suffix == ".tsv" else ","
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def _column(columns: Iterable[str], aliases: Iterable[str]) -> str | None:
|
| 207 |
+
exact = {str(col).strip().lower(): str(col) for col in columns}
|
| 208 |
+
for alias in aliases:
|
| 209 |
+
if alias.lower() in exact:
|
| 210 |
+
return exact[alias.lower()]
|
| 211 |
+
return None
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def _number(value: Any) -> float | None:
|
| 215 |
+
text = str(value or "").strip().replace(",", "")
|
| 216 |
+
if not text:
|
| 217 |
+
return None
|
| 218 |
+
try:
|
| 219 |
+
return float(text)
|
| 220 |
+
except Exception:
|
| 221 |
+
return None
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _year_month(row: dict[str, str], columns: list[str]) -> tuple[int | None, int | None]:
|
| 225 |
+
year_col = _column(columns, YEAR_ALIASES)
|
| 226 |
+
month_col = _column(columns, MONTH_ALIASES)
|
| 227 |
+
date_col = _column(columns, DATE_ALIASES)
|
| 228 |
+
year = None
|
| 229 |
+
month = None
|
| 230 |
+
if year_col:
|
| 231 |
+
value = _number(row.get(year_col))
|
| 232 |
+
if value is not None and 1800 <= int(value) <= 2200:
|
| 233 |
+
year = int(value)
|
| 234 |
+
if month_col:
|
| 235 |
+
value = _number(row.get(month_col))
|
| 236 |
+
if value is not None and 1 <= int(value) <= 12:
|
| 237 |
+
month = int(value)
|
| 238 |
+
if date_col and (year is None or month is None):
|
| 239 |
+
text = str(row.get(date_col) or "")
|
| 240 |
+
match = re.search(r"(19\d{2}|20\d{2}|21\d{2})[-/]?(0?[1-9]|1[0-2])?", text)
|
| 241 |
+
if match:
|
| 242 |
+
year = year or int(match.group(1))
|
| 243 |
+
month = month or (int(match.group(2)) if match.group(2) else None)
|
| 244 |
+
return year, month
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
def _metric_columns(columns: list[str], requested: str | None = None) -> list[str]:
|
| 248 |
+
if requested:
|
| 249 |
+
wanted = [x.strip() for x in requested.split(",") if x.strip()]
|
| 250 |
+
missing = [x for x in wanted if x not in columns]
|
| 251 |
+
if missing:
|
| 252 |
+
raise ValueError(f"requested metric columns not found: {missing}")
|
| 253 |
+
return wanted
|
| 254 |
+
keys = (
|
| 255 |
+
"effort", "fishing_hour", "fishing hours", "apparent_fishing",
|
| 256 |
+
"catch", "harvest", "landing", "cpue", "value",
|
| 257 |
+
)
|
| 258 |
+
excluded = set(YEAR_ALIASES + MONTH_ALIASES + LON_ALIASES + LAT_ALIASES)
|
| 259 |
+
return [
|
| 260 |
+
col for col in columns
|
| 261 |
+
if str(col).strip().lower() not in excluded
|
| 262 |
+
and any(key in str(col).strip().lower() for key in keys)
|
| 263 |
+
]
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def _matches_filters(
|
| 267 |
+
row: dict[str, str],
|
| 268 |
+
columns: list[str],
|
| 269 |
+
*,
|
| 270 |
+
year: int | None,
|
| 271 |
+
lon_min: float | None,
|
| 272 |
+
lon_max: float | None,
|
| 273 |
+
lat_min: float | None,
|
| 274 |
+
lat_max: float | None,
|
| 275 |
+
) -> tuple[bool, int | None, int | None]:
|
| 276 |
+
row_year, row_month = _year_month(row, columns)
|
| 277 |
+
if year is not None and row_year != int(year):
|
| 278 |
+
return False, row_year, row_month
|
| 279 |
+
lon_col = _column(columns, LON_ALIASES)
|
| 280 |
+
lat_col = _column(columns, LAT_ALIASES)
|
| 281 |
+
if any(v is not None for v in (lon_min, lon_max)):
|
| 282 |
+
if not lon_col:
|
| 283 |
+
raise ValueError("经度筛选已请求,但文件中未识别到经度字段。")
|
| 284 |
+
lon = _number(row.get(lon_col))
|
| 285 |
+
if lon is None or (lon_min is not None and lon < lon_min) or (lon_max is not None and lon > lon_max):
|
| 286 |
+
return False, row_year, row_month
|
| 287 |
+
if any(v is not None for v in (lat_min, lat_max)):
|
| 288 |
+
if not lat_col:
|
| 289 |
+
raise ValueError("纬度筛选已请求,但文件中未识别到纬度字段。")
|
| 290 |
+
lat = _number(row.get(lat_col))
|
| 291 |
+
if lat is None or (lat_min is not None and lat < lat_min) or (lat_max is not None and lat > lat_max):
|
| 292 |
+
return False, row_year, row_month
|
| 293 |
+
return True, row_year, row_month
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def _create_export(filename: str, rows: Iterable[dict[str, Any]], columns: list[str]) -> dict[str, str]:
|
| 297 |
+
now = time.time()
|
| 298 |
+
for item in list(EXPORT_ROOT.iterdir())[:2000]:
|
| 299 |
+
if not item.is_dir():
|
| 300 |
+
continue
|
| 301 |
+
try:
|
| 302 |
+
meta = json.loads((item / "meta.json").read_text(encoding="utf-8"))
|
| 303 |
+
expired = float(meta.get("expires_ts") or 0) < now
|
| 304 |
+
except Exception:
|
| 305 |
+
expired = True
|
| 306 |
+
if expired:
|
| 307 |
+
shutil.rmtree(item, ignore_errors=True)
|
| 308 |
+
|
| 309 |
+
token = secrets.token_urlsafe(24)
|
| 310 |
+
safe_name = re.sub(r"[^A-Za-z0-9._-]+", "_", filename).strip("._") or "fisheries_export.csv"
|
| 311 |
+
folder = EXPORT_ROOT / token
|
| 312 |
+
folder.mkdir(parents=True, exist_ok=False)
|
| 313 |
+
target = folder / safe_name
|
| 314 |
+
with target.open("w", encoding="utf-8-sig", newline="") as stream:
|
| 315 |
+
writer = csv.DictWriter(stream, fieldnames=columns, extrasaction="ignore")
|
| 316 |
+
writer.writeheader()
|
| 317 |
+
writer.writerows(rows)
|
| 318 |
+
meta = {
|
| 319 |
+
"filename": safe_name,
|
| 320 |
+
"content_type": "text/csv; charset=utf-8",
|
| 321 |
+
"size_bytes": target.stat().st_size,
|
| 322 |
+
"created_ts": now,
|
| 323 |
+
"expires_ts": now + EXPORT_TTL_SECONDS,
|
| 324 |
+
}
|
| 325 |
+
(folder / "meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2), encoding="utf-8")
|
| 326 |
+
path = f"/api/fisheries/download/{token}"
|
| 327 |
+
return {
|
| 328 |
+
"token": token,
|
| 329 |
+
"filename": safe_name,
|
| 330 |
+
"download_path": path,
|
| 331 |
+
"download_url": f"{PUBLIC_BASE_URL}{path}" if PUBLIC_BASE_URL else path,
|
| 332 |
+
}
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
def _process_reader(
|
| 336 |
+
reader: csv.DictReader,
|
| 337 |
+
*,
|
| 338 |
+
source_label: str,
|
| 339 |
+
year: int | None,
|
| 340 |
+
lon_min: float | None,
|
| 341 |
+
lon_max: float | None,
|
| 342 |
+
lat_min: float | None,
|
| 343 |
+
lat_max: float | None,
|
| 344 |
+
metric_columns: str | None,
|
| 345 |
+
max_rows: int,
|
| 346 |
+
state: dict[str, Any],
|
| 347 |
+
) -> None:
|
| 348 |
+
columns = [str(x or "").strip() for x in (reader.fieldnames or [])]
|
| 349 |
+
if not columns:
|
| 350 |
+
return
|
| 351 |
+
if not state["columns"]:
|
| 352 |
+
state["columns"] = columns
|
| 353 |
+
state["metrics"] = _metric_columns(columns, metric_columns)
|
| 354 |
+
elif columns != state["columns"]:
|
| 355 |
+
state["schema_variants"].append({"source": source_label, "columns": columns})
|
| 356 |
+
|
| 357 |
+
seen = state["seen"]
|
| 358 |
+
for raw in reader:
|
| 359 |
+
if state["scanned"] >= max_rows:
|
| 360 |
+
state["truncated"] = True
|
| 361 |
+
return
|
| 362 |
+
state["scanned"] += 1
|
| 363 |
+
row = {str(k or "").strip(): "" if v is None else str(v).strip() for k, v in raw.items()}
|
| 364 |
+
matched, row_year, row_month = _matches_filters(
|
| 365 |
+
row,
|
| 366 |
+
columns,
|
| 367 |
+
year=year,
|
| 368 |
+
lon_min=lon_min,
|
| 369 |
+
lon_max=lon_max,
|
| 370 |
+
lat_min=lat_min,
|
| 371 |
+
lat_max=lat_max,
|
| 372 |
+
)
|
| 373 |
+
if not matched:
|
| 374 |
+
continue
|
| 375 |
+
# Quality counts are reported for the requested result set. This
|
| 376 |
+
# keeps large year/bbox ZIP queries bounded instead of retaining a
|
| 377 |
+
# duplicate key for every global row that was scanned and rejected.
|
| 378 |
+
key = tuple(row.get(col, "") for col in columns)
|
| 379 |
+
if key in seen:
|
| 380 |
+
state["duplicates"] += 1
|
| 381 |
+
else:
|
| 382 |
+
seen.add(key)
|
| 383 |
+
for col in columns:
|
| 384 |
+
if not row.get(col, "").strip():
|
| 385 |
+
state["missing"][col] += 1
|
| 386 |
+
state["matched"] += 1
|
| 387 |
+
state["rows"].append(row)
|
| 388 |
+
if row_year is not None:
|
| 389 |
+
state["years"].add(row_year)
|
| 390 |
+
if row_month is not None:
|
| 391 |
+
state["months"][row_month] += 1
|
| 392 |
+
if row_year is not None:
|
| 393 |
+
state["annual_counts"][row_year] += 1
|
| 394 |
+
for col in state["metrics"]:
|
| 395 |
+
value = _number(row.get(col))
|
| 396 |
+
if value is not None:
|
| 397 |
+
state["annual_values"][(row_year, col)] += value
|
| 398 |
+
|
| 399 |
+
|
| 400 |
+
def analyze_and_export(
|
| 401 |
+
local_path: Path,
|
| 402 |
+
*,
|
| 403 |
+
dataset_path: str,
|
| 404 |
+
revision: str,
|
| 405 |
+
repository: str | None = None,
|
| 406 |
+
year: int | None = None,
|
| 407 |
+
lon_min: float | None = None,
|
| 408 |
+
lon_max: float | None = None,
|
| 409 |
+
lat_min: float | None = None,
|
| 410 |
+
lat_max: float | None = None,
|
| 411 |
+
metric_columns: str | None = None,
|
| 412 |
+
max_rows: int = 2_000_000,
|
| 413 |
+
) -> dict[str, Any]:
|
| 414 |
+
if lon_min is not None and not -180 <= float(lon_min) <= 180:
|
| 415 |
+
raise ValueError("lon_min 必须位于 -180 至 180。")
|
| 416 |
+
if lon_max is not None and not -180 <= float(lon_max) <= 180:
|
| 417 |
+
raise ValueError("lon_max 必须位于 -180 至 180。")
|
| 418 |
+
if lat_min is not None and not -90 <= float(lat_min) <= 90:
|
| 419 |
+
raise ValueError("lat_min 必须位于 -90 至 90。")
|
| 420 |
+
if lat_max is not None and not -90 <= float(lat_max) <= 90:
|
| 421 |
+
raise ValueError("lat_max 必须位于 -90 至 90。")
|
| 422 |
+
if lon_min is not None and lon_max is not None and float(lon_min) > float(lon_max):
|
| 423 |
+
raise ValueError("lon_min 不能大于 lon_max。")
|
| 424 |
+
if lat_min is not None and lat_max is not None and float(lat_min) > float(lat_max):
|
| 425 |
+
raise ValueError("lat_min 不能大于 lat_max。")
|
| 426 |
+
|
| 427 |
+
has_filter = year is not None or any(
|
| 428 |
+
value is not None for value in (lon_min, lon_max, lat_min, lat_max)
|
| 429 |
+
)
|
| 430 |
+
if (
|
| 431 |
+
local_path.suffix.lower() == ".zip"
|
| 432 |
+
and local_path.stat().st_size > 200 * 1024 * 1024
|
| 433 |
+
and not has_filter
|
| 434 |
+
):
|
| 435 |
+
raise ValueError(
|
| 436 |
+
"大型 ZIP 查询必须提供 year 或经纬度范围,避免无边界解压扫描;"
|
| 437 |
+
"请补充筛选条件后重试。"
|
| 438 |
+
)
|
| 439 |
+
|
| 440 |
+
max_rows = max(1, min(int(max_rows or 2_000_000), 5_000_000))
|
| 441 |
+
state: dict[str, Any] = {
|
| 442 |
+
"columns": [], "metrics": [], "schema_variants": [], "scanned": 0,
|
| 443 |
+
"matched": 0, "duplicates": 0, "missing": defaultdict(int),
|
| 444 |
+
"years": set(), "months": defaultdict(int), "annual_counts": defaultdict(int),
|
| 445 |
+
"annual_values": defaultdict(float), "rows": [], "seen": set(), "truncated": False,
|
| 446 |
+
}
|
| 447 |
+
|
| 448 |
+
suffix = local_path.suffix.lower()
|
| 449 |
+
members: list[str] = []
|
| 450 |
+
if suffix in {".csv", ".tsv"}:
|
| 451 |
+
text, encoding = _decode_text(local_path)
|
| 452 |
+
delimiter = _dialect(text, suffix)
|
| 453 |
+
_process_reader(
|
| 454 |
+
csv.DictReader(io.StringIO(text), delimiter=delimiter),
|
| 455 |
+
source_label=dataset_path,
|
| 456 |
+
year=year, lon_min=lon_min, lon_max=lon_max,
|
| 457 |
+
lat_min=lat_min, lat_max=lat_max,
|
| 458 |
+
metric_columns=metric_columns, max_rows=max_rows, state=state,
|
| 459 |
+
)
|
| 460 |
+
else:
|
| 461 |
+
encoding = "utf-8-replace"
|
| 462 |
+
delimiter = "auto-by-member-extension"
|
| 463 |
+
with zipfile.ZipFile(local_path) as archive:
|
| 464 |
+
candidates = [
|
| 465 |
+
info for info in archive.infolist()
|
| 466 |
+
if not info.is_dir() and Path(info.filename).suffix.lower() in {".csv", ".tsv"}
|
| 467 |
+
]
|
| 468 |
+
if year is not None:
|
| 469 |
+
preferred = [info for info in candidates if str(year) in info.filename]
|
| 470 |
+
if preferred:
|
| 471 |
+
candidates = preferred
|
| 472 |
+
for info in candidates:
|
| 473 |
+
members.append(info.filename)
|
| 474 |
+
with archive.open(info) as binary:
|
| 475 |
+
text_stream = io.TextIOWrapper(binary, encoding="utf-8-sig", errors="replace", newline="")
|
| 476 |
+
sample_lines = list(itertools.islice(text_stream, 50))
|
| 477 |
+
member_delimiter = _dialect(
|
| 478 |
+
"".join(sample_lines),
|
| 479 |
+
Path(info.filename).suffix.lower(),
|
| 480 |
+
)
|
| 481 |
+
_process_reader(
|
| 482 |
+
csv.DictReader(
|
| 483 |
+
itertools.chain(sample_lines, text_stream),
|
| 484 |
+
delimiter=member_delimiter,
|
| 485 |
+
),
|
| 486 |
+
source_label=info.filename,
|
| 487 |
+
year=year, lon_min=lon_min, lon_max=lon_max,
|
| 488 |
+
lat_min=lat_min, lat_max=lat_max,
|
| 489 |
+
metric_columns=metric_columns, max_rows=max_rows, state=state,
|
| 490 |
+
)
|
| 491 |
+
if state["truncated"]:
|
| 492 |
+
break
|
| 493 |
+
|
| 494 |
+
if not state["columns"]:
|
| 495 |
+
raise ValueError("未在文件中发现可读取的 CSV/TSV 表格。")
|
| 496 |
+
|
| 497 |
+
annual_summary = []
|
| 498 |
+
for yr in sorted(state["annual_counts"]):
|
| 499 |
+
item: dict[str, Any] = {"year": yr, "record_count": state["annual_counts"][yr]}
|
| 500 |
+
for col in state["metrics"]:
|
| 501 |
+
item[f"sum_{col}"] = state["annual_values"].get((yr, col), 0.0)
|
| 502 |
+
annual_summary.append(item)
|
| 503 |
+
|
| 504 |
+
# Return three distinct, truthfully labelled artifacts when the source has
|
| 505 |
+
# a usable year field. Keep the raw-detail export as the backwards-
|
| 506 |
+
# compatible primary download_url/token/filename.
|
| 507 |
+
stamp = int(time.time())
|
| 508 |
+
source_stem = Path(dataset_path).stem or "fisheries_query"
|
| 509 |
+
raw_export = _create_export(
|
| 510 |
+
f"{source_stem}_{stamp}_filtered_raw.csv",
|
| 511 |
+
state["rows"],
|
| 512 |
+
state["columns"],
|
| 513 |
+
)
|
| 514 |
+
deduplicated_rows = []
|
| 515 |
+
exported_keys = set()
|
| 516 |
+
for row in state["rows"]:
|
| 517 |
+
key = tuple(row.get(col, "") for col in state["columns"])
|
| 518 |
+
if key in exported_keys:
|
| 519 |
+
continue
|
| 520 |
+
exported_keys.add(key)
|
| 521 |
+
deduplicated_rows.append(row)
|
| 522 |
+
deduplicated_export = _create_export(
|
| 523 |
+
f"{source_stem}_{stamp}_deduplicated.csv",
|
| 524 |
+
deduplicated_rows,
|
| 525 |
+
state["columns"],
|
| 526 |
+
)
|
| 527 |
+
exports = [
|
| 528 |
+
{"kind": "filtered_raw", **raw_export},
|
| 529 |
+
{
|
| 530 |
+
"kind": "deduplicated",
|
| 531 |
+
"record_count": len(deduplicated_rows),
|
| 532 |
+
**deduplicated_export,
|
| 533 |
+
},
|
| 534 |
+
]
|
| 535 |
+
if annual_summary:
|
| 536 |
+
annual_columns = list(annual_summary[0])
|
| 537 |
+
annual_export = _create_export(
|
| 538 |
+
f"{source_stem}_{stamp}_annual_summary.csv",
|
| 539 |
+
annual_summary,
|
| 540 |
+
annual_columns,
|
| 541 |
+
)
|
| 542 |
+
exports.append({"kind": "annual_summary", **annual_export})
|
| 543 |
+
|
| 544 |
+
return {
|
| 545 |
+
"status": "ok",
|
| 546 |
+
"repository": normalize_repository(repository),
|
| 547 |
+
"revision": revision,
|
| 548 |
+
"dataset_path": dataset_path,
|
| 549 |
+
"encoding": encoding,
|
| 550 |
+
"delimiter": delimiter,
|
| 551 |
+
"columns": state["columns"],
|
| 552 |
+
"metric_columns": state["metrics"],
|
| 553 |
+
"scanned_row_count": state["scanned"],
|
| 554 |
+
"matched_row_count": state["matched"],
|
| 555 |
+
"scan_truncated": state["truncated"],
|
| 556 |
+
"time_range": {
|
| 557 |
+
"min_year": min(state["years"]) if state["years"] else None,
|
| 558 |
+
"max_year": max(state["years"]) if state["years"] else None,
|
| 559 |
+
},
|
| 560 |
+
"monthly_record_counts": [
|
| 561 |
+
{"month": month, "record_count": state["months"][month]}
|
| 562 |
+
for month in sorted(state["months"])
|
| 563 |
+
],
|
| 564 |
+
"missing_values_by_column": {
|
| 565 |
+
col: int(state["missing"].get(col, 0)) for col in state["columns"]
|
| 566 |
+
},
|
| 567 |
+
"exact_duplicate_count": state["duplicates"],
|
| 568 |
+
"deduplicated_record_count": len(deduplicated_rows),
|
| 569 |
+
"annual_summary": annual_summary[:100],
|
| 570 |
+
"exports": exports,
|
| 571 |
+
"download_urls": [item["download_url"] for item in exports],
|
| 572 |
+
"zip_members_processed": members[:200],
|
| 573 |
+
"schema_variants": state["schema_variants"][:20],
|
| 574 |
+
**raw_export,
|
| 575 |
+
}
|
harness_safe.cordis.yml
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
- id: sdk-jsonrpc-server
|
| 2 |
+
name: '@deepseek-ai/dsh-sdk-jsonrpc-server'
|
| 3 |
+
config:
|
| 4 |
+
maxTokensAsSuccess: true
|
| 5 |
+
|
| 6 |
+
- id: llm-deepseek
|
| 7 |
+
name: '@deepseek-ai/dsh-llm-deepseek'
|
| 8 |
+
config:
|
| 9 |
+
apiKeyEnv: DEEPSEEK_API_KEY
|
| 10 |
+
streamIdleTimeoutMs: 300000
|
| 11 |
+
models:
|
| 12 |
+
- id: !!js process.env.DSH_MODEL ?? 'glm-5.2'
|
| 13 |
+
contextWindow: !!js Number(process.env.DSH_CONTEXT_WINDOW ?? 128000)
|
| 14 |
+
|
| 15 |
+
- id: agent-spine
|
| 16 |
+
name: '@deepseek-ai/dsh-agent-spine-demo'
|
| 17 |
+
config:
|
| 18 |
+
includeHarnessIdentity: true
|
| 19 |
+
includeRuntimeContext: false
|
| 20 |
+
persona: !!js process.env.DSH_SYSTEM_PROMPT ?? 'You are Global Marine Foundation Data Agent. Reply in Chinese by default. Only output the final answer.'
|
| 21 |
+
workspaceContext: false
|
| 22 |
+
skills:
|
| 23 |
+
enabled: false
|
| 24 |
+
toolBash: false
|
| 25 |
+
toolJobs: false
|
| 26 |
+
|
| 27 |
+
- id: sessions
|
| 28 |
+
name: '@deepseek-ai/dsh-session-persistence-jsonl'
|
| 29 |
+
config:
|
| 30 |
+
root: !!js process.env.DSH_SESSION_ROOT ?? '/tmp/deepseek-harness-sessions'
|
| 31 |
+
compression: none
|
marine_mcp.py
ADDED
|
@@ -0,0 +1,755 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Marine MCP bridge to the school Marine Data FastAPI."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import os, re, time
|
| 4 |
+
from typing import Any
|
| 5 |
+
import httpx
|
| 6 |
+
from mcp.server.mcpserver import MCPServer
|
| 7 |
+
from fisheries_hf import analyze_and_export, download_dataset_file
|
| 8 |
+
|
| 9 |
+
API_URL = os.environ.get("MARINE_API_URL", "").strip().rstrip("/")
|
| 10 |
+
if not API_URL:
|
| 11 |
+
raise RuntimeError("MARINE_API_URL is not configured")
|
| 12 |
+
|
| 13 |
+
mcp = MCPServer(
|
| 14 |
+
"Marine Data",
|
| 15 |
+
instructions=(
|
| 16 |
+
"Gateway to the user's school Marine Data Server. "
|
| 17 |
+
"Use health/domains/status for live state. "
|
| 18 |
+
"Use marine_query and marine_subset for real data retrieval. "
|
| 19 |
+
"School-server Ocean data and Hugging Face fisheries data are separate data planes. "
|
| 20 |
+
"Search both configured fisheries repositories and preserve repository provenance. "
|
| 21 |
+
"Use fisheries_analyze_export with both repository and path for actual CSV/TSV/ZIP content, filtering, statistics and CSV export. "
|
| 22 |
+
"Never invent files or values."
|
| 23 |
+
),
|
| 24 |
+
)
|
| 25 |
+
|
| 26 |
+
def _get(path: str) -> dict[str, Any]:
|
| 27 |
+
with httpx.Client(timeout=60.0, follow_redirects=True) as client:
|
| 28 |
+
r = client.get(f"{API_URL}{path}")
|
| 29 |
+
r.raise_for_status()
|
| 30 |
+
return r.json()
|
| 31 |
+
|
| 32 |
+
def _post(path: str, payload: dict[str, Any]) -> dict[str, Any]:
|
| 33 |
+
with httpx.Client(timeout=30.0, follow_redirects=True) as client:
|
| 34 |
+
response = client.post(
|
| 35 |
+
f"{API_URL}{path}",
|
| 36 |
+
json=payload,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
if response.is_error:
|
| 40 |
+
try:
|
| 41 |
+
body = response.json()
|
| 42 |
+
detail = (
|
| 43 |
+
body.get("detail")
|
| 44 |
+
if isinstance(body, dict)
|
| 45 |
+
else None
|
| 46 |
+
)
|
| 47 |
+
except Exception:
|
| 48 |
+
detail = None
|
| 49 |
+
|
| 50 |
+
if not detail:
|
| 51 |
+
detail = (
|
| 52 |
+
response.text.strip()
|
| 53 |
+
or response.reason_phrase
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
return {
|
| 57 |
+
"status": "error",
|
| 58 |
+
"http_status": response.status_code,
|
| 59 |
+
"detail": detail,
|
| 60 |
+
}
|
| 61 |
+
|
| 62 |
+
return response.json()
|
| 63 |
+
|
| 64 |
+
def _domain(value: str) -> str:
|
| 65 |
+
value = value.strip().lower()
|
| 66 |
+
if value not in {"ocean", "tuna", "squid"}:
|
| 67 |
+
raise ValueError("domain must be one of: ocean, tuna, squid")
|
| 68 |
+
return value
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def _norm_domain(value: str) -> str:
|
| 72 |
+
value = value.strip().lower()
|
| 73 |
+
if value not in {"ocean", "tuna", "squid"}:
|
| 74 |
+
raise ValueError("domain must be one of: ocean, tuna, squid")
|
| 75 |
+
return value
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
@mcp.tool()
|
| 79 |
+
def marine_health() -> dict[str, Any]:
|
| 80 |
+
"""Check whether the school Marine Data Server is reachable."""
|
| 81 |
+
return _get("/health")
|
| 82 |
+
|
| 83 |
+
@mcp.tool()
|
| 84 |
+
def marine_domains() -> dict[str, Any]:
|
| 85 |
+
"""Return live overview for ocean, tuna and squid."""
|
| 86 |
+
return _get("/domains")
|
| 87 |
+
|
| 88 |
+
@mcp.tool()
|
| 89 |
+
def marine_status(domain: str = "ocean") -> dict[str, Any]:
|
| 90 |
+
"""Return detailed live status for one data center."""
|
| 91 |
+
d = _domain(domain)
|
| 92 |
+
return _get("/status" if d == "ocean" else f"/status/{d}")
|
| 93 |
+
|
| 94 |
+
@mcp.tool()
|
| 95 |
+
def marine_catalog() -> dict[str, Any]:
|
| 96 |
+
'Return the live Ocean data catalog from the school server.'
|
| 97 |
+
return _get("/catalog")
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
@mcp.tool()
|
| 101 |
+
def marine_query(
|
| 102 |
+
date: str,
|
| 103 |
+
variable: str,
|
| 104 |
+
source: str,
|
| 105 |
+
domain: str = "ocean",
|
| 106 |
+
) -> dict[str, Any]:
|
| 107 |
+
'Check whether a source/variable/date exists on the school Ocean server.'
|
| 108 |
+
return _post(
|
| 109 |
+
"/data/query",
|
| 110 |
+
{
|
| 111 |
+
"domain": _norm_domain(domain),
|
| 112 |
+
"source": source.strip().lower(),
|
| 113 |
+
"date": date.strip(),
|
| 114 |
+
"variable": variable.strip().lower(),
|
| 115 |
+
},
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
@mcp.tool()
|
| 119 |
+
def marine_subset(
|
| 120 |
+
date: str,
|
| 121 |
+
lon_min: float,
|
| 122 |
+
lon_max: float,
|
| 123 |
+
lat_min: float,
|
| 124 |
+
lat_max: float,
|
| 125 |
+
variable: str,
|
| 126 |
+
source: str,
|
| 127 |
+
domain: str = "ocean",
|
| 128 |
+
depth: float | None = None,
|
| 129 |
+
) -> dict[str, Any]:
|
| 130 |
+
'Create a NetCDF subset from any supported Ocean source.'
|
| 131 |
+
payload = {
|
| 132 |
+
"domain": _norm_domain(domain),
|
| 133 |
+
"source": source.strip().lower(),
|
| 134 |
+
"date": date.strip(),
|
| 135 |
+
"variable": variable.strip().lower(),
|
| 136 |
+
"lon_min": float(lon_min),
|
| 137 |
+
"lon_max": float(lon_max),
|
| 138 |
+
"lat_min": float(lat_min),
|
| 139 |
+
"lat_max": float(lat_max),
|
| 140 |
+
"format": "netcdf",
|
| 141 |
+
}
|
| 142 |
+
if depth is not None:
|
| 143 |
+
payload["depth"] = float(depth)
|
| 144 |
+
result = _post("/data/export", payload)
|
| 145 |
+
path = result.get("download_path")
|
| 146 |
+
if isinstance(path, str) and path.startswith("/download/"):
|
| 147 |
+
result["download_url"] = f"{API_URL}{path}"
|
| 148 |
+
return result
|
| 149 |
+
|
| 150 |
+
@mcp.tool()
|
| 151 |
+
def marine_download(token: str) -> dict[str, Any]:
|
| 152 |
+
"""Convert an export token into a browser HTTPS download URL."""
|
| 153 |
+
token = token.strip()
|
| 154 |
+
if not re.fullmatch(r"[A-Za-z0-9_-]{20,160}", token):
|
| 155 |
+
raise ValueError("invalid download token")
|
| 156 |
+
return {"download_url": f"{API_URL}/download/{token}"}
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
@mcp.tool()
|
| 161 |
+
def marine_fisheries_catalog() -> dict:
|
| 162 |
+
"""Compatibility alias for the live Hugging Face squid catalog."""
|
| 163 |
+
return fisheries_catalog("squid")
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
@mcp.tool()
|
| 167 |
+
def marine_export(
|
| 168 |
+
date: str,
|
| 169 |
+
lon_min: float,
|
| 170 |
+
lon_max: float,
|
| 171 |
+
lat_min: float,
|
| 172 |
+
lat_max: float,
|
| 173 |
+
variable: str,
|
| 174 |
+
source: str,
|
| 175 |
+
format: str = "netcdf",
|
| 176 |
+
domain: str = "ocean",
|
| 177 |
+
depth: float | None = None,
|
| 178 |
+
) -> dict[str, Any]:
|
| 179 |
+
'Export Ocean data as netcdf/csv/xlsx/json/geotiff/png.'
|
| 180 |
+
payload = {
|
| 181 |
+
"domain": _norm_domain(domain),
|
| 182 |
+
"source": source.strip().lower(),
|
| 183 |
+
"date": date.strip(),
|
| 184 |
+
"variable": variable.strip().lower(),
|
| 185 |
+
"lon_min": float(lon_min),
|
| 186 |
+
"lon_max": float(lon_max),
|
| 187 |
+
"lat_min": float(lat_min),
|
| 188 |
+
"lat_max": float(lat_max),
|
| 189 |
+
"format": format.strip().lower(),
|
| 190 |
+
}
|
| 191 |
+
if depth is not None:
|
| 192 |
+
payload["depth"] = float(depth)
|
| 193 |
+
result = _post("/data/export", payload)
|
| 194 |
+
path = result.get("download_path")
|
| 195 |
+
if isinstance(path, str) and path.startswith("/download/"):
|
| 196 |
+
result["download_url"] = f"{API_URL}{path}"
|
| 197 |
+
return result
|
| 198 |
+
|
| 199 |
+
# ============================================================================
|
| 200 |
+
# Hugging Face fisheries data bridge
|
| 201 |
+
# ============================================================================
|
| 202 |
+
HF_SQUID_DATASET_REPO = (
|
| 203 |
+
os.environ.get("HF_SQUID_DATASET_REPO")
|
| 204 |
+
or os.environ.get("HF_DATASET_REPO")
|
| 205 |
+
or "globalsquiddatabase/squid_dataset"
|
| 206 |
+
).strip()
|
| 207 |
+
HF_TUNA_DATASET_REPO = (
|
| 208 |
+
os.environ.get("HF_TUNA_DATASET_REPO")
|
| 209 |
+
or "globalsquiddatabase/Tuna-Fisheries-Dataset"
|
| 210 |
+
).strip()
|
| 211 |
+
HF_DATASET_REPOS = {
|
| 212 |
+
"squid": HF_SQUID_DATASET_REPO,
|
| 213 |
+
"tuna": HF_TUNA_DATASET_REPO,
|
| 214 |
+
}
|
| 215 |
+
HF_DATASET_REPO = HF_SQUID_DATASET_REPO
|
| 216 |
+
|
| 217 |
+
_HF_TREE_CACHE: dict[str, dict[str, Any]] = {}
|
| 218 |
+
|
| 219 |
+
_SQUID_CATALOG = [
|
| 220 |
+
{
|
| 221 |
+
"source": "FAO FishStatJ",
|
| 222 |
+
"resource": "全球柔鱼科捕捞量",
|
| 223 |
+
"variables": ["catch", "species", "country_or_area", "year"],
|
| 224 |
+
"time_range": "1998-2024(正式资源清单口径;实际标准层以live inventory为准)",
|
| 225 |
+
"spatial_scale": "全球;无经纬度网格",
|
| 226 |
+
"temporal_scale": "年",
|
| 227 |
+
"science_uses": ["长期捕捞量变化", "国家/地区贡献结构", "物种捕捞组成变化"],
|
| 228 |
+
"caveats": ["不能用于精细渔场位置分析", "没有努力量时不能直接得到CPUE"],
|
| 229 |
+
},
|
| 230 |
+
{
|
| 231 |
+
"source": "Sea Around Us",
|
| 232 |
+
"resource": "全球柔鱼科重建捕捞量",
|
| 233 |
+
"variables": ["reconstructed_catch", "species", "area", "year"],
|
| 234 |
+
"time_range": "1950-2019",
|
| 235 |
+
"spatial_scale": "0.5°×0.5°",
|
| 236 |
+
"temporal_scale": "年",
|
| 237 |
+
"science_uses": ["历史空间捕捞格局", "渔场重心变化", "区域热点年代际变化"],
|
| 238 |
+
"caveats": ["属于重建数据", "使用时必须说明重建口径"],
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"source": "SPRFMO",
|
| 242 |
+
"resource": "南太平洋捕捞量与努力量",
|
| 243 |
+
"variables": ["catch", "effort", "year", "grid"],
|
| 244 |
+
"time_range": "2007-2021(后续补充以live inventory为准)",
|
| 245 |
+
"spatial_scale": "5°×5°",
|
| 246 |
+
"temporal_scale": "年/仓库后续标准层可能含月",
|
| 247 |
+
"science_uses": ["区域作业格局", "捕捞强度变化", "重算CPUE后做相对丰度分析"],
|
| 248 |
+
"caveats": ["CPUE必须用总catch÷总effort重算", "不同努力量单位不可直接相加"],
|
| 249 |
+
},
|
| 250 |
+
{
|
| 251 |
+
"source": "WCPFC",
|
| 252 |
+
"resource": "中西太平洋月度捕捞数据",
|
| 253 |
+
"variables": ["catch", "year", "month", "grid", "coverage"],
|
| 254 |
+
"time_range": "1967-2024",
|
| 255 |
+
"spatial_scale": "1°×1°",
|
| 256 |
+
"temporal_scale": "月",
|
| 257 |
+
"science_uses": ["月尺度捕捞热点", "渔场季节迁移", "与SST/锋面/ENSO做时空匹配"],
|
| 258 |
+
"caveats": ["需结合coverage解释缺测", "缺测不能直接解释为零捕捞"],
|
| 259 |
+
},
|
| 260 |
+
{
|
| 261 |
+
"source": "RAM Legacy",
|
| 262 |
+
"resource": "茎柔鱼资源评估数据",
|
| 263 |
+
"variables": ["catch", "biomass", "recruitment", "CPUE"],
|
| 264 |
+
"time_range": "1950-2024(不同种群覆盖不同)",
|
| 265 |
+
"spatial_scale": "评估种群/stock",
|
| 266 |
+
"temporal_scale": "年",
|
| 267 |
+
"science_uses": ["资源量长期变化", "补充量变化", "资源状态与捕捞压力分析"],
|
| 268 |
+
"caveats": ["不同评估模型单位/标准化口径不同", "跨种群比较前需统一数据字典"],
|
| 269 |
+
},
|
| 270 |
+
{
|
| 271 |
+
"source": "Global Fishing Watch",
|
| 272 |
+
"resource": "全球AIS表观渔船作业努力量",
|
| 273 |
+
"variables": ["apparent_fishing_hours", "vessel_presence", "flag", "gear_type"],
|
| 274 |
+
"time_range": "2012-2024",
|
| 275 |
+
"spatial_scale": "0.1°×0.1°",
|
| 276 |
+
"temporal_scale": "月",
|
| 277 |
+
"science_uses": ["渔船活动强度", "作业努力热点迁移", "与渔获/CPUE联合分析捕捞压力"],
|
| 278 |
+
"caveats": ["AIS+模型推断的表观努力量", "不能等同于捕捞量、日志努力量或资源丰度"],
|
| 279 |
+
},
|
| 280 |
+
{
|
| 281 |
+
"source": "VIIRS VBD",
|
| 282 |
+
"resource": "夜��船探测三变量",
|
| 283 |
+
"variables": ["n_detect", "avg_rade9", "pct_detect"],
|
| 284 |
+
"time_range": "2017-2024",
|
| 285 |
+
"spatial_scale": "15 arcsec 原始;仓库可能含1°标准层",
|
| 286 |
+
"temporal_scale": "月",
|
| 287 |
+
"science_uses": ["夜光作业船热点", "灯光强度与探测稳定性", "补充AIS不足区的活动证据"],
|
| 288 |
+
"caveats": ["夜光探测不是捕捞量", "必须结合cvg评估观测机会"],
|
| 289 |
+
},
|
| 290 |
+
{
|
| 291 |
+
"source": "VIIRS CVG",
|
| 292 |
+
"resource": "卫星覆盖次数/观测机会",
|
| 293 |
+
"variables": ["cvg"],
|
| 294 |
+
"time_range": "2017-2024",
|
| 295 |
+
"spatial_scale": "15 arcsec",
|
| 296 |
+
"temporal_scale": "月",
|
| 297 |
+
"science_uses": ["夜光质量控制", "覆盖偏差校正", "区域/月际可比性评估"],
|
| 298 |
+
"caveats": ["cvg不是渔船活动量", "不能当作捕捞努力量"],
|
| 299 |
+
},
|
| 300 |
+
]
|
| 301 |
+
|
| 302 |
+
_TUNA_SOURCE_TERMS = {
|
| 303 |
+
"WCPFC": ["wcpfc"],
|
| 304 |
+
"IATTC": ["iattc"],
|
| 305 |
+
"ICCAT": ["iccat"],
|
| 306 |
+
"IOTC": ["iotc"],
|
| 307 |
+
"CCSBT": ["ccsbt"],
|
| 308 |
+
"FAO": ["fao"],
|
| 309 |
+
"GFW": ["global fishing watch", "gfw"],
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
_DOMAIN_TERMS = {
|
| 313 |
+
"squid": [
|
| 314 |
+
"柔鱼", "鱿鱼", "squid", "ommastre", "dosidicus", "illex", "todarodes",
|
| 315 |
+
"sprfmo", "npfc", "ram legacy", "viirs", "vbd", "sea around", "sea_around", "gfw",
|
| 316 |
+
],
|
| 317 |
+
"tuna": [
|
| 318 |
+
"金枪鱼", "tuna", "wcpfc", "iattc", "iccat", "iotc", "ccsbt",
|
| 319 |
+
"yellowfin", "bigeye", "skipjack", "albacore", "bluefin", "yft", "bet", "skj",
|
| 320 |
+
],
|
| 321 |
+
}
|
| 322 |
+
|
| 323 |
+
def _hf_headers() -> dict[str, str]:
|
| 324 |
+
token = os.environ.get("HF_TOKEN", "").strip()
|
| 325 |
+
return {"Authorization": f"Bearer {token}"} if token else {}
|
| 326 |
+
|
| 327 |
+
def _hf_tree(repo: str, force: bool = False) -> list[dict[str, Any]]:
|
| 328 |
+
repo = repo.strip()
|
| 329 |
+
now = time.time()
|
| 330 |
+
cache = _HF_TREE_CACHE.get(repo) or {}
|
| 331 |
+
if (
|
| 332 |
+
not force
|
| 333 |
+
and now - float(cache.get("ts") or 0) < 300
|
| 334 |
+
and cache.get("items")
|
| 335 |
+
):
|
| 336 |
+
return list(cache["items"])
|
| 337 |
+
|
| 338 |
+
next_url = f"https://huggingface.co/api/datasets/{repo}/tree/main"
|
| 339 |
+
params: dict[str, Any] | None = {
|
| 340 |
+
"recursive": "true",
|
| 341 |
+
"expand": "false",
|
| 342 |
+
"limit": 1000,
|
| 343 |
+
}
|
| 344 |
+
items: list[dict[str, Any]] = []
|
| 345 |
+
pages = 0
|
| 346 |
+
with httpx.Client(timeout=30.0, follow_redirects=True) as client:
|
| 347 |
+
while next_url and pages < 50:
|
| 348 |
+
r = client.get(next_url, params=params, headers=_hf_headers())
|
| 349 |
+
params = None
|
| 350 |
+
pages += 1
|
| 351 |
+
if r.status_code in {401, 403}:
|
| 352 |
+
raise RuntimeError(
|
| 353 |
+
f"无法读取 Hugging Face Dataset {repo}。请确认 Space Secret 中存在具有 Dataset 读取权限的 HF_TOKEN,"
|
| 354 |
+
"且运行时配置已将 HF_TOKEN 传给 marine MCP 子进程。"
|
| 355 |
+
)
|
| 356 |
+
if r.status_code >= 400:
|
| 357 |
+
raise RuntimeError(
|
| 358 |
+
f"Hugging Face Dataset tree request failed: HTTP {r.status_code} ({repo}): {r.text[:300]}"
|
| 359 |
+
)
|
| 360 |
+
data = r.json()
|
| 361 |
+
if not isinstance(data, list):
|
| 362 |
+
raise RuntimeError(f"Hugging Face Dataset tree returned an unexpected response: {repo}")
|
| 363 |
+
items.extend(x for x in data if isinstance(x, dict))
|
| 364 |
+
next_url = (r.links.get("next") or {}).get("url")
|
| 365 |
+
|
| 366 |
+
if next_url:
|
| 367 |
+
raise RuntimeError("Hugging Face Dataset 文件树超过在线分页安全上限。")
|
| 368 |
+
_HF_TREE_CACHE[repo] = {"ts": now, "items": items}
|
| 369 |
+
return items
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def _repos_for_domain(domain: str) -> list[tuple[str, str]]:
|
| 373 |
+
d = (domain or "all").strip().lower()
|
| 374 |
+
if d == "squid":
|
| 375 |
+
return [("squid", HF_SQUID_DATASET_REPO)]
|
| 376 |
+
if d == "tuna":
|
| 377 |
+
return [("tuna", HF_TUNA_DATASET_REPO)]
|
| 378 |
+
if d in {"all", "fisheries", "fishery"}:
|
| 379 |
+
return list(HF_DATASET_REPOS.items())
|
| 380 |
+
raise ValueError("domain must be one of: squid, tuna, all")
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
def _hf_files(
|
| 384 |
+
domain: str = "all",
|
| 385 |
+
force: bool = False,
|
| 386 |
+
) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
| 387 |
+
files: list[dict[str, Any]] = []
|
| 388 |
+
errors: dict[str, str] = {}
|
| 389 |
+
for repo_domain, repo in _repos_for_domain(domain):
|
| 390 |
+
try:
|
| 391 |
+
items = _files_only(_hf_tree(repo, force=force))
|
| 392 |
+
except Exception as exc:
|
| 393 |
+
errors[repo] = str(exc)[:500]
|
| 394 |
+
continue
|
| 395 |
+
for item in items:
|
| 396 |
+
row = dict(item)
|
| 397 |
+
row["repository"] = repo
|
| 398 |
+
row["repository_domain"] = repo_domain
|
| 399 |
+
files.append(row)
|
| 400 |
+
return files, errors
|
| 401 |
+
|
| 402 |
+
def _files_only(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
| 403 |
+
return [
|
| 404 |
+
x for x in items
|
| 405 |
+
if str(x.get("type") or "").lower() in {"file", "blob"}
|
| 406 |
+
or (
|
| 407 |
+
not str(x.get("type") or "").strip()
|
| 408 |
+
and "path" in x
|
| 409 |
+
and "size" in x
|
| 410 |
+
)
|
| 411 |
+
]
|
| 412 |
+
|
| 413 |
+
def _human_bytes(value: Any) -> str:
|
| 414 |
+
try:
|
| 415 |
+
n = float(value or 0)
|
| 416 |
+
except Exception:
|
| 417 |
+
n = 0.0
|
| 418 |
+
units = ["B", "KB", "MB", "GB", "TB"]
|
| 419 |
+
i = 0
|
| 420 |
+
while n >= 1024 and i < len(units) - 1:
|
| 421 |
+
n /= 1024.0
|
| 422 |
+
i += 1
|
| 423 |
+
return f"{n:.2f} {units[i]}"
|
| 424 |
+
|
| 425 |
+
def _domain_match(path: str, domain: str) -> bool:
|
| 426 |
+
d = (domain or "all").strip().lower()
|
| 427 |
+
if d in {"all", "fisheries", "fishery"}:
|
| 428 |
+
return True
|
| 429 |
+
terms = _DOMAIN_TERMS.get(d)
|
| 430 |
+
if not terms:
|
| 431 |
+
raise ValueError("domain must be one of: squid, tuna, all")
|
| 432 |
+
p = path.lower()
|
| 433 |
+
return any(term in p for term in terms)
|
| 434 |
+
|
| 435 |
+
def _query_terms(query: str) -> list[str]:
|
| 436 |
+
q = (query or "").strip().lower()
|
| 437 |
+
aliases = {
|
| 438 |
+
"柔鱼": ["柔鱼", "鱿鱼", "squid"],
|
| 439 |
+
"鱿鱼": ["柔鱼", "鱿鱼", "squid"],
|
| 440 |
+
"金枪鱼": ["金枪鱼", "tuna"],
|
| 441 |
+
"捕捞量": ["捕捞", "catch"],
|
| 442 |
+
"努力量": ["努力", "effort", "fishing_hours", "fishing hours"],
|
| 443 |
+
"cpue": ["cpue"],
|
| 444 |
+
"渔船": ["gfw", "vessel", "ais", "viirs", "vbd"],
|
| 445 |
+
"夜光": ["viirs", "vbd", "cvg", "n_detect", "rade"],
|
| 446 |
+
"资源评估": ["ram", "assessment", "biomass", "recruitment"],
|
| 447 |
+
}
|
| 448 |
+
terms = [q] if q else []
|
| 449 |
+
for key, vals in aliases.items():
|
| 450 |
+
if key in q:
|
| 451 |
+
terms.extend(vals)
|
| 452 |
+
for token in re.split(r"[\s,,/、;;]+", q):
|
| 453 |
+
if len(token) >= 2:
|
| 454 |
+
terms.append(token)
|
| 455 |
+
out = []
|
| 456 |
+
for t in terms:
|
| 457 |
+
if t and t not in out:
|
| 458 |
+
out.append(t)
|
| 459 |
+
return out
|
| 460 |
+
|
| 461 |
+
@mcp.tool()
|
| 462 |
+
def fisheries_catalog(domain: str = "squid") -> dict[str, Any]:
|
| 463 |
+
"""Return fisheries resources and the scientific questions they can support."""
|
| 464 |
+
d = (domain or "squid").strip().lower()
|
| 465 |
+
if d not in {"squid", "tuna", "all"}:
|
| 466 |
+
raise ValueError("domain must be one of: squid, tuna, all")
|
| 467 |
+
|
| 468 |
+
result: dict[str, Any] = {
|
| 469 |
+
"status": "ok",
|
| 470 |
+
"repositories": [repo for _, repo in _repos_for_domain(d)],
|
| 471 |
+
"data_plane": "Hugging Face Dataset",
|
| 472 |
+
"important_distinction": (
|
| 473 |
+
"HF fisheries Dataset is separate from the school-server tuna_data/squid_data task databases. "
|
| 474 |
+
"Empty school-server task databases do not mean the HF fisheries Dataset is empty."
|
| 475 |
+
),
|
| 476 |
+
"aggregation_rules": {
|
| 477 |
+
"catch": "SUM over time/space; preserve units",
|
| 478 |
+
"effort": "SUM only within compatible units",
|
| 479 |
+
"CPUE": "recompute aggregated total catch / aggregated total effort; never average monthly CPUE",
|
| 480 |
+
"GFW": "AIS/model-derived apparent fishing effort; not catch or stock abundance",
|
| 481 |
+
"VIIRS": "night-light vessel activity evidence; use CVG for observation-opportunity QC",
|
| 482 |
+
},
|
| 483 |
+
}
|
| 484 |
+
|
| 485 |
+
if d in {"squid", "all"}:
|
| 486 |
+
result["squid_semantic_catalog"] = _SQUID_CATALOG
|
| 487 |
+
|
| 488 |
+
try:
|
| 489 |
+
items, repo_errors = _hf_files(d)
|
| 490 |
+
live = []
|
| 491 |
+
for x in items:
|
| 492 |
+
path = str(x.get("path") or "")
|
| 493 |
+
live.append({
|
| 494 |
+
"path": path,
|
| 495 |
+
"repository": x.get("repository", ""),
|
| 496 |
+
"repository_domain": x.get("repository_domain", ""),
|
| 497 |
+
"size_bytes": int(x.get("size") or 0),
|
| 498 |
+
"size": _human_bytes(x.get("size") or 0),
|
| 499 |
+
})
|
| 500 |
+
result["repository_errors"] = repo_errors
|
| 501 |
+
|
| 502 |
+
total_bytes = sum(x["size_bytes"] for x in live)
|
| 503 |
+
result["live_inventory"] = {
|
| 504 |
+
"matched_file_count": len(live),
|
| 505 |
+
"matched_size_bytes": total_bytes,
|
| 506 |
+
"matched_size": _human_bytes(total_bytes),
|
| 507 |
+
"path_preview": live[:40],
|
| 508 |
+
"preview_truncated": len(live) > 40,
|
| 509 |
+
}
|
| 510 |
+
|
| 511 |
+
if d in {"tuna", "all"}:
|
| 512 |
+
groups = {}
|
| 513 |
+
for source, terms in _TUNA_SOURCE_TERMS.items():
|
| 514 |
+
matched = [x for x in live if any(t in x["path"].lower() for t in terms)]
|
| 515 |
+
if matched:
|
| 516 |
+
groups[source] = {
|
| 517 |
+
"file_count": len(matched),
|
| 518 |
+
"size": _human_bytes(sum(x["size_bytes"] for x in matched)),
|
| 519 |
+
"examples": [x["path"] for x in matched[:6]],
|
| 520 |
+
}
|
| 521 |
+
result["tuna_live_groups"] = groups
|
| 522 |
+
result["tuna_note"] = (
|
| 523 |
+
"Tuna availability is derived from the live HF repository tree. "
|
| 524 |
+
"Do not use a planned download list as proof that a tuna dataset is already present."
|
| 525 |
+
)
|
| 526 |
+
except Exception as exc:
|
| 527 |
+
result["live_inventory"] = {"status": "error", "detail": str(exc)}
|
| 528 |
+
|
| 529 |
+
return result
|
| 530 |
+
|
| 531 |
+
@mcp.tool()
|
| 532 |
+
def fisheries_inventory(
|
| 533 |
+
domain: str = "all",
|
| 534 |
+
keyword: str | None = None,
|
| 535 |
+
max_results: int = 80,
|
| 536 |
+
refresh: bool = False,
|
| 537 |
+
query: str | None = None,
|
| 538 |
+
source: str | None = None,
|
| 539 |
+
) -> dict[str, Any]:
|
| 540 |
+
"""Inspect both live fisheries trees.
|
| 541 |
+
|
| 542 |
+
Preferred arguments are ``domain`` (squid/tuna/all) and ``keyword``.
|
| 543 |
+
``query`` and ``source`` are accepted as compatibility aliases because
|
| 544 |
+
some chat runtimes emit those names for inventory searches.
|
| 545 |
+
"""
|
| 546 |
+
if query and not keyword:
|
| 547 |
+
keyword = str(query).strip()
|
| 548 |
+
if source:
|
| 549 |
+
source_text = str(source).strip()
|
| 550 |
+
source_lower = source_text.lower()
|
| 551 |
+
if source_lower in {"squid", "tuna", "all", "fisheries", "fishery"}:
|
| 552 |
+
domain = source_lower
|
| 553 |
+
elif source_text == HF_SQUID_DATASET_REPO:
|
| 554 |
+
domain = "squid"
|
| 555 |
+
elif source_text == HF_TUNA_DATASET_REPO:
|
| 556 |
+
domain = "tuna"
|
| 557 |
+
elif not keyword:
|
| 558 |
+
keyword = source_text
|
| 559 |
+
try:
|
| 560 |
+
items, repo_errors = _hf_files(domain, force=bool(refresh))
|
| 561 |
+
except Exception as exc:
|
| 562 |
+
return {"status": "error", "repositories": HF_DATASET_REPOS, "detail": str(exc)}
|
| 563 |
+
|
| 564 |
+
d = (domain or "all").strip().lower()
|
| 565 |
+
limit = max(1, min(int(max_results or 80), 200))
|
| 566 |
+
qterms = _query_terms(keyword or "")
|
| 567 |
+
|
| 568 |
+
matches = []
|
| 569 |
+
for x in items:
|
| 570 |
+
path = str(x.get("path") or "")
|
| 571 |
+
if d not in {"all", "fisheries", "fishery"} and x.get("repository_domain") != d:
|
| 572 |
+
continue
|
| 573 |
+
plow = path.lower()
|
| 574 |
+
if qterms and not any(t in plow for t in qterms):
|
| 575 |
+
continue
|
| 576 |
+
matches.append({
|
| 577 |
+
"path": path,
|
| 578 |
+
"repository": x.get("repository", ""),
|
| 579 |
+
"repository_domain": x.get("repository_domain", ""),
|
| 580 |
+
"size_bytes": int(x.get("size") or 0),
|
| 581 |
+
"size": _human_bytes(x.get("size") or 0),
|
| 582 |
+
})
|
| 583 |
+
|
| 584 |
+
total_bytes = sum(x["size_bytes"] for x in matches)
|
| 585 |
+
return {
|
| 586 |
+
"status": "ok",
|
| 587 |
+
"repositories": [repo for _, repo in _repos_for_domain(d)],
|
| 588 |
+
"repository_errors": repo_errors,
|
| 589 |
+
"branch": "main",
|
| 590 |
+
"domain": d,
|
| 591 |
+
"keyword": keyword,
|
| 592 |
+
"matched_file_count": len(matches),
|
| 593 |
+
"matched_size_bytes": total_bytes,
|
| 594 |
+
"matched_size": _human_bytes(total_bytes),
|
| 595 |
+
"results": matches[:limit],
|
| 596 |
+
"results_truncated": len(matches) > limit,
|
| 597 |
+
"cache_seconds": 300,
|
| 598 |
+
}
|
| 599 |
+
|
| 600 |
+
@mcp.tool()
|
| 601 |
+
def fisheries_search(query: str, max_results: int = 40) -> dict[str, Any]:
|
| 602 |
+
"""Search real HF fisheries files by source/species/metric/path keywords."""
|
| 603 |
+
q = (query or "").strip()
|
| 604 |
+
if not q:
|
| 605 |
+
raise ValueError("query is required")
|
| 606 |
+
|
| 607 |
+
try:
|
| 608 |
+
items, repo_errors = _hf_files("all")
|
| 609 |
+
except Exception as exc:
|
| 610 |
+
return {"status": "error", "repositories": HF_DATASET_REPOS, "detail": str(exc)}
|
| 611 |
+
|
| 612 |
+
terms = _query_terms(q)
|
| 613 |
+
scored = []
|
| 614 |
+
for x in items:
|
| 615 |
+
path = str(x.get("path") or "")
|
| 616 |
+
plow = path.lower()
|
| 617 |
+
score = sum(1 for t in terms if t in plow)
|
| 618 |
+
if score:
|
| 619 |
+
scored.append((
|
| 620 |
+
score,
|
| 621 |
+
{
|
| 622 |
+
"path": path,
|
| 623 |
+
"repository": x.get("repository", ""),
|
| 624 |
+
"repository_domain": x.get("repository_domain", ""),
|
| 625 |
+
"size_bytes": int(x.get("size") or 0),
|
| 626 |
+
"size": _human_bytes(x.get("size") or 0),
|
| 627 |
+
},
|
| 628 |
+
))
|
| 629 |
+
|
| 630 |
+
scored.sort(key=lambda z: (-z[0], z[1]["path"]))
|
| 631 |
+
limit = max(1, min(int(max_results or 40), 100))
|
| 632 |
+
return {
|
| 633 |
+
"status": "ok",
|
| 634 |
+
"repositories": list(HF_DATASET_REPOS.values()),
|
| 635 |
+
"repository_errors": repo_errors,
|
| 636 |
+
"query": q,
|
| 637 |
+
"matched_file_count": len(scored),
|
| 638 |
+
"results": [x for _, x in scored[:limit]],
|
| 639 |
+
"results_truncated": len(scored) > limit,
|
| 640 |
+
}
|
| 641 |
+
|
| 642 |
+
|
| 643 |
+
def _live_file(path: str, repository: str | None = None) -> dict[str, Any]:
|
| 644 |
+
clean = str(path or "").strip().lstrip("/")
|
| 645 |
+
if not clean:
|
| 646 |
+
raise ValueError("path is required")
|
| 647 |
+
selector = str(repository or "all").strip()
|
| 648 |
+
lowered = selector.lower()
|
| 649 |
+
if lowered in HF_DATASET_REPOS:
|
| 650 |
+
domain = lowered
|
| 651 |
+
elif selector in HF_DATASET_REPOS.values():
|
| 652 |
+
domain = next(k for k, v in HF_DATASET_REPOS.items() if v == selector)
|
| 653 |
+
elif lowered in {"", "all"}:
|
| 654 |
+
domain = "all"
|
| 655 |
+
else:
|
| 656 |
+
raise ValueError("repository must be squid, tuna, all, or an exact configured repository id")
|
| 657 |
+
items, repo_errors = _hf_files(domain)
|
| 658 |
+
exact = [item for item in items if str(item.get("path") or "") == clean]
|
| 659 |
+
if not exact:
|
| 660 |
+
raise ValueError(
|
| 661 |
+
"请求的文件不在所选 Hugging Face main 实时文件树中;"
|
| 662 |
+
"请先使用 fisheries_search 或 fisheries_inventory 确认精确路径。"
|
| 663 |
+
)
|
| 664 |
+
if len(exact) > 1:
|
| 665 |
+
repos = ", ".join(str(item.get("repository") or "") for item in exact)
|
| 666 |
+
raise ValueError(f"同一路径存在于多个仓库({repos}),请显式指定 repository。")
|
| 667 |
+
item = exact[0]
|
| 668 |
+
return {
|
| 669 |
+
"path": clean,
|
| 670 |
+
"size_bytes": int(item.get("size") or 0),
|
| 671 |
+
"repository": str(item.get("repository") or ""),
|
| 672 |
+
"repository_domain": str(item.get("repository_domain") or ""),
|
| 673 |
+
"repository_errors": repo_errors,
|
| 674 |
+
}
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
@mcp.tool()
|
| 678 |
+
def fisheries_analyze_export(
|
| 679 |
+
path: str,
|
| 680 |
+
repository: str | None = None,
|
| 681 |
+
year: int | None = None,
|
| 682 |
+
lon_min: float | None = None,
|
| 683 |
+
lon_max: float | None = None,
|
| 684 |
+
lat_min: float | None = None,
|
| 685 |
+
lat_max: float | None = None,
|
| 686 |
+
metric_columns: str | None = None,
|
| 687 |
+
max_rows: int = 2_000_000,
|
| 688 |
+
) -> dict[str, Any]:
|
| 689 |
+
"""Read a validated HF fisheries CSV/TSV/ZIP, analyze/filter it, and export a real CSV.
|
| 690 |
+
|
| 691 |
+
The path must exactly match one configured Dataset live tree. Repository
|
| 692 |
+
may be squid, tuna, or an exact configured repository id. The tool
|
| 693 |
+
accepts optional year and bounding-box filters, reports actual columns,
|
| 694 |
+
scanned/matched rows, missing values, exact duplicates, monthly counts and
|
| 695 |
+
annual metric sums, then returns a tokenized HTTPS download URL. It never
|
| 696 |
+
accepts arbitrary URLs, repositories, shell commands, or local paths.
|
| 697 |
+
"""
|
| 698 |
+
try:
|
| 699 |
+
item = _live_file(path, repository=repository)
|
| 700 |
+
local_path, revision = download_dataset_file(
|
| 701 |
+
item["path"],
|
| 702 |
+
item["size_bytes"],
|
| 703 |
+
repository=item["repository"],
|
| 704 |
+
)
|
| 705 |
+
return analyze_and_export(
|
| 706 |
+
local_path,
|
| 707 |
+
dataset_path=item["path"],
|
| 708 |
+
revision=revision,
|
| 709 |
+
repository=item["repository"],
|
| 710 |
+
year=year,
|
| 711 |
+
lon_min=lon_min,
|
| 712 |
+
lon_max=lon_max,
|
| 713 |
+
lat_min=lat_min,
|
| 714 |
+
lat_max=lat_max,
|
| 715 |
+
metric_columns=metric_columns,
|
| 716 |
+
max_rows=max_rows,
|
| 717 |
+
)
|
| 718 |
+
except Exception as exc:
|
| 719 |
+
return {
|
| 720 |
+
"status": "error",
|
| 721 |
+
"repository": str(repository or "all"),
|
| 722 |
+
"path": str(path or ""),
|
| 723 |
+
"detail": str(exc),
|
| 724 |
+
}
|
| 725 |
+
|
| 726 |
+
@mcp.tool()
|
| 727 |
+
def fisheries_data_rules() -> dict[str, Any]:
|
| 728 |
+
"""Return fisheries aggregation and interpretation rules."""
|
| 729 |
+
return {
|
| 730 |
+
"catch": {
|
| 731 |
+
"aggregation": "sum",
|
| 732 |
+
"rule": "时间/空间聚合采用求和,并保留原始单位。",
|
| 733 |
+
},
|
| 734 |
+
"effort": {
|
| 735 |
+
"aggregation": "sum",
|
| 736 |
+
"rule": "时间/空间聚合采用求和;fishing hours 与 vessel-days 等不同单位不可直接相加。",
|
| 737 |
+
},
|
| 738 |
+
"CPUE": {
|
| 739 |
+
"aggregation": "recompute",
|
| 740 |
+
"rule": "CPUE = 聚合后的总catch / 聚合后的总effort;禁止直接平均月度或格点CPUE。",
|
| 741 |
+
},
|
| 742 |
+
"GFW": {
|
| 743 |
+
"rule": "apparent fishing hours 是AIS+模型推断的表观作业努力量,不等同于真实捕捞量或资源丰度。",
|
| 744 |
+
},
|
| 745 |
+
"VIIRS": {
|
| 746 |
+
"rule": "n_detect/avg_rade9/pct_detect是夜光船活动指标;cvg是观测机会/覆盖质量控制变量。",
|
| 747 |
+
},
|
| 748 |
+
"missing_time": {
|
| 749 |
+
"rule": "不得把月度/年度数据伪装成逐日数据;缺失月份必须显式报告。",
|
| 750 |
+
},
|
| 751 |
+
}
|
| 752 |
+
|
| 753 |
+
|
| 754 |
+
if __name__ == "__main__":
|
| 755 |
+
mcp.run()
|
query_tools/__init__.py
DELETED
|
@@ -1,21 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
查询工具模块包
|
| 3 |
-
|
| 4 |
-
包含以下查询工具:
|
| 5 |
-
- logbook_availability_query: Logbook 数据可用性查询
|
| 6 |
-
- query_gfw: GFW 渔船作业努力量查询(2012-2024)
|
| 7 |
-
- literature_cpue_query: 文献 CPUE 数据查询
|
| 8 |
-
- query_sprfmo: SPRFMO 南太平洋数据查询
|
| 9 |
-
"""
|
| 10 |
-
|
| 11 |
-
from .logbook_availability_query import query_logbook_availability
|
| 12 |
-
from .query_gfw import query_gfw
|
| 13 |
-
from .literature_cpue_query import query_literature_cpue
|
| 14 |
-
from .query_sprfmo import query_sprfmo
|
| 15 |
-
|
| 16 |
-
__all__ = [
|
| 17 |
-
'query_logbook_availability',
|
| 18 |
-
'query_gfw',
|
| 19 |
-
'query_literature_cpue',
|
| 20 |
-
'query_sprfmo',
|
| 21 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
query_tools/literature_cpue_query.py
DELETED
|
@@ -1,384 +0,0 @@
|
|
| 1 |
-
import pandas as pd
|
| 2 |
-
import os
|
| 3 |
-
from datetime import datetime
|
| 4 |
-
|
| 5 |
-
# ============================
|
| 6 |
-
# 项目路径
|
| 7 |
-
# ============================
|
| 8 |
-
|
| 9 |
-
BASE_DIR = os.path.dirname(
|
| 10 |
-
os.path.dirname(
|
| 11 |
-
os.path.abspath(__file__)
|
| 12 |
-
)
|
| 13 |
-
)
|
| 14 |
-
|
| 15 |
-
DATA_PATH = os.path.join(
|
| 16 |
-
BASE_DIR,
|
| 17 |
-
"data",
|
| 18 |
-
"literature_cpue.csv"
|
| 19 |
-
)
|
| 20 |
-
|
| 21 |
-
OUTPUT_DIR = os.path.join(
|
| 22 |
-
BASE_DIR,
|
| 23 |
-
"output"
|
| 24 |
-
)
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
# ============================
|
| 28 |
-
# Markdown生成
|
| 29 |
-
# ============================
|
| 30 |
-
|
| 31 |
-
def dataframe_to_markdown(df, max_rows=10):
|
| 32 |
-
if df.empty:
|
| 33 |
-
return "暂无数据"
|
| 34 |
-
|
| 35 |
-
df = df.head(max_rows)
|
| 36 |
-
|
| 37 |
-
columns = df.columns.tolist()
|
| 38 |
-
|
| 39 |
-
text = "| " + " | ".join(columns) + " |\n"
|
| 40 |
-
|
| 41 |
-
text += "| " + " | ".join(
|
| 42 |
-
["---"] * len(columns)
|
| 43 |
-
) + " |\n"
|
| 44 |
-
|
| 45 |
-
for _, row in df.iterrows():
|
| 46 |
-
text += "| " + " | ".join(
|
| 47 |
-
str(x)
|
| 48 |
-
for x in row.tolist()
|
| 49 |
-
) + " |\n"
|
| 50 |
-
|
| 51 |
-
return text
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
# ============================
|
| 55 |
-
# 年份检查
|
| 56 |
-
# ============================
|
| 57 |
-
|
| 58 |
-
def check_year(value, name):
|
| 59 |
-
if value is None:
|
| 60 |
-
return None
|
| 61 |
-
|
| 62 |
-
try:
|
| 63 |
-
|
| 64 |
-
return int(value)
|
| 65 |
-
|
| 66 |
-
except:
|
| 67 |
-
|
| 68 |
-
raise ValueError(
|
| 69 |
-
f"{name}必须为数字,例如2020"
|
| 70 |
-
)
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
# ============================
|
| 74 |
-
# CPUE文献查询函数
|
| 75 |
-
# ============================
|
| 76 |
-
|
| 77 |
-
def query_literature_cpue(
|
| 78 |
-
region=None,
|
| 79 |
-
year_start=None,
|
| 80 |
-
year_end=None,
|
| 81 |
-
species=None,
|
| 82 |
-
response_variable=None,
|
| 83 |
-
paper_type=None,
|
| 84 |
-
output_format="markdown"
|
| 85 |
-
):
|
| 86 |
-
"""
|
| 87 |
-
查询CPUE相关论文
|
| 88 |
-
|
| 89 |
-
参数:
|
| 90 |
-
----------------
|
| 91 |
-
region:
|
| 92 |
-
海区
|
| 93 |
-
|
| 94 |
-
year_start:
|
| 95 |
-
起始年份
|
| 96 |
-
|
| 97 |
-
year_end:
|
| 98 |
-
结束年份
|
| 99 |
-
|
| 100 |
-
species:
|
| 101 |
-
物种
|
| 102 |
-
|
| 103 |
-
response_variable:
|
| 104 |
-
响应变量
|
| 105 |
-
例如 CPUE biomass
|
| 106 |
-
|
| 107 |
-
paper_type:
|
| 108 |
-
论文类型
|
| 109 |
-
|
| 110 |
-
output_format:
|
| 111 |
-
markdown/csv/excel
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
返回:
|
| 115 |
-
----------------
|
| 116 |
-
|
| 117 |
-
summary
|
| 118 |
-
|
| 119 |
-
records
|
| 120 |
-
|
| 121 |
-
preview_markdown
|
| 122 |
-
|
| 123 |
-
csv_path
|
| 124 |
-
|
| 125 |
-
excel_path
|
| 126 |
-
|
| 127 |
-
source_files
|
| 128 |
-
|
| 129 |
-
"""
|
| 130 |
-
|
| 131 |
-
try:
|
| 132 |
-
|
| 133 |
-
# ======================
|
| 134 |
-
# 参数处理
|
| 135 |
-
# ======================
|
| 136 |
-
|
| 137 |
-
year_start = check_year(
|
| 138 |
-
year_start,
|
| 139 |
-
"year_start"
|
| 140 |
-
)
|
| 141 |
-
|
| 142 |
-
year_end = check_year(
|
| 143 |
-
year_end,
|
| 144 |
-
"year_end"
|
| 145 |
-
)
|
| 146 |
-
|
| 147 |
-
if year_start and year_end:
|
| 148 |
-
|
| 149 |
-
if year_start > year_end:
|
| 150 |
-
return {
|
| 151 |
-
|
| 152 |
-
"summary":
|
| 153 |
-
"开始年份不能大于结束年份",
|
| 154 |
-
|
| 155 |
-
"records": []
|
| 156 |
-
|
| 157 |
-
}
|
| 158 |
-
|
| 159 |
-
# ======================
|
| 160 |
-
# 数据检查
|
| 161 |
-
# ======================
|
| 162 |
-
|
| 163 |
-
if not os.path.exists(DATA_PATH):
|
| 164 |
-
return {
|
| 165 |
-
|
| 166 |
-
"summary":
|
| 167 |
-
"未找到CPUE文献数据库",
|
| 168 |
-
|
| 169 |
-
"records": []
|
| 170 |
-
|
| 171 |
-
}
|
| 172 |
-
|
| 173 |
-
df = pd.read_csv(
|
| 174 |
-
DATA_PATH
|
| 175 |
-
)
|
| 176 |
-
|
| 177 |
-
# 年份转换
|
| 178 |
-
|
| 179 |
-
if "year" in df.columns:
|
| 180 |
-
df["year"] = pd.to_numeric(
|
| 181 |
-
df["year"],
|
| 182 |
-
errors="coerce"
|
| 183 |
-
)
|
| 184 |
-
|
| 185 |
-
result = df.copy()
|
| 186 |
-
|
| 187 |
-
# ======================
|
| 188 |
-
# 条件过滤
|
| 189 |
-
# ======================
|
| 190 |
-
|
| 191 |
-
filters = {
|
| 192 |
-
|
| 193 |
-
"region": region,
|
| 194 |
-
|
| 195 |
-
"species": species,
|
| 196 |
-
|
| 197 |
-
"response_variable":
|
| 198 |
-
response_variable,
|
| 199 |
-
|
| 200 |
-
"paper_type":
|
| 201 |
-
paper_type
|
| 202 |
-
|
| 203 |
-
}
|
| 204 |
-
|
| 205 |
-
for column, value in filters.items():
|
| 206 |
-
|
| 207 |
-
if value:
|
| 208 |
-
result = result[
|
| 209 |
-
result[column]
|
| 210 |
-
.astype(str)
|
| 211 |
-
.str.contains(
|
| 212 |
-
value,
|
| 213 |
-
na=False,
|
| 214 |
-
case=False
|
| 215 |
-
)
|
| 216 |
-
]
|
| 217 |
-
|
| 218 |
-
if year_start:
|
| 219 |
-
result = result[
|
| 220 |
-
result.year >= year_start
|
| 221 |
-
]
|
| 222 |
-
|
| 223 |
-
if year_end:
|
| 224 |
-
result = result[
|
| 225 |
-
result.year <= year_end
|
| 226 |
-
]
|
| 227 |
-
|
| 228 |
-
# ======================
|
| 229 |
-
# 无结果
|
| 230 |
-
# ======================
|
| 231 |
-
|
| 232 |
-
if result.empty:
|
| 233 |
-
return {
|
| 234 |
-
|
| 235 |
-
"summary":
|
| 236 |
-
"没有找到符合条件的CPUE论文",
|
| 237 |
-
|
| 238 |
-
"records": [],
|
| 239 |
-
|
| 240 |
-
"preview_markdown":
|
| 241 |
-
"暂无数据",
|
| 242 |
-
|
| 243 |
-
"source_files":
|
| 244 |
-
[
|
| 245 |
-
DATA_PATH
|
| 246 |
-
]
|
| 247 |
-
|
| 248 |
-
}
|
| 249 |
-
|
| 250 |
-
# ======================
|
| 251 |
-
# 统计摘要
|
| 252 |
-
# ======================
|
| 253 |
-
|
| 254 |
-
summary = {
|
| 255 |
-
|
| 256 |
-
"paper_count":
|
| 257 |
-
len(result),
|
| 258 |
-
|
| 259 |
-
"year_range":
|
| 260 |
-
[
|
| 261 |
-
int(result.year.min()),
|
| 262 |
-
int(result.year.max())
|
| 263 |
-
],
|
| 264 |
-
|
| 265 |
-
"regions":
|
| 266 |
-
result.region.unique()
|
| 267 |
-
.tolist(),
|
| 268 |
-
|
| 269 |
-
"species":
|
| 270 |
-
result.species.unique()
|
| 271 |
-
.tolist(),
|
| 272 |
-
|
| 273 |
-
"models":
|
| 274 |
-
result.model.unique()
|
| 275 |
-
.tolist()
|
| 276 |
-
|
| 277 |
-
}
|
| 278 |
-
|
| 279 |
-
# ======================
|
| 280 |
-
# 文件输出
|
| 281 |
-
# ======================
|
| 282 |
-
|
| 283 |
-
os.makedirs(
|
| 284 |
-
OUTPUT_DIR,
|
| 285 |
-
exist_ok=True
|
| 286 |
-
)
|
| 287 |
-
|
| 288 |
-
timestamp = datetime.now() \
|
| 289 |
-
.strftime(
|
| 290 |
-
"%Y%m%d_%H%M%S"
|
| 291 |
-
)
|
| 292 |
-
|
| 293 |
-
csv_path = None
|
| 294 |
-
|
| 295 |
-
excel_path = None
|
| 296 |
-
|
| 297 |
-
if output_format in [
|
| 298 |
-
"csv",
|
| 299 |
-
"excel"
|
| 300 |
-
]:
|
| 301 |
-
csv_path = os.path.join(
|
| 302 |
-
OUTPUT_DIR,
|
| 303 |
-
f"cpue_literature_{timestamp}.csv"
|
| 304 |
-
)
|
| 305 |
-
|
| 306 |
-
result.to_csv(
|
| 307 |
-
csv_path,
|
| 308 |
-
index=False,
|
| 309 |
-
encoding="utf-8-sig"
|
| 310 |
-
)
|
| 311 |
-
|
| 312 |
-
if output_format == "excel":
|
| 313 |
-
excel_path = os.path.join(
|
| 314 |
-
OUTPUT_DIR,
|
| 315 |
-
f"cpue_literature_{timestamp}.xlsx"
|
| 316 |
-
)
|
| 317 |
-
|
| 318 |
-
result.to_excel(
|
| 319 |
-
excel_path,
|
| 320 |
-
index=False
|
| 321 |
-
)
|
| 322 |
-
|
| 323 |
-
# ======================
|
| 324 |
-
# 返回Agent
|
| 325 |
-
# ======================
|
| 326 |
-
|
| 327 |
-
return {
|
| 328 |
-
|
| 329 |
-
"summary":
|
| 330 |
-
summary,
|
| 331 |
-
|
| 332 |
-
"records":
|
| 333 |
-
result.to_dict(
|
| 334 |
-
orient="records"
|
| 335 |
-
),
|
| 336 |
-
|
| 337 |
-
"preview_markdown":
|
| 338 |
-
dataframe_to_markdown(
|
| 339 |
-
result
|
| 340 |
-
),
|
| 341 |
-
|
| 342 |
-
"csv_path":
|
| 343 |
-
csv_path,
|
| 344 |
-
|
| 345 |
-
"excel_path":
|
| 346 |
-
excel_path,
|
| 347 |
-
|
| 348 |
-
"source_files":
|
| 349 |
-
[
|
| 350 |
-
DATA_PATH
|
| 351 |
-
]
|
| 352 |
-
|
| 353 |
-
}
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
except Exception as e:
|
| 358 |
-
|
| 359 |
-
return {
|
| 360 |
-
|
| 361 |
-
"summary":
|
| 362 |
-
f"查询CPUE文献失败:{str(e)}",
|
| 363 |
-
|
| 364 |
-
"records": []
|
| 365 |
-
|
| 366 |
-
}
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
# 这是一个示例 Python 脚本。
|
| 370 |
-
|
| 371 |
-
# 按 ⌃R 执行或将其替换为您的代码。
|
| 372 |
-
# 按 双击 ⇧ 在所有地方搜索类、文件、工具窗口、操作和设置。
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
def print_hi(name):
|
| 376 |
-
# 在下面的代码行中使用断点来调试脚本。
|
| 377 |
-
print(f'Hi, {name}') # 按 ⌘F8 切换断点。
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
# 按装订区域中的绿色按钮以运行脚本。
|
| 381 |
-
if __name__ == '__main__':
|
| 382 |
-
print_hi('PyCharm')
|
| 383 |
-
|
| 384 |
-
# 访问 https://www.jetbrains.com/help/pycharm/ 获取 PyCharm 帮助
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
query_tools/logbook_availability_query.py
DELETED
|
@@ -1,401 +0,0 @@
|
|
| 1 |
-
import pandas as pd
|
| 2 |
-
import os
|
| 3 |
-
from datetime import datetime
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
# ============================
|
| 7 |
-
# 自动定位项目路径
|
| 8 |
-
# ============================
|
| 9 |
-
|
| 10 |
-
BASE_DIR = os.path.dirname(
|
| 11 |
-
os.path.dirname(
|
| 12 |
-
os.path.abspath(__file__)
|
| 13 |
-
)
|
| 14 |
-
)
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
DATA_PATH = os.path.join(
|
| 18 |
-
BASE_DIR,
|
| 19 |
-
"data",
|
| 20 |
-
"logbook_availability.csv"
|
| 21 |
-
)
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
OUTPUT_DIR = os.path.join(
|
| 25 |
-
BASE_DIR,
|
| 26 |
-
"output"
|
| 27 |
-
)
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
# ============================
|
| 32 |
-
# markdown生成函数
|
| 33 |
-
# 替代 pandas.to_markdown()
|
| 34 |
-
# ============================
|
| 35 |
-
|
| 36 |
-
def dataframe_to_markdown(df, max_rows=10):
|
| 37 |
-
|
| 38 |
-
if df.empty:
|
| 39 |
-
return "暂无数据"
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
df=df.head(max_rows)
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
columns=df.columns.tolist()
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
md="| " + " | ".join(columns)+" |\n"
|
| 49 |
-
|
| 50 |
-
md+="| "+" | ".join(
|
| 51 |
-
["---"]*len(columns)
|
| 52 |
-
)+" |\n"
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
for _,row in df.iterrows():
|
| 56 |
-
|
| 57 |
-
md+="| "+" | ".join(
|
| 58 |
-
str(x)
|
| 59 |
-
for x in row.tolist()
|
| 60 |
-
)+" |\n"
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
return md
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
# ============================
|
| 68 |
-
# 参数检查
|
| 69 |
-
# ============================
|
| 70 |
-
|
| 71 |
-
def check_year(value,name):
|
| 72 |
-
|
| 73 |
-
if value is None:
|
| 74 |
-
return None
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
try:
|
| 78 |
-
return int(value)
|
| 79 |
-
|
| 80 |
-
except:
|
| 81 |
-
|
| 82 |
-
raise ValueError(
|
| 83 |
-
f"{name}必须是年份数字,例如2020"
|
| 84 |
-
)
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
# ============================
|
| 90 |
-
# 主查询函数
|
| 91 |
-
# ============================
|
| 92 |
-
|
| 93 |
-
def query_logbook_availability(
|
| 94 |
-
region=None,
|
| 95 |
-
year_start=None,
|
| 96 |
-
year_end=None,
|
| 97 |
-
species=None,
|
| 98 |
-
data_type=None,
|
| 99 |
-
output_format="markdown"
|
| 100 |
-
):
|
| 101 |
-
|
| 102 |
-
"""
|
| 103 |
-
查询logbook数据可用性
|
| 104 |
-
|
| 105 |
-
返回:
|
| 106 |
-
summary
|
| 107 |
-
records
|
| 108 |
-
preview_markdown
|
| 109 |
-
csv_path
|
| 110 |
-
excel_path
|
| 111 |
-
source_files
|
| 112 |
-
|
| 113 |
-
"""
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
try:
|
| 117 |
-
|
| 118 |
-
# ------------------
|
| 119 |
-
# 参数校验
|
| 120 |
-
# ------------------
|
| 121 |
-
|
| 122 |
-
year_start = check_year(
|
| 123 |
-
year_start,
|
| 124 |
-
"year_start"
|
| 125 |
-
)
|
| 126 |
-
|
| 127 |
-
year_end = check_year(
|
| 128 |
-
year_end,
|
| 129 |
-
"year_end"
|
| 130 |
-
)
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
if year_start and year_end:
|
| 135 |
-
|
| 136 |
-
if year_start > year_end:
|
| 137 |
-
|
| 138 |
-
return {
|
| 139 |
-
|
| 140 |
-
"summary":
|
| 141 |
-
"错误:开始年份不能大于结束年份",
|
| 142 |
-
|
| 143 |
-
"records":[]
|
| 144 |
-
|
| 145 |
-
}
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
# ------------------
|
| 150 |
-
# 文件检查
|
| 151 |
-
# ------------------
|
| 152 |
-
|
| 153 |
-
if not os.path.exists(DATA_PATH):
|
| 154 |
-
|
| 155 |
-
return {
|
| 156 |
-
|
| 157 |
-
"summary":
|
| 158 |
-
f"找不到数据文件:{DATA_PATH}",
|
| 159 |
-
|
| 160 |
-
"records":[]
|
| 161 |
-
|
| 162 |
-
}
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
# ------------------
|
| 167 |
-
# 读取数据
|
| 168 |
-
# ------------------
|
| 169 |
-
|
| 170 |
-
df=pd.read_csv(
|
| 171 |
-
DATA_PATH
|
| 172 |
-
)
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
# year强制转换
|
| 176 |
-
|
| 177 |
-
if "year" in df.columns:
|
| 178 |
-
|
| 179 |
-
df["year"]=pd.to_numeric(
|
| 180 |
-
df["year"],
|
| 181 |
-
errors="coerce"
|
| 182 |
-
)
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
result=df.copy()
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
# ------------------
|
| 190 |
-
# 条件过滤
|
| 191 |
-
# ------------------
|
| 192 |
-
|
| 193 |
-
if region:
|
| 194 |
-
|
| 195 |
-
result=result[
|
| 196 |
-
result["region"]
|
| 197 |
-
.astype(str)
|
| 198 |
-
.str.contains(
|
| 199 |
-
region,
|
| 200 |
-
na=False
|
| 201 |
-
)
|
| 202 |
-
]
|
| 203 |
-
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
if species:
|
| 207 |
-
|
| 208 |
-
result=result[
|
| 209 |
-
result["species"]
|
| 210 |
-
.astype(str)
|
| 211 |
-
.str.contains(
|
| 212 |
-
species,
|
| 213 |
-
na=False
|
| 214 |
-
)
|
| 215 |
-
]
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
if data_type:
|
| 220 |
-
|
| 221 |
-
result=result[
|
| 222 |
-
result["data_type"]
|
| 223 |
-
.astype(str)
|
| 224 |
-
.str.contains(
|
| 225 |
-
data_type,
|
| 226 |
-
na=False
|
| 227 |
-
)
|
| 228 |
-
]
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
if year_start:
|
| 233 |
-
|
| 234 |
-
result=result[
|
| 235 |
-
result.year>=year_start
|
| 236 |
-
]
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
if year_end:
|
| 241 |
-
|
| 242 |
-
result=result[
|
| 243 |
-
result.year<=year_end
|
| 244 |
-
]
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
# ------------------
|
| 249 |
-
# 无结果
|
| 250 |
-
# ------------------
|
| 251 |
-
|
| 252 |
-
if result.empty:
|
| 253 |
-
|
| 254 |
-
return {
|
| 255 |
-
|
| 256 |
-
"summary":
|
| 257 |
-
"没有找到符合条件的logbook数据",
|
| 258 |
-
|
| 259 |
-
"records":[],
|
| 260 |
-
|
| 261 |
-
"preview_markdown":
|
| 262 |
-
"暂无数据",
|
| 263 |
-
|
| 264 |
-
"source_files":
|
| 265 |
-
[
|
| 266 |
-
DATA_PATH
|
| 267 |
-
]
|
| 268 |
-
|
| 269 |
-
}
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
# ------------------
|
| 274 |
-
# 统计信息
|
| 275 |
-
# ------------------
|
| 276 |
-
|
| 277 |
-
summary={
|
| 278 |
-
|
| 279 |
-
"records_count":
|
| 280 |
-
len(result),
|
| 281 |
-
|
| 282 |
-
"year_range":
|
| 283 |
-
[
|
| 284 |
-
int(result.year.min()),
|
| 285 |
-
int(result.year.max())
|
| 286 |
-
],
|
| 287 |
-
|
| 288 |
-
"regions":
|
| 289 |
-
result.region.unique().tolist(),
|
| 290 |
-
|
| 291 |
-
"species":
|
| 292 |
-
result.species.unique().tolist()
|
| 293 |
-
|
| 294 |
-
}
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
# ------------------
|
| 299 |
-
# 输出文件
|
| 300 |
-
# ------------------
|
| 301 |
-
|
| 302 |
-
os.makedirs(
|
| 303 |
-
OUTPUT_DIR,
|
| 304 |
-
exist_ok=True
|
| 305 |
-
)
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
timestamp=datetime.now()\
|
| 309 |
-
.strftime("%Y%m%d_%H%M%S")
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
csv_path=None
|
| 313 |
-
excel_path=None
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
if output_format in [
|
| 318 |
-
"csv",
|
| 319 |
-
"excel"
|
| 320 |
-
]:
|
| 321 |
-
|
| 322 |
-
csv_path=os.path.join(
|
| 323 |
-
OUTPUT_DIR,
|
| 324 |
-
f"logbook_{timestamp}.csv"
|
| 325 |
-
)
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
result.to_csv(
|
| 329 |
-
csv_path,
|
| 330 |
-
index=False,
|
| 331 |
-
encoding="utf-8-sig"
|
| 332 |
-
)
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
if output_format=="excel":
|
| 337 |
-
|
| 338 |
-
excel_path=os.path.join(
|
| 339 |
-
OUTPUT_DIR,
|
| 340 |
-
f"logbook_{timestamp}.xlsx"
|
| 341 |
-
)
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
result.to_excel(
|
| 345 |
-
excel_path,
|
| 346 |
-
index=False
|
| 347 |
-
)
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
# ------------------
|
| 352 |
-
# Agent标准返回
|
| 353 |
-
# ------------------
|
| 354 |
-
|
| 355 |
-
return {
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
"summary":
|
| 359 |
-
summary,
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
"records":
|
| 363 |
-
result.to_dict(
|
| 364 |
-
orient="records"
|
| 365 |
-
),
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
"preview_markdown":
|
| 369 |
-
dataframe_to_markdown(
|
| 370 |
-
result
|
| 371 |
-
),
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
"csv_path":
|
| 375 |
-
csv_path,
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
"excel_path":
|
| 379 |
-
excel_path,
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
"source_files":
|
| 383 |
-
[
|
| 384 |
-
DATA_PATH
|
| 385 |
-
]
|
| 386 |
-
|
| 387 |
-
}
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
except Exception as e:
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
return {
|
| 395 |
-
|
| 396 |
-
"summary":
|
| 397 |
-
f"查询失败:{str(e)}",
|
| 398 |
-
|
| 399 |
-
"records":[]
|
| 400 |
-
|
| 401 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
query_tools/query_gfw.py
DELETED
|
@@ -1,310 +0,0 @@
|
|
| 1 |
-
import argparse
|
| 2 |
-
import io
|
| 3 |
-
import tempfile
|
| 4 |
-
import zipfile
|
| 5 |
-
from pathlib import Path
|
| 6 |
-
|
| 7 |
-
import pandas as pd
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
def find_gfw_zip(gfw_zip=None):
|
| 11 |
-
"""
|
| 12 |
-
查找GFW压缩包。
|
| 13 |
-
|
| 14 |
-
如果运行时指定了压缩包路径,就直接使用;
|
| 15 |
-
如果没有指定,就依次在脚本所在文件夹、当前运行目录、
|
| 16 |
-
"""
|
| 17 |
-
if gfw_zip:
|
| 18 |
-
path = Path(gfw_zip).expanduser()
|
| 19 |
-
if not path.exists():
|
| 20 |
-
raise FileNotFoundError(f"指定的GFW压缩包不存在:{path}")
|
| 21 |
-
return path
|
| 22 |
-
|
| 23 |
-
search_dirs = [
|
| 24 |
-
Path(__file__).resolve().parent,
|
| 25 |
-
Path.cwd(),
|
| 26 |
-
Path.home() / "Desktop" / "资料下载" / "柔鱼数据",
|
| 27 |
-
]
|
| 28 |
-
|
| 29 |
-
for folder in search_dirs:
|
| 30 |
-
if not folder.exists():
|
| 31 |
-
continue
|
| 32 |
-
|
| 33 |
-
files = sorted(folder.glob("GFW*.zip"))
|
| 34 |
-
if files:
|
| 35 |
-
return files[0]
|
| 36 |
-
|
| 37 |
-
raise FileNotFoundError(
|
| 38 |
-
"没有找到GFW压缩包。请把GFW zip文件与本脚本放在同一文件夹,"
|
| 39 |
-
"或运行时使用 --gfw-zip 指定文件路径。"
|
| 40 |
-
)
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
def _open_year_zip_streaming(outer_zip, year_zip_path):
|
| 44 |
-
year_zip_file = tempfile.NamedTemporaryFile(delete=False, suffix=".zip")
|
| 45 |
-
try:
|
| 46 |
-
with outer_zip.open(year_zip_path, "r") as source, year_zip_file as target:
|
| 47 |
-
while True:
|
| 48 |
-
chunk = source.read(4 * 1024 * 1024)
|
| 49 |
-
if not chunk:
|
| 50 |
-
break
|
| 51 |
-
target.write(chunk)
|
| 52 |
-
except Exception:
|
| 53 |
-
year_zip_file.close()
|
| 54 |
-
Path(year_zip_file.name).unlink(missing_ok=True)
|
| 55 |
-
raise
|
| 56 |
-
|
| 57 |
-
return zipfile.ZipFile(year_zip_file.name, "r")
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
def read_gfw_month(year, month, gfw_zip=None):
|
| 61 |
-
"""读取指定年份和月份的GFW月尺度数据。"""
|
| 62 |
-
if month < 1 or month > 12:
|
| 63 |
-
raise ValueError("月份必须在1到12之间")
|
| 64 |
-
if not 2012 <= year <= 2024:
|
| 65 |
-
raise ValueError("年份只允许2012到2024年")
|
| 66 |
-
|
| 67 |
-
gfw_zip = find_gfw_zip(gfw_zip)
|
| 68 |
-
|
| 69 |
-
month_csv_name = (
|
| 70 |
-
f"fleet-monthly-csvs-10-v3-{year}-{month:02d}-01.csv"
|
| 71 |
-
)
|
| 72 |
-
year_zip_name = f"fleet-monthly-csvs-10-v3-{year}.zip"
|
| 73 |
-
|
| 74 |
-
with zipfile.ZipFile(gfw_zip, "r") as outer_zip:
|
| 75 |
-
file_list = outer_zip.namelist()
|
| 76 |
-
|
| 77 |
-
# 外层压缩包中直接存在月份CSV
|
| 78 |
-
for file_name in file_list:
|
| 79 |
-
if file_name.endswith(month_csv_name):
|
| 80 |
-
with outer_zip.open(file_name) as file:
|
| 81 |
-
return pd.read_csv(file), gfw_zip
|
| 82 |
-
|
| 83 |
-
# 月份CSV位于年度压缩包中
|
| 84 |
-
year_zip_path = None
|
| 85 |
-
for file_name in file_list:
|
| 86 |
-
if file_name.endswith(year_zip_name):
|
| 87 |
-
year_zip_path = file_name
|
| 88 |
-
break
|
| 89 |
-
|
| 90 |
-
if year_zip_path is None:
|
| 91 |
-
raise FileNotFoundError(f"没有找到{year}年的GFW数据")
|
| 92 |
-
|
| 93 |
-
with _open_year_zip_streaming(outer_zip, year_zip_path) as year_zip:
|
| 94 |
-
for file_name in year_zip.namelist():
|
| 95 |
-
if file_name.endswith(month_csv_name):
|
| 96 |
-
with year_zip.open(file_name) as file:
|
| 97 |
-
return pd.read_csv(file), gfw_zip
|
| 98 |
-
|
| 99 |
-
raise FileNotFoundError(f"没有找到{year}年{month}月的GFW数据")
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
def query_gfw(
|
| 103 |
-
year,
|
| 104 |
-
month,
|
| 105 |
-
lat_min,
|
| 106 |
-
lat_max,
|
| 107 |
-
lon_min,
|
| 108 |
-
lon_max,
|
| 109 |
-
flag=None,
|
| 110 |
-
geartype=None,
|
| 111 |
-
min_fishing_hours=None,
|
| 112 |
-
gfw_zip=None,
|
| 113 |
-
):
|
| 114 |
-
"""
|
| 115 |
-
按时间和空间范围查询GFW渔船作业努力量。
|
| 116 |
-
|
| 117 |
-
lon_min大于lon_max时,表示查询范围跨越180°经线。
|
| 118 |
-
例如145°E至175°W可写为lon_min=145、lon_max=-175。
|
| 119 |
-
"""
|
| 120 |
-
if lat_min > lat_max:
|
| 121 |
-
raise ValueError("最小纬度不能大于最大纬度")
|
| 122 |
-
|
| 123 |
-
data, gfw_zip = read_gfw_month(
|
| 124 |
-
year=year,
|
| 125 |
-
month=month,
|
| 126 |
-
gfw_zip=gfw_zip,
|
| 127 |
-
)
|
| 128 |
-
|
| 129 |
-
required_columns = [
|
| 130 |
-
"date",
|
| 131 |
-
"year",
|
| 132 |
-
"month",
|
| 133 |
-
"cell_ll_lat",
|
| 134 |
-
"cell_ll_lon",
|
| 135 |
-
"flag",
|
| 136 |
-
"geartype",
|
| 137 |
-
"hours",
|
| 138 |
-
"fishing_hours",
|
| 139 |
-
"mmsi_present",
|
| 140 |
-
]
|
| 141 |
-
|
| 142 |
-
missing_columns = [
|
| 143 |
-
column for column in required_columns
|
| 144 |
-
if column not in data.columns
|
| 145 |
-
]
|
| 146 |
-
|
| 147 |
-
if missing_columns:
|
| 148 |
-
raise ValueError(f"数据中缺少必要字段:{missing_columns}")
|
| 149 |
-
|
| 150 |
-
numeric_columns = [
|
| 151 |
-
"cell_ll_lat",
|
| 152 |
-
"cell_ll_lon",
|
| 153 |
-
"hours",
|
| 154 |
-
"fishing_hours",
|
| 155 |
-
"mmsi_present",
|
| 156 |
-
]
|
| 157 |
-
|
| 158 |
-
invalid_value_warnings = []
|
| 159 |
-
for column in numeric_columns:
|
| 160 |
-
original_non_null_count = data[column].notna().sum()
|
| 161 |
-
data[column] = pd.to_numeric(
|
| 162 |
-
data[column],
|
| 163 |
-
errors="coerce",
|
| 164 |
-
)
|
| 165 |
-
coerced_null_count = data[column].isna().sum()
|
| 166 |
-
invalid_count = coerced_null_count - (len(data) - original_non_null_count)
|
| 167 |
-
if invalid_count > 0:
|
| 168 |
-
invalid_value_warnings.append((column, int(invalid_count)))
|
| 169 |
-
|
| 170 |
-
if invalid_value_warnings:
|
| 171 |
-
warning_parts = [
|
| 172 |
-
f"{column}: {count} 个值无法转换为数值"
|
| 173 |
-
for column, count in invalid_value_warnings
|
| 174 |
-
]
|
| 175 |
-
print("数值转换警告:以下列存在无法解析的脏值,已按 NaN 处理 -> " + ";".join(warning_parts))
|
| 176 |
-
|
| 177 |
-
result = data[
|
| 178 |
-
(data["cell_ll_lat"] >= lat_min)
|
| 179 |
-
& (data["cell_ll_lat"] <= lat_max)
|
| 180 |
-
].copy()
|
| 181 |
-
|
| 182 |
-
if lon_min <= lon_max:
|
| 183 |
-
result = result[
|
| 184 |
-
(result["cell_ll_lon"] >= lon_min)
|
| 185 |
-
& (result["cell_ll_lon"] <= lon_max)
|
| 186 |
-
]
|
| 187 |
-
else:
|
| 188 |
-
result = result[
|
| 189 |
-
(result["cell_ll_lon"] >= lon_min)
|
| 190 |
-
| (result["cell_ll_lon"] <= lon_max)
|
| 191 |
-
]
|
| 192 |
-
|
| 193 |
-
if flag:
|
| 194 |
-
result = result[
|
| 195 |
-
result["flag"].notna()
|
| 196 |
-
& (result["flag"].astype(str).str.upper() == flag.upper())
|
| 197 |
-
]
|
| 198 |
-
|
| 199 |
-
if geartype:
|
| 200 |
-
result = result[
|
| 201 |
-
result["geartype"].notna()
|
| 202 |
-
& (result["geartype"].astype(str).str.lower() == geartype.lower())
|
| 203 |
-
]
|
| 204 |
-
|
| 205 |
-
if min_fishing_hours is not None:
|
| 206 |
-
result = result[
|
| 207 |
-
result["fishing_hours"] >= min_fishing_hours
|
| 208 |
-
]
|
| 209 |
-
|
| 210 |
-
result = result.sort_values(
|
| 211 |
-
by=["fishing_hours", "hours"],
|
| 212 |
-
ascending=[False, False],
|
| 213 |
-
).reset_index(drop=True)
|
| 214 |
-
|
| 215 |
-
summary = {
|
| 216 |
-
"data_file": str(gfw_zip),
|
| 217 |
-
"year": year,
|
| 218 |
-
"month": month,
|
| 219 |
-
"lat_range": [lat_min, lat_max],
|
| 220 |
-
"lon_range": [lon_min, lon_max],
|
| 221 |
-
"flag": flag if flag else "all",
|
| 222 |
-
"geartype": geartype if geartype else "all",
|
| 223 |
-
"record_count": len(result),
|
| 224 |
-
"total_hours": round(
|
| 225 |
-
result["hours"].fillna(0).sum(),
|
| 226 |
-
4,
|
| 227 |
-
),
|
| 228 |
-
"total_fishing_hours": round(
|
| 229 |
-
result["fishing_hours"].fillna(0).sum(),
|
| 230 |
-
4,
|
| 231 |
-
),
|
| 232 |
-
}
|
| 233 |
-
|
| 234 |
-
return result, summary
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
def save_result(result, output_file):
|
| 238 |
-
"""保存查询结果。"""
|
| 239 |
-
output_path = Path(output_file).expanduser()
|
| 240 |
-
output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 241 |
-
|
| 242 |
-
result.to_csv(
|
| 243 |
-
output_path,
|
| 244 |
-
index=False,
|
| 245 |
-
encoding="utf-8-sig",
|
| 246 |
-
)
|
| 247 |
-
|
| 248 |
-
return output_path
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
def main():
|
| 252 |
-
parser = argparse.ArgumentParser(
|
| 253 |
-
description="查询GFW月尺度渔船作业努力量数据"
|
| 254 |
-
)
|
| 255 |
-
|
| 256 |
-
parser.add_argument("--gfw-zip", default=None)
|
| 257 |
-
parser.add_argument("--year", type=int, default=2012)
|
| 258 |
-
parser.add_argument("--month", type=int, default=1)
|
| 259 |
-
parser.add_argument("--lat-min", type=float, default=-10)
|
| 260 |
-
parser.add_argument("--lat-max", type=float, default=10)
|
| 261 |
-
parser.add_argument("--lon-min", type=float, default=145)
|
| 262 |
-
parser.add_argument("--lon-max", type=float, default=-175)
|
| 263 |
-
parser.add_argument("--flag", default=None)
|
| 264 |
-
parser.add_argument("--geartype", default=None)
|
| 265 |
-
parser.add_argument("--min-fishing-hours", type=float, default=None)
|
| 266 |
-
parser.add_argument("--output", default="GFW查询结果.csv")
|
| 267 |
-
|
| 268 |
-
args = parser.parse_args()
|
| 269 |
-
|
| 270 |
-
if args.year is not None and not 2012 <= args.year <= 2024:
|
| 271 |
-
raise ValueError("年份只允许2012到2024年")
|
| 272 |
-
|
| 273 |
-
if args.month is not None and not 1 <= args.month <= 12:
|
| 274 |
-
raise ValueError("月份只允许1到12月")
|
| 275 |
-
|
| 276 |
-
result, summary = query_gfw(
|
| 277 |
-
year=args.year,
|
| 278 |
-
month=args.month,
|
| 279 |
-
lat_min=args.lat_min,
|
| 280 |
-
lat_max=args.lat_max,
|
| 281 |
-
lon_min=args.lon_min,
|
| 282 |
-
lon_max=args.lon_max,
|
| 283 |
-
flag=args.flag,
|
| 284 |
-
geartype=args.geartype,
|
| 285 |
-
min_fishing_hours=args.min_fishing_hours,
|
| 286 |
-
gfw_zip=args.gfw_zip,
|
| 287 |
-
)
|
| 288 |
-
|
| 289 |
-
print("\n查询汇总")
|
| 290 |
-
print("-" * 40)
|
| 291 |
-
|
| 292 |
-
for key, value in summary.items():
|
| 293 |
-
print(f"{key}: {value}")
|
| 294 |
-
|
| 295 |
-
print("\n捕捞小时数最高的前10条记录")
|
| 296 |
-
print("-" * 40)
|
| 297 |
-
|
| 298 |
-
if result.empty:
|
| 299 |
-
print("没有查询到符合条件的数据")
|
| 300 |
-
else:
|
| 301 |
-
print(result.head(10).to_string(index=False))
|
| 302 |
-
|
| 303 |
-
output_path = save_result(result, args.output)
|
| 304 |
-
|
| 305 |
-
print("\n结果已保存到:")
|
| 306 |
-
print(output_path)
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
if __name__ == "__main__":
|
| 310 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
query_tools/query_sprfmo.py
DELETED
|
@@ -1,311 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
query_sprfmo.py - SPRFMO 查询函数(最简稳定版)
|
| 3 |
-
"""
|
| 4 |
-
|
| 5 |
-
from pathlib import Path
|
| 6 |
-
import pandas as pd
|
| 7 |
-
|
| 8 |
-
# ============================================================
|
| 9 |
-
# 1. 路径配置
|
| 10 |
-
# ============================================================
|
| 11 |
-
DATA_DIR = Path(r"C:\Users\niniy\Desktop\PythonProject\SPRFMO南太平洋 5×5")
|
| 12 |
-
|
| 13 |
-
# ============================================================
|
| 14 |
-
# 2. 字段映射(原始 Excel 字段 → 标准字段)
|
| 15 |
-
# ============================================================
|
| 16 |
-
CATCH_MAP = {
|
| 17 |
-
"Year": "year",
|
| 18 |
-
"Flag": "country",
|
| 19 |
-
"Latitude": "lat",
|
| 20 |
-
"Longitude": "lon",
|
| 21 |
-
"Species": "species",
|
| 22 |
-
"NumVesels": "numvessels",
|
| 23 |
-
"Harvest_kg": "catch",
|
| 24 |
-
"Discard_kg": "discard",
|
| 25 |
-
}
|
| 26 |
-
|
| 27 |
-
EFFORT_MAP = {
|
| 28 |
-
"Year": "year",
|
| 29 |
-
"Fishery": "gear_type",
|
| 30 |
-
"NumDays": "effort",
|
| 31 |
-
"NumEvents": "num_events",
|
| 32 |
-
"NumVessels": "numvessels",
|
| 33 |
-
}
|
| 34 |
-
|
| 35 |
-
# ============================================================
|
| 36 |
-
# 3. 国家代码映射
|
| 37 |
-
# ============================================================
|
| 38 |
-
COUNTRY_MAP = {
|
| 39 |
-
"中国": "CHN", "China": "CHN", "CHN": "CHN",
|
| 40 |
-
"日本": "JPN", "Japan": "JPN", "JPN": "JPN",
|
| 41 |
-
"韩国": "KOR", "Korea": "KOR", "KOR": "KOR",
|
| 42 |
-
"中国台湾": "TWN", "Taiwan": "TWN", "TWN": "TWN",
|
| 43 |
-
"智利": "CHL", "Chile": "CHL", "CHL": "CHL",
|
| 44 |
-
"秘鲁": "PER", "Peru": "PER", "PER": "PER",
|
| 45 |
-
"俄罗斯": "RUS", "Russia": "RUS", "RUS": "RUS",
|
| 46 |
-
"美国": "USA", "USA": "USA", "United States": "USA",
|
| 47 |
-
"新西兰": "NZL", "New Zealand": "NZL", "NZL": "NZL",
|
| 48 |
-
"澳大利亚": "AUS", "Australia": "AUS", "AUS": "AUS",
|
| 49 |
-
}
|
| 50 |
-
|
| 51 |
-
# ============================================================
|
| 52 |
-
# 4. 主查询入口
|
| 53 |
-
# ============================================================
|
| 54 |
-
def query_sprfmo(filters=None, group_by=None, metrics="catch",
|
| 55 |
-
data_type="catch", output_format="markdown"):
|
| 56 |
-
"""SPRFMO 数据查询入口"""
|
| 57 |
-
if filters is None:
|
| 58 |
-
filters = {}
|
| 59 |
-
if group_by is None:
|
| 60 |
-
group_by = []
|
| 61 |
-
elif isinstance(group_by, str):
|
| 62 |
-
group_by = [group_by]
|
| 63 |
-
if isinstance(metrics, str):
|
| 64 |
-
metrics = [metrics]
|
| 65 |
-
|
| 66 |
-
warnings = []
|
| 67 |
-
|
| 68 |
-
print("\n" + "=" * 60)
|
| 69 |
-
print(f"【query_sprfmo】data_type={data_type}")
|
| 70 |
-
print(f" filters={filters}")
|
| 71 |
-
print(f" group_by={group_by}, metrics={metrics}")
|
| 72 |
-
print("=" * 60)
|
| 73 |
-
|
| 74 |
-
# 诊断信息
|
| 75 |
-
print(f"\n📂 数据文件夹: {DATA_DIR}")
|
| 76 |
-
print(f" 文件夹存在: {DATA_DIR.exists()}")
|
| 77 |
-
if DATA_DIR.exists():
|
| 78 |
-
all_files = list(DATA_DIR.glob("*.xlsx"))
|
| 79 |
-
print(f" 所有 Excel: {[f.name for f in all_files]}")
|
| 80 |
-
if not all_files:
|
| 81 |
-
warnings.append(f"⚠️ 文件夹下没有任何 .xlsx 文件")
|
| 82 |
-
else:
|
| 83 |
-
warnings.append(f"⚠️ 文件夹不存在: {DATA_DIR}")
|
| 84 |
-
return _error_result(f"文件夹不存在: {DATA_DIR}", warnings)
|
| 85 |
-
|
| 86 |
-
# 加载数据
|
| 87 |
-
if data_type == "catch":
|
| 88 |
-
df, source = _load_catch_data()
|
| 89 |
-
elif data_type == "effort":
|
| 90 |
-
df, source = _load_effort_data()
|
| 91 |
-
else:
|
| 92 |
-
return _error_result(f"未知 data_type: {data_type}", warnings)
|
| 93 |
-
|
| 94 |
-
# 字段翻译
|
| 95 |
-
df = _translate(df, data_type)
|
| 96 |
-
print(f" 翻译后字段: {df.columns.tolist()}")
|
| 97 |
-
|
| 98 |
-
# 国家代码翻译
|
| 99 |
-
if "country" in filters and filters["country"]:
|
| 100 |
-
c = str(filters["country"])
|
| 101 |
-
translated = COUNTRY_MAP.get(c, c)
|
| 102 |
-
filters["country"] = translated
|
| 103 |
-
if translated != c:
|
| 104 |
-
print(f" 国家翻译: '{c}' → '{translated}'")
|
| 105 |
-
|
| 106 |
-
# 筛选
|
| 107 |
-
df = _apply_filters(df, filters, data_type)
|
| 108 |
-
|
| 109 |
-
if len(df) == 0:
|
| 110 |
-
warnings.append("⚠️ 筛选后无数据,请检查筛选条件")
|
| 111 |
-
|
| 112 |
-
# 聚合
|
| 113 |
-
df = _aggregate(df, group_by, metrics, data_type)
|
| 114 |
-
|
| 115 |
-
# 返回
|
| 116 |
-
return {
|
| 117 |
-
"preview_markdown": df.to_markdown(index=False, floatfmt=".2f") if len(df) > 0 else "*(无数据)*",
|
| 118 |
-
"records": df.to_dict(orient="records"),
|
| 119 |
-
"csv_path": None,
|
| 120 |
-
"excel_path": None,
|
| 121 |
-
"summary": _build_summary(df, data_type),
|
| 122 |
-
"source_files": [source],
|
| 123 |
-
"warnings": warnings,
|
| 124 |
-
"metadata": {
|
| 125 |
-
"spatial_resolution": "5x5 degree",
|
| 126 |
-
"time_resolution": "annual",
|
| 127 |
-
"region": "South Pacific",
|
| 128 |
-
"unit": "kg" if data_type == "catch" else "days",
|
| 129 |
-
}
|
| 130 |
-
}
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
# ============================================================
|
| 134 |
-
# 5. 加载数据
|
| 135 |
-
# ============================================================
|
| 136 |
-
def _load_catch_data():
|
| 137 |
-
"""加载捕捞量 Excel"""
|
| 138 |
-
files = []
|
| 139 |
-
for kw in ["捕捞", "Catch", "catch"]:
|
| 140 |
-
files.extend(DATA_DIR.glob(f"*{kw}*.xlsx"))
|
| 141 |
-
files = list(set(files))
|
| 142 |
-
|
| 143 |
-
if not files:
|
| 144 |
-
all_files = [f.name for f in DATA_DIR.glob("*.xlsx")]
|
| 145 |
-
raise FileNotFoundError(
|
| 146 |
-
f"找不到捕捞量文件(需含 '捕捞' 或 'Catch')\n"
|
| 147 |
-
f"文件夹下所有 Excel: {all_files}"
|
| 148 |
-
)
|
| 149 |
-
|
| 150 |
-
df = pd.read_excel(files[0])
|
| 151 |
-
print(f" ✅ 加载: {files[0].name} ({len(df)} 条)")
|
| 152 |
-
return df, str(files[0])
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
def _load_effort_data():
|
| 156 |
-
"""加载努力量 Excel"""
|
| 157 |
-
files = []
|
| 158 |
-
for kw in ["努力", "Effort", "effort"]:
|
| 159 |
-
files.extend(DATA_DIR.glob(f"*{kw}*.xlsx"))
|
| 160 |
-
files = list(set(files))
|
| 161 |
-
|
| 162 |
-
if not files:
|
| 163 |
-
all_files = [f.name for f in DATA_DIR.glob("*.xlsx")]
|
| 164 |
-
raise FileNotFoundError(
|
| 165 |
-
f"找不到努力量文件(需含 '努力' 或 'Effort')\n"
|
| 166 |
-
f"文件夹下所有 Excel: {all_files}"
|
| 167 |
-
)
|
| 168 |
-
|
| 169 |
-
df = pd.read_excel(files[0])
|
| 170 |
-
print(f" ✅ 加载: {files[0].name} ({len(df)} 条)")
|
| 171 |
-
return df, str(files[0])
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
# ============================================================
|
| 175 |
-
# 6. 字段翻译
|
| 176 |
-
# ============================================================
|
| 177 |
-
def _translate(df, data_type):
|
| 178 |
-
"""原始字段 → 标准字段"""
|
| 179 |
-
fmap = CATCH_MAP if data_type == "catch" else EFFORT_MAP
|
| 180 |
-
used = {k: v for k, v in fmap.items() if k in df.columns}
|
| 181 |
-
return df.rename(columns=used)
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
# ============================================================
|
| 185 |
-
# 7. 筛选
|
| 186 |
-
# ============================================================
|
| 187 |
-
def _apply_filters(df, filters, data_type):
|
| 188 |
-
n0 = len(df)
|
| 189 |
-
|
| 190 |
-
if "year_start" in filters and filters["year_start"] is not None:
|
| 191 |
-
df = df[df["year"] >= filters["year_start"]]
|
| 192 |
-
if "year_end" in filters and filters["year_end"] is not None:
|
| 193 |
-
df = df[df["year"] <= filters["year_end"]]
|
| 194 |
-
|
| 195 |
-
if data_type == "catch":
|
| 196 |
-
if "country" in filters and filters["country"]:
|
| 197 |
-
c = str(filters["country"]).upper()
|
| 198 |
-
df = df[df["country"].astype(str).str.upper() == c]
|
| 199 |
-
if "species" in filters and filters["species"]:
|
| 200 |
-
kw = filters["species"]
|
| 201 |
-
df = df[df["species"].astype(str).str.contains(kw, case=False, na=False)]
|
| 202 |
-
|
| 203 |
-
elif data_type == "effort":
|
| 204 |
-
if "gear_type" in filters and filters["gear_type"]:
|
| 205 |
-
kw = filters["gear_type"]
|
| 206 |
-
df = df[df["gear_type"].astype(str).str.contains(kw, case=False, na=False)]
|
| 207 |
-
|
| 208 |
-
print(f" 筛选: {n0} → {len(df)} 条")
|
| 209 |
-
return df
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
# ============================================================
|
| 213 |
-
# 8. 聚合
|
| 214 |
-
# ============================================================
|
| 215 |
-
def _aggregate(df, group_by, metrics, data_type):
|
| 216 |
-
if not group_by:
|
| 217 |
-
return df
|
| 218 |
-
|
| 219 |
-
valid_g = [g for g in group_by if g in df.columns]
|
| 220 |
-
if not valid_g:
|
| 221 |
-
return df
|
| 222 |
-
|
| 223 |
-
valid_m = [m for m in metrics if m in df.columns]
|
| 224 |
-
if not valid_m:
|
| 225 |
-
return df
|
| 226 |
-
|
| 227 |
-
agg = {m: ("first" if m == "numvessels" else "sum") for m in valid_m}
|
| 228 |
-
df2 = df.groupby(valid_g, as_index=False).agg(agg)
|
| 229 |
-
|
| 230 |
-
print(f" 聚合: {len(df2)} 条")
|
| 231 |
-
return df2
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
# ============================================================
|
| 235 |
-
# 9. 摘要
|
| 236 |
-
# ============================================================
|
| 237 |
-
def _build_summary(df, data_type):
|
| 238 |
-
summary = {
|
| 239 |
-
"data_type": data_type,
|
| 240 |
-
"total_records": len(df),
|
| 241 |
-
"data_source": "SPRFMO",
|
| 242 |
-
}
|
| 243 |
-
if "year" in df.columns and len(df) > 0:
|
| 244 |
-
years = df["year"].dropna()
|
| 245 |
-
if len(years) > 0:
|
| 246 |
-
summary["year_range"] = f"{int(years.min())}-{int(years.max())}"
|
| 247 |
-
if data_type == "catch" and "catch" in df.columns:
|
| 248 |
-
summary["total_catch"] = float(df["catch"].sum())
|
| 249 |
-
elif data_type == "effort" and "effort" in df.columns:
|
| 250 |
-
summary["total_effort"] = float(df["effort"].sum())
|
| 251 |
-
return summary
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
def _error_result(msg, warnings=None):
|
| 255 |
-
if warnings is None:
|
| 256 |
-
warnings = [msg]
|
| 257 |
-
return {
|
| 258 |
-
"preview_markdown": f"**错误**: {msg}",
|
| 259 |
-
"records": [],
|
| 260 |
-
"csv_path": None,
|
| 261 |
-
"excel_path": None,
|
| 262 |
-
"summary": {"data_type": None, "total_records": 0},
|
| 263 |
-
"source_files": [],
|
| 264 |
-
"warnings": warnings,
|
| 265 |
-
"metadata": {},
|
| 266 |
-
}
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
# ============================================================
|
| 270 |
-
# 10. 测试代码
|
| 271 |
-
# ============================================================
|
| 272 |
-
if __name__ == "__main__":
|
| 273 |
-
print("\n" + "🧪" * 30)
|
| 274 |
-
print("测试 query_sprfmo")
|
| 275 |
-
print("🧪" * 30)
|
| 276 |
-
|
| 277 |
-
# 测试 1
|
| 278 |
-
print("\n【测试 1】中国 2015-2020 年渔获量(按年)")
|
| 279 |
-
print("-" * 60)
|
| 280 |
-
r = query_sprfmo(
|
| 281 |
-
filters={"country": "CHN", "year_start": 2015, "year_end": 2020},
|
| 282 |
-
group_by=["year"],
|
| 283 |
-
metrics=["catch"],
|
| 284 |
-
data_type="catch",
|
| 285 |
-
)
|
| 286 |
-
print(r["preview_markdown"])
|
| 287 |
-
print("\n摘要:", r["summary"])
|
| 288 |
-
if r["warnings"]:
|
| 289 |
-
print("警告:")
|
| 290 |
-
for w in r["warnings"]:
|
| 291 |
-
print(f" {w}")
|
| 292 |
-
|
| 293 |
-
# 测试 2
|
| 294 |
-
print("\n\n【测试 2】2018 年努力量(按渔业类型)")
|
| 295 |
-
print("-" * 60)
|
| 296 |
-
r2 = query_sprfmo(
|
| 297 |
-
filters={"year_start": 2018, "year_end": 2018},
|
| 298 |
-
group_by=["gear_type"],
|
| 299 |
-
metrics=["effort"],
|
| 300 |
-
data_type="effort",
|
| 301 |
-
)
|
| 302 |
-
print(r2["preview_markdown"])
|
| 303 |
-
print("\n摘要:", r2["summary"])
|
| 304 |
-
if r2["warnings"]:
|
| 305 |
-
print("警告:")
|
| 306 |
-
for w in r2["warnings"]:
|
| 307 |
-
print(f" {w}")
|
| 308 |
-
|
| 309 |
-
print("\n" + "✅" * 30)
|
| 310 |
-
print("测试完成")
|
| 311 |
-
print("✅" * 30)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
|
|
|
| 1 |
+
mcp[cli]>=2,<3
|
| 2 |
+
httpx>=0.27,<1
|
| 3 |
+
fastapi>=0.116,<1
|
| 4 |
+
uvicorn[standard]>=0.35,<1
|
| 5 |
+
|
| 6 |
+
deepseek-harness-sdk==0.1.0rc6
|
scripts/preflight.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import re
|
| 4 |
+
import sys
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 8 |
+
errors: list[str] = []
|
| 9 |
+
warnings: list[str] = []
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def fail(msg: str) -> None:
|
| 13 |
+
errors.append(msg)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def warn(msg: str) -> None:
|
| 17 |
+
warnings.append(msg)
|
| 18 |
+
|
| 19 |
+
version_file = ROOT / "VERSION"
|
| 20 |
+
version = version_file.read_text(encoding="utf-8").strip() if version_file.exists() else ""
|
| 21 |
+
if not re.fullmatch(r"\d+\.\d+\.\d+", version):
|
| 22 |
+
fail("VERSION missing or not semantic x.y.z")
|
| 23 |
+
|
| 24 |
+
readme = (ROOT / "README.md").read_text(encoding="utf-8") if (ROOT / "README.md").exists() else ""
|
| 25 |
+
if not readme.startswith("---\n"):
|
| 26 |
+
fail("README.md must start with Hugging Face YAML front matter")
|
| 27 |
+
else:
|
| 28 |
+
end = readme.find("\n---", 4)
|
| 29 |
+
if end < 0:
|
| 30 |
+
fail("README.md YAML front matter is not closed")
|
| 31 |
+
else:
|
| 32 |
+
meta = readme[4:end]
|
| 33 |
+
for required in ("title:", "sdk: docker", "app_port: 7860"):
|
| 34 |
+
if required not in meta:
|
| 35 |
+
fail(f"README.md metadata missing: {required}")
|
| 36 |
+
|
| 37 |
+
app_html = ROOT / "app.html"
|
| 38 |
+
if not app_html.exists():
|
| 39 |
+
fail("root app.html is missing")
|
| 40 |
+
else:
|
| 41 |
+
html = app_html.read_text(encoding="utf-8")
|
| 42 |
+
if '__APP_VERSION__' not in html:
|
| 43 |
+
fail("app.html must use __APP_VERSION__ placeholder")
|
| 44 |
+
if 'data-view="projectPackage"' not in html:
|
| 45 |
+
fail("project data package sidebar entry is missing")
|
| 46 |
+
|
| 47 |
+
ui = ROOT / "ui_server.py"
|
| 48 |
+
if not ui.exists():
|
| 49 |
+
fail("ui_server.py is missing")
|
| 50 |
+
else:
|
| 51 |
+
text = ui.read_text(encoding="utf-8")
|
| 52 |
+
if 'with_name("VERSION")' not in text:
|
| 53 |
+
fail("ui_server.py must load VERSION as the single version source")
|
| 54 |
+
if 'with_name("app.html")' not in text:
|
| 55 |
+
fail("ui_server.py must serve root app.html")
|
| 56 |
+
if 'templates/app.html' in text:
|
| 57 |
+
fail("ui_server.py still references templates/app.html")
|
| 58 |
+
|
| 59 |
+
if (ROOT / "templates" / "app.html").exists():
|
| 60 |
+
fail("duplicate templates/app.html must not be shipped")
|
| 61 |
+
|
| 62 |
+
for forbidden in (".pytest_cache", "__pycache__"):
|
| 63 |
+
matches = [p for p in ROOT.rglob(forbidden) if p.is_dir()]
|
| 64 |
+
if matches:
|
| 65 |
+
fail(f"forbidden cache directory present: {matches[0].relative_to(ROOT)}")
|
| 66 |
+
if list(ROOT.rglob("*.pyc")):
|
| 67 |
+
fail("compiled .pyc files must not be shipped")
|
| 68 |
+
|
| 69 |
+
for name in ("Dockerfile", "start.sh", "requirements.txt"):
|
| 70 |
+
if not (ROOT / name).exists():
|
| 71 |
+
fail(f"deployment file missing: {name}")
|
| 72 |
+
|
| 73 |
+
if version and f"Current UI release: **v{version}**" not in readme:
|
| 74 |
+
warn("README current release line is not synchronized with VERSION")
|
| 75 |
+
|
| 76 |
+
for msg in warnings:
|
| 77 |
+
print(f"[preflight] WARNING: {msg}")
|
| 78 |
+
if errors:
|
| 79 |
+
for msg in errors:
|
| 80 |
+
print(f"[preflight] ERROR: {msg}", file=sys.stderr)
|
| 81 |
+
sys.exit(1)
|
| 82 |
+
print(f"[preflight] OK: v{version}; single template; clean deployment tree")
|
sidebar_catalog.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sidebar dataset catalog and Chinese display metadata.
|
| 2 |
+
|
| 3 |
+
Keep static routing/catalog data outside ``ui_server.py`` so UI work does not
|
| 4 |
+
accidentally alter the Ocean/Fisheries data-plane rules.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
FISHERY_TERMS = (
|
| 8 |
+
"柔鱼", "鱿鱼", "squid", "金枪鱼", "tuna", "wcpfc", "sprfmo", "npfc",
|
| 9 |
+
"iattc", "iccat", "iotc", "ccsbt", "effdis", "金枪鱼委员会",
|
| 10 |
+
"fao", "sea around", "ram legacy", "ram", "gfw", "global fishing watch",
|
| 11 |
+
"viirs", "vbd", "捕捞", "努力量", "cpue", "渔船", "资源评估", "biomass",
|
| 12 |
+
"recruitment",
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
OCEAN_TOOL_TERMS = (
|
| 16 |
+
"cmems", "era5", "oisst", "oc-cci", "occci", "sst", "海温", "温度",
|
| 17 |
+
"盐度", "thetao", "流速", "uo", "vo", "mlotst", "混合层", "zos",
|
| 18 |
+
"海面高度", "bgc", "叶绿素", "chlor", "chlor_a", "spco2", "碳酸盐",
|
| 19 |
+
"v10", "u10", "t2m", "msl", "slhf", "sshf", "ssrd", "strd", "tp",
|
| 20 |
+
"no3", "nppv", "o2", "po4", "anom", "10米风", "10 米风", "经向风", "纬向风",
|
| 21 |
+
"ocean 数据",
|
| 22 |
+
"ocean数据", "海洋环境", "学校海洋数据", "海洋数据服务器",
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
OCEAN_STATUS_TERMS = (
|
| 26 |
+
"三个数据中心", "ocean 状态", "ocean状态", "下载进度", "当前状态和存储量",
|
| 27 |
+
"海洋数据中心",
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
HF_SOURCE_ALIASES = {
|
| 31 |
+
"WCPFC": ("wcpfc", "中西太平洋"),
|
| 32 |
+
"SPRFMO": ("sprfmo", "南太平洋区域渔业"),
|
| 33 |
+
"NPFC": ("npfc", "北太平洋渔业"),
|
| 34 |
+
"FAO": ("fao",),
|
| 35 |
+
"Sea Around Us": ("sea around", "sea_around"),
|
| 36 |
+
"RAM Legacy": ("ram legacy", "ram_legacy", "/ram", "ram "),
|
| 37 |
+
"GFW": ("gfw", "global fishing watch"),
|
| 38 |
+
"VIIRS": ("viirs", "vbd"),
|
| 39 |
+
"IATTC": ("iattc", "美洲热带金枪鱼"),
|
| 40 |
+
"ICCAT": ("iccat", "大西洋金枪鱼"),
|
| 41 |
+
"IOTC": ("iotc", "印度洋金枪鱼"),
|
| 42 |
+
"CCSBT": ("ccsbt", "南方蓝鳍金枪鱼"),
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
OCEAN_CATALOG = (
|
| 46 |
+
("cmems_physics", "CMEMS Physics", "CMEMS 海洋物理", ("so", "thetao", "uo", "vo")),
|
| 47 |
+
("cmems_surface", "CMEMS Surface", "CMEMS 海表参数", ("mlotst", "zos")),
|
| 48 |
+
("cmems_bgc", "CMEMS BGC", "CMEMS 生物地球化学", ("chl", "no3", "nppv", "o2", "po4", "si")),
|
| 49 |
+
("cmems_carbonate", "CMEMS Carbonate", "CMEMS 碳酸盐系统", ("ph", "spco2")),
|
| 50 |
+
("era5", "ERA5", "ERA5 大气再分析", ("msl", "t2m", "u10", "v10")),
|
| 51 |
+
("era5_accum", "ERA5 Accumulation", "ERA5 累积量与通量", ("e", "slhf", "sshf", "ssrd", "strd", "tp")),
|
| 52 |
+
("occci", "OC-CCI", "海色气候变化倡议", ("chlor_a", "chlor_a_log10_bias", "chlor_a_log10_rmsd", "total_nobs_sum")),
|
| 53 |
+
("oisst", "OISST", "最优插值海表温度", ("sst", "anom", "err", "ice")),
|
| 54 |
+
)
|
| 55 |
+
|
| 56 |
+
OCEAN_SOURCE_DETAILS = {
|
| 57 |
+
"cmems_physics": {"description": "CMEMS 海洋物理三维场,供温度、盐度与海流分析。", "data_shape": "三维海洋场;部分导出格式需要指定深度"},
|
| 58 |
+
"cmems_surface": {"description": "CMEMS 海表与上层海洋参数,供混合层和海面高度分析。", "data_shape": "二维海表 / 上层海洋场"},
|
| 59 |
+
"cmems_bgc": {"description": "CMEMS 生物地球化学场,包含营养盐、叶绿素、溶解氧和初级生产力。", "data_shape": "三维生物地球化学场;部分导出格式需要指定深度"},
|
| 60 |
+
"cmems_carbonate": {"description": "CMEMS 海洋碳酸盐系统数据,供海洋酸化与碳循环分析。", "data_shape": "海表 / 三维变量以学校服务器实际产品为准"},
|
| 61 |
+
"era5": {"description": "ERA5 大气再分析瞬时场,包含气压、近地面气温和风场。", "data_shape": "二维大气表面场"},
|
| 62 |
+
"era5_accum": {"description": "ERA5 累积量与通量,包含降水、蒸发、辐射和热通量。", "data_shape": "二维大气表面累积 / 通量场"},
|
| 63 |
+
"occci": {"description": "ESA Ocean Colour CCI 海色数据,供叶绿素及观测质量分析。", "data_shape": "二维海表遥感场"},
|
| 64 |
+
"oisst": {"description": "NOAA 最优插值海表温度,包含海温、距平、误差和海冰浓度。", "data_shape": "二维海表格点场"},
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
FISHERIES_SOURCE_DETAILS = {
|
| 68 |
+
"WCPFC": "中西部太平洋金枪鱼及相关渔业统计与空间数据。",
|
| 69 |
+
"SPRFMO": "南太平洋公海渔业捕捞量与努力量资料。",
|
| 70 |
+
"NPFC": "北太平洋公海渔业统计与管理资料。",
|
| 71 |
+
"FAO": "联合国粮农组织全球捕捞产量统计。",
|
| 72 |
+
"Sea Around Us": "全球历史重建捕捞量及空间分布数据。",
|
| 73 |
+
"RAM Legacy": "全球渔业种群评估、资源量与补充量数据。",
|
| 74 |
+
"GFW": "基于 AIS 与模型推断的全球表观捕捞活动数据。",
|
| 75 |
+
"VIIRS": "夜光船探测和卫星观测机会数据;不能直接等同于捕捞量。",
|
| 76 |
+
"IATTC": "东太平洋金枪鱼区域渔业管理资料。",
|
| 77 |
+
"ICCAT": "大西洋金枪鱼及相关物种渔业资料。",
|
| 78 |
+
"IOTC": "印度洋金枪鱼及相关物种渔业资料。",
|
| 79 |
+
"CCSBT": "南方蓝鳍金枪鱼养护与资源评估资料。",
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
OCEAN_VARIABLE_NAMES_ZH = {
|
| 83 |
+
"so": "盐度", "thetao": "海水温度", "uo": "东向流速", "vo": "北向流速",
|
| 84 |
+
"mlotst": "混合层深度", "zos": "海面高度", "chl": "叶绿素", "no3": "硝酸盐",
|
| 85 |
+
"nppv": "净初级生产力", "o2": "溶解氧", "po4": "磷酸盐", "si": "硅酸盐",
|
| 86 |
+
"ph": "酸碱度", "spco2": "海表二氧化碳分压", "msl": "海平面气压",
|
| 87 |
+
"t2m": "2米气温", "u10": "10米东向风", "v10": "10米北向风", "e": "蒸发",
|
| 88 |
+
"slhf": "潜热通量", "sshf": "感热通量", "ssrd": "地表太阳辐射",
|
| 89 |
+
"strd": "地表热辐射", "tp": "总降水", "chlor_a": "叶绿素a",
|
| 90 |
+
"chlor_a_log10_bias": "叶绿素偏差", "chlor_a_log10_rmsd": "叶绿素均方根差",
|
| 91 |
+
"total_nobs_sum": "有效观测数", "sst": "海表温度", "anom": "温度距平",
|
| 92 |
+
"err": "估计误差", "ice": "海冰浓度",
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
HF_SOURCE_NAMES_ZH = {
|
| 96 |
+
"WCPFC": "中西部太平洋渔业委员会", "SPRFMO": "南太平洋区域渔业管理组织",
|
| 97 |
+
"NPFC": "北太平洋渔业委员会", "FAO": "联合国粮农组织",
|
| 98 |
+
"Sea Around Us": "海洋环绕我们数据库", "RAM Legacy": "RAM 渔业资源评估数据库",
|
| 99 |
+
"GFW": "全球渔业观察", "VIIRS": "可见光红外成像辐射仪",
|
| 100 |
+
"IATTC": "美洲热带金枪鱼委员会", "ICCAT": "大西洋金枪鱼养护委员会",
|
| 101 |
+
"IOTC": "印度洋金枪鱼委员会", "CCSBT": "南方蓝鳍金枪鱼养护委员会",
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
HF_SOURCE_CATEGORIES = {
|
| 105 |
+
"WCPFC": ("tuna", "金枪鱼区域组织"), "IATTC": ("tuna", "金枪鱼区域组织"),
|
| 106 |
+
"ICCAT": ("tuna", "金枪鱼区域组织"), "IOTC": ("tuna", "金枪鱼区域组织"),
|
| 107 |
+
"CCSBT": ("tuna", "金枪鱼区域组织"), "SPRFMO": ("squid", "柔鱼及区域渔业组织"),
|
| 108 |
+
"NPFC": ("squid", "柔鱼及区域渔业组织"), "FAO": ("general", "综合渔业统计"),
|
| 109 |
+
"Sea Around Us": ("general", "综合渔业统计"), "RAM Legacy": ("general", "资源评估数据"),
|
| 110 |
+
"GFW": ("observation", "捕捞活动观测"), "VIIRS": ("observation", "船舶活动遥感观测"),
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
EXPECTED_METADATA_FIELDS = (
|
| 114 |
+
"time_range", "temporal_resolution", "spatial_resolution", "spatial_coverage",
|
| 115 |
+
"depth_range", "units", "updated_at",
|
| 116 |
+
)
|
skills/marine-data/SKILL.md
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Marine Data
|
| 2 |
+
|
| 3 |
+
Use Marine MCP for the user's school marine data platform.
|
| 4 |
+
|
| 5 |
+
Status:
|
| 6 |
+
- `mcp_marine_marine_health`
|
| 7 |
+
- `mcp_marine_marine_domains`
|
| 8 |
+
- `mcp_marine_marine_status`
|
| 9 |
+
|
| 10 |
+
Retrieval:
|
| 11 |
+
- `mcp_marine_marine_query`
|
| 12 |
+
- `mcp_marine_marine_subset`
|
| 13 |
+
- `mcp_marine_marine_download`
|
| 14 |
+
|
| 15 |
+
Current retrieval MVP: ocean/OISST, `sst`, `anom`, `err`, `ice`, geographic NetCDF subset with expiring HTTPS URL.
|
| 16 |
+
For “1998-07-01, 120–130E, 20–30N SST, NetCDF”, query first, then subset, then return the resulting `download_url`.
|
| 17 |
+
Do not inspect the HF container for school-server data and do not invent files.
|
| 18 |
+
|
| 19 |
+
Multi-format export:
|
| 20 |
+
- `mcp_marine_marine_export`
|
| 21 |
+
- formats: netcdf, csv, xlsx, json, geotiff, png
|
| 22 |
+
- return the absolute HTTPS `download_url`
|
| 23 |
+
|
| 24 |
+
## Ocean multi-source V3
|
| 25 |
+
Use mcp_marine_marine_catalog for source/variable discovery.
|
| 26 |
+
Sources: cmems_physics, cmems_surface, cmems_bgc, cmems_carbonate, era5, era5_accum, occci, oisst.
|
| 27 |
+
Formats: netcdf, csv, xlsx, json, geotiff, png.
|
| 28 |
+
Use depth for GeoTIFF/PNG when the variable has a depth dimension.
|
| 29 |
+
Return the absolute HTTPS download_url.
|
| 30 |
+
|
| 31 |
+
## HF fisheries data plane
|
| 32 |
+
Use `mcp_marine_fisheries_catalog`, `mcp_marine_fisheries_inventory`,
|
| 33 |
+
`mcp_marine_fisheries_search`, `mcp_marine_fisheries_data_rules`, and
|
| 34 |
+
`mcp_marine_fisheries_analyze_export` for
|
| 35 |
+
squid/tuna catch, effort, CPUE, vessel activity, stock assessment and RFMO data.
|
| 36 |
+
|
| 37 |
+
Use `mcp_marine_fisheries_analyze_export` whenever the user asks for actual
|
| 38 |
+
columns, row counts, missing/duplicate checks, filters, aggregation or CSV
|
| 39 |
+
export. It validates the exact path against the live HF tree, pins the current
|
| 40 |
+
commit SHA, supports CSV/TSV and CSV/TSV members in ZIP, and returns a tokenized
|
| 41 |
+
download URL. Do not ask for Shell/Run/JS approval as a substitute.
|
| 42 |
+
|
| 43 |
+
HF fisheries Dataset and the school-server tuna/squid task databases are separate.
|
| 44 |
+
Do not infer HF absence from an empty school-server task DB.
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
## Fisheries live inventory fast path
|
| 48 |
+
|
| 49 |
+
For squid/tuna fisheries inventory, catalog, source availability, and "what files
|
| 50 |
+
are already present" questions, the UI backend injects a
|
| 51 |
+
`[HF_FISHERIES_LIVE_CONTEXT]` block from the live Hugging Face Dataset main tree.
|
| 52 |
+
|
| 53 |
+
When that context is present:
|
| 54 |
+
- answer inventory/presence questions directly from it;
|
| 55 |
+
- do not call `start_mcp_server`;
|
| 56 |
+
- use the content tool for actual data reads/analysis/export;
|
| 57 |
+
- do not infer availability from planning spreadsheets.
|
| 58 |
+
|
| 59 |
+
Marine MCP is reserved for school-server Ocean data and is connected lazily only
|
| 60 |
+
when an Ocean request needs it.
|
start.sh
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -euo pipefail
|
| 3 |
+
|
| 4 |
+
# Fail fast on broken Space metadata, duplicate templates, or stale cache artifacts.
|
| 5 |
+
python3 /home/user/app/scripts/preflight.py
|
| 6 |
+
|
| 7 |
+
# ============================================================
|
| 8 |
+
# Required non-model settings
|
| 9 |
+
# ============================================================
|
| 10 |
+
|
| 11 |
+
: "${CODEWHALE_RUNTIME_TOKEN:?Missing CODEWHALE_RUNTIME_TOKEN secret}"
|
| 12 |
+
: "${MARINE_API_URL:?Missing MARINE_API_URL variable}"
|
| 13 |
+
|
| 14 |
+
# ============================================================
|
| 15 |
+
# Model key
|
| 16 |
+
#
|
| 17 |
+
# We do NOT require or use DEEPSEEK_API_KEY.
|
| 18 |
+
# Prefer the dedicated OpenCode Go secret.
|
| 19 |
+
# Fall back to OPENAI_API_KEY only if you chose to store the same
|
| 20 |
+
# OpenCode Go key under that name in Hugging Face.
|
| 21 |
+
# ============================================================
|
| 22 |
+
|
| 23 |
+
MODEL_API_KEY="${OPENCODE_GO_API_KEY:-${OPENAI_API_KEY:-}}"
|
| 24 |
+
|
| 25 |
+
if [ -z "${MODEL_API_KEY}" ]; then
|
| 26 |
+
echo "[startup] ERROR: no OpenCode Go key was injected."
|
| 27 |
+
echo "[startup] Expected one of: OPENCODE_GO_API_KEY or OPENAI_API_KEY"
|
| 28 |
+
echo "[startup] Available environment variable names containing KEY/API:"
|
| 29 |
+
env | cut -d= -f1 | grep -E 'KEY|API' | sort || true
|
| 30 |
+
exit 1
|
| 31 |
+
fi
|
| 32 |
+
|
| 33 |
+
# ============================================================
|
| 34 |
+
# Force CodeWhale to OpenCode Go -> GLM-5.3
|
| 35 |
+
# ============================================================
|
| 36 |
+
|
| 37 |
+
export OPENCODE_GO_API_KEY="${MODEL_API_KEY}"
|
| 38 |
+
export CODEWHALE_PROVIDER="opencode-go"
|
| 39 |
+
export CODEWHALE_MODEL="glm-5.2"
|
| 40 |
+
export OPENCODE_GO_MODEL="glm-5.2"
|
| 41 |
+
|
| 42 |
+
unset OPENAI_BASE_URL OPENAI_MODEL CODEWHALE_BASE_URL || true
|
| 43 |
+
export OPENCODE_GO_BASE_URL="https://opencode.ai/zen/go/v1"
|
| 44 |
+
|
| 45 |
+
# ============================================================
|
| 46 |
+
# Web / Runtime
|
| 47 |
+
# ============================================================
|
| 48 |
+
|
| 49 |
+
export PORT="${PORT:-7860}"
|
| 50 |
+
export CODEWHALE_INTERNAL_URL="http://127.0.0.1:7878"
|
| 51 |
+
export FISHERIES_EXPORT_ROOT="${FISHERIES_EXPORT_ROOT:-/tmp/squid_fisheries_exports}"
|
| 52 |
+
export HF_FISHERIES_CACHE_ROOT="${HF_FISHERIES_CACHE_ROOT:-/tmp/squid_hf_fisheries_cache}"
|
| 53 |
+
if [ -z "${PUBLIC_BASE_URL:-}" ] && [ -n "${SPACE_HOST:-}" ]; then
|
| 54 |
+
export PUBLIC_BASE_URL="https://${SPACE_HOST}"
|
| 55 |
+
fi
|
| 56 |
+
|
| 57 |
+
# ============================================================
|
| 58 |
+
# Build private Marine MCP runtime configuration
|
| 59 |
+
# ============================================================
|
| 60 |
+
|
| 61 |
+
RUNTIME_MCP_CONFIG="/tmp/codewhale-marine-mcp.json"
|
| 62 |
+
export RUNTIME_MCP_CONFIG
|
| 63 |
+
|
| 64 |
+
python3 - <<'PY_MCP_CONFIG'
|
| 65 |
+
import json
|
| 66 |
+
import os
|
| 67 |
+
from pathlib import Path
|
| 68 |
+
|
| 69 |
+
env = {
|
| 70 |
+
"MARINE_API_URL": os.environ["MARINE_API_URL"],
|
| 71 |
+
"HF_SQUID_DATASET_REPO": (
|
| 72 |
+
os.environ.get("HF_SQUID_DATASET_REPO")
|
| 73 |
+
or os.environ.get("HF_DATASET_REPO")
|
| 74 |
+
or "globalsquiddatabase/squid_dataset"
|
| 75 |
+
),
|
| 76 |
+
"HF_TUNA_DATASET_REPO": (
|
| 77 |
+
os.environ.get("HF_TUNA_DATASET_REPO")
|
| 78 |
+
or "globalsquiddatabase/Tuna-Fisheries-Dataset"
|
| 79 |
+
),
|
| 80 |
+
"FISHERIES_EXPORT_ROOT": os.environ.get(
|
| 81 |
+
"FISHERIES_EXPORT_ROOT", "/tmp/squid_fisheries_exports"
|
| 82 |
+
),
|
| 83 |
+
"HF_FISHERIES_CACHE_ROOT": os.environ.get(
|
| 84 |
+
"HF_FISHERIES_CACHE_ROOT", "/tmp/squid_hf_fisheries_cache"
|
| 85 |
+
),
|
| 86 |
+
"HF_SQUID_DATASET_REVISION": (
|
| 87 |
+
os.environ.get("HF_SQUID_DATASET_REVISION")
|
| 88 |
+
or os.environ.get("HF_DATASET_REVISION", "")
|
| 89 |
+
),
|
| 90 |
+
"HF_TUNA_DATASET_REVISION": os.environ.get("HF_TUNA_DATASET_REVISION", ""),
|
| 91 |
+
"PUBLIC_BASE_URL": os.environ.get("PUBLIC_BASE_URL", ""),
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
hf_token = os.environ.get("HF_TOKEN", "").strip()
|
| 95 |
+
if hf_token:
|
| 96 |
+
env["HF_TOKEN"] = hf_token
|
| 97 |
+
|
| 98 |
+
cfg = {
|
| 99 |
+
"timeouts": {
|
| 100 |
+
"connect_timeout": 15,
|
| 101 |
+
"execute_timeout": 900,
|
| 102 |
+
"read_timeout": 900,
|
| 103 |
+
},
|
| 104 |
+
"servers": {
|
| 105 |
+
"marine": {
|
| 106 |
+
"command": "python3",
|
| 107 |
+
"args": ["/home/user/app/marine_mcp.py"],
|
| 108 |
+
"disabled": False,
|
| 109 |
+
"required": False,
|
| 110 |
+
"env": env,
|
| 111 |
+
}
|
| 112 |
+
},
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
p = Path(os.environ["RUNTIME_MCP_CONFIG"])
|
| 116 |
+
p.write_text(
|
| 117 |
+
json.dumps(cfg, ensure_ascii=False, indent=2) + "\n",
|
| 118 |
+
encoding="utf-8",
|
| 119 |
+
)
|
| 120 |
+
p.chmod(0o600)
|
| 121 |
+
|
| 122 |
+
print("[startup] private Marine MCP runtime config ready")
|
| 123 |
+
PY_MCP_CONFIG
|
| 124 |
+
|
| 125 |
+
# Legacy CodeWhale MCP-config variable name.
|
| 126 |
+
# This only points to the MCP config file; it does NOT select DeepSeek.
|
| 127 |
+
export DEEPSEEK_MCP_CONFIG="${RUNTIME_MCP_CONFIG}"
|
| 128 |
+
|
| 129 |
+
# ============================================================
|
| 130 |
+
# Startup information
|
| 131 |
+
# ============================================================
|
| 132 |
+
|
| 133 |
+
echo "[startup] CodeWhale:"
|
| 134 |
+
codewhale --version
|
| 135 |
+
echo "[startup] provider=${CODEWHALE_PROVIDER}"
|
| 136 |
+
echo "[startup] model=${CODEWHALE_MODEL}"
|
| 137 |
+
echo "[startup] endpoint=${OPENAI_BASE_URL:-}"
|
| 138 |
+
echo "[startup] OpenCode Go key present=yes"
|
| 139 |
+
|
| 140 |
+
# ============================================================
|
| 141 |
+
# Start CodeWhale Runtime
|
| 142 |
+
# ============================================================
|
| 143 |
+
|
| 144 |
+
codewhale app-server --http \
|
| 145 |
+
--host 127.0.0.1 \
|
| 146 |
+
--port 7878 \
|
| 147 |
+
--auth-token "${CODEWHALE_RUNTIME_TOKEN}" &
|
| 148 |
+
|
| 149 |
+
CW_PID=$!
|
| 150 |
+
trap 'kill "$CW_PID" 2>/dev/null || true' EXIT INT TERM
|
| 151 |
+
|
| 152 |
+
for i in $(seq 1 60); do
|
| 153 |
+
if curl -fsS --max-time 2 \
|
| 154 |
+
http://127.0.0.1:7878/health \
|
| 155 |
+
>/dev/null 2>&1; then
|
| 156 |
+
echo "[startup] private CodeWhale Runtime ready"
|
| 157 |
+
break
|
| 158 |
+
fi
|
| 159 |
+
|
| 160 |
+
kill -0 "$CW_PID" 2>/dev/null || {
|
| 161 |
+
echo "[startup] CodeWhale exited"
|
| 162 |
+
exit 1
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
sleep 1
|
| 166 |
+
done
|
| 167 |
+
|
| 168 |
+
# ============================================================
|
| 169 |
+
# Marine API health check
|
| 170 |
+
# ============================================================
|
| 171 |
+
|
| 172 |
+
curl -fsS --max-time 12 "${MARINE_API_URL%/}/health" || true
|
| 173 |
+
echo
|
| 174 |
+
|
| 175 |
+
# ============================================================
|
| 176 |
+
# Start Squid Agent Web UI
|
| 177 |
+
# ============================================================
|
| 178 |
+
|
| 179 |
+
exec uvicorn ui_server:app \
|
| 180 |
+
--host 0.0.0.0 \
|
| 181 |
+
--port "${PORT}" \
|
| 182 |
+
--proxy-headers
|
tests/test_data_routing_and_export_guard.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ast
|
| 2 |
+
import re
|
| 3 |
+
import unittest
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def load_helpers():
|
| 11 |
+
source = ROOT / "ui_server.py"
|
| 12 |
+
tree = ast.parse(source.read_text(encoding="utf-8"))
|
| 13 |
+
names = {
|
| 14 |
+
"_is_fisheries_prompt",
|
| 15 |
+
"_needs_ocean_mcp",
|
| 16 |
+
"_is_ocean_export_request",
|
| 17 |
+
"_apply_ocean_export_defaults",
|
| 18 |
+
"_ocean_export_execution_error",
|
| 19 |
+
}
|
| 20 |
+
functions = [
|
| 21 |
+
node for node in tree.body
|
| 22 |
+
if isinstance(node, ast.FunctionDef) and node.name in names
|
| 23 |
+
]
|
| 24 |
+
catalog = {}
|
| 25 |
+
exec((ROOT / "sidebar_catalog.py").read_text(encoding="utf-8"), catalog)
|
| 26 |
+
namespace = {
|
| 27 |
+
"re": re,
|
| 28 |
+
"_FISHERY_TERMS": catalog["FISHERY_TERMS"],
|
| 29 |
+
"_OCEAN_TOOL_TERMS": catalog["OCEAN_TOOL_TERMS"],
|
| 30 |
+
"_OCEAN_STATUS_TERMS": catalog["OCEAN_STATUS_TERMS"],
|
| 31 |
+
}
|
| 32 |
+
exec(
|
| 33 |
+
compile(ast.Module(body=functions, type_ignores=[]), str(source), "exec"),
|
| 34 |
+
namespace,
|
| 35 |
+
)
|
| 36 |
+
return namespace
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
helpers = load_helpers()
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class DataRoutingTests(unittest.TestCase):
|
| 43 |
+
def test_tuna_commissions_route_to_fisheries(self):
|
| 44 |
+
route = helpers["_is_fisheries_prompt"]
|
| 45 |
+
for source in ("IATTC", "ICCAT", "IOTC", "CCSBT", "EFFDIS_LL2000-2024"):
|
| 46 |
+
self.assertTrue(route(f"查询 {source} 当前已入库数据"), source)
|
| 47 |
+
|
| 48 |
+
def test_variable_only_v10_routes_to_ocean(self):
|
| 49 |
+
route = helpers["_needs_ocean_mcp"]
|
| 50 |
+
self.assertTrue(route("导出1998年1月4日135E-140E的v10数据"))
|
| 51 |
+
self.assertFalse(route("provide a normal conversation response"))
|
| 52 |
+
|
| 53 |
+
def test_variable_only_v10_gets_era5_and_netcdf_defaults(self):
|
| 54 |
+
routed = helpers["_apply_ocean_export_defaults"](
|
| 55 |
+
"导出1998年1月4日135E-140E的v10数据"
|
| 56 |
+
)
|
| 57 |
+
self.assertIn("source=era5", routed)
|
| 58 |
+
self.assertIn("format=netcdf", routed)
|
| 59 |
+
self.assertIn("mcp_marine_marine_export", routed)
|
| 60 |
+
|
| 61 |
+
def test_export_without_real_tool_completion_is_rejected(self):
|
| 62 |
+
guard = helpers["_ocean_export_execution_error"]
|
| 63 |
+
error = guard(
|
| 64 |
+
"导出1998年1月4日的v10数据",
|
| 65 |
+
export_tool_completed=False,
|
| 66 |
+
tool_result_text="",
|
| 67 |
+
final_answer="正在向 Ocean 服务器提交,请稍后查询状态。",
|
| 68 |
+
)
|
| 69 |
+
self.assertIn("未实际执行", error)
|
| 70 |
+
|
| 71 |
+
def test_export_with_real_download_url_is_accepted(self):
|
| 72 |
+
guard = helpers["_ocean_export_execution_error"]
|
| 73 |
+
error = guard(
|
| 74 |
+
"导出1998年1月4日ERA5 v10数据",
|
| 75 |
+
export_tool_completed=True,
|
| 76 |
+
tool_result_text=(
|
| 77 |
+
'{"status":"ok","download_url":'
|
| 78 |
+
'"https://ocean.example/download/abc-123"}'
|
| 79 |
+
),
|
| 80 |
+
final_answer="导出完成。",
|
| 81 |
+
)
|
| 82 |
+
self.assertEqual(error, "")
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
if __name__ == "__main__":
|
| 86 |
+
unittest.main()
|
tests/test_fisheries_hf.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import csv
|
| 2 |
+
import sys
|
| 3 |
+
import tempfile
|
| 4 |
+
import unittest
|
| 5 |
+
import zipfile
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from unittest.mock import MagicMock
|
| 8 |
+
|
| 9 |
+
sys.modules.setdefault("httpx", MagicMock())
|
| 10 |
+
import fisheries_hf
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
CSV_TEXT = """year,month,lon,lat,effort,flag
|
| 14 |
+
2022,12,141,2,9,A
|
| 15 |
+
2023,1,141,2,1.5,A
|
| 16 |
+
2023,2,142,3,2.5,B
|
| 17 |
+
2023,2,142,3,2.5,B
|
| 18 |
+
2023,3,150,3,8,C
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class FisheriesHFTests(unittest.TestCase):
|
| 23 |
+
def setUp(self):
|
| 24 |
+
self.tmp = tempfile.TemporaryDirectory()
|
| 25 |
+
self.root = Path(self.tmp.name)
|
| 26 |
+
fisheries_hf.EXPORT_ROOT = self.root / "exports"
|
| 27 |
+
fisheries_hf.EXPORT_ROOT.mkdir()
|
| 28 |
+
|
| 29 |
+
def tearDown(self):
|
| 30 |
+
self.tmp.cleanup()
|
| 31 |
+
|
| 32 |
+
def _check(self, path: Path):
|
| 33 |
+
result = fisheries_hf.analyze_and_export(
|
| 34 |
+
path,
|
| 35 |
+
dataset_path="current/test" + path.suffix,
|
| 36 |
+
revision="a" * 40,
|
| 37 |
+
repository="tuna",
|
| 38 |
+
year=2023,
|
| 39 |
+
lon_min=140,
|
| 40 |
+
lon_max=145,
|
| 41 |
+
lat_min=0,
|
| 42 |
+
lat_max=5,
|
| 43 |
+
)
|
| 44 |
+
self.assertEqual(result["status"], "ok")
|
| 45 |
+
self.assertEqual(
|
| 46 |
+
result["repository"],
|
| 47 |
+
"globalsquiddatabase/Tuna-Fisheries-Dataset",
|
| 48 |
+
)
|
| 49 |
+
self.assertEqual(result["scanned_row_count"], 5)
|
| 50 |
+
self.assertEqual(result["matched_row_count"], 3)
|
| 51 |
+
self.assertEqual(result["time_range"], {"min_year": 2023, "max_year": 2023})
|
| 52 |
+
self.assertEqual(result["exact_duplicate_count"], 1)
|
| 53 |
+
self.assertEqual(result["missing_values_by_column"]["effort"], 0)
|
| 54 |
+
self.assertEqual(result["monthly_record_counts"], [
|
| 55 |
+
{"month": 1, "record_count": 1},
|
| 56 |
+
{"month": 2, "record_count": 2},
|
| 57 |
+
])
|
| 58 |
+
self.assertEqual(result["annual_summary"][0]["record_count"], 3)
|
| 59 |
+
self.assertAlmostEqual(result["annual_summary"][0]["sum_effort"], 6.5)
|
| 60 |
+
self.assertEqual(result["deduplicated_record_count"], 2)
|
| 61 |
+
self.assertEqual(
|
| 62 |
+
[item["kind"] for item in result["exports"]],
|
| 63 |
+
["filtered_raw", "deduplicated", "annual_summary"],
|
| 64 |
+
)
|
| 65 |
+
self.assertEqual(len(result["download_urls"]), 3)
|
| 66 |
+
export = fisheries_hf.EXPORT_ROOT / result["token"] / result["filename"]
|
| 67 |
+
self.assertTrue(export.is_file())
|
| 68 |
+
with export.open(encoding="utf-8-sig", newline="") as stream:
|
| 69 |
+
rows = list(csv.DictReader(stream))
|
| 70 |
+
self.assertEqual(len(rows), 3)
|
| 71 |
+
for item in result["exports"]:
|
| 72 |
+
exported = fisheries_hf.EXPORT_ROOT / item["token"] / item["filename"]
|
| 73 |
+
self.assertTrue(exported.is_file())
|
| 74 |
+
|
| 75 |
+
def test_csv_filter_quality_and_export(self):
|
| 76 |
+
path = self.root / "sample.csv"
|
| 77 |
+
path.write_text(CSV_TEXT, encoding="utf-8")
|
| 78 |
+
self._check(path)
|
| 79 |
+
|
| 80 |
+
def test_zip_csv_member_filter_and_export(self):
|
| 81 |
+
path = self.root / "sample.zip"
|
| 82 |
+
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
| 83 |
+
archive.writestr("monthly_2023.csv", CSV_TEXT)
|
| 84 |
+
self._check(path)
|
| 85 |
+
|
| 86 |
+
def test_rejects_invalid_bbox(self):
|
| 87 |
+
path = self.root / "sample.csv"
|
| 88 |
+
path.write_text(CSV_TEXT, encoding="utf-8")
|
| 89 |
+
with self.assertRaisesRegex(ValueError, "lon_min"):
|
| 90 |
+
fisheries_hf.analyze_and_export(
|
| 91 |
+
path,
|
| 92 |
+
dataset_path="current/test.csv",
|
| 93 |
+
revision="a" * 40,
|
| 94 |
+
lon_min=181,
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
def test_repository_allowlist(self):
|
| 98 |
+
self.assertEqual(
|
| 99 |
+
fisheries_hf.normalize_repository("squid"),
|
| 100 |
+
"globalsquiddatabase/squid_dataset",
|
| 101 |
+
)
|
| 102 |
+
self.assertEqual(
|
| 103 |
+
fisheries_hf.normalize_repository("tuna"),
|
| 104 |
+
"globalsquiddatabase/Tuna-Fisheries-Dataset",
|
| 105 |
+
)
|
| 106 |
+
with self.assertRaisesRegex(ValueError, "repository"):
|
| 107 |
+
fisheries_hf.normalize_repository("someone/other-dataset")
|
| 108 |
+
|
| 109 |
+
def test_iccat_yearc_alias(self):
|
| 110 |
+
path = self.root / "iccat.csv"
|
| 111 |
+
path.write_text(
|
| 112 |
+
"YearC,effort\n2023,2\n2023,3\n2024,4\n",
|
| 113 |
+
encoding="utf-8",
|
| 114 |
+
)
|
| 115 |
+
result = fisheries_hf.analyze_and_export(
|
| 116 |
+
path,
|
| 117 |
+
dataset_path="ICCAT/processed/EFFDIS_LL2000-2024.csv",
|
| 118 |
+
revision="b" * 40,
|
| 119 |
+
repository="tuna",
|
| 120 |
+
)
|
| 121 |
+
self.assertEqual(result["time_range"], {"min_year": 2023, "max_year": 2024})
|
| 122 |
+
self.assertEqual(result["annual_summary"][0]["year"], 2023)
|
| 123 |
+
self.assertEqual(result["annual_summary"][0]["sum_effort"], 5.0)
|
| 124 |
+
self.assertEqual(len(result["exports"]), 3)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
if __name__ == "__main__":
|
| 128 |
+
unittest.main()
|
tests/test_followup_routing.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ast
|
| 2 |
+
import re
|
| 3 |
+
import unittest
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def load_helpers():
|
| 8 |
+
source = Path(__file__).resolve().parents[1] / "ui_server.py"
|
| 9 |
+
tree = ast.parse(source.read_text(encoding="utf-8"))
|
| 10 |
+
names = {
|
| 11 |
+
"_is_confirmation_prompt",
|
| 12 |
+
"_is_ocean_export_request",
|
| 13 |
+
"_apply_ocean_export_defaults",
|
| 14 |
+
}
|
| 15 |
+
functions = [
|
| 16 |
+
node for node in tree.body
|
| 17 |
+
if isinstance(node, ast.FunctionDef) and node.name in names
|
| 18 |
+
]
|
| 19 |
+
namespace = {
|
| 20 |
+
"re": re,
|
| 21 |
+
"_needs_ocean_mcp": lambda prompt: any(
|
| 22 |
+
term in str(prompt).lower()
|
| 23 |
+
for term in ("era5", "ocean", "sst", "cmems", "oisst")
|
| 24 |
+
),
|
| 25 |
+
}
|
| 26 |
+
exec(
|
| 27 |
+
compile(ast.Module(body=functions, type_ignores=[]), str(source), "exec"),
|
| 28 |
+
namespace,
|
| 29 |
+
)
|
| 30 |
+
return namespace
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
helpers = load_helpers()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class FollowupRoutingTests(unittest.TestCase):
|
| 37 |
+
def test_short_confirmation_variants(self):
|
| 38 |
+
confirm = helpers["_is_confirmation_prompt"]
|
| 39 |
+
for value in ("确认", "好的", "继续", "OK", "yes"):
|
| 40 |
+
self.assertTrue(confirm(value))
|
| 41 |
+
self.assertFalse(confirm("确认一下ERA5数据范围"))
|
| 42 |
+
|
| 43 |
+
def test_missing_format_defaults_to_netcdf(self):
|
| 44 |
+
apply_defaults = helpers["_apply_ocean_export_defaults"]
|
| 45 |
+
prompt = "导出1998年1月4日ERA5 v10数据"
|
| 46 |
+
routed = apply_defaults(prompt)
|
| 47 |
+
self.assertIn("format=netcdf", routed)
|
| 48 |
+
self.assertIn("mcp_marine_marine_export", routed)
|
| 49 |
+
|
| 50 |
+
def test_explicit_format_is_preserved(self):
|
| 51 |
+
apply_defaults = helpers["_apply_ocean_export_defaults"]
|
| 52 |
+
prompt = "导出1998年1月4日ERA5 v10数据,格式csv"
|
| 53 |
+
routed = apply_defaults(prompt)
|
| 54 |
+
self.assertIn(prompt, routed)
|
| 55 |
+
self.assertNotIn("format=netcdf", routed)
|
| 56 |
+
self.assertIn("mcp_marine_marine_export", routed)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
if __name__ == "__main__":
|
| 60 |
+
unittest.main()
|
tests/test_marine_inventory_aliases.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import types
|
| 4 |
+
import unittest
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
os.environ.setdefault("MARINE_API_URL", "http://example.invalid")
|
| 8 |
+
|
| 9 |
+
if "httpx" not in sys.modules:
|
| 10 |
+
httpx = types.ModuleType("httpx")
|
| 11 |
+
httpx.Client = object
|
| 12 |
+
httpx.Timeout = object
|
| 13 |
+
sys.modules["httpx"] = httpx
|
| 14 |
+
|
| 15 |
+
if "mcp.server.mcpserver" not in sys.modules:
|
| 16 |
+
mcp_package = types.ModuleType("mcp")
|
| 17 |
+
mcp_server_package = types.ModuleType("mcp.server")
|
| 18 |
+
mcp_server_module = types.ModuleType("mcp.server.mcpserver")
|
| 19 |
+
|
| 20 |
+
class MCPServer:
|
| 21 |
+
def __init__(self, *args, **kwargs):
|
| 22 |
+
pass
|
| 23 |
+
|
| 24 |
+
def tool(self):
|
| 25 |
+
return lambda function: function
|
| 26 |
+
|
| 27 |
+
def run(self):
|
| 28 |
+
pass
|
| 29 |
+
|
| 30 |
+
mcp_server_module.MCPServer = MCPServer
|
| 31 |
+
sys.modules["mcp"] = mcp_package
|
| 32 |
+
sys.modules["mcp.server"] = mcp_server_package
|
| 33 |
+
sys.modules["mcp.server.mcpserver"] = mcp_server_module
|
| 34 |
+
|
| 35 |
+
import marine_mcp
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class FisheriesInventoryAliasTests(unittest.TestCase):
|
| 39 |
+
def setUp(self):
|
| 40 |
+
self.original_hf_files = marine_mcp._hf_files
|
| 41 |
+
|
| 42 |
+
def fake_hf_files(domain="all", force=False):
|
| 43 |
+
rows = [
|
| 44 |
+
{
|
| 45 |
+
"type": "file",
|
| 46 |
+
"path": "IATTC/sample.csv",
|
| 47 |
+
"size": 120,
|
| 48 |
+
"repository": marine_mcp.HF_TUNA_DATASET_REPO,
|
| 49 |
+
"repository_domain": "tuna",
|
| 50 |
+
},
|
| 51 |
+
{
|
| 52 |
+
"type": "file",
|
| 53 |
+
"path": "current/sprfmo_effort.csv",
|
| 54 |
+
"size": 80,
|
| 55 |
+
"repository": marine_mcp.HF_SQUID_DATASET_REPO,
|
| 56 |
+
"repository_domain": "squid",
|
| 57 |
+
},
|
| 58 |
+
]
|
| 59 |
+
if domain in {"tuna", "squid"}:
|
| 60 |
+
rows = [x for x in rows if x["repository_domain"] == domain]
|
| 61 |
+
return rows, {}
|
| 62 |
+
|
| 63 |
+
marine_mcp._hf_files = fake_hf_files
|
| 64 |
+
|
| 65 |
+
def tearDown(self):
|
| 66 |
+
marine_mcp._hf_files = self.original_hf_files
|
| 67 |
+
|
| 68 |
+
def test_query_alias_becomes_keyword(self):
|
| 69 |
+
result = marine_mcp.fisheries_inventory(query="IATTC")
|
| 70 |
+
self.assertEqual(result["matched_file_count"], 1)
|
| 71 |
+
self.assertEqual(result["keyword"], "IATTC")
|
| 72 |
+
|
| 73 |
+
def test_source_alias_becomes_keyword(self):
|
| 74 |
+
result = marine_mcp.fisheries_inventory(source="IATTC")
|
| 75 |
+
self.assertEqual(result["matched_file_count"], 1)
|
| 76 |
+
self.assertEqual(result["keyword"], "IATTC")
|
| 77 |
+
|
| 78 |
+
def test_source_tuna_becomes_domain(self):
|
| 79 |
+
result = marine_mcp.fisheries_inventory(source="tuna")
|
| 80 |
+
self.assertEqual(result["domain"], "tuna")
|
| 81 |
+
self.assertEqual(result["matched_file_count"], 1)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
if __name__ == "__main__":
|
| 85 |
+
unittest.main()
|
tests/test_output_sanitizer.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import ast
|
| 2 |
+
import re
|
| 3 |
+
import unittest
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def load_sanitizer():
|
| 8 |
+
source = Path(__file__).resolve().parents[1] / "ui_server.py"
|
| 9 |
+
tree = ast.parse(source.read_text(encoding="utf-8"))
|
| 10 |
+
function = next(
|
| 11 |
+
node
|
| 12 |
+
for node in tree.body
|
| 13 |
+
if isinstance(node, ast.FunctionDef) and node.name == "_sanitize_final_answer"
|
| 14 |
+
)
|
| 15 |
+
namespace = {"re": re}
|
| 16 |
+
exec(compile(ast.Module(body=[function], type_ignores=[]), str(source), "exec"), namespace)
|
| 17 |
+
return namespace["_sanitize_final_answer"]
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
sanitize = load_sanitizer()
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class OutputSanitizerTests(unittest.TestCase):
|
| 24 |
+
def test_rejects_literal_function_call_markup(self):
|
| 25 |
+
raw = '''我需要查询数据,让我调用工具。<function_calls>
|
| 26 |
+
<invoke name="mcp_marine_fisheries_inventory">
|
| 27 |
+
<parameter name="query">IATTC</parameter>
|
| 28 |
+
</invoke>
|
| 29 |
+
</function_calls>'''
|
| 30 |
+
self.assertEqual(sanitize(raw), "")
|
| 31 |
+
|
| 32 |
+
def test_keeps_final_answer_and_removes_trailing_markup(self):
|
| 33 |
+
raw = '''【FINAL】IATTC 数据已确认存在。
|
| 34 |
+
<function_calls><invoke name="x"></invoke></function_calls>'''
|
| 35 |
+
self.assertEqual(sanitize(raw), "IATTC 数据已确认存在。")
|
| 36 |
+
|
| 37 |
+
def test_keeps_normal_answer(self):
|
| 38 |
+
self.assertEqual(sanitize("【FINAL】正常回答"), "正常回答")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
if __name__ == "__main__":
|
| 42 |
+
unittest.main()
|
tests/test_release_stability.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from pathlib import Path
|
| 2 |
+
|
| 3 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def test_version_is_single_source():
|
| 7 |
+
version = (ROOT / "VERSION").read_text(encoding="utf-8").strip()
|
| 8 |
+
ui = (ROOT / "ui_server.py").read_text(encoding="utf-8")
|
| 9 |
+
html = (ROOT / "app.html").read_text(encoding="utf-8")
|
| 10 |
+
assert version == "3.3.0"
|
| 11 |
+
assert 'with_name("VERSION")' in ui
|
| 12 |
+
assert "__APP_VERSION__" in html
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def test_single_frontend_template():
|
| 16 |
+
assert (ROOT / "app.html").exists()
|
| 17 |
+
assert not (ROOT / "templates" / "app.html").exists()
|
| 18 |
+
ui = (ROOT / "ui_server.py").read_text(encoding="utf-8")
|
| 19 |
+
assert 'with_name("app.html")' in ui
|
| 20 |
+
assert "templates/app.html" not in ui
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_hf_space_metadata_is_valid_shape():
|
| 24 |
+
readme = (ROOT / "README.md").read_text(encoding="utf-8")
|
| 25 |
+
assert readme.startswith("---\n")
|
| 26 |
+
meta_end = readme.find("\n---", 4)
|
| 27 |
+
assert meta_end > 0
|
| 28 |
+
meta = readme[4:meta_end]
|
| 29 |
+
assert "sdk: docker" in meta
|
| 30 |
+
assert "app_port: 7860" in meta
|
ui_server.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|