Add 3-seed device-contract stress validation
Browse files- MANIFEST.json +21 -11
- MODEL_INDEX.json +12 -0
- README.md +15 -0
- reports/RESEARCH_REPORT.md +10 -0
- reports/p_boundary_device_stress_3seed.json +335 -0
- scripts/evaluate_math_ink_06_p_boundary_device_stress.py +214 -0
MANIFEST.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
| 1 |
-
{
|
| 2 |
-
"schema": "aiflow-hf-research-snapshot-
|
| 3 |
-
"generated_at": "2026-07-23T19:
|
| 4 |
"track": "R_noncommercial_plus_P_proxy",
|
| 5 |
"product_validation": false,
|
| 6 |
"public_release": true,
|
| 7 |
"contains_raw_dataset": false,
|
| 8 |
-
"tests": "
|
| 9 |
"files": [
|
| 10 |
{
|
| 11 |
"path": "artifacts/boundary_behavior_guard.joblib",
|
|
@@ -24,8 +24,8 @@
|
|
| 24 |
},
|
| 25 |
{
|
| 26 |
"path": "MODEL_INDEX.json",
|
| 27 |
-
"bytes":
|
| 28 |
-
"sha256": "
|
| 29 |
},
|
| 30 |
{
|
| 31 |
"path": "models/auxiliary/boundary_auxiliary_head.pt",
|
|
@@ -104,8 +104,8 @@
|
|
| 104 |
},
|
| 105 |
{
|
| 106 |
"path": "README.md",
|
| 107 |
-
"bytes":
|
| 108 |
-
"sha256": "
|
| 109 |
},
|
| 110 |
{
|
| 111 |
"path": "reports/behavior_role_3seed_summary.json",
|
|
@@ -172,6 +172,11 @@
|
|
| 172 |
"bytes": 16685,
|
| 173 |
"sha256": "9d250e5c75d4f104971551c07a702810bf94fb5b6e5e587035a624933766356e"
|
| 174 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 175 |
{
|
| 176 |
"path": "reports/p_boundary_joint_3seed_summary.json",
|
| 177 |
"bytes": 2645,
|
|
@@ -194,8 +199,8 @@
|
|
| 194 |
},
|
| 195 |
{
|
| 196 |
"path": "reports/RESEARCH_REPORT.md",
|
| 197 |
-
"bytes":
|
| 198 |
-
"sha256": "
|
| 199 |
},
|
| 200 |
{
|
| 201 |
"path": "scripts/audit_math_ink_06_local_baseline_overmerge.py",
|
|
@@ -207,6 +212,11 @@
|
|
| 207 |
"bytes": 11671,
|
| 208 |
"sha256": "58c175a95cdeb9fc764fb96594a76888122b8f118f2f26da1283e743b1a7cd04"
|
| 209 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 210 |
{
|
| 211 |
"path": "scripts/summarize_math_ink_06_p_boundary_joint.py",
|
| 212 |
"bytes": 4196,
|
|
@@ -258,4 +268,4 @@
|
|
| 258 |
"sha256": "160c2dd8db3dd4da77bdd21209a898b73fb0cb5a1b24e59d2c8bc54e159ea3e0"
|
| 259 |
}
|
| 260 |
]
|
| 261 |
-
}
|
|
|
|
| 1 |
+
๏ปฟ{
|
| 2 |
+
"schema": "aiflow-hf-research-snapshot-v6",
|
| 3 |
+
"generated_at": "2026-07-23T19:35:13.8098972Z",
|
| 4 |
"track": "R_noncommercial_plus_P_proxy",
|
| 5 |
"product_validation": false,
|
| 6 |
"public_release": true,
|
| 7 |
"contains_raw_dataset": false,
|
| 8 |
+
"tests": "276 passed",
|
| 9 |
"files": [
|
| 10 |
{
|
| 11 |
"path": "artifacts/boundary_behavior_guard.joblib",
|
|
|
|
| 24 |
},
|
| 25 |
{
|
| 26 |
"path": "MODEL_INDEX.json",
|
| 27 |
+
"bytes": 2496,
|
| 28 |
+
"sha256": "b787ca6e82f893c28c5f828587352afdc9de444e515a80578df7891c49cd9444"
|
| 29 |
},
|
| 30 |
{
|
| 31 |
"path": "models/auxiliary/boundary_auxiliary_head.pt",
|
|
|
|
| 104 |
},
|
| 105 |
{
|
| 106 |
"path": "README.md",
|
| 107 |
+
"bytes": 11718,
|
| 108 |
+
"sha256": "2ba69db6d4d51c09c23d33cbc6ccf5dee087a3eadb02192e36002e5ac28162fa"
|
| 109 |
},
|
| 110 |
{
|
| 111 |
"path": "reports/behavior_role_3seed_summary.json",
|
|
|
|
| 172 |
"bytes": 16685,
|
| 173 |
"sha256": "9d250e5c75d4f104971551c07a702810bf94fb5b6e5e587035a624933766356e"
|
| 174 |
},
|
| 175 |
+
{
|
| 176 |
+
"path": "reports/p_boundary_device_stress_3seed.json",
|
| 177 |
+
"bytes": 10805,
|
| 178 |
+
"sha256": "31d1fd522fce274cedba00cfcc748b7479898a0509b403120d825a26a83fa6f2"
|
| 179 |
+
},
|
| 180 |
{
|
| 181 |
"path": "reports/p_boundary_joint_3seed_summary.json",
|
| 182 |
"bytes": 2645,
|
|
|
|
| 199 |
},
|
| 200 |
{
|
| 201 |
"path": "reports/RESEARCH_REPORT.md",
|
| 202 |
+
"bytes": 28194,
|
| 203 |
+
"sha256": "3f128ea412f68061afe013154b783a16585db5c62302eadc486c6872852a7e29"
|
| 204 |
},
|
| 205 |
{
|
| 206 |
"path": "scripts/audit_math_ink_06_local_baseline_overmerge.py",
|
|
|
|
| 212 |
"bytes": 11671,
|
| 213 |
"sha256": "58c175a95cdeb9fc764fb96594a76888122b8f118f2f26da1283e743b1a7cd04"
|
| 214 |
},
|
| 215 |
+
{
|
| 216 |
+
"path": "scripts/evaluate_math_ink_06_p_boundary_device_stress.py",
|
| 217 |
+
"bytes": 9597,
|
| 218 |
+
"sha256": "249006dd3d8462a1247288066534f8108828c622aeba2834489961ea10ca68f1"
|
| 219 |
+
},
|
| 220 |
{
|
| 221 |
"path": "scripts/summarize_math_ink_06_p_boundary_joint.py",
|
| 222 |
"bytes": 4196,
|
|
|
|
| 268 |
"sha256": "160c2dd8db3dd4da77bdd21209a898b73fb0cb5a1b24e59d2c8bc54e159ea3e0"
|
| 269 |
}
|
| 270 |
]
|
| 271 |
+
}
|
MODEL_INDEX.json
CHANGED
|
@@ -58,6 +58,18 @@
|
|
| 58 |
"training_track": "P_with_obligations",
|
| 59 |
"product_validation": false
|
| 60 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
"release_state": {
|
| 62 |
"track": "R_noncommercial_only",
|
| 63 |
"product_validation": false,
|
|
|
|
| 58 |
"training_track": "P_with_obligations",
|
| 59 |
"product_validation": false
|
| 60 |
},
|
| 61 |
+
"device_contract_stress": {
|
| 62 |
+
"report": "reports/p_boundary_device_stress_3seed.json",
|
| 63 |
+
"modes": [
|
| 64 |
+
"coordinate_jitter",
|
| 65 |
+
"timestamp_missing",
|
| 66 |
+
"sparse_sampling_recanonicalized",
|
| 67 |
+
"affine_device"
|
| 68 |
+
],
|
| 69 |
+
"all_seed_mode_gates_passed": true,
|
| 70 |
+
"android_hardware_validation": false,
|
| 71 |
+
"product_validation": false
|
| 72 |
+
},
|
| 73 |
"release_state": {
|
| 74 |
"track": "R_noncommercial_only",
|
| 75 |
"product_validation": false,
|
README.md
CHANGED
|
@@ -205,6 +205,19 @@ Same-row, superscript, subscript, fraction slots, wide infix sides์ ๋ค์ฏ bou
|
|
| 205 |
|
| 206 |
Delta checkpoint๋ ํด๋น seed์ `base_378.pt`์ `online_adapter.pt` ์์๋ง ์ ์ฉํ๋ค. Seed ๊ฐ delta๋ฅผ ๊ต์ฐจ ์ ์ฉํ๋ฉด ์ ๋๋ค.
|
| 207 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
## ์ถ๋ ฅ ๋ฒ์
|
| 209 |
|
| 210 |
์๋ํ ๋ชจ๋ฐ์ผ API:
|
|
@@ -231,6 +244,7 @@ SymbolResult:
|
|
| 231 |
- boundary/behavior head๋ CROHME R-track ํ์ต๋ฌผ์ด๋ฏ๋ก ์ ํ weight๋ก distillํ ์ ์๋ค.
|
| 232 |
- P boundary auxiliary head๋ ํฉ์ฑ ๋ฐฐ์น proxy์ด๋ฉฐ validation์์ ์ ํํ threshold 0.70์ ์ฌ์ฉํ๋ค. ์ค์ ์์ ๊ฒ์ฆ ์ ๊น์ง ๊ธฐ๋ณธ ์ถ๋ก ์์๋ ๋นํ์ฑ์ด๋ค.
|
| 233 |
- Joint delta๋ ๋ถ๋ฆฌ๋ ๊ณ ๋ฆฝ๊ธฐํธ์ ์์ ๋ฐฐ์น proxy๋ก ํ์ต๋์ผ๋ฉฐ ์ค์ ์ฌ์ฉ์์ ์ฐ์ stroke rhythm์ ์์ง ๊ฒ์ฆํ์ง ์์๋ค.
|
|
|
|
| 234 |
- Android LiteRT ๋ณํ, PyTorch/LiteRT logit parity, ์ ๊ฐยท์ค๊ธยท๊ณ ๊ธ ๊ธฐ๊ธฐ benchmark๊ฐ ๋จ์ ์๋ค.
|
| 235 |
|
| 236 |
## ๋ฐ์ดํฐ์ ๊ถ๋ฆฌ
|
|
@@ -255,4 +269,5 @@ PyTorch checkpoint์ joblib/pickle์ ์ ๋ขฐํ ์ ์๋ ์ถ์ฒ์์ ๋ก๋
|
|
| 255 |
- boundary ํ์ต ๊ฒฐ๊ณผ: [`reports/boundary_behavior_guard_report.json`](reports/boundary_behavior_guard_report.json)
|
| 256 |
- 3-seed ํ๋ head: [`reports/behavior_role_3seed_summary.json`](reports/behavior_role_3seed_summary.json)
|
| 257 |
- P boundary joint 3-seed: [`reports/p_boundary_joint_3seed_summary.json`](reports/p_boundary_joint_3seed_summary.json)
|
|
|
|
| 258 |
- ํ์ผ checksum: [`MANIFEST.json`](MANIFEST.json)
|
|
|
|
| 205 |
|
| 206 |
Delta checkpoint๋ ํด๋น seed์ `base_378.pt`์ `online_adapter.pt` ์์๋ง ์ ์ฉํ๋ค. Seed ๊ฐ delta๋ฅผ ๊ต์ฐจ ์ ์ฉํ๋ฉด ์ ๋๋ค.
|
| 207 |
|
| 208 |
+
### ์
๋ ฅ ๊ณ์ฝ device stress
|
| 209 |
+
|
| 210 |
+
์ฑํํ ์ธ delta์ ์ขํ jitter, timestamp ๊ฒฐ์ธก, sparse raw-event ์ฌ์ ๊ทํ, affine device ๋ณํ์ ์ ์ฉํ๋ค. Sparse ํ๊ฐ๋ canonical point๋ฅผ ์์ ์ญ์ ํ์ง ์๊ณ ์์ยท๋ยทpen-up anchor๋ฅผ ๋ณด์กดํ raw event๋ฅผ 6Hz timeline์ผ๋ก ๋ค์ ๋ณด๊ฐํ๋ค.
|
| 211 |
+
|
| 212 |
+
| stress mode | ์ต์
exact ๋ณํ | ์ต์
family ๋ณํ |
|
| 213 |
+
|---|---:|---:|
|
| 214 |
+
| coordinate jitter | -0.37%p | -0.37%p |
|
| 215 |
+
| timestamp missing | -0.26%p | -0.32%p |
|
| 216 |
+
| sparse sampling + recanonicalization | -0.48%p | -0.74%p |
|
| 217 |
+
| affine device | -1.53%p | -1.48%p |
|
| 218 |
+
|
| 219 |
+
์ธ seed์ ๋ชจ๋ mode๊ฐ proxy gate๋ฅผ ํต๊ณผํ๊ณ stressed single/cross-boundary recall ์ต์ ์น๋ 96.67%์๋ค. ์ด๋ ์
๋ ฅ ์ ๊ทํ ๊ณ์ฝ์ ๊ฐ๊ฑด์ฑ ๊ฒ์ฌ์ด๋ฉฐ ์ค์ Android ๊ธฐ๊ธฐ์ latency, digitizer, touch-driver ๊ฒ์ฆ์ ์๋๋ค.
|
| 220 |
+
|
| 221 |
## ์ถ๋ ฅ ๋ฒ์
|
| 222 |
|
| 223 |
์๋ํ ๋ชจ๋ฐ์ผ API:
|
|
|
|
| 244 |
- boundary/behavior head๋ CROHME R-track ํ์ต๋ฌผ์ด๋ฏ๋ก ์ ํ weight๋ก distillํ ์ ์๋ค.
|
| 245 |
- P boundary auxiliary head๋ ํฉ์ฑ ๋ฐฐ์น proxy์ด๋ฉฐ validation์์ ์ ํํ threshold 0.70์ ์ฌ์ฉํ๋ค. ์ค์ ์์ ๊ฒ์ฆ ์ ๊น์ง ๊ธฐ๋ณธ ์ถ๋ก ์์๋ ๋นํ์ฑ์ด๋ค.
|
| 246 |
- Joint delta๋ ๋ถ๋ฆฌ๋ ๊ณ ๋ฆฝ๊ธฐํธ์ ์์ ๋ฐฐ์น proxy๋ก ํ์ต๋์ผ๋ฉฐ ์ค์ ์ฌ์ฉ์์ ์ฐ์ stroke rhythm์ ์์ง ๊ฒ์ฆํ์ง ์์๋ค.
|
| 247 |
+
- Device stress๋ software perturbation ๊ฒฐ๊ณผ์ด๋ฉฐ ์ค์ stylus/device-disjoint ์ฑ๋ฅ ๊ทผ๊ฑฐ๊ฐ ์๋๋ค.
|
| 248 |
- Android LiteRT ๋ณํ, PyTorch/LiteRT logit parity, ์ ๊ฐยท์ค๊ธยท๊ณ ๊ธ ๊ธฐ๊ธฐ benchmark๊ฐ ๋จ์ ์๋ค.
|
| 249 |
|
| 250 |
## ๋ฐ์ดํฐ์ ๊ถ๋ฆฌ
|
|
|
|
| 269 |
- boundary ํ์ต ๊ฒฐ๊ณผ: [`reports/boundary_behavior_guard_report.json`](reports/boundary_behavior_guard_report.json)
|
| 270 |
- 3-seed ํ๋ head: [`reports/behavior_role_3seed_summary.json`](reports/behavior_role_3seed_summary.json)
|
| 271 |
- P boundary joint 3-seed: [`reports/p_boundary_joint_3seed_summary.json`](reports/p_boundary_joint_3seed_summary.json)
|
| 272 |
+
- P boundary device stress: [`reports/p_boundary_device_stress_3seed.json`](reports/p_boundary_device_stress_3seed.json)
|
| 273 |
- ํ์ผ checksum: [`MANIFEST.json`](MANIFEST.json)
|
reports/RESEARCH_REPORT.md
CHANGED
|
@@ -362,6 +362,14 @@ Authentic 4,800๊ฐ์๋ exact CEยทfamily CEยทsingle-symbol boundary loss๋ฅผ ์ฃผ
|
|
| 362 |
|
| 363 |
์ธ seed ๋ชจ๋ validation๊ณผ paired test gate๋ฅผ ๊ฐ๋ณ ํต๊ณผํ๊ณ exact/family๋ ๋ชจ๋ seed์์ ๊ฐ์ ๋ผ P formula-layout proxy delta๋ก ์ฑํํ๋ค. ์ด๋ R-track head๋ฅผ distillํ ๊ฒฐ๊ณผ๊ฐ ์๋๋ผ ์น์ธ P trajectory์ ์๋ ค์ง ํฉ์ฑ boundary๋ง ์ฌ์ฉํ ๊ฒฐ๊ณผ๋ค. ๋ค๋ง ๋ถ๋ฆฌ๋ ๊ณ ๋ฆฝ๊ธฐํธ๋ฅผ ๋ฐฐ์นํ proxy์ด๋ฏ๋ก ์ค์ ์ฌ์ฉ์์ ์ฐ์์ stroke rhythmยทco-articulationยทdevice sampling ๋ณ๋์ ํฌํจํ์ง ์๋๋ค. ์ ํ ๊ฒ์ฆ์ false์ด๋ฉฐ ๋ค์ ํ์ gate๋ ์ค์ P ์ฐ์์ writer/device-disjoint boundary ํ๊ฐ๋ค.
|
| 364 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 365 |
## ์ฐ์ถ๋ฌผ
|
| 366 |
|
| 367 |
- `src/math_grid_drawer/research/behavior_context06.py`
|
|
@@ -383,6 +391,7 @@ Authentic 4,800๊ฐ์๋ exact CEยทfamily CEยทsingle-symbol boundary loss๋ฅผ ์ฃผ
|
|
| 383 |
- `scripts/train_math_ink_06_p_boundary_auxiliary.py`
|
| 384 |
- `scripts/train_math_ink_06_p_boundary_joint.py`
|
| 385 |
- `scripts/summarize_math_ink_06_p_boundary_joint.py`
|
|
|
|
| 386 |
- `scripts/analyze_crohme_lattice_failures.py`
|
| 387 |
- `tests/test_behavior_context06.py`
|
| 388 |
- `tests/test_behavior_role_head06.py`
|
|
@@ -408,3 +417,4 @@ Authentic 4,800๊ฐ์๋ exact CEยทfamily CEยทsingle-symbol boundary loss๋ฅผ ์ฃผ
|
|
| 408 |
- `research/runs/math_ink_06_p_boundary_auxiliary_smoke_20260724/report.json`
|
| 409 |
- `research/runs/math_ink_06_p_boundary_auxiliary_fullsmoke_20260724/report.json`
|
| 410 |
- `research/runs/math_ink_06_p_boundary_joint_3seed_20260724/run_summary.json`
|
|
|
|
|
|
| 362 |
|
| 363 |
์ธ seed ๋ชจ๋ validation๊ณผ paired test gate๋ฅผ ๊ฐ๋ณ ํต๊ณผํ๊ณ exact/family๋ ๋ชจ๋ seed์์ ๊ฐ์ ๋ผ P formula-layout proxy delta๋ก ์ฑํํ๋ค. ์ด๋ R-track head๋ฅผ distillํ ๊ฒฐ๊ณผ๊ฐ ์๋๋ผ ์น์ธ P trajectory์ ์๋ ค์ง ํฉ์ฑ boundary๋ง ์ฌ์ฉํ ๊ฒฐ๊ณผ๋ค. ๋ค๋ง ๋ถ๋ฆฌ๋ ๊ณ ๋ฆฝ๊ธฐํธ๋ฅผ ๋ฐฐ์นํ proxy์ด๋ฏ๋ก ์ค์ ์ฌ์ฉ์์ ์ฐ์์ stroke rhythmยทco-articulationยทdevice sampling ๋ณ๋์ ํฌํจํ์ง ์๋๋ค. ์ ํ ๊ฒ์ฆ์ false์ด๋ฉฐ ๋ค์ ํ์ gate๋ ์ค์ P ์ฐ์์ writer/device-disjoint boundary ํ๊ฐ๋ค.
|
| 364 |
|
| 365 |
+
### 3-seed device contract stress
|
| 366 |
+
|
| 367 |
+
์ฑํ delta๋ฅผ ๋ณ๋ ํ์ต ์์ด ์ขํ jitter, timestamp/speed ์ ์ฒด ๊ฒฐ์ธก, raw event ์ ๋ฐ ํฌ์ํ ํ 6Hz ์ฌ๋ณด๊ฐ, x 1.12/y 0.88 affine ์กฐ๊ฑด์์ ํ๊ฐํ๋ค. ์ต์ด sparse ์คํ์ canonical tensor๋ฅผ ์ง์ ์ ๋ฐ ์ญ์ ํด ์
๋ ฅ ๊ณ์ฝ์ ์๋ฐํ์ผ๋ฏ๋ก ํ๊ธฐํ๊ณ , start/endยทpen-up anchor๋ฅผ ๋ณด์กดํ ์ฌ๋ณด๊ฐ ๊ฒฝ๋ก๋ก ๋ค์ ๊ณ ์ ํ๋ค.
|
| 368 |
+
|
| 369 |
+
๋ชจ๋ seedยทstress๊ฐ clean ๋๋น exact/family -3%p ์ด๋ด, single/boundary recall 90% ์ด์ gate๋ฅผ ํต๊ณผํ๋ค. ์ต์
exact ํ๋ฝ์ seed-47 affine `-1.53%p`, ์ต์
family ํ๋ฝ์ seed-31 affine `-1.48%p`์๋ค. Stress ์ ์ฒด์ single-symbol/cross-boundary recall ์ต์ ๋ ๊ฐ๊ฐ 96.67%๋ค. Timestamp ๊ฒฐ์ธก๊ณผ 2๋ฐฐ ํฌ์ sampling์ ์ต์
exact ํ๋ฝ์ด ๊ฐ๊ฐ -0.26%p์ -0.48%p ์ด๋ด์๋ค.
|
| 370 |
+
|
| 371 |
+
์ด๋ canonical ์
๋ ฅ ๊ณ์ฝ์ ๋ณํ ๋ด์ฑ ์ฆ๊ฑฐ์ด์ง ์ค์ Android digitizerยทWDDMยทbattery/latency ๊ฒ์ฆ์ด ์๋๋ค. ๋ค์ ์ค์ gate๋ raw MotionEvent๋ฅผ ๊ฐ์ง P writer/device-disjoint ์ฐ์์์ด๋ค. ์ ์ฒด Python ํ๊ท๋ 276๊ฐ๊ฐ ํต๊ณผํ๋ค.
|
| 372 |
+
|
| 373 |
## ์ฐ์ถ๋ฌผ
|
| 374 |
|
| 375 |
- `src/math_grid_drawer/research/behavior_context06.py`
|
|
|
|
| 391 |
- `scripts/train_math_ink_06_p_boundary_auxiliary.py`
|
| 392 |
- `scripts/train_math_ink_06_p_boundary_joint.py`
|
| 393 |
- `scripts/summarize_math_ink_06_p_boundary_joint.py`
|
| 394 |
+
- `scripts/evaluate_math_ink_06_p_boundary_device_stress.py`
|
| 395 |
- `scripts/analyze_crohme_lattice_failures.py`
|
| 396 |
- `tests/test_behavior_context06.py`
|
| 397 |
- `tests/test_behavior_role_head06.py`
|
|
|
|
| 417 |
- `research/runs/math_ink_06_p_boundary_auxiliary_smoke_20260724/report.json`
|
| 418 |
- `research/runs/math_ink_06_p_boundary_auxiliary_fullsmoke_20260724/report.json`
|
| 419 |
- `research/runs/math_ink_06_p_boundary_joint_3seed_20260724/run_summary.json`
|
| 420 |
+
- `research/runs/math_ink_06_p_boundary_device_stress_3seed_20260724/report.json`
|
reports/p_boundary_device_stress_3seed.json
ADDED
|
@@ -0,0 +1,335 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "P-MATH-INK-06-BOUNDARY-DEVICE-STRESS-001",
|
| 3 |
+
"generated_at": "2026-07-23T19:31:55.957852+00:00",
|
| 4 |
+
"device": "cuda",
|
| 5 |
+
"cuda_device": "NVIDIA GeForce GTX 1650",
|
| 6 |
+
"test_cache_key": "26bee2f320c6f0a7eca3c6ca1c9c71dd0cbe89cfaee8d8c415af74ae84c0d879",
|
| 7 |
+
"stress_contract": {
|
| 8 |
+
"modes": [
|
| 9 |
+
"clean",
|
| 10 |
+
"coordinate_jitter",
|
| 11 |
+
"timestamp_missing",
|
| 12 |
+
"sparse_sampling",
|
| 13 |
+
"affine_device"
|
| 14 |
+
],
|
| 15 |
+
"maximum_exact_regression_pp": 3.0,
|
| 16 |
+
"maximum_family_regression_pp": 3.0,
|
| 17 |
+
"minimum_single_symbol_recall": 0.9,
|
| 18 |
+
"minimum_cross_boundary_recall": 0.9
|
| 19 |
+
},
|
| 20 |
+
"seeds": [
|
| 21 |
+
{
|
| 22 |
+
"seed": 17,
|
| 23 |
+
"threshold": 0.525,
|
| 24 |
+
"modes": {
|
| 25 |
+
"clean": {
|
| 26 |
+
"authentic": {
|
| 27 |
+
"samples": 3782,
|
| 28 |
+
"exact_top1": 0.7765732416710736,
|
| 29 |
+
"family_top1": 0.8622421998942359
|
| 30 |
+
},
|
| 31 |
+
"boundary": {
|
| 32 |
+
"threshold": 0.525,
|
| 33 |
+
"accuracy": 0.9783333333333334,
|
| 34 |
+
"f1": 0.9785655399835119,
|
| 35 |
+
"roc_auc": 0.9964729166666666,
|
| 36 |
+
"single_symbol_recall": 0.9675,
|
| 37 |
+
"cross_boundary_recall": 0.9891666666666666
|
| 38 |
+
}
|
| 39 |
+
},
|
| 40 |
+
"coordinate_jitter": {
|
| 41 |
+
"authentic": {
|
| 42 |
+
"samples": 3782,
|
| 43 |
+
"exact_top1": 0.775251189846642,
|
| 44 |
+
"family_top1": 0.8622421998942359
|
| 45 |
+
},
|
| 46 |
+
"boundary": {
|
| 47 |
+
"threshold": 0.525,
|
| 48 |
+
"accuracy": 0.9783333333333334,
|
| 49 |
+
"f1": 0.9785831960461285,
|
| 50 |
+
"roc_auc": 0.9964930555555556,
|
| 51 |
+
"single_symbol_recall": 0.9666666666666667,
|
| 52 |
+
"cross_boundary_recall": 0.99
|
| 53 |
+
},
|
| 54 |
+
"deltas": {
|
| 55 |
+
"exact_top1_pp": -0.13220518244315693,
|
| 56 |
+
"family_top1_pp": 0.0
|
| 57 |
+
},
|
| 58 |
+
"gate_passed": true
|
| 59 |
+
},
|
| 60 |
+
"timestamp_missing": {
|
| 61 |
+
"authentic": {
|
| 62 |
+
"samples": 3782,
|
| 63 |
+
"exact_top1": 0.7800105764145955,
|
| 64 |
+
"family_top1": 0.8553675304071919
|
| 65 |
+
},
|
| 66 |
+
"boundary": {
|
| 67 |
+
"threshold": 0.525,
|
| 68 |
+
"accuracy": 0.97875,
|
| 69 |
+
"f1": 0.9789517127527858,
|
| 70 |
+
"roc_auc": 0.9966340277777778,
|
| 71 |
+
"single_symbol_recall": 0.9691666666666666,
|
| 72 |
+
"cross_boundary_recall": 0.9883333333333333
|
| 73 |
+
},
|
| 74 |
+
"deltas": {
|
| 75 |
+
"exact_top1_pp": 0.3437334743521925,
|
| 76 |
+
"family_top1_pp": -0.6874669487043961
|
| 77 |
+
},
|
| 78 |
+
"gate_passed": true
|
| 79 |
+
},
|
| 80 |
+
"sparse_sampling": {
|
| 81 |
+
"authentic": {
|
| 82 |
+
"samples": 3782,
|
| 83 |
+
"exact_top1": 0.7723426758328926,
|
| 84 |
+
"family_top1": 0.8548387096774194
|
| 85 |
+
},
|
| 86 |
+
"boundary": {
|
| 87 |
+
"threshold": 0.525,
|
| 88 |
+
"accuracy": 0.9791666666666666,
|
| 89 |
+
"f1": 0.9794069192751236,
|
| 90 |
+
"roc_auc": 0.99676875,
|
| 91 |
+
"single_symbol_recall": 0.9675,
|
| 92 |
+
"cross_boundary_recall": 0.9908333333333333
|
| 93 |
+
},
|
| 94 |
+
"deltas": {
|
| 95 |
+
"exact_top1_pp": -0.4230565838180933,
|
| 96 |
+
"family_top1_pp": -0.7403490216816522
|
| 97 |
+
},
|
| 98 |
+
"gate_passed": true
|
| 99 |
+
},
|
| 100 |
+
"affine_device": {
|
| 101 |
+
"authentic": {
|
| 102 |
+
"samples": 3782,
|
| 103 |
+
"exact_top1": 0.7675832892649392,
|
| 104 |
+
"family_top1": 0.8527234267583289
|
| 105 |
+
},
|
| 106 |
+
"boundary": {
|
| 107 |
+
"threshold": 0.525,
|
| 108 |
+
"accuracy": 0.9720833333333333,
|
| 109 |
+
"f1": 0.9722337339411521,
|
| 110 |
+
"roc_auc": 0.9952104166666667,
|
| 111 |
+
"single_symbol_recall": 0.9666666666666667,
|
| 112 |
+
"cross_boundary_recall": 0.9775
|
| 113 |
+
},
|
| 114 |
+
"deltas": {
|
| 115 |
+
"exact_top1_pp": -0.8989952406134316,
|
| 116 |
+
"family_top1_pp": -0.9518773135906988
|
| 117 |
+
},
|
| 118 |
+
"gate_passed": true
|
| 119 |
+
}
|
| 120 |
+
},
|
| 121 |
+
"all_stress_gates_passed": true
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"seed": 31,
|
| 125 |
+
"threshold": 0.625,
|
| 126 |
+
"modes": {
|
| 127 |
+
"clean": {
|
| 128 |
+
"authentic": {
|
| 129 |
+
"samples": 3782,
|
| 130 |
+
"exact_top1": 0.7937599153886833,
|
| 131 |
+
"family_top1": 0.8770491803278688
|
| 132 |
+
},
|
| 133 |
+
"boundary": {
|
| 134 |
+
"threshold": 0.625,
|
| 135 |
+
"accuracy": 0.9808333333333333,
|
| 136 |
+
"f1": 0.9808013355592654,
|
| 137 |
+
"roc_auc": 0.9979034722222222,
|
| 138 |
+
"single_symbol_recall": 0.9825,
|
| 139 |
+
"cross_boundary_recall": 0.9791666666666666
|
| 140 |
+
}
|
| 141 |
+
},
|
| 142 |
+
"coordinate_jitter": {
|
| 143 |
+
"authentic": {
|
| 144 |
+
"samples": 3782,
|
| 145 |
+
"exact_top1": 0.7942887361184559,
|
| 146 |
+
"family_top1": 0.8778424114225277
|
| 147 |
+
},
|
| 148 |
+
"boundary": {
|
| 149 |
+
"threshold": 0.625,
|
| 150 |
+
"accuracy": 0.98125,
|
| 151 |
+
"f1": 0.9811951525282073,
|
| 152 |
+
"roc_auc": 0.9979090277777778,
|
| 153 |
+
"single_symbol_recall": 0.9841666666666666,
|
| 154 |
+
"cross_boundary_recall": 0.9783333333333334
|
| 155 |
+
},
|
| 156 |
+
"deltas": {
|
| 157 |
+
"exact_top1_pp": 0.05288207297725611,
|
| 158 |
+
"family_top1_pp": 0.07932310946588972
|
| 159 |
+
},
|
| 160 |
+
"gate_passed": true
|
| 161 |
+
},
|
| 162 |
+
"timestamp_missing": {
|
| 163 |
+
"authentic": {
|
| 164 |
+
"samples": 3782,
|
| 165 |
+
"exact_top1": 0.7911158117398202,
|
| 166 |
+
"family_top1": 0.8693812797461661
|
| 167 |
+
},
|
| 168 |
+
"boundary": {
|
| 169 |
+
"threshold": 0.625,
|
| 170 |
+
"accuracy": 0.9791666666666666,
|
| 171 |
+
"f1": 0.9790444258172674,
|
| 172 |
+
"roc_auc": 0.9979673611111111,
|
| 173 |
+
"single_symbol_recall": 0.985,
|
| 174 |
+
"cross_boundary_recall": 0.9733333333333334
|
| 175 |
+
},
|
| 176 |
+
"deltas": {
|
| 177 |
+
"exact_top1_pp": -0.26441036488631386,
|
| 178 |
+
"family_top1_pp": -0.7667900581702747
|
| 179 |
+
},
|
| 180 |
+
"gate_passed": true
|
| 181 |
+
},
|
| 182 |
+
"sparse_sampling": {
|
| 183 |
+
"authentic": {
|
| 184 |
+
"samples": 3782,
|
| 185 |
+
"exact_top1": 0.7890005288207298,
|
| 186 |
+
"family_top1": 0.8725542041248017
|
| 187 |
+
},
|
| 188 |
+
"boundary": {
|
| 189 |
+
"threshold": 0.625,
|
| 190 |
+
"accuracy": 0.98125,
|
| 191 |
+
"f1": 0.9812889812889813,
|
| 192 |
+
"roc_auc": 0.9979597222222221,
|
| 193 |
+
"single_symbol_recall": 0.9791666666666666,
|
| 194 |
+
"cross_boundary_recall": 0.9833333333333333
|
| 195 |
+
},
|
| 196 |
+
"deltas": {
|
| 197 |
+
"exact_top1_pp": -0.4759386567953494,
|
| 198 |
+
"family_top1_pp": -0.4494976203067158
|
| 199 |
+
},
|
| 200 |
+
"gate_passed": true
|
| 201 |
+
},
|
| 202 |
+
"affine_device": {
|
| 203 |
+
"authentic": {
|
| 204 |
+
"samples": 3782,
|
| 205 |
+
"exact_top1": 0.7842411422527763,
|
| 206 |
+
"family_top1": 0.8622421998942359
|
| 207 |
+
},
|
| 208 |
+
"boundary": {
|
| 209 |
+
"threshold": 0.625,
|
| 210 |
+
"accuracy": 0.9745833333333334,
|
| 211 |
+
"f1": 0.9743805123897522,
|
| 212 |
+
"roc_auc": 0.9969861111111111,
|
| 213 |
+
"single_symbol_recall": 0.9825,
|
| 214 |
+
"cross_boundary_recall": 0.9666666666666667
|
| 215 |
+
},
|
| 216 |
+
"deltas": {
|
| 217 |
+
"exact_top1_pp": -0.9518773135906988,
|
| 218 |
+
"family_top1_pp": -1.4806980433632932
|
| 219 |
+
},
|
| 220 |
+
"gate_passed": true
|
| 221 |
+
}
|
| 222 |
+
},
|
| 223 |
+
"all_stress_gates_passed": true
|
| 224 |
+
},
|
| 225 |
+
{
|
| 226 |
+
"seed": 47,
|
| 227 |
+
"threshold": 0.625,
|
| 228 |
+
"modes": {
|
| 229 |
+
"clean": {
|
| 230 |
+
"authentic": {
|
| 231 |
+
"samples": 3782,
|
| 232 |
+
"exact_top1": 0.7784241142252777,
|
| 233 |
+
"family_top1": 0.8646218931782126
|
| 234 |
+
},
|
| 235 |
+
"boundary": {
|
| 236 |
+
"threshold": 0.625,
|
| 237 |
+
"accuracy": 0.9795833333333334,
|
| 238 |
+
"f1": 0.9797269342159702,
|
| 239 |
+
"roc_auc": 0.9966916666666666,
|
| 240 |
+
"single_symbol_recall": 0.9725,
|
| 241 |
+
"cross_boundary_recall": 0.9866666666666667
|
| 242 |
+
}
|
| 243 |
+
},
|
| 244 |
+
"coordinate_jitter": {
|
| 245 |
+
"authentic": {
|
| 246 |
+
"samples": 3782,
|
| 247 |
+
"exact_top1": 0.7786885245901639,
|
| 248 |
+
"family_top1": 0.8654151242728715
|
| 249 |
+
},
|
| 250 |
+
"boundary": {
|
| 251 |
+
"threshold": 0.625,
|
| 252 |
+
"accuracy": 0.9804166666666667,
|
| 253 |
+
"f1": 0.9805704836709384,
|
| 254 |
+
"roc_auc": 0.9966652777777778,
|
| 255 |
+
"single_symbol_recall": 0.9725,
|
| 256 |
+
"cross_boundary_recall": 0.9883333333333333
|
| 257 |
+
},
|
| 258 |
+
"deltas": {
|
| 259 |
+
"exact_top1_pp": 0.026441036488622505,
|
| 260 |
+
"family_top1_pp": 0.07932310946588972
|
| 261 |
+
},
|
| 262 |
+
"gate_passed": true
|
| 263 |
+
},
|
| 264 |
+
"timestamp_missing": {
|
| 265 |
+
"authentic": {
|
| 266 |
+
"samples": 3782,
|
| 267 |
+
"exact_top1": 0.777895293495505,
|
| 268 |
+
"family_top1": 0.8622421998942359
|
| 269 |
+
},
|
| 270 |
+
"boundary": {
|
| 271 |
+
"threshold": 0.625,
|
| 272 |
+
"accuracy": 0.9791666666666666,
|
| 273 |
+
"f1": 0.9792874896437448,
|
| 274 |
+
"roc_auc": 0.997129861111111,
|
| 275 |
+
"single_symbol_recall": 0.9733333333333334,
|
| 276 |
+
"cross_boundary_recall": 0.985
|
| 277 |
+
},
|
| 278 |
+
"deltas": {
|
| 279 |
+
"exact_top1_pp": -0.052882072977267214,
|
| 280 |
+
"family_top1_pp": -0.23796932839766916
|
| 281 |
+
},
|
| 282 |
+
"gate_passed": true
|
| 283 |
+
},
|
| 284 |
+
"sparse_sampling": {
|
| 285 |
+
"authentic": {
|
| 286 |
+
"samples": 3782,
|
| 287 |
+
"exact_top1": 0.7757800105764145,
|
| 288 |
+
"family_top1": 0.8590692755156002
|
| 289 |
+
},
|
| 290 |
+
"boundary": {
|
| 291 |
+
"threshold": 0.625,
|
| 292 |
+
"accuracy": 0.97875,
|
| 293 |
+
"f1": 0.9788994621431527,
|
| 294 |
+
"roc_auc": 0.9966645833333333,
|
| 295 |
+
"single_symbol_recall": 0.9716666666666667,
|
| 296 |
+
"cross_boundary_recall": 0.9858333333333333
|
| 297 |
+
},
|
| 298 |
+
"deltas": {
|
| 299 |
+
"exact_top1_pp": -0.26441036488631386,
|
| 300 |
+
"family_top1_pp": -0.5552617662612391
|
| 301 |
+
},
|
| 302 |
+
"gate_passed": true
|
| 303 |
+
},
|
| 304 |
+
"affine_device": {
|
| 305 |
+
"authentic": {
|
| 306 |
+
"samples": 3782,
|
| 307 |
+
"exact_top1": 0.763088313061872,
|
| 308 |
+
"family_top1": 0.8500793231094659
|
| 309 |
+
},
|
| 310 |
+
"boundary": {
|
| 311 |
+
"threshold": 0.625,
|
| 312 |
+
"accuracy": 0.9708333333333333,
|
| 313 |
+
"f1": 0.9708090075062552,
|
| 314 |
+
"roc_auc": 0.99540625,
|
| 315 |
+
"single_symbol_recall": 0.9716666666666667,
|
| 316 |
+
"cross_boundary_recall": 0.97
|
| 317 |
+
},
|
| 318 |
+
"deltas": {
|
| 319 |
+
"exact_top1_pp": -1.5335801163405716,
|
| 320 |
+
"family_top1_pp": -1.4542570068746707
|
| 321 |
+
},
|
| 322 |
+
"gate_passed": true
|
| 323 |
+
}
|
| 324 |
+
},
|
| 325 |
+
"all_stress_gates_passed": true
|
| 326 |
+
}
|
| 327 |
+
],
|
| 328 |
+
"decision": {
|
| 329 |
+
"all_seed_stress_gates_passed": true,
|
| 330 |
+
"failures": [],
|
| 331 |
+
"product_validation": false
|
| 332 |
+
},
|
| 333 |
+
"track": "P_with_obligations",
|
| 334 |
+
"product_validation": false
|
| 335 |
+
}
|
scripts/evaluate_math_ink_06_p_boundary_device_stress.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""์ฑํ๋ P boundary joint delta๋ฅผ ๋ฏธ๊ด์ธก device/sampling stress์์ ํ๊ฐํ๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from datetime import datetime, timezone
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import sys
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 14 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 15 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 16 |
+
if str(path) not in sys.path:
|
| 17 |
+
sys.path.insert(0, str(path))
|
| 18 |
+
|
| 19 |
+
from scripts.train_math_ink_06_p_boundary_auxiliary import (
|
| 20 |
+
_balanced_boundary_set06,
|
| 21 |
+
_load_encoder06,
|
| 22 |
+
_load_feature_cache06,
|
| 23 |
+
_metrics06,
|
| 24 |
+
)
|
| 25 |
+
from scripts.train_math_ink_06_p_boundary_joint import (
|
| 26 |
+
_authentic_metrics06,
|
| 27 |
+
_boundary_logits06,
|
| 28 |
+
_family_target_map06,
|
| 29 |
+
)
|
| 30 |
+
from scripts.train_math_ink_06_skeleton_adapter import _resolve_device06
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _parse_args() -> argparse.Namespace:
|
| 34 |
+
"""ํ์ ๋ณ์: seed๋ณ base/adapter/head/delta. ์๋ ์๋ฆฌ: ๊ณ ์ 3-seed stress ํ๊ฐ CLI๋ฅผ ๋ง๋ ๋ค."""
|
| 35 |
+
|
| 36 |
+
parser = argparse.ArgumentParser(description="Evaluate Math Ink 0.6 P boundary device stress")
|
| 37 |
+
parser.add_argument(
|
| 38 |
+
"--test-cache", type=Path,
|
| 39 |
+
default=Path(r"D:\Aiflow-CUDA\ink06_feature_cache\paired-paired-test-26bee2f320c6f0a7eca3.pt"),
|
| 40 |
+
)
|
| 41 |
+
parser.add_argument("--samples-per-class", type=int, default=1200)
|
| 42 |
+
parser.add_argument("--batch-size", type=int, default=256)
|
| 43 |
+
parser.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto")
|
| 44 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 45 |
+
return parser.parse_args()
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _sparse_sequence06(sequence: torch.Tensor) -> torch.Tensor:
|
| 49 |
+
"""ํ์ ๋ณ์: 128ร19 canonical sequence. ์๋ ์๋ฆฌ: raw event ์ ๋ฐ์ ์ ๊ฑฐํ ๋ค 6Hz canonical timeline์ผ๋ก ์ฌ๋ณด๊ฐํ๋ค."""
|
| 50 |
+
|
| 51 |
+
valid = sequence[sequence[:, 8] >= 0]
|
| 52 |
+
if len(valid) <= 2:
|
| 53 |
+
return sequence.clone()
|
| 54 |
+
anchors = torch.nonzero(valid[:, 7] > 0.5, as_tuple=False).flatten()
|
| 55 |
+
indices = torch.unique(torch.cat((
|
| 56 |
+
torch.arange(0, len(valid), 2), anchors, torch.tensor([0, len(valid) - 1]),
|
| 57 |
+
)), sorted=True)
|
| 58 |
+
kept = valid[indices].clone()
|
| 59 |
+
# ์ขํยท๋ฐฉํฅยท์๊ฐ ๋ฑ ์ฐ์ channel์ ํฌ์ raw event์์ canonical tick์ผ๋ก ์ ํ ๋ณต์ํ๋ค.
|
| 60 |
+
restored = torch.nn.functional.interpolate(
|
| 61 |
+
kept.transpose(0, 1).unsqueeze(0), size=len(valid),
|
| 62 |
+
mode="linear", align_corners=True,
|
| 63 |
+
).squeeze(0).transpose(0, 1)
|
| 64 |
+
# Pen state์ modality/missing ๊ณ์ฝ์ ์ถ์ interpolation ๊ฐ์ด ์๋๋ผ ๋ช
์์ anchor๋ฅผ ์ ์งํ๋ค.
|
| 65 |
+
restored[:, 7] = 0.0
|
| 66 |
+
restored[torch.nonzero(valid[:, 7] > 0.5, as_tuple=False).flatten(), 7] = 1.0
|
| 67 |
+
restored[:, 17] = valid[:, 17]
|
| 68 |
+
restored[:, 18] = valid[:, 18]
|
| 69 |
+
restored[:, 8] = valid[:, 8]
|
| 70 |
+
output = torch.zeros_like(sequence)
|
| 71 |
+
output[:, 8] = -1.0
|
| 72 |
+
output[:len(restored)] = restored
|
| 73 |
+
return output
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _stress06(features: torch.Tensor, mode: str, *, seed: int) -> torch.Tensor:
|
| 77 |
+
"""ํ์ ๋ณ์: canonical feature batchยทstress mode. ์๋ ์๋ฆฌ: label์ ๋ณด์ง ์๊ณ device/sampling ๋ณ๋๋ง ์ ์ฉํ๋ค."""
|
| 78 |
+
|
| 79 |
+
if mode == "clean":
|
| 80 |
+
return features.clone()
|
| 81 |
+
output = features.clone()
|
| 82 |
+
valid = output[:, :, 8] >= 0
|
| 83 |
+
if mode == "coordinate_jitter":
|
| 84 |
+
generator = torch.Generator().manual_seed(seed)
|
| 85 |
+
noise = torch.randn(output.shape[:2] + (2,), generator=generator) * 0.008
|
| 86 |
+
for columns in ((0, 1), (2, 3)):
|
| 87 |
+
values = output[:, :, list(columns)]
|
| 88 |
+
values[valid] = (values[valid] + noise[valid]).clamp(0.0, 1.0)
|
| 89 |
+
output[:, :, list(columns)] = values
|
| 90 |
+
elif mode == "timestamp_missing":
|
| 91 |
+
output[:, :, 15:17] = 0.0
|
| 92 |
+
output[:, :, 17] = 1.0
|
| 93 |
+
elif mode == "sparse_sampling":
|
| 94 |
+
output = torch.stack([_sparse_sequence06(sequence) for sequence in output])
|
| 95 |
+
elif mode == "affine_device":
|
| 96 |
+
x = ((output[:, :, 2] - 0.5) * 1.12 + 0.5).clamp(0.0, 1.0)
|
| 97 |
+
y = ((output[:, :, 3] - 0.5) * 0.88 + 0.5).clamp(0.0, 1.0)
|
| 98 |
+
output[:, :, 2] = torch.where(valid, x, output[:, :, 2])
|
| 99 |
+
output[:, :, 3] = torch.where(valid, y, output[:, :, 3])
|
| 100 |
+
output[:, :, 9] = output[:, :, 9] * (1.12 / 0.88)
|
| 101 |
+
else:
|
| 102 |
+
raise ValueError(f"์ง์ํ์ง ์๋ stress mode์
๋๋ค: {mode}")
|
| 103 |
+
return output
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _seed_paths06(seed: int) -> dict[str, Path]:
|
| 107 |
+
"""ํ์ ๋ณ์: seed. ์๋ ์๋ฆฌ: ์ฑํ๋ baseยทadapterยท์ด๊ธฐ headยทjoint delta ๊ฒฝ๋ก๋ฅผ ๊ฒฐ์ ํ๋ค."""
|
| 108 |
+
|
| 109 |
+
boundary_run = (
|
| 110 |
+
"math_ink_06_p_boundary_auxiliary_layout5_20260724"
|
| 111 |
+
if seed == 17 else f"math_ink_06_p_boundary_auxiliary_layout5_seed{seed}_20260724"
|
| 112 |
+
)
|
| 113 |
+
return {
|
| 114 |
+
"base": PROJECT_ROOT / f"research/runs/math_ink_06_federated_virtual_ce025_family010_seed{seed}_20260723/math_ink_06_candidate.pt",
|
| 115 |
+
"adapter": PROJECT_ROOT / f"research/runs/math_ink_06_online_casecontext_refined_seed{seed}_20260723/skeleton_adapter.pt",
|
| 116 |
+
"boundary": PROJECT_ROOT / f"research/runs/{boundary_run}/boundary_auxiliary_head.pt",
|
| 117 |
+
"joint": PROJECT_ROOT / f"research/runs/math_ink_06_p_boundary_joint_seed{seed}_20260724/boundary_joint_delta.pt",
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def main() -> None:
|
| 122 |
+
"""ํ์ ๋ณ์: paired test์ 3-seed delta. ์๋ ์๋ฆฌ: clean ๋๋น stress ํ๊ท๋ฅผ seed๋ณ๋ก ๊ณ์ฐํ๊ณ ์ต์ gate๋ฅผ ํ์ ํ๋ค."""
|
| 123 |
+
|
| 124 |
+
args = _parse_args()
|
| 125 |
+
device = _resolve_device06(args.device)
|
| 126 |
+
features, targets, cache_key = _load_feature_cache06(args.test_cache)
|
| 127 |
+
modes = ("clean", "coordinate_jitter", "timestamp_missing", "sparse_sampling", "affine_device")
|
| 128 |
+
seed_reports = []
|
| 129 |
+
for seed in (17, 31, 47):
|
| 130 |
+
paths = _seed_paths06(seed)
|
| 131 |
+
model, adapter, base, _adapter_payload = _load_encoder06(paths["base"], paths["adapter"], device)
|
| 132 |
+
boundary = torch.load(paths["boundary"], map_location="cpu", weights_only=True)
|
| 133 |
+
joint = torch.load(paths["joint"], map_location="cpu", weights_only=True)
|
| 134 |
+
if not joint["adopted"]:
|
| 135 |
+
raise ValueError(f"์ฑํ๋์ง ์์ joint delta์
๋๋ค: seed {seed}")
|
| 136 |
+
if model.boundary_head is None:
|
| 137 |
+
raise RuntimeError("boundary head๊ฐ ์์ต๋๋ค.")
|
| 138 |
+
model.boundary_head.load_state_dict(boundary["state_dict"])
|
| 139 |
+
model.load_state_dict(joint["state_dict"], strict=False)
|
| 140 |
+
family_map = _family_target_map06(base)
|
| 141 |
+
boundary_x, boundary_y = _balanced_boundary_set06(
|
| 142 |
+
features, targets, samples_per_class=args.samples_per_class, seed=seed + 2,
|
| 143 |
+
)
|
| 144 |
+
rows = {}
|
| 145 |
+
for mode_index, mode in enumerate(modes):
|
| 146 |
+
authentic_stressed = _stress06(features, mode, seed=seed * 100 + mode_index)
|
| 147 |
+
boundary_stressed = _stress06(boundary_x, mode, seed=seed * 1000 + mode_index)
|
| 148 |
+
authentic_metrics = _authentic_metrics06(
|
| 149 |
+
model, adapter, authentic_stressed, targets, family_map,
|
| 150 |
+
device=device, batch_size=args.batch_size,
|
| 151 |
+
)
|
| 152 |
+
boundary_logits = _boundary_logits06(
|
| 153 |
+
model, adapter, boundary_stressed, device=device, batch_size=args.batch_size,
|
| 154 |
+
)
|
| 155 |
+
boundary_metrics = _metrics06(
|
| 156 |
+
boundary_logits, boundary_y, threshold=float(joint["threshold"]),
|
| 157 |
+
)
|
| 158 |
+
rows[mode] = {"authentic": authentic_metrics, "boundary": boundary_metrics}
|
| 159 |
+
clean = rows["clean"]
|
| 160 |
+
for mode in modes[1:]:
|
| 161 |
+
rows[mode]["deltas"] = {
|
| 162 |
+
"exact_top1_pp": (
|
| 163 |
+
rows[mode]["authentic"]["exact_top1"] - clean["authentic"]["exact_top1"]
|
| 164 |
+
) * 100.0,
|
| 165 |
+
"family_top1_pp": (
|
| 166 |
+
rows[mode]["authentic"]["family_top1"] - clean["authentic"]["family_top1"]
|
| 167 |
+
) * 100.0,
|
| 168 |
+
}
|
| 169 |
+
rows[mode]["gate_passed"] = bool(
|
| 170 |
+
rows[mode]["deltas"]["exact_top1_pp"] >= -3.0
|
| 171 |
+
and rows[mode]["deltas"]["family_top1_pp"] >= -3.0
|
| 172 |
+
and rows[mode]["boundary"]["single_symbol_recall"] >= 0.90
|
| 173 |
+
and rows[mode]["boundary"]["cross_boundary_recall"] >= 0.90
|
| 174 |
+
)
|
| 175 |
+
seed_reports.append({
|
| 176 |
+
"seed": seed, "threshold": float(joint["threshold"]), "modes": rows,
|
| 177 |
+
"all_stress_gates_passed": all(rows[mode]["gate_passed"] for mode in modes[1:]),
|
| 178 |
+
})
|
| 179 |
+
failures = [
|
| 180 |
+
{"seed": row["seed"], "mode": mode}
|
| 181 |
+
for row in seed_reports for mode in modes[1:]
|
| 182 |
+
if not row["modes"][mode]["gate_passed"]
|
| 183 |
+
]
|
| 184 |
+
report = {
|
| 185 |
+
"experiment": "P-MATH-INK-06-BOUNDARY-DEVICE-STRESS-001",
|
| 186 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 187 |
+
"device": str(device),
|
| 188 |
+
"cuda_device": torch.cuda.get_device_name(device) if device.type == "cuda" else None,
|
| 189 |
+
"test_cache_key": cache_key,
|
| 190 |
+
"stress_contract": {
|
| 191 |
+
"modes": list(modes),
|
| 192 |
+
"maximum_exact_regression_pp": 3.0,
|
| 193 |
+
"maximum_family_regression_pp": 3.0,
|
| 194 |
+
"minimum_single_symbol_recall": 0.90,
|
| 195 |
+
"minimum_cross_boundary_recall": 0.90,
|
| 196 |
+
},
|
| 197 |
+
"seeds": seed_reports,
|
| 198 |
+
"decision": {
|
| 199 |
+
"all_seed_stress_gates_passed": not failures,
|
| 200 |
+
"failures": failures,
|
| 201 |
+
"product_validation": False,
|
| 202 |
+
},
|
| 203 |
+
"track": "P_with_obligations",
|
| 204 |
+
"product_validation": False,
|
| 205 |
+
}
|
| 206 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 207 |
+
args.output.write_text(
|
| 208 |
+
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8",
|
| 209 |
+
)
|
| 210 |
+
print(json.dumps(report, ensure_ascii=False, indent=2))
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
if __name__ == "__main__":
|
| 214 |
+
main()
|