Add fail-closed P formula student distillation path
Browse files- MANIFEST.json +69 -64
- MODEL_INDEX.json +10 -0
- README.md +8 -1
- configs/MATH-INK-06-P-FORMULA-v1.json +16 -0
- reports/RESEARCH_REPORT.md +15 -2
- scripts/distill_math_ink_06_p_formula_student.py +552 -0
- scripts/summarize_math_ink_06_p_formula_seeds.py +4 -0
- scripts/train_math_ink_06_p_formula_adapter.py +14 -0
MANIFEST.json
CHANGED
|
@@ -1,11 +1,11 @@
|
|
| 1 |
-
|
| 2 |
-
"schema": "aiflow-hf-research-snapshot-
|
| 3 |
-
"generated_at": "2026-07-
|
| 4 |
"track": "R_noncommercial_plus_rejected_P_proxy",
|
| 5 |
"product_validation": false,
|
| 6 |
"public_release": true,
|
| 7 |
"contains_raw_dataset": false,
|
| 8 |
-
"tests": "
|
| 9 |
"retracted_paths": [
|
| 10 |
"models/auxiliary/boundary_auxiliary_head.pt",
|
| 11 |
"models/auxiliary/seed17/boundary_joint_delta.pt",
|
|
@@ -13,15 +13,25 @@
|
|
| 13 |
"models/auxiliary/seed47/boundary_joint_delta.pt"
|
| 14 |
],
|
| 15 |
"files": [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
{
|
| 17 |
"path": "scripts/audit_math_ink_06_formula_context_contract.py",
|
| 18 |
"bytes": 12602,
|
| 19 |
"sha256": "1a81c46465958e2ae5f8f8ef1c09d7cf664e6ecb951d7706db5ccce4084d4220"
|
| 20 |
},
|
| 21 |
{
|
| 22 |
-
"path": "
|
| 23 |
-
"bytes":
|
| 24 |
-
"sha256": "
|
| 25 |
},
|
| 26 |
{
|
| 27 |
"path": "scripts/audit_math_ink_06_behavior_exact_gate.py",
|
|
@@ -29,19 +39,19 @@
|
|
| 29 |
"sha256": "79feee00110eebb9429c1577df43e8747831cec897b0955e64dc5b5c5da1a018"
|
| 30 |
},
|
| 31 |
{
|
| 32 |
-
"path": "
|
| 33 |
-
"bytes":
|
| 34 |
-
"sha256": "
|
| 35 |
},
|
| 36 |
{
|
| 37 |
-
"path": "
|
| 38 |
-
"bytes":
|
| 39 |
-
"sha256": "
|
| 40 |
},
|
| 41 |
{
|
| 42 |
-
"path": "scripts/
|
| 43 |
-
"bytes":
|
| 44 |
-
"sha256": "
|
| 45 |
},
|
| 46 |
{
|
| 47 |
"path": "scripts/calibrate_math_ink_06_online_family_fusion.py",
|
|
@@ -49,9 +59,9 @@
|
|
| 49 |
"sha256": "535c2b6197148fada9a1762f2c6db9a494e89b703a9e60b913348b7c7ffdc056"
|
| 50 |
},
|
| 51 |
{
|
| 52 |
-
"path": "scripts/
|
| 53 |
-
"bytes":
|
| 54 |
-
"sha256": "
|
| 55 |
},
|
| 56 |
{
|
| 57 |
"path": "scripts/build_math_ink_06_litert_colab_bundle.py",
|
|
@@ -59,19 +69,19 @@
|
|
| 59 |
"sha256": "f008635ab59344005f65f1394f56275e4c92610cd25781a2d95243d060010718"
|
| 60 |
},
|
| 61 |
{
|
| 62 |
-
"path": "
|
| 63 |
-
"bytes":
|
| 64 |
-
"sha256": "
|
| 65 |
},
|
| 66 |
{
|
| 67 |
-
"path": "
|
| 68 |
-
"bytes":
|
| 69 |
-
"sha256": "
|
| 70 |
},
|
| 71 |
{
|
| 72 |
-
"path": "reports/
|
| 73 |
-
"bytes":
|
| 74 |
-
"sha256": "
|
| 75 |
},
|
| 76 |
{
|
| 77 |
"path": "reports/online_casecontext_seed47.json",
|
|
@@ -79,9 +89,9 @@
|
|
| 79 |
"sha256": "c3dfd328d9ff0a55eed2cb74657ad87e5e164968fddd7fae8c8fd92e137a4557"
|
| 80 |
},
|
| 81 |
{
|
| 82 |
-
"path": "reports/
|
| 83 |
-
"bytes":
|
| 84 |
-
"sha256": "
|
| 85 |
},
|
| 86 |
{
|
| 87 |
"path": "reports/online_casecontext_seed31.json",
|
|
@@ -89,19 +99,19 @@
|
|
| 89 |
"sha256": "30e6ca83aa4c04d1d257b0221fea7ac726f8e09e91906fdecbc12a2bd4d4d34b"
|
| 90 |
},
|
| 91 |
{
|
| 92 |
-
"path": "reports/
|
| 93 |
-
"bytes":
|
| 94 |
-
"sha256": "
|
| 95 |
},
|
| 96 |
{
|
| 97 |
-
"path": "reports/
|
| 98 |
-
"bytes":
|
| 99 |
-
"sha256": "
|
| 100 |
},
|
| 101 |
{
|
| 102 |
-
"path": "reports/
|
| 103 |
-
"bytes":
|
| 104 |
-
"sha256": "
|
| 105 |
},
|
| 106 |
{
|
| 107 |
"path": "reports/p_boundary_joint_seed31.json",
|
|
@@ -109,25 +119,15 @@
|
|
| 109 |
"sha256": "4f842852093ca46b1942c942fd5e475ca6c19f044ba9e27b6d372f1aed56056e"
|
| 110 |
},
|
| 111 |
{
|
| 112 |
-
"path": "reports/
|
| 113 |
-
"bytes":
|
| 114 |
-
"sha256": "
|
| 115 |
},
|
| 116 |
{
|
| 117 |
"path": "reports/p_boundary_joint_seed17.json",
|
| 118 |
"bytes": 6920,
|
| 119 |
"sha256": "7af3bb995041e5ea621d3c9535f3552e264860f12a7e6145637849f5cd0220a8"
|
| 120 |
},
|
| 121 |
-
{
|
| 122 |
-
"path": "reports/p_boundary_auxiliary_smoke.json",
|
| 123 |
-
"bytes": 16685,
|
| 124 |
-
"sha256": "9d250e5c75d4f104971551c07a702810bf94fb5b6e5e587035a624933766356e"
|
| 125 |
-
},
|
| 126 |
-
{
|
| 127 |
-
"path": "reports/p_boundary_device_stress_sharedfix_3seed.json",
|
| 128 |
-
"bytes": 10867,
|
| 129 |
-
"sha256": "3cad0c0e6aed77ebf2bd19a915fc3d898beb87d7b8a5e5ab29f2013b4184826e"
|
| 130 |
-
},
|
| 131 |
{
|
| 132 |
"path": "src/ink06_export.py",
|
| 133 |
"bytes": 5348,
|
|
@@ -140,8 +140,8 @@
|
|
| 140 |
},
|
| 141 |
{
|
| 142 |
"path": "scripts/train_math_ink_06_p_formula_adapter.py",
|
| 143 |
-
"bytes":
|
| 144 |
-
"sha256": "
|
| 145 |
},
|
| 146 |
{
|
| 147 |
"path": "scripts/train_math_ink_06_p_boundary_auxiliary.py",
|
|
@@ -235,14 +235,19 @@
|
|
| 235 |
},
|
| 236 |
{
|
| 237 |
"path": "scripts/summarize_math_ink_06_p_formula_seeds.py",
|
| 238 |
-
"bytes":
|
| 239 |
-
"sha256": "
|
| 240 |
},
|
| 241 |
{
|
| 242 |
"path": "scripts/sweep_math_ink_06_component_competition_guard.py",
|
| 243 |
"bytes": 9084,
|
| 244 |
"sha256": "e2dee616e2c6ebdc32281624889017685648b7076515242e04a1452d98d1c7da"
|
| 245 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
{
|
| 247 |
"path": "exports/seed17/export_manifest.json",
|
| 248 |
"bytes": 1069,
|
|
@@ -260,8 +265,8 @@
|
|
| 260 |
},
|
| 261 |
{
|
| 262 |
"path": "configs/MATH-INK-06-P-FORMULA-v1.json",
|
| 263 |
-
"bytes":
|
| 264 |
-
"sha256": "
|
| 265 |
},
|
| 266 |
{
|
| 267 |
"path": "contracts/aiflow_p_formula_annotation_v1.schema.json",
|
|
@@ -310,8 +315,8 @@
|
|
| 310 |
},
|
| 311 |
{
|
| 312 |
"path": "README.md",
|
| 313 |
-
"bytes":
|
| 314 |
-
"sha256": "
|
| 315 |
},
|
| 316 |
{
|
| 317 |
"path": "NOTICE.md",
|
|
@@ -325,8 +330,8 @@
|
|
| 325 |
},
|
| 326 |
{
|
| 327 |
"path": "MODEL_INDEX.json",
|
| 328 |
-
"bytes":
|
| 329 |
-
"sha256": "
|
| 330 |
},
|
| 331 |
{
|
| 332 |
"path": "artifacts/boundary_behavior_guard.joblib",
|
|
@@ -474,4 +479,4 @@
|
|
| 474 |
"sha256": "7b0890e13164f170be02f519e3ffc56930486e2018cde050434286f769bfce83"
|
| 475 |
}
|
| 476 |
]
|
| 477 |
-
}
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": "aiflow-hf-research-snapshot-v19",
|
| 3 |
+
"generated_at": "2026-07-23T22:08:08.2837334Z",
|
| 4 |
"track": "R_noncommercial_plus_rejected_P_proxy",
|
| 5 |
"product_validation": false,
|
| 6 |
"public_release": true,
|
| 7 |
"contains_raw_dataset": false,
|
| 8 |
+
"tests": "318 passed",
|
| 9 |
"retracted_paths": [
|
| 10 |
"models/auxiliary/boundary_auxiliary_head.pt",
|
| 11 |
"models/auxiliary/seed17/boundary_joint_delta.pt",
|
|
|
|
| 13 |
"models/auxiliary/seed47/boundary_joint_delta.pt"
|
| 14 |
],
|
| 15 |
"files": [
|
| 16 |
+
{
|
| 17 |
+
"path": "scripts/audit_math_ink_06_local_baseline_overmerge.py",
|
| 18 |
+
"bytes": 11900,
|
| 19 |
+
"sha256": "f32a18c8e7d3a0ea36379293e6be88bfb509b7df99846c38149fb8458e6cc994"
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"path": "scripts/audit_math_ink_06_online_error_consensus.py",
|
| 23 |
+
"bytes": 12174,
|
| 24 |
+
"sha256": "c562a8d0300aa358f9c3faaed274992c3548224347e1e875401eec93953d0807"
|
| 25 |
+
},
|
| 26 |
{
|
| 27 |
"path": "scripts/audit_math_ink_06_formula_context_contract.py",
|
| 28 |
"bytes": 12602,
|
| 29 |
"sha256": "1a81c46465958e2ae5f8f8ef1c09d7cf664e6ecb951d7706db5ccce4084d4220"
|
| 30 |
},
|
| 31 |
{
|
| 32 |
+
"path": "reports/RESEARCH_REPORT.md",
|
| 33 |
+
"bytes": 49820,
|
| 34 |
+
"sha256": "d275203b0a1c094fbb6d8977a2f6d658998e67fe0794f1f012566fe68f3014a2"
|
| 35 |
},
|
| 36 |
{
|
| 37 |
"path": "scripts/audit_math_ink_06_behavior_exact_gate.py",
|
|
|
|
| 39 |
"sha256": "79feee00110eebb9429c1577df43e8747831cec897b0955e64dc5b5c5da1a018"
|
| 40 |
},
|
| 41 |
{
|
| 42 |
+
"path": "scripts/audit_math_ink_06_p_proxy_shift.py",
|
| 43 |
+
"bytes": 9362,
|
| 44 |
+
"sha256": "f34ca3b9809eeea06a1557073cae89f31ec58dcb7ecb9073517b4730f4311686"
|
| 45 |
},
|
| 46 |
{
|
| 47 |
+
"path": "scripts/distill_math_ink_06_p_formula_student.py",
|
| 48 |
+
"bytes": 21388,
|
| 49 |
+
"sha256": "6596f9a4114431dad97d64616c24ddec6a37c3de934562b7438e4fc900ad080f"
|
| 50 |
},
|
| 51 |
{
|
| 52 |
+
"path": "scripts/evaluate_crohme_tray_joint_selector.py",
|
| 53 |
+
"bytes": 11671,
|
| 54 |
+
"sha256": "58c175a95cdeb9fc764fb96594a76888122b8f118f2f26da1283e743b1a7cd04"
|
| 55 |
},
|
| 56 |
{
|
| 57 |
"path": "scripts/calibrate_math_ink_06_online_family_fusion.py",
|
|
|
|
| 59 |
"sha256": "535c2b6197148fada9a1762f2c6db9a494e89b703a9e60b913348b7c7ffdc056"
|
| 60 |
},
|
| 61 |
{
|
| 62 |
+
"path": "scripts/benchmark_math_ink_06_composite.py",
|
| 63 |
+
"bytes": 7618,
|
| 64 |
+
"sha256": "6c5347fe68114116e0dfd29e55e3682f47b3ce42af1412009ab7883e2076887c"
|
| 65 |
},
|
| 66 |
{
|
| 67 |
"path": "scripts/build_math_ink_06_litert_colab_bundle.py",
|
|
|
|
| 69 |
"sha256": "f008635ab59344005f65f1394f56275e4c92610cd25781a2d95243d060010718"
|
| 70 |
},
|
| 71 |
{
|
| 72 |
+
"path": "reports/p_proxy_shift_seed17.json",
|
| 73 |
+
"bytes": 10788,
|
| 74 |
+
"sha256": "a2dac8be7a0fadf58a11e2c2d737ac6876e8d89310570ce0159d1b593039a99a"
|
| 75 |
},
|
| 76 |
{
|
| 77 |
+
"path": "reports/online_error_consensus_paired_test_3seed.json",
|
| 78 |
+
"bytes": 12267,
|
| 79 |
+
"sha256": "ec624e9c96d7fa767b438ca2d555b5b0930edc0b80716ee740229ae894711f52"
|
| 80 |
},
|
| 81 |
{
|
| 82 |
+
"path": "reports/online_family_fusion_calibration_3seed.json",
|
| 83 |
+
"bytes": 2861,
|
| 84 |
+
"sha256": "50b06e65e9220639c68a2686c4b123697e3f4f8411755b9d9e9222e7f39f3042"
|
| 85 |
},
|
| 86 |
{
|
| 87 |
"path": "reports/online_casecontext_seed47.json",
|
|
|
|
| 89 |
"sha256": "c3dfd328d9ff0a55eed2cb74657ad87e5e164968fddd7fae8c8fd92e137a4557"
|
| 90 |
},
|
| 91 |
{
|
| 92 |
+
"path": "reports/online_casecontext_seed17.json",
|
| 93 |
+
"bytes": 33805,
|
| 94 |
+
"sha256": "050c00c14df4712f55b0c76118dd11615b9129b078d56bd987e329c9edee1bda"
|
| 95 |
},
|
| 96 |
{
|
| 97 |
"path": "reports/online_casecontext_seed31.json",
|
|
|
|
| 99 |
"sha256": "30e6ca83aa4c04d1d257b0221fea7ac726f8e09e91906fdecbc12a2bd4d4d34b"
|
| 100 |
},
|
| 101 |
{
|
| 102 |
+
"path": "reports/p_boundary_auxiliary_smoke.json",
|
| 103 |
+
"bytes": 16685,
|
| 104 |
+
"sha256": "9d250e5c75d4f104971551c07a702810bf94fb5b6e5e587035a624933766356e"
|
| 105 |
},
|
| 106 |
{
|
| 107 |
+
"path": "reports/p_boundary_joint_seed47.json",
|
| 108 |
+
"bytes": 6809,
|
| 109 |
+
"sha256": "fd4ddcf71209cf59443e7a252a8337d7812ff66fae9482867f6eb0a21666e064"
|
| 110 |
},
|
| 111 |
{
|
| 112 |
+
"path": "reports/p_boundary_joint_sharedfix_3seed_summary.json",
|
| 113 |
+
"bytes": 3056,
|
| 114 |
+
"sha256": "1ad9192d45218d4644f69a8f076487bd5fb36d5c2348f5225ee4ae06e9f4b127"
|
| 115 |
},
|
| 116 |
{
|
| 117 |
"path": "reports/p_boundary_joint_seed31.json",
|
|
|
|
| 119 |
"sha256": "4f842852093ca46b1942c942fd5e475ca6c19f044ba9e27b6d372f1aed56056e"
|
| 120 |
},
|
| 121 |
{
|
| 122 |
+
"path": "reports/p_boundary_device_stress_sharedfix_3seed.json",
|
| 123 |
+
"bytes": 10867,
|
| 124 |
+
"sha256": "3cad0c0e6aed77ebf2bd19a915fc3d898beb87d7b8a5e5ab29f2013b4184826e"
|
| 125 |
},
|
| 126 |
{
|
| 127 |
"path": "reports/p_boundary_joint_seed17.json",
|
| 128 |
"bytes": 6920,
|
| 129 |
"sha256": "7af3bb995041e5ea621d3c9535f3552e264860f12a7e6145637849f5cd0220a8"
|
| 130 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
{
|
| 132 |
"path": "src/ink06_export.py",
|
| 133 |
"bytes": 5348,
|
|
|
|
| 140 |
},
|
| 141 |
{
|
| 142 |
"path": "scripts/train_math_ink_06_p_formula_adapter.py",
|
| 143 |
+
"bytes": 15489,
|
| 144 |
+
"sha256": "6aac2058f6cc69a577f59ab5e4746fe910e2595f11ecd7f1a72339330db8da8e"
|
| 145 |
},
|
| 146 |
{
|
| 147 |
"path": "scripts/train_math_ink_06_p_boundary_auxiliary.py",
|
|
|
|
| 235 |
},
|
| 236 |
{
|
| 237 |
"path": "scripts/summarize_math_ink_06_p_formula_seeds.py",
|
| 238 |
+
"bytes": 4633,
|
| 239 |
+
"sha256": "fe4658a877489398c4f31fd8dfe76aecac34d13443fdc356587c6e6999e6ac73"
|
| 240 |
},
|
| 241 |
{
|
| 242 |
"path": "scripts/sweep_math_ink_06_component_competition_guard.py",
|
| 243 |
"bytes": 9084,
|
| 244 |
"sha256": "e2dee616e2c6ebdc32281624889017685648b7076515242e04a1452d98d1c7da"
|
| 245 |
},
|
| 246 |
+
{
|
| 247 |
+
"path": "reports/multistroke_family_guard_report.json",
|
| 248 |
+
"bytes": 15740,
|
| 249 |
+
"sha256": "27b31da71b9f39f7ded38aa1ce3694911af140d22854543d4ae2b5f9e709b479"
|
| 250 |
+
},
|
| 251 |
{
|
| 252 |
"path": "exports/seed17/export_manifest.json",
|
| 253 |
"bytes": 1069,
|
|
|
|
| 265 |
},
|
| 266 |
{
|
| 267 |
"path": "configs/MATH-INK-06-P-FORMULA-v1.json",
|
| 268 |
+
"bytes": 1985,
|
| 269 |
+
"sha256": "4d365d18ef92ad01c4dccc26c37e8752a239cfb19ad9ec3387af65f1936ce0d2"
|
| 270 |
},
|
| 271 |
{
|
| 272 |
"path": "contracts/aiflow_p_formula_annotation_v1.schema.json",
|
|
|
|
| 315 |
},
|
| 316 |
{
|
| 317 |
"path": "README.md",
|
| 318 |
+
"bytes": 19979,
|
| 319 |
+
"sha256": "0d3376547a35761c1004811d4dc76c654bb46f282ba0906ba0b3411e6599964f"
|
| 320 |
},
|
| 321 |
{
|
| 322 |
"path": "NOTICE.md",
|
|
|
|
| 330 |
},
|
| 331 |
{
|
| 332 |
"path": "MODEL_INDEX.json",
|
| 333 |
+
"bytes": 7349,
|
| 334 |
+
"sha256": "7272a5e11b08e391698fd68b88fb04da5e35b9665d6131d272dee923a086c535"
|
| 335 |
},
|
| 336 |
{
|
| 337 |
"path": "artifacts/boundary_behavior_guard.joblib",
|
|
|
|
| 479 |
"sha256": "7b0890e13164f170be02f519e3ffc56930486e2018cde050434286f769bfce83"
|
| 480 |
}
|
| 481 |
]
|
| 482 |
+
}
|
MODEL_INDEX.json
CHANGED
|
@@ -157,6 +157,7 @@
|
|
| 157 |
"dataset": "src/p_formula_dataset06.py",
|
| 158 |
"trainer": "scripts/train_math_ink_06_p_formula_adapter.py",
|
| 159 |
"three_seed_summary": "scripts/summarize_math_ink_06_p_formula_seeds.py",
|
|
|
|
| 160 |
"recipe": "configs/MATH-INK-06-P-FORMULA-v1.json",
|
| 161 |
"seeds": [17, 31, 47],
|
| 162 |
"formula_adapter_hidden_size": 64,
|
|
@@ -170,6 +171,15 @@
|
|
| 170 |
"missing_metadata_drop_maximum_pp": 3.0
|
| 171 |
},
|
| 172 |
"student_distillation_requires_all_seeds": true,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
"teacher_ensemble_mobile_deployment_allowed": false,
|
| 174 |
"fixture_smoke_passed": true,
|
| 175 |
"real_p_training_completed": false,
|
|
|
|
| 157 |
"dataset": "src/p_formula_dataset06.py",
|
| 158 |
"trainer": "scripts/train_math_ink_06_p_formula_adapter.py",
|
| 159 |
"three_seed_summary": "scripts/summarize_math_ink_06_p_formula_seeds.py",
|
| 160 |
+
"student_distiller": "scripts/distill_math_ink_06_p_formula_student.py",
|
| 161 |
"recipe": "configs/MATH-INK-06-P-FORMULA-v1.json",
|
| 162 |
"seeds": [17, 31, 47],
|
| 163 |
"formula_adapter_hidden_size": 64,
|
|
|
|
| 171 |
"missing_metadata_drop_maximum_pp": 3.0
|
| 172 |
},
|
| 173 |
"student_distillation_requires_all_seeds": true,
|
| 174 |
+
"same_data_sha256_required_across_seeds": true,
|
| 175 |
+
"distillation": {
|
| 176 |
+
"teacher_probability_temperature": 2.0,
|
| 177 |
+
"teacher_weights_embedded": false,
|
| 178 |
+
"maximum_teacher_to_student_regression_pp": 1.0,
|
| 179 |
+
"cuda_fixture_path_passed": true,
|
| 180 |
+
"fixture_release_gate_passed": false,
|
| 181 |
+
"litert_exported": false
|
| 182 |
+
},
|
| 183 |
"teacher_ensemble_mobile_deployment_allowed": false,
|
| 184 |
"fixture_smoke_passed": true,
|
| 185 |
"real_p_training_completed": false,
|
README.md
CHANGED
|
@@ -302,10 +302,16 @@ inverse-sqrt(source frequency ร exact-label frequency) sampler
|
|
| 302 |
validation-only checkpoint selection
|
| 303 |
```
|
| 304 |
|
| 305 |
-
๊ฐ seed๋ test exact top-1 92%, top-5 99%, macro-F1 90%, writer floor 75%, ๊ฒฐ์ธก metadata slice ํ๋ฝ 3%p ์ดํ๋ฅผ ๋ชจ๋ ํต๊ณผํด์ผ ํ๋ค. Seed 17ยท31ยท47์ด ๊ฐ๋ณ ํต๊ณผ
|
| 306 |
|
| 307 |
์ค์ seed-17 composite์ GTX 1650์ ์ฌ์ฉํ 1-epoch fixture smoke์์ CUDA ํ์ต๋ถํฐ report/checkpoint ์์ฑ๊น์ง ํต๊ณผํ๋ค. Fixture checkpoint๋ ์ฑ๋ฅ ๊ทผ๊ฑฐ๊ฐ ์๋๋ฏ๋ก ์ด ๊ณต๊ฐ ์ ์ฅ์์ ํฌํจํ์ง ์์๋ค. ๊ณ ์ recipe๋ [`configs/MATH-INK-06-P-FORMULA-v1.json`](configs/MATH-INK-06-P-FORMULA-v1.json)์ ์๋ค.
|
| 308 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 309 |
## ์๋ ค์ง ํ๊ณ
|
| 310 |
|
| 311 |
- 378-label paired writer/device-disjoint top-1 ๋ชฉํ 92%์ ์์ง ๋ฏธ๋ฌํ๋ค.
|
|
@@ -356,4 +362,5 @@ PyTorch checkpoint์ joblib/pickle์ ์ ๋ขฐํ ์ ์๋ ์ถ์ฒ์์ ๋ก๋
|
|
| 356 |
- ์ค์ P formula schema: [`contracts/aiflow_p_formula_v1.schema.json`](contracts/aiflow_p_formula_v1.schema.json)
|
| 357 |
- P formula ์ฌ๋ annotation schema: [`contracts/aiflow_p_formula_annotation_v1.schema.json`](contracts/aiflow_p_formula_annotation_v1.schema.json)
|
| 358 |
- P-only formula ํ์ต recipe: [`configs/MATH-INK-06-P-FORMULA-v1.json`](configs/MATH-INK-06-P-FORMULA-v1.json)
|
|
|
|
| 359 |
- ํ์ผ checksum: [`MANIFEST.json`](MANIFEST.json)
|
|
|
|
| 302 |
validation-only checkpoint selection
|
| 303 |
```
|
| 304 |
|
| 305 |
+
๊ฐ seed๋ test exact top-1 92%, top-5 99%, macro-F1 90%, writer floor 75%, ๊ฒฐ์ธก metadata slice ํ๋ฝ 3%p ์ดํ๋ฅผ ๋ชจ๋ ํต๊ณผํด์ผ ํ๋ค. Seed 17ยท31ยท47์ด ๊ฐ๋ณ ํต๊ณผํ๊ณ ์ธ run์ ์๋ณธ P Formula JSONL SHA-256์ด ์ ํํ ๊ฐ์ ๊ฒฝ์ฐ์๋ง single mobile student distillation์ ํ์ฉํ๋ค. Teacher ensemble ์์ฒด๋ ๊ธฐ๊ธฐ์ ํ์ฌํ์ง ์๋๋ค.
|
| 306 |
|
| 307 |
์ค์ seed-17 composite์ GTX 1650์ ์ฌ์ฉํ 1-epoch fixture smoke์์ CUDA ํ์ต๋ถํฐ report/checkpoint ์์ฑ๊น์ง ํต๊ณผํ๋ค. Fixture checkpoint๋ ์ฑ๋ฅ ๊ทผ๊ฑฐ๊ฐ ์๋๋ฏ๋ก ์ด ๊ณต๊ฐ ์ ์ฅ์์ ํฌํจํ์ง ์์๋ค. ๊ณ ์ recipe๋ [`configs/MATH-INK-06-P-FORMULA-v1.json`](configs/MATH-INK-06-P-FORMULA-v1.json)์ ์๋ค.
|
| 308 |
|
| 309 |
+
### Single-student distillation
|
| 310 |
+
|
| 311 |
+
ํต๊ณผํ ์ธ teacher๋ base โ shared online adapter โ P formula adapter ์์๋ก ํฉ์ฑํ๋ค. Temperature 2.0์ exact/family ํ๋ฅ ์ seed ์ฌ์ด์์ ํ๊ท ํ๊ณ KL + hard-label CE๋ก hidden-64 formula adapter ํ๋๋ง ํ์ตํ๋ค. Student checkpoint์๋ teacher weight๋ฅผ ํฌํจํ์ง ์๋๋ค.
|
| 312 |
+
|
| 313 |
+
Student๋ ์์ฒด 92/99ยทmacro-F1ยทwriter/missing gate๋ฟ ์๋๋ผ teacher ensemble ๋๋น exact top-1ยทtop-5ยทvisual-family ํ๋ฝ 1%p ์ดํ๋ฅผ ๋ชจ๋ ๋ง์กฑํด์ผ ํ๋ค. 2/2/2-symbol fixture์ 3-teacherโstudent CUDA ์คํ ๊ฒฝ๋ก๋ ํต๊ณผํ์ง๋ง ์ ์ student gate๋ ์คํจํ๋ค. Fixture์ checkpoint๋ ์ด ๊ณต๊ฐ ์ ์ฅ์์ ์์ผ๋ฉฐ, ์ค์ P ๋ฐ์ดํฐ ์ฑ๋ฅ์ด๋ ์ ํ ๊ฒ์ฆ์ ๋ปํ์ง ์๋๋ค. LiteRT ๋ณํ๋ ์์ง ์ํํ์ง ์์๋ค.
|
| 314 |
+
|
| 315 |
## ์๋ ค์ง ํ๊ณ
|
| 316 |
|
| 317 |
- 378-label paired writer/device-disjoint top-1 ๋ชฉํ 92%์ ์์ง ๋ฏธ๋ฌํ๋ค.
|
|
|
|
| 362 |
- ์ค์ P formula schema: [`contracts/aiflow_p_formula_v1.schema.json`](contracts/aiflow_p_formula_v1.schema.json)
|
| 363 |
- P formula ์ฌ๋ annotation schema: [`contracts/aiflow_p_formula_annotation_v1.schema.json`](contracts/aiflow_p_formula_annotation_v1.schema.json)
|
| 364 |
- P-only formula ํ์ต recipe: [`configs/MATH-INK-06-P-FORMULA-v1.json`](configs/MATH-INK-06-P-FORMULA-v1.json)
|
| 365 |
+
- P-only formula student distiller: [`scripts/distill_math_ink_06_p_formula_student.py`](scripts/distill_math_ink_06_p_formula_student.py)
|
| 366 |
- ํ์ผ checksum: [`MANIFEST.json`](MANIFEST.json)
|
configs/MATH-INK-06-P-FORMULA-v1.json
CHANGED
|
@@ -7,6 +7,7 @@
|
|
| 7 |
"commercial_training_allowed": true,
|
| 8 |
"minimum_independent_sources": 2,
|
| 9 |
"required_splits": ["training", "validation", "test"],
|
|
|
|
| 10 |
"identity_overlap_required": {
|
| 11 |
"origin_id": 0,
|
| 12 |
"writer_id": 0,
|
|
@@ -48,5 +49,20 @@
|
|
| 48 |
"student_distillation_allowed_after_pass": true,
|
| 49 |
"product_validation": false,
|
| 50 |
"next_gate": "LiteRT parity and Android low/mid/high tier benchmark"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
}
|
| 52 |
}
|
|
|
|
| 7 |
"commercial_training_allowed": true,
|
| 8 |
"minimum_independent_sources": 2,
|
| 9 |
"required_splits": ["training", "validation", "test"],
|
| 10 |
+
"same_data_sha256_required_across_seeds": true,
|
| 11 |
"identity_overlap_required": {
|
| 12 |
"origin_id": 0,
|
| 13 |
"writer_id": 0,
|
|
|
|
| 49 |
"student_distillation_allowed_after_pass": true,
|
| 50 |
"product_validation": false,
|
| 51 |
"next_gate": "LiteRT parity and Android low/mid/high tier benchmark"
|
| 52 |
+
},
|
| 53 |
+
"distillation": {
|
| 54 |
+
"teacher_ensemble": "probability_mean",
|
| 55 |
+
"temperature": 2.0,
|
| 56 |
+
"teacher_exact_weight": 0.7,
|
| 57 |
+
"teacher_family_weight": 1.0,
|
| 58 |
+
"hard_exact_weight": 0.2,
|
| 59 |
+
"hard_family_weight": 0.5,
|
| 60 |
+
"student_seed": 17,
|
| 61 |
+
"student_formula_adapter_hidden_size": 64,
|
| 62 |
+
"teacher_weights_embedded": false,
|
| 63 |
+
"maximum_teacher_to_student_regression_pp": 1.0,
|
| 64 |
+
"student_must_pass_seed_gate": true,
|
| 65 |
+
"litert_exported": false,
|
| 66 |
+
"product_validation": false
|
| 67 |
}
|
| 68 |
}
|
reports/RESEARCH_REPORT.md
CHANGED
|
@@ -519,13 +519,24 @@ Materialize๋ P Formula v1์ ๋ณ๋ ๊ฐ๊ณต ์์ด ์๋นํ๋ GPU trainer์
|
|
| 519 |
- writer floor โฅ75%
|
| 520 |
- timestampยทpressure ๊ฒฐ์ธก slice ํ๋ฝ โค3%p
|
| 521 |
|
| 522 |
-
Seed 17ยท31ยท47 report๊ฐ ์ ํํ ํ๋์ฉ ์๊ณ ๋ชจ๋ ๊ฐ๋ณ gate๋ฅผ ํต๊ณผํ ๊ฒฝ์ฐ์๋ง single mobile student distillation์ ํ์ฉํ๋ค. Teacher ensemble ์์ฒด์ ๋ชจ๋ฐ์ผ ํ์ฌ๋ ํญ์ ๊ธ์งํ๋ฉฐ, ์ธ seed๊ฐ ํต๊ณผํด๋ LiteRT parity์ Android 3-tier gate ์ ๊น์ง `product_validation=false`๋ค.
|
| 523 |
|
| 524 |
์ค์ seed-17 composite์ GTX 1650์์ 2/2/2-symbol P fixture๋ก 1-epoch end-to-end smoke๋ฅผ ์คํํ๋ค. CUDA ํ์ต, validation selection, test metric, ๊ฒฐ์ธก slice, checkpoint/report ์์ฑ์ด ์๋ฃ๋๋ค. ๊ทน์ fixture์ seed gate๋ ์์๋๋ก ์คํจํ๊ณ checkpoint๋ ํ๋ก์ ํธ ์ฐ์ถ๋ฌผยท๊ณต๊ฐ๋ณธ์ ํฌํจํ์ง ์์๋ค. ์ด๋ ์คํ ๊ณ์ฝ ์ฆ๊ฑฐ์ด๋ฉฐ ์ฑ๋ฅ ๊ทผ๊ฑฐ๊ฐ ์๋๋ค.
|
| 525 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 526 |
๊ณ ์ recipe๋ `research/configs/MATH-INK-06-P-FORMULA-v1.json`์ ์ ์ฅํ๋ค.
|
| 527 |
|
| 528 |
-
์ ์ฒด ํ๊ท๋
|
| 529 |
|
| 530 |
## ์ฐ์ถ๋ฌผ
|
| 531 |
|
|
@@ -562,12 +573,14 @@ Seed 17ยท31ยท47 report๊ฐ ์ ํํ ํ๋์ฉ ์๊ณ ๋ชจ๋ ๊ฐ๋ณ gate๋ฅผ ํต
|
|
| 562 |
- `scripts/materialize_math_ink_06_p_formula.py`
|
| 563 |
- `scripts/train_math_ink_06_p_formula_adapter.py`
|
| 564 |
- `scripts/summarize_math_ink_06_p_formula_seeds.py`
|
|
|
|
| 565 |
- `scripts/preflight_math_ink_06_p_formula.py`
|
| 566 |
- `scripts/analyze_crohme_lattice_failures.py`
|
| 567 |
- `tests/test_behavior_context06.py`
|
| 568 |
- `tests/test_behavior_role_head06.py`
|
| 569 |
- `tests/test_p_formula_gate06.py`
|
| 570 |
- `tests/test_p_formula_intake06.py`
|
|
|
|
| 571 |
- `src/math_grid_drawer/research/p_formula_intake06.py`
|
| 572 |
- `src/math_grid_drawer/research/p_formula_dataset06.py`
|
| 573 |
- `research/contracts/aiflow_p_formula_v1.schema.json`
|
|
|
|
| 519 |
- writer floor โฅ75%
|
| 520 |
- timestampยทpressure ๊ฒฐ์ธก slice ํ๋ฝ โค3%p
|
| 521 |
|
| 522 |
+
Seed 17ยท31ยท47 report๊ฐ ์ ํํ ํ๋์ฉ ์๊ณ ๋ชจ๋ ๊ฐ๋ณ gate๋ฅผ ํต๊ณผํ ๊ฒฝ์ฐ์๋ง single mobile student distillation์ ํ์ฉํ๋ค. ์ธ run์ ์๋ณธ P Formula JSONL byte-level SHA-256๋ ๋ฐ๋์ ๊ฐ์์ผ ํ๋ฉฐ, ๋น fingerprint๋ ์๋ก ๋ค๋ฅธ corpus๋ ์์ฝ ๋จ๊ณ์์ ๊ฑฐ๋ถํ๋ค. Teacher ensemble ์์ฒด์ ๋ชจ๋ฐ์ผ ํ์ฌ๋ ํญ์ ๊ธ์งํ๋ฉฐ, ์ธ seed๊ฐ ํต๊ณผํด๋ LiteRT parity์ Android 3-tier gate ์ ๊น์ง `product_validation=false`๋ค.
|
| 523 |
|
| 524 |
์ค์ seed-17 composite์ GTX 1650์์ 2/2/2-symbol P fixture๋ก 1-epoch end-to-end smoke๋ฅผ ์คํํ๋ค. CUDA ํ์ต, validation selection, test metric, ๊ฒฐ์ธก slice, checkpoint/report ์์ฑ์ด ์๋ฃ๋๋ค. ๊ทน์ fixture์ seed gate๋ ์์๋๋ก ์คํจํ๊ณ checkpoint๋ ํ๋ก์ ํธ ์ฐ์ถ๋ฌผยท๊ณต๊ฐ๋ณธ์ ํฌํจํ์ง ์์๋ค. ์ด๋ ์คํ ๊ณ์ฝ ์ฆ๊ฑฐ์ด๋ฉฐ ์ฑ๋ฅ ๊ทผ๊ฑฐ๊ฐ ์๋๋ค.
|
| 525 |
|
| 526 |
+
### 3-teacher ํ๋ฅ ์ฆ๋ฅยท๋จ์ผ ๋ชจ๋ฐ์ผ student
|
| 527 |
+
|
| 528 |
+
์ธ seed๊ฐ ๋์ผ P corpus์ ๊ฐ๋ณ gate๋ฅผ ํต๊ณผํ ๋ค ์คํ๋๋ fail-closed distiller๋ฅผ ์ถ๊ฐํ๋ค. ๊ฐ teacher๋ base โ shared online adapter โ P formula adapter ์์๋ก ํฉ์ฑํ๋ฉฐ, exact vocabulary์ data SHA-256์ ๋ค์ ๋์กฐํ๋ค. Temperature 2.0์์ seed๋ณ softmax ํ๋ฅ ์ ํ๊ท ํ๊ณ exact/family KL๊ณผ hard-label CE๋ฅผ ํจ๊ป ์ฌ์ฉํด hidden-64 formula adapter ํ๋๋ง ํ์ตํ๋ค.
|
| 529 |
+
|
| 530 |
+
Student๋ validation visual-family/exact ์์๋ก ํ ๋ฒ ์ ํํ๊ณ test์ ํ ๋ฒ๋ง ์ ์ฉํ๋ค. ์ต์ข
checkpoint๋ teacher weight๋ฅผ ํฌํจํ์ง ์์ผ๋ฉฐ ๋ค์ ๋ ์กฐ๊ฑด์ ๋ชจ๋ ์๊ตฌํ๋ค.
|
| 531 |
+
|
| 532 |
+
- student ์์ฒด P seed gate: exact top-1 92%, top-5 99%, macro-F1 90%, writer floor 75%, ๊ฒฐ์ธก slice ํ๋ฝ 3%p ์ดํ
|
| 533 |
+
- teacher ํ๋ฅ ensemble ๋๋น exact top-1ยทtop-5ยทvisual-family ํ๋ฝ ๊ฐ๊ฐ 1%p ์ดํ
|
| 534 |
+
|
| 535 |
+
๋์ผ 2/2/2-symbol fixture๋ฅผ seed 17ยท31ยท47๋ก ๊ฐ๊ฐ ํ์ตํ๊ณ ์์ฝํ ๋ค GTX 1650์์ student๊น์ง CUDA smoke๋ฅผ ์๋ฃํ๋ค. Data fingerprintยทteacher load orderยทprobability ensembleยทstudent checkpoint/report ์์ฑ์ ํต๊ณผํ๋ค. ๊ทน์ fixture์ ์ ์ student gate๋ ์์๋๋ก ์คํจํ์ผ๋ฉฐ checkpoint๋ `D:\AiflowTools` ๊ฒฉ๋ฆฌ ๊ฒฝ๋ก์๋ง ๋จ๊ณ ํ๋ก์ ํธยท๊ณต๊ฐ ์ ์ฅ์์๋ ํฌํจํ์ง ์๋๋ค. `litert_exported=false`, `product_validation=false`์ด๋ฉฐ ์ค์ P ์ฑ๋ฅ ๊ทผ๊ฑฐ๊ฐ ์๋๋ค.
|
| 536 |
+
|
| 537 |
๊ณ ์ recipe๋ `research/configs/MATH-INK-06-P-FORMULA-v1.json`์ ์ ์ฅํ๋ค.
|
| 538 |
|
| 539 |
+
์ ์ฒด ํ๊ท๋ 318๊ฐ๊ฐ ํต๊ณผํ๋ค.
|
| 540 |
|
| 541 |
## ์ฐ์ถ๋ฌผ
|
| 542 |
|
|
|
|
| 573 |
- `scripts/materialize_math_ink_06_p_formula.py`
|
| 574 |
- `scripts/train_math_ink_06_p_formula_adapter.py`
|
| 575 |
- `scripts/summarize_math_ink_06_p_formula_seeds.py`
|
| 576 |
+
- `scripts/distill_math_ink_06_p_formula_student.py`
|
| 577 |
- `scripts/preflight_math_ink_06_p_formula.py`
|
| 578 |
- `scripts/analyze_crohme_lattice_failures.py`
|
| 579 |
- `tests/test_behavior_context06.py`
|
| 580 |
- `tests/test_behavior_role_head06.py`
|
| 581 |
- `tests/test_p_formula_gate06.py`
|
| 582 |
- `tests/test_p_formula_intake06.py`
|
| 583 |
+
- `tests/test_distill_math_ink_06_p_formula_student.py`
|
| 584 |
- `src/math_grid_drawer/research/p_formula_intake06.py`
|
| 585 |
- `src/math_grid_drawer/research/p_formula_dataset06.py`
|
| 586 |
- `research/contracts/aiflow_p_formula_v1.schema.json`
|
scripts/distill_math_ink_06_p_formula_student.py
ADDED
|
@@ -0,0 +1,552 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ํต๊ณผํ P Formula 3-seed teacher๋ฅผ ํ๋์ ๋ชจ๋ฐ์ผ formula-adapter student๋ก ์ฆ๋ฅํ๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from collections import Counter
|
| 7 |
+
from copy import deepcopy
|
| 8 |
+
from datetime import datetime, timezone
|
| 9 |
+
import json
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
import random
|
| 12 |
+
import sys
|
| 13 |
+
from typing import Any, Sequence
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
import torch
|
| 17 |
+
from torch import Tensor
|
| 18 |
+
from torch.utils.data import DataLoader, TensorDataset, WeightedRandomSampler
|
| 19 |
+
|
| 20 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 21 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 22 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 23 |
+
if str(path) not in sys.path:
|
| 24 |
+
sys.path.insert(0, str(path))
|
| 25 |
+
|
| 26 |
+
from math_grid_drawer.research.external_corpus import read_jsonl
|
| 27 |
+
from math_grid_drawer.research.p_formula_dataset06 import (
|
| 28 |
+
PFormulaTensorBatch06,
|
| 29 |
+
materialize_p_formula_split06,
|
| 30 |
+
p_formula_release_metrics06,
|
| 31 |
+
p_formula_seed_gate06,
|
| 32 |
+
)
|
| 33 |
+
from math_grid_drawer.research.p_formula_gate06 import audit_p_formula_records06
|
| 34 |
+
from math_grid_drawer.research.skeleton_adapter06 import SkeletonTrajectoryAdapter06
|
| 35 |
+
from scripts.audit_math_ink_06_case_context import _load_model06
|
| 36 |
+
from scripts.train_math_ink_06_formula_adapter import _forward06, _targets06
|
| 37 |
+
from scripts.train_math_ink_06_p_formula_adapter import _file_sha25606
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
REQUIRED_SEEDS06 = frozenset({17, 31, 47})
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def validate_p_formula_distillation_inputs06(
|
| 44 |
+
summary: dict[str, Any],
|
| 45 |
+
reports: Sequence[dict[str, Any]],
|
| 46 |
+
*,
|
| 47 |
+
data_sha256: str,
|
| 48 |
+
) -> list[dict[str, Any]]:
|
| 49 |
+
"""ํ์ ๋ณ์: 3-seed summaryยทteacher reportยทํ์ฌ data hash. ์๋ ์๋ฆฌ: ๋์ผ P corpus์ ์ seed ํต๊ณผ๋ฅผ AND๋ก ๊ฒ์ฆํ๋ค."""
|
| 50 |
+
|
| 51 |
+
if summary.get("track") != "P_approved_formula_only":
|
| 52 |
+
raise ValueError("P Formula distillation์๋ P-track summary๋ง ํ์ฉํฉ๋๋ค.")
|
| 53 |
+
decision = summary.get("decision") or {}
|
| 54 |
+
if decision.get("student_distillation_allowed") is not True:
|
| 55 |
+
raise ValueError("3-seed summary๊ฐ student distillation์ ํ์ฉํ์ง ์์์ต๋๋ค.")
|
| 56 |
+
if str(summary.get("data_sha256") or "") != data_sha256:
|
| 57 |
+
raise ValueError("ํ์ฌ P Formula data SHA-256์ด 3-seed summary์ ๋ค๋ฆ
๋๋ค.")
|
| 58 |
+
if len(reports) != 3 or {int(report["seed"]) for report in reports} != REQUIRED_SEEDS06:
|
| 59 |
+
raise ValueError("Teacher report๋ seed 17ยท31ยท47์ด ์ ํํ ํ๋์ฉ ํ์ํฉ๋๋ค.")
|
| 60 |
+
ordered = sorted(reports, key=lambda report: int(report["seed"]))
|
| 61 |
+
for report in ordered:
|
| 62 |
+
if report.get("track") != "P_approved_formula_only":
|
| 63 |
+
raise ValueError("R-track teacher๋ฅผ P student์ ์ฆ๋ฅํ ์ ์์ต๋๋ค.")
|
| 64 |
+
if str(report.get("data_sha256") or "") != data_sha256:
|
| 65 |
+
raise ValueError("Teacher report์ P Formula data SHA-256์ด ๋ค๋ฆ
๋๋ค.")
|
| 66 |
+
if report.get("seed_gate", {}).get("passed") is not True:
|
| 67 |
+
raise ValueError(f"seed {report['seed']} teacher๊ฐ ๊ฐ๋ณ release gate๋ฅผ ํต๊ณผํ์ง ์์์ต๋๋ค.")
|
| 68 |
+
return ordered
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def ensemble_teacher_probability06(
|
| 72 |
+
logits: Sequence[Tensor],
|
| 73 |
+
*,
|
| 74 |
+
temperature: float,
|
| 75 |
+
) -> Tensor:
|
| 76 |
+
"""ํ์ ๋ณ์: seed๋ณ ๋์ผ shape logitsยทtemperature. ์๋ ์๋ฆฌ: logit ํ๊ท ๋์ ํ๋ฅ ํ๊ท ์ผ๋ก teacher target์ ๋ง๋ ๋ค."""
|
| 77 |
+
|
| 78 |
+
if not logits or temperature <= 0.0:
|
| 79 |
+
raise ValueError("Teacher logit๊ณผ ์์ temperature๊ฐ ํ์ํฉ๋๋ค.")
|
| 80 |
+
shape = logits[0].shape
|
| 81 |
+
if any(value.shape != shape for value in logits):
|
| 82 |
+
raise ValueError("Teacher logit shape๊ฐ ์๋ก ๋ค๋ฆ
๋๋ค.")
|
| 83 |
+
return torch.stack([
|
| 84 |
+
(value / temperature).softmax(dim=1)
|
| 85 |
+
for value in logits
|
| 86 |
+
]).mean(dim=0)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _parse_args() -> argparse.Namespace:
|
| 90 |
+
"""ํ์ ๋ณ์: P corpusยทteacher reports/summaryยทstudent main adapter. ์๋ ์๋ฆฌ: fail-closed distillation CLI๋ฅผ ๋ง๋ ๋ค."""
|
| 91 |
+
|
| 92 |
+
parser = argparse.ArgumentParser(description="Distill Math Ink 0.6 P formula student")
|
| 93 |
+
parser.add_argument("--data", type=Path, required=True)
|
| 94 |
+
parser.add_argument("--teacher-report", type=Path, action="append", required=True)
|
| 95 |
+
parser.add_argument("--summary", type=Path, required=True)
|
| 96 |
+
parser.add_argument("--student-adapter", type=Path, required=True)
|
| 97 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 98 |
+
parser.add_argument("--seed", type=int, default=17)
|
| 99 |
+
parser.add_argument("--epochs", type=int, default=20)
|
| 100 |
+
parser.add_argument("--batch-size", type=int, default=128)
|
| 101 |
+
parser.add_argument("--learning-rate", type=float, default=4e-4)
|
| 102 |
+
parser.add_argument("--weight-decay", type=float, default=2e-3)
|
| 103 |
+
parser.add_argument("--hidden-size", type=int, default=64)
|
| 104 |
+
parser.add_argument("--temperature", type=float, default=2.0)
|
| 105 |
+
parser.add_argument("--teacher-exact-weight", type=float, default=0.70)
|
| 106 |
+
parser.add_argument("--teacher-family-weight", type=float, default=1.00)
|
| 107 |
+
parser.add_argument("--hard-exact-weight", type=float, default=0.20)
|
| 108 |
+
parser.add_argument("--hard-family-weight", type=float, default=0.50)
|
| 109 |
+
parser.add_argument("--patience", type=int, default=5)
|
| 110 |
+
parser.add_argument("--minimum-independent-sources", type=int, default=2)
|
| 111 |
+
parser.add_argument("--distillation-regression-maximum-pp", type=float, default=1.0)
|
| 112 |
+
parser.add_argument("--device", choices=("cuda", "cpu"), default="cuda")
|
| 113 |
+
return parser.parse_args()
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def _seed06(seed: int) -> None:
|
| 117 |
+
"""ํ์ ๋ณ์: student seed. ์๋ ์๋ฆฌ: PythonยทNumPyยทPyTorch ์ด๊ธฐํ๋ฅผ ๊ณ ์ ํ๋ค."""
|
| 118 |
+
|
| 119 |
+
random.seed(seed)
|
| 120 |
+
np.random.seed(seed)
|
| 121 |
+
torch.manual_seed(seed)
|
| 122 |
+
if torch.cuda.is_available():
|
| 123 |
+
torch.cuda.manual_seed_all(seed)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _resolve_project_path06(value: str | Path, *, parent: Path | None = None) -> Path:
|
| 127 |
+
"""ํ์ ๋ณ์: checkpoint lineage ๊ฒฝ๋กยท์ ํ report parent. ์๋ ์๋ฆฌ: ์ ๋/์๋ ๊ฒฝ๋ก๋ฅผ ์กด์ฌํ๋ ์ค์ ํ์ผ๋ก ํด์ํ๋ค."""
|
| 128 |
+
|
| 129 |
+
path = Path(value)
|
| 130 |
+
candidates = [path] if path.is_absolute() else [
|
| 131 |
+
*((parent / path,) if parent is not None else ()),
|
| 132 |
+
PROJECT_ROOT / path,
|
| 133 |
+
]
|
| 134 |
+
for candidate in candidates:
|
| 135 |
+
if candidate.is_file():
|
| 136 |
+
return candidate
|
| 137 |
+
raise FileNotFoundError(f"checkpoint ๊ฒฝ๋ก๋ฅผ ์ฐพ์ ์ ์์ต๋๋ค: {value}")
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _load_teacher06(
|
| 141 |
+
report: dict[str, Any],
|
| 142 |
+
report_path: Path,
|
| 143 |
+
*,
|
| 144 |
+
device: torch.device,
|
| 145 |
+
) -> tuple[Any, torch.nn.Module, SkeletonTrajectoryAdapter06, tuple[str, ...]]:
|
| 146 |
+
"""ํ์ ๋ณ์: ํต๊ณผ teacher report/path. ์๋ ์๋ฆฌ: baseโshared onlineโP formula adapter ์์๋ก ํฉ์ฑํ๋ค."""
|
| 147 |
+
|
| 148 |
+
formula_checkpoint = _resolve_project_path06(
|
| 149 |
+
str(report["checkpoint"]),
|
| 150 |
+
parent=report_path.parent,
|
| 151 |
+
)
|
| 152 |
+
payload = torch.load(formula_checkpoint, map_location="cpu", weights_only=False)
|
| 153 |
+
if payload.get("track") != "P_approved_formula_only":
|
| 154 |
+
raise ValueError("P Formula teacher checkpoint track์ด ์ฌ๋ฐ๋ฅด์ง ์์ต๋๋ค.")
|
| 155 |
+
if payload.get("seed_gate_passed") is not True:
|
| 156 |
+
raise ValueError("๊ฐ๋ณ gate๋ฅผ ํต๊ณผํ์ง ์์ teacher checkpoint์
๋๋ค.")
|
| 157 |
+
if str(payload.get("data_sha256") or "") != str(report["data_sha256"]):
|
| 158 |
+
raise ValueError("Teacher checkpoint/report data SHA-256์ด ๋ค๋ฆ
๋๋ค.")
|
| 159 |
+
online_path = _resolve_project_path06(str(payload["online_adapter"]))
|
| 160 |
+
base_path = _resolve_project_path06(str(payload["base_checkpoint"]))
|
| 161 |
+
engine, online_adapter = _load_model06(base_path, online_path, device)
|
| 162 |
+
formula_adapter = SkeletonTrajectoryAdapter06(
|
| 163 |
+
hidden_size=int(payload["hidden_size"]),
|
| 164 |
+
).to(device)
|
| 165 |
+
formula_adapter.load_state_dict(payload["state_dict"])
|
| 166 |
+
engine.model.eval()
|
| 167 |
+
online_adapter.eval()
|
| 168 |
+
formula_adapter.eval()
|
| 169 |
+
return engine, online_adapter, formula_adapter, tuple(str(label) for label in engine.labels)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _teacher_targets06(
|
| 173 |
+
teachers: Sequence[tuple[Any, torch.nn.Module, SkeletonTrajectoryAdapter06]],
|
| 174 |
+
features: Tensor,
|
| 175 |
+
*,
|
| 176 |
+
temperature: float,
|
| 177 |
+
device: torch.device,
|
| 178 |
+
batch_size: int,
|
| 179 |
+
) -> tuple[Tensor, Tensor]:
|
| 180 |
+
"""ํ์ ๋ณ์: ์ธ teacherยทํ split feature. ์๋ ์๋ฆฌ: seed๋ณ exact/family probability๋ฅผ CPU์์ ํ๊ท ํ๋ค."""
|
| 181 |
+
|
| 182 |
+
exact_rows, family_rows = [], []
|
| 183 |
+
for engine, online_adapter, formula_adapter in teachers:
|
| 184 |
+
exact, family = _forward06(
|
| 185 |
+
engine.model,
|
| 186 |
+
online_adapter,
|
| 187 |
+
formula_adapter,
|
| 188 |
+
features,
|
| 189 |
+
device=device,
|
| 190 |
+
batch_size=batch_size,
|
| 191 |
+
)
|
| 192 |
+
exact_rows.append(exact)
|
| 193 |
+
family_rows.append(family)
|
| 194 |
+
return (
|
| 195 |
+
ensemble_teacher_probability06(exact_rows, temperature=temperature),
|
| 196 |
+
ensemble_teacher_probability06(family_rows, temperature=temperature),
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _balanced_loader06(
|
| 201 |
+
batch: PFormulaTensorBatch06,
|
| 202 |
+
tensors: Sequence[Tensor],
|
| 203 |
+
*,
|
| 204 |
+
batch_size: int,
|
| 205 |
+
seed: int,
|
| 206 |
+
) -> DataLoader:
|
| 207 |
+
"""ํ์ ๋ณ์: P batchยทํ์ต tensor. ์๋ ์๋ฆฌ: sourceรlabel ์ญ์ ๊ณฑ๊ทผ sampler๋ก distillation batch๋ฅผ ๋ง๋ ๋ค."""
|
| 208 |
+
|
| 209 |
+
exact_targets = tensors[0]
|
| 210 |
+
label_counts = Counter(int(value) for value in exact_targets.tolist())
|
| 211 |
+
source_counts = Counter(batch.source_ids)
|
| 212 |
+
weights = torch.tensor([
|
| 213 |
+
1.0 / (
|
| 214 |
+
max(label_counts[int(label)], 1) ** 0.5
|
| 215 |
+
* max(source_counts[source], 1) ** 0.5
|
| 216 |
+
)
|
| 217 |
+
for label, source in zip(
|
| 218 |
+
exact_targets.tolist(),
|
| 219 |
+
batch.source_ids,
|
| 220 |
+
strict=True,
|
| 221 |
+
)
|
| 222 |
+
])
|
| 223 |
+
sampler = WeightedRandomSampler(
|
| 224 |
+
weights,
|
| 225 |
+
num_samples=len(weights),
|
| 226 |
+
replacement=True,
|
| 227 |
+
generator=torch.Generator().manual_seed(seed),
|
| 228 |
+
)
|
| 229 |
+
return DataLoader(
|
| 230 |
+
TensorDataset(batch.features, *tensors),
|
| 231 |
+
batch_size=batch_size,
|
| 232 |
+
sampler=sampler,
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def _release_metrics06(
|
| 237 |
+
logits: Tensor,
|
| 238 |
+
targets: Tensor,
|
| 239 |
+
batch: PFormulaTensorBatch06,
|
| 240 |
+
labels: Sequence[str],
|
| 241 |
+
) -> dict[str, Any]:
|
| 242 |
+
"""ํ์ ๋ณ์: student/teacher exact logitยทsplit metadata. ์๋ ์๋ฆฌ: ๊ณตํต P release metric์ ํธ์ถํ๋ค."""
|
| 243 |
+
|
| 244 |
+
return p_formula_release_metrics06(
|
| 245 |
+
logits,
|
| 246 |
+
targets,
|
| 247 |
+
labels=labels,
|
| 248 |
+
writer_ids=batch.writer_ids,
|
| 249 |
+
source_ids=batch.source_ids,
|
| 250 |
+
timestamp_missing=batch.timestamp_missing,
|
| 251 |
+
pressure_missing=batch.pressure_missing,
|
| 252 |
+
)
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def main() -> None:
|
| 256 |
+
"""ํ์ ๋ณ์: ํต๊ณผํ ์ธ teacher์ ๋์ผ P corpus. ์๋ ์๋ฆฌ: ํ๋์ formula adapter student๋ฅผ ํ์ตํ๊ณ test regression์ ํ์ ํ๋ค."""
|
| 257 |
+
|
| 258 |
+
args = _parse_args()
|
| 259 |
+
device = torch.device(args.device)
|
| 260 |
+
if device.type == "cuda" and not torch.cuda.is_available():
|
| 261 |
+
raise RuntimeError("CUDA distillation์ ์์ฒญํ์ง๋ง ์ฌ์ฉํ ์ ์์ต๋๋ค.")
|
| 262 |
+
_seed06(args.seed)
|
| 263 |
+
data_sha256 = _file_sha25606(args.data)
|
| 264 |
+
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
| 265 |
+
reports = [
|
| 266 |
+
json.loads(path.read_text(encoding="utf-8"))
|
| 267 |
+
for path in args.teacher_report
|
| 268 |
+
]
|
| 269 |
+
ordered_reports = validate_p_formula_distillation_inputs06(
|
| 270 |
+
summary,
|
| 271 |
+
reports,
|
| 272 |
+
data_sha256=data_sha256,
|
| 273 |
+
)
|
| 274 |
+
report_paths = {
|
| 275 |
+
int(json.loads(path.read_text(encoding="utf-8"))["seed"]): path
|
| 276 |
+
for path in args.teacher_report
|
| 277 |
+
}
|
| 278 |
+
loaded = [
|
| 279 |
+
_load_teacher06(
|
| 280 |
+
report,
|
| 281 |
+
report_paths[int(report["seed"])],
|
| 282 |
+
device=device,
|
| 283 |
+
)
|
| 284 |
+
for report in ordered_reports
|
| 285 |
+
]
|
| 286 |
+
label_contracts = {labels for *_modules, labels in loaded}
|
| 287 |
+
if len(label_contracts) != 1:
|
| 288 |
+
raise ValueError("์ธ teacher์ exact vocabulary๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 289 |
+
labels = next(iter(label_contracts))
|
| 290 |
+
teachers = [(engine, online, formula) for engine, online, formula, _labels in loaded]
|
| 291 |
+
|
| 292 |
+
records = list(read_jsonl(args.data))
|
| 293 |
+
audit = audit_p_formula_records06(
|
| 294 |
+
records,
|
| 295 |
+
minimum_independent_sources=args.minimum_independent_sources,
|
| 296 |
+
)
|
| 297 |
+
if not audit["eligible_for_product_evaluation"]:
|
| 298 |
+
raise ValueError("ํ์ฌ P Formula corpus๊ฐ product preflight๋ฅผ ํต๊ณผํ์ง ๋ชปํ์ต๋๋ค.")
|
| 299 |
+
split_records = {
|
| 300 |
+
split: [record for record in records if str(record["split"]) == split]
|
| 301 |
+
for split in ("training", "validation", "test")
|
| 302 |
+
}
|
| 303 |
+
batches = {
|
| 304 |
+
split: materialize_p_formula_split06(values, allowed_labels=labels)
|
| 305 |
+
for split, values in split_records.items()
|
| 306 |
+
}
|
| 307 |
+
targets = {
|
| 308 |
+
split: _targets06(batch.truths, labels, loaded[0][0].family_labels)
|
| 309 |
+
for split, batch in batches.items()
|
| 310 |
+
}
|
| 311 |
+
teacher_targets = {
|
| 312 |
+
split: _teacher_targets06(
|
| 313 |
+
teachers,
|
| 314 |
+
batch.features,
|
| 315 |
+
temperature=args.temperature,
|
| 316 |
+
device=device,
|
| 317 |
+
batch_size=args.batch_size,
|
| 318 |
+
)
|
| 319 |
+
for split, batch in batches.items()
|
| 320 |
+
}
|
| 321 |
+
|
| 322 |
+
student_adapter_path = _resolve_project_path06(args.student_adapter)
|
| 323 |
+
student_payload = torch.load(
|
| 324 |
+
student_adapter_path,
|
| 325 |
+
map_location="cpu",
|
| 326 |
+
weights_only=False,
|
| 327 |
+
)
|
| 328 |
+
student_base = _resolve_project_path06(str(student_payload["base_checkpoint"]))
|
| 329 |
+
student_engine, student_online = _load_model06(
|
| 330 |
+
student_base,
|
| 331 |
+
student_adapter_path,
|
| 332 |
+
device,
|
| 333 |
+
)
|
| 334 |
+
if tuple(str(label) for label in student_engine.labels) != labels:
|
| 335 |
+
raise ValueError("Student main vocabulary๊ฐ teacher์ ๋ค๋ฆ
๋๋ค.")
|
| 336 |
+
for parameter in student_engine.model.parameters():
|
| 337 |
+
parameter.requires_grad_(False)
|
| 338 |
+
for parameter in student_online.parameters():
|
| 339 |
+
parameter.requires_grad_(False)
|
| 340 |
+
student_formula = SkeletonTrajectoryAdapter06(
|
| 341 |
+
hidden_size=args.hidden_size,
|
| 342 |
+
).to(device)
|
| 343 |
+
train_exact, train_family = targets["training"]
|
| 344 |
+
train_teacher_exact, train_teacher_family = teacher_targets["training"]
|
| 345 |
+
loader = _balanced_loader06(
|
| 346 |
+
batches["training"],
|
| 347 |
+
(
|
| 348 |
+
train_exact,
|
| 349 |
+
train_family,
|
| 350 |
+
train_teacher_exact,
|
| 351 |
+
train_teacher_family,
|
| 352 |
+
),
|
| 353 |
+
batch_size=args.batch_size,
|
| 354 |
+
seed=args.seed,
|
| 355 |
+
)
|
| 356 |
+
optimizer = torch.optim.AdamW(
|
| 357 |
+
student_formula.parameters(),
|
| 358 |
+
lr=args.learning_rate,
|
| 359 |
+
weight_decay=args.weight_decay,
|
| 360 |
+
)
|
| 361 |
+
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
|
| 362 |
+
optimizer,
|
| 363 |
+
T_max=max(args.epochs, 1),
|
| 364 |
+
eta_min=args.learning_rate * 0.1,
|
| 365 |
+
)
|
| 366 |
+
best_key = (-1.0, -1.0)
|
| 367 |
+
best_state: dict[str, Tensor] | None = None
|
| 368 |
+
best_epoch = 0
|
| 369 |
+
stale = 0
|
| 370 |
+
history = []
|
| 371 |
+
for epoch in range(1, args.epochs + 1):
|
| 372 |
+
student_formula.train()
|
| 373 |
+
losses = []
|
| 374 |
+
for (
|
| 375 |
+
features,
|
| 376 |
+
exact_target,
|
| 377 |
+
family_target,
|
| 378 |
+
teacher_exact,
|
| 379 |
+
teacher_family,
|
| 380 |
+
) in loader:
|
| 381 |
+
features = features.to(device)
|
| 382 |
+
exact_target = exact_target.to(device)
|
| 383 |
+
family_target = family_target.to(device)
|
| 384 |
+
teacher_exact = teacher_exact.to(device)
|
| 385 |
+
teacher_family = teacher_family.to(device)
|
| 386 |
+
optimizer.zero_grad(set_to_none=True)
|
| 387 |
+
with torch.no_grad():
|
| 388 |
+
online = student_online(features)
|
| 389 |
+
exact_logits, family_logits = student_engine.model.classify_trajectory(
|
| 390 |
+
student_formula(online),
|
| 391 |
+
)
|
| 392 |
+
temperature = args.temperature
|
| 393 |
+
exact_distill = torch.nn.functional.kl_div(
|
| 394 |
+
(exact_logits / temperature).log_softmax(dim=1),
|
| 395 |
+
teacher_exact,
|
| 396 |
+
reduction="batchmean",
|
| 397 |
+
) * temperature ** 2
|
| 398 |
+
family_distill = torch.nn.functional.kl_div(
|
| 399 |
+
(family_logits / temperature).log_softmax(dim=1),
|
| 400 |
+
teacher_family,
|
| 401 |
+
reduction="batchmean",
|
| 402 |
+
) * temperature ** 2
|
| 403 |
+
loss = (
|
| 404 |
+
args.teacher_exact_weight * exact_distill
|
| 405 |
+
+ args.teacher_family_weight * family_distill
|
| 406 |
+
+ args.hard_exact_weight
|
| 407 |
+
* torch.nn.functional.cross_entropy(exact_logits, exact_target)
|
| 408 |
+
+ args.hard_family_weight
|
| 409 |
+
* torch.nn.functional.cross_entropy(family_logits, family_target)
|
| 410 |
+
)
|
| 411 |
+
loss.backward()
|
| 412 |
+
torch.nn.utils.clip_grad_norm_(student_formula.parameters(), 2.0)
|
| 413 |
+
optimizer.step()
|
| 414 |
+
losses.append(float(loss.detach()))
|
| 415 |
+
scheduler.step()
|
| 416 |
+
validation_logits = _forward06(
|
| 417 |
+
student_engine.model,
|
| 418 |
+
student_online,
|
| 419 |
+
student_formula,
|
| 420 |
+
batches["validation"].features,
|
| 421 |
+
device=device,
|
| 422 |
+
batch_size=args.batch_size,
|
| 423 |
+
)
|
| 424 |
+
validation_metrics = _release_metrics06(
|
| 425 |
+
validation_logits[0],
|
| 426 |
+
targets["validation"][0],
|
| 427 |
+
batches["validation"],
|
| 428 |
+
labels,
|
| 429 |
+
)
|
| 430 |
+
row = {
|
| 431 |
+
"epoch": epoch,
|
| 432 |
+
"loss": sum(losses) / max(len(losses), 1),
|
| 433 |
+
"validation": validation_metrics,
|
| 434 |
+
}
|
| 435 |
+
history.append(row)
|
| 436 |
+
print(json.dumps(row, ensure_ascii=False), flush=True)
|
| 437 |
+
key = (
|
| 438 |
+
float(validation_metrics["visual_family_top1"]),
|
| 439 |
+
float(validation_metrics["exact_top1"]),
|
| 440 |
+
)
|
| 441 |
+
if key > best_key:
|
| 442 |
+
best_key = key
|
| 443 |
+
best_epoch = epoch
|
| 444 |
+
best_state = deepcopy({
|
| 445 |
+
name: value.detach().cpu()
|
| 446 |
+
for name, value in student_formula.state_dict().items()
|
| 447 |
+
})
|
| 448 |
+
stale = 0
|
| 449 |
+
else:
|
| 450 |
+
stale += 1
|
| 451 |
+
if stale >= args.patience:
|
| 452 |
+
break
|
| 453 |
+
if best_state is None:
|
| 454 |
+
raise RuntimeError("Distilled student checkpoint๊ฐ ์ ํ๋์ง ์์์ต๋๋ค.")
|
| 455 |
+
student_formula.load_state_dict(best_state)
|
| 456 |
+
student_test_logits = _forward06(
|
| 457 |
+
student_engine.model,
|
| 458 |
+
student_online,
|
| 459 |
+
student_formula,
|
| 460 |
+
batches["test"].features,
|
| 461 |
+
device=device,
|
| 462 |
+
batch_size=args.batch_size,
|
| 463 |
+
)[0]
|
| 464 |
+
student_test = _release_metrics06(
|
| 465 |
+
student_test_logits,
|
| 466 |
+
targets["test"][0],
|
| 467 |
+
batches["test"],
|
| 468 |
+
labels,
|
| 469 |
+
)
|
| 470 |
+
teacher_exact_probability = teacher_targets["test"][0]
|
| 471 |
+
teacher_test = _release_metrics06(
|
| 472 |
+
teacher_exact_probability.clamp_min(1e-9).log(),
|
| 473 |
+
targets["test"][0],
|
| 474 |
+
batches["test"],
|
| 475 |
+
labels,
|
| 476 |
+
)
|
| 477 |
+
seed_gate = p_formula_seed_gate06(student_test)
|
| 478 |
+
regression = {
|
| 479 |
+
metric: (
|
| 480 |
+
float(teacher_test[metric]) - float(student_test[metric])
|
| 481 |
+
) * 100.0
|
| 482 |
+
for metric in ("exact_top1", "exact_top5", "visual_family_top1")
|
| 483 |
+
}
|
| 484 |
+
regression_passed = all(
|
| 485 |
+
drop <= args.distillation_regression_maximum_pp
|
| 486 |
+
for drop in regression.values()
|
| 487 |
+
)
|
| 488 |
+
distillation_gate_passed = bool(seed_gate["passed"] and regression_passed)
|
| 489 |
+
args.output.mkdir(parents=True, exist_ok=True)
|
| 490 |
+
checkpoint = args.output / "p_formula_student_adapter.pt"
|
| 491 |
+
torch.save({
|
| 492 |
+
"schema": "aiflow-math-ink-06-p-formula-student-v1",
|
| 493 |
+
"state_dict": best_state,
|
| 494 |
+
"hidden_size": args.hidden_size,
|
| 495 |
+
"student_base_checkpoint": str(student_base),
|
| 496 |
+
"student_online_adapter": str(student_adapter_path),
|
| 497 |
+
"teacher_seeds": [17, 31, 47],
|
| 498 |
+
"data_sha256": data_sha256,
|
| 499 |
+
"selected_epoch": best_epoch,
|
| 500 |
+
"distillation_gate_passed": distillation_gate_passed,
|
| 501 |
+
"track": "P_approved_formula_only",
|
| 502 |
+
"teacher_weights_embedded": False,
|
| 503 |
+
"litert_exported": False,
|
| 504 |
+
"product_validation": False,
|
| 505 |
+
}, checkpoint)
|
| 506 |
+
report = {
|
| 507 |
+
"experiment": "P-MATH-INK-06-FORMULA-STUDENT-DISTILL-001",
|
| 508 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 509 |
+
"student_seed": args.seed,
|
| 510 |
+
"teacher_seeds": [17, 31, 47],
|
| 511 |
+
"data": str(args.data),
|
| 512 |
+
"data_sha256": data_sha256,
|
| 513 |
+
"preflight": audit,
|
| 514 |
+
"temperature": args.temperature,
|
| 515 |
+
"loss_weights": {
|
| 516 |
+
"teacher_exact": args.teacher_exact_weight,
|
| 517 |
+
"teacher_family": args.teacher_family_weight,
|
| 518 |
+
"hard_exact": args.hard_exact_weight,
|
| 519 |
+
"hard_family": args.hard_family_weight,
|
| 520 |
+
},
|
| 521 |
+
"selected_epoch": best_epoch,
|
| 522 |
+
"teacher_test": teacher_test,
|
| 523 |
+
"student_test": student_test,
|
| 524 |
+
"student_seed_gate": seed_gate,
|
| 525 |
+
"teacher_to_student_drop_pp": regression,
|
| 526 |
+
"distillation_regression_maximum_pp": args.distillation_regression_maximum_pp,
|
| 527 |
+
"distillation_regression_passed": regression_passed,
|
| 528 |
+
"distillation_gate_passed": distillation_gate_passed,
|
| 529 |
+
"history": history,
|
| 530 |
+
"checkpoint": checkpoint.name,
|
| 531 |
+
"checkpoint_bytes": checkpoint.stat().st_size,
|
| 532 |
+
"teacher_weights_embedded": False,
|
| 533 |
+
"track": "P_approved_formula_only",
|
| 534 |
+
"litert_exported": False,
|
| 535 |
+
"product_validation": False,
|
| 536 |
+
"next_gate": "composite torch.exportโLiteRT parityโAndroid 3-tier benchmark",
|
| 537 |
+
}
|
| 538 |
+
(args.output / "report.json").write_text(
|
| 539 |
+
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
| 540 |
+
encoding="utf-8",
|
| 541 |
+
)
|
| 542 |
+
print(json.dumps({
|
| 543 |
+
"student_test": student_test,
|
| 544 |
+
"teacher_test": teacher_test,
|
| 545 |
+
"distillation_gate_passed": distillation_gate_passed,
|
| 546 |
+
"checkpoint": str(checkpoint),
|
| 547 |
+
"product_validation": False,
|
| 548 |
+
}, ensure_ascii=False, indent=2))
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
if __name__ == "__main__":
|
| 552 |
+
main()
|
scripts/summarize_math_ink_06_p_formula_seeds.py
CHANGED
|
@@ -23,6 +23,9 @@ def summarize_p_formula_seeds06(reports: list[dict[str, Any]]) -> dict[str, Any]
|
|
| 23 |
raise ValueError("R-track ๋๋ ์ ์ ์๋ formula adapter report๋ฅผ distillation ์์ฝ์ ์์ ์ ์์ต๋๋ค.")
|
| 24 |
if any(report.get("official_test") is None for report in reports):
|
| 25 |
raise ValueError("official test๋ฅผ ์๋ตํ seed๋ 3-seed gate์ ์ฌ์ฉํ ์ ์์ต๋๋ค.")
|
|
|
|
|
|
|
|
|
|
| 26 |
ordered = sorted(reports, key=lambda report: int(report["seed"]))
|
| 27 |
|
| 28 |
def aggregate(path: tuple[str, ...]) -> dict[str, Any]:
|
|
@@ -62,6 +65,7 @@ def summarize_p_formula_seeds06(reports: list[dict[str, Any]]) -> dict[str, Any]
|
|
| 62 |
"experiment": "P-MATH-INK-06-FORMULA-ADAPTER-3SEED-001",
|
| 63 |
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 64 |
"seeds": [17, 31, 47],
|
|
|
|
| 65 |
"metrics": metrics,
|
| 66 |
"individual_seed_gates": individual,
|
| 67 |
"decision": {
|
|
|
|
| 23 |
raise ValueError("R-track ๋๋ ์ ์ ์๋ formula adapter report๋ฅผ distillation ์์ฝ์ ์์ ์ ์์ต๋๋ค.")
|
| 24 |
if any(report.get("official_test") is None for report in reports):
|
| 25 |
raise ValueError("official test๋ฅผ ์๋ตํ seed๋ 3-seed gate์ ์ฌ์ฉํ ์ ์์ต๋๋ค.")
|
| 26 |
+
fingerprints = {str(report.get("data_sha256") or "") for report in reports}
|
| 27 |
+
if len(fingerprints) != 1 or "" in fingerprints:
|
| 28 |
+
raise ValueError("์ธ seed๋ ๋์ผํ ๋น์ด ์์ง ์์ P Formula data_sha256์ ๊ฐ์ ธ์ผ ํฉ๋๋ค.")
|
| 29 |
ordered = sorted(reports, key=lambda report: int(report["seed"]))
|
| 30 |
|
| 31 |
def aggregate(path: tuple[str, ...]) -> dict[str, Any]:
|
|
|
|
| 65 |
"experiment": "P-MATH-INK-06-FORMULA-ADAPTER-3SEED-001",
|
| 66 |
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 67 |
"seeds": [17, 31, 47],
|
| 68 |
+
"data_sha256": next(iter(fingerprints)),
|
| 69 |
"metrics": metrics,
|
| 70 |
"individual_seed_gates": individual,
|
| 71 |
"decision": {
|
scripts/train_math_ink_06_p_formula_adapter.py
CHANGED
|
@@ -6,6 +6,7 @@ import argparse
|
|
| 6 |
from collections import Counter
|
| 7 |
from copy import deepcopy
|
| 8 |
from datetime import datetime, timezone
|
|
|
|
| 9 |
import json
|
| 10 |
from pathlib import Path
|
| 11 |
import random
|
|
@@ -77,6 +78,16 @@ def _seed06(seed: int) -> None:
|
|
| 77 |
torch.cuda.manual_seed_all(seed)
|
| 78 |
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
def _source_label_loader06(
|
| 81 |
batch: PFormulaTensorBatch06,
|
| 82 |
exact_targets: Tensor,
|
|
@@ -146,6 +157,7 @@ def main() -> None:
|
|
| 146 |
_seed06(args.seed)
|
| 147 |
|
| 148 |
records = list(read_jsonl(args.data))
|
|
|
|
| 149 |
audit = audit_p_formula_records06(
|
| 150 |
records,
|
| 151 |
minimum_independent_sources=args.minimum_independent_sources,
|
|
@@ -360,6 +372,7 @@ def main() -> None:
|
|
| 360 |
"seed_gate_passed": bool(seed_gate and seed_gate["passed"]),
|
| 361 |
"product_validation": False,
|
| 362 |
"distillation_allowed": False,
|
|
|
|
| 363 |
}, checkpoint)
|
| 364 |
report = {
|
| 365 |
"experiment": "P-MATH-INK-06-FORMULA-ADAPTER-001",
|
|
@@ -370,6 +383,7 @@ def main() -> None:
|
|
| 370 |
torch.cuda.get_device_name(device) if device.type == "cuda" else None
|
| 371 |
),
|
| 372 |
"data": str(args.data),
|
|
|
|
| 373 |
"preflight": audit,
|
| 374 |
"samples": {
|
| 375 |
"training": len(train_batch.truths),
|
|
|
|
| 6 |
from collections import Counter
|
| 7 |
from copy import deepcopy
|
| 8 |
from datetime import datetime, timezone
|
| 9 |
+
from hashlib import sha256
|
| 10 |
import json
|
| 11 |
from pathlib import Path
|
| 12 |
import random
|
|
|
|
| 78 |
torch.cuda.manual_seed_all(seed)
|
| 79 |
|
| 80 |
|
| 81 |
+
def _file_sha25606(path: Path) -> str:
|
| 82 |
+
"""ํ์ ๋ณ์: P Formula JSONL. ์๋ ์๋ฆฌ: seed ๊ฐ ๋์ผ corpus๋ฅผ ์ฆ๋ช
ํ byte-level SHA-256์ ๊ณ์ฐํ๋ค."""
|
| 83 |
+
|
| 84 |
+
digest = sha256()
|
| 85 |
+
with path.open("rb") as file:
|
| 86 |
+
for chunk in iter(lambda: file.read(1024 * 1024), b""):
|
| 87 |
+
digest.update(chunk)
|
| 88 |
+
return digest.hexdigest()
|
| 89 |
+
|
| 90 |
+
|
| 91 |
def _source_label_loader06(
|
| 92 |
batch: PFormulaTensorBatch06,
|
| 93 |
exact_targets: Tensor,
|
|
|
|
| 157 |
_seed06(args.seed)
|
| 158 |
|
| 159 |
records = list(read_jsonl(args.data))
|
| 160 |
+
data_sha256 = _file_sha25606(args.data)
|
| 161 |
audit = audit_p_formula_records06(
|
| 162 |
records,
|
| 163 |
minimum_independent_sources=args.minimum_independent_sources,
|
|
|
|
| 372 |
"seed_gate_passed": bool(seed_gate and seed_gate["passed"]),
|
| 373 |
"product_validation": False,
|
| 374 |
"distillation_allowed": False,
|
| 375 |
+
"data_sha256": data_sha256,
|
| 376 |
}, checkpoint)
|
| 377 |
report = {
|
| 378 |
"experiment": "P-MATH-INK-06-FORMULA-ADAPTER-001",
|
|
|
|
| 383 |
torch.cuda.get_device_name(device) if device.type == "cuda" else None
|
| 384 |
),
|
| 385 |
"data": str(args.data),
|
| 386 |
+
"data_sha256": data_sha256,
|
| 387 |
"preflight": audit,
|
| 388 |
"samples": {
|
| 389 |
"training": len(train_batch.truths),
|