Add AIFlow Math Ink 0.6 intermediate research snapshot
Browse files- MANIFEST.json +99 -0
- README.md +64 -0
- artifacts/boundary_behavior_guard.joblib +3 -0
- pyproject.snapshot.toml +25 -0
- reports/RESEARCH_REPORT.md +363 -0
- reports/behavior_role_3seed_summary.json +132 -0
- reports/boundary_behavior_guard_report.json +2735 -0
- reports/component_competition_report.json +843 -0
- reports/local_baseline_guard_report.json +915 -0
- reports/local_baseline_overmerge_audit.json +0 -0
- reports/multistroke_family_guard_report.json +565 -0
- scripts/audit_math_ink_06_local_baseline_overmerge.py +260 -0
- scripts/evaluate_crohme_tray_joint_selector.py +203 -0
- scripts/sweep_math_ink_06_component_competition_guard.py +236 -0
- scripts/sweep_math_ink_06_local_baseline_guard.py +207 -0
- scripts/train_math_ink_06_boundary_behavior_guard.py +283 -0
- src/segmentation_lattice.py +256 -0
- src/structure_relations.py +203 -0
- src/tray_joint.py +393 -0
MANIFEST.json
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema": "aiflow-hf-research-snapshot-v1",
|
| 3 |
+
"generated_at": "2026-07-23T18:48:05.4863509Z",
|
| 4 |
+
"track": "R_noncommercial_only",
|
| 5 |
+
"product_validation": false,
|
| 6 |
+
"contains_raw_dataset": false,
|
| 7 |
+
"files": [
|
| 8 |
+
{
|
| 9 |
+
"path": "artifacts/boundary_behavior_guard.joblib",
|
| 10 |
+
"bytes": 824191,
|
| 11 |
+
"sha256": "18aff472c1bdb25a519b1553364229c9869aed1f36a9d663cf88a1bc808fab8f"
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"path": "pyproject.snapshot.toml",
|
| 15 |
+
"bytes": 669,
|
| 16 |
+
"sha256": "e9571da506b96d9ab6baa8f8e2c1d87cd8ee9fd0c880f3cf6646a99086ae3f09"
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"path": "README.md",
|
| 20 |
+
"bytes": 2800,
|
| 21 |
+
"sha256": "49ed73662471bdd8180f561c3b318aef49b3849ab71e541cbb88062530a27193"
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"path": "reports/behavior_role_3seed_summary.json",
|
| 25 |
+
"bytes": 3567,
|
| 26 |
+
"sha256": "b16049218d2d006d952f59f55642fa9f3fbfbc3cc3a730fbd7b075f507c3b251"
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"path": "reports/boundary_behavior_guard_report.json",
|
| 30 |
+
"bytes": 72053,
|
| 31 |
+
"sha256": "f55a230a9f0892b9676c71db46e9e527862b3d8da776dc43db767ce7a0f6a5c7"
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"path": "reports/component_competition_report.json",
|
| 35 |
+
"bytes": 22284,
|
| 36 |
+
"sha256": "4d3db5f71e5008381dd072f2c10aafb22e08f96e23f2a32f82a3a46abd9c70b9"
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"path": "reports/local_baseline_guard_report.json",
|
| 40 |
+
"bytes": 24052,
|
| 41 |
+
"sha256": "8f209b2890245f01f437c26e7556e19fee76c057d449c9382e2d64a70a3764d6"
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"path": "reports/local_baseline_overmerge_audit.json",
|
| 45 |
+
"bytes": 1237409,
|
| 46 |
+
"sha256": "33a9ccafebd559ca8b173f011e9a0a6f495d37b8fdb2693673663479d3457071"
|
| 47 |
+
},
|
| 48 |
+
{
|
| 49 |
+
"path": "reports/multistroke_family_guard_report.json",
|
| 50 |
+
"bytes": 15740,
|
| 51 |
+
"sha256": "27b31da71b9f39f7ded38aa1ce3694911af140d22854543d4ae2b5f9e709b479"
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"path": "reports/RESEARCH_REPORT.md",
|
| 55 |
+
"bytes": 22993,
|
| 56 |
+
"sha256": "ef83f8145422cdf341ec14ebc5dab3d799e7fd3a0536398c70c0e67640cc1c59"
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"path": "scripts/audit_math_ink_06_local_baseline_overmerge.py",
|
| 60 |
+
"bytes": 11900,
|
| 61 |
+
"sha256": "f32a18c8e7d3a0ea36379293e6be88bfb509b7df99846c38149fb8458e6cc994"
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
"path": "scripts/evaluate_crohme_tray_joint_selector.py",
|
| 65 |
+
"bytes": 11671,
|
| 66 |
+
"sha256": "58c175a95cdeb9fc764fb96594a76888122b8f118f2f26da1283e743b1a7cd04"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"path": "scripts/sweep_math_ink_06_component_competition_guard.py",
|
| 70 |
+
"bytes": 9084,
|
| 71 |
+
"sha256": "e2dee616e2c6ebdc32281624889017685648b7076515242e04a1452d98d1c7da"
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"path": "scripts/sweep_math_ink_06_local_baseline_guard.py",
|
| 75 |
+
"bytes": 9063,
|
| 76 |
+
"sha256": "28cfc7b9b695256cfa9199d912d1d938d402292b88e9411d42fddb3508446d3b"
|
| 77 |
+
},
|
| 78 |
+
{
|
| 79 |
+
"path": "scripts/train_math_ink_06_boundary_behavior_guard.py",
|
| 80 |
+
"bytes": 12461,
|
| 81 |
+
"sha256": "9fd410b402976416c12057a196babf6c98d971c41c23ae1a0e5f183c06a42b6e"
|
| 82 |
+
},
|
| 83 |
+
{
|
| 84 |
+
"path": "src/segmentation_lattice.py",
|
| 85 |
+
"bytes": 13772,
|
| 86 |
+
"sha256": "8cc2dc1e13b56c21d53a76b9b863cdf5c8f8dd315da6eb18c1bd2b5b87c1a616"
|
| 87 |
+
},
|
| 88 |
+
{
|
| 89 |
+
"path": "src/structure_relations.py",
|
| 90 |
+
"bytes": 9671,
|
| 91 |
+
"sha256": "59edbdcbfebaab029612beed132f38b69c84487f5ba930fd169f3278e10323bb"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"path": "src/tray_joint.py",
|
| 95 |
+
"bytes": 19404,
|
| 96 |
+
"sha256": "160c2dd8db3dd4da77bdd21209a898b73fb0cb5a1b24e59d2c8bc54e159ea3e0"
|
| 97 |
+
}
|
| 98 |
+
]
|
| 99 |
+
}
|
README.md
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
language:
|
| 3 |
+
- ko
|
| 4 |
+
- en
|
| 5 |
+
library_name: scikit-learn
|
| 6 |
+
license: other
|
| 7 |
+
tags:
|
| 8 |
+
- handwriting
|
| 9 |
+
- mathematical-expression-recognition
|
| 10 |
+
- online-handwriting
|
| 11 |
+
- segmentation
|
| 12 |
+
- research
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# AIFlow Math Ink 0.6 โ Intermediate Research Snapshot
|
| 16 |
+
|
| 17 |
+
์ด ์ ์ฅ์๋ AIFlow Math Ink 0.6์ **์ค๊ฐ R-track ์ฐ๊ตฌ ์ค๋
์ท**์ด๋ค. ์ ์ฒด LaTeX OCR ๋๋ ์์ฉ ODA ๋ชจ๋ธ์ด ์๋๋ฉฐ, ์์ stroke grouping๊ณผ ํ๋ ๋ฌธ๋งฅ ์คํ์ ์ฌํ ์๋ฃ๋ฅผ ๋ณด์กดํ๋ค.
|
| 18 |
+
|
| 19 |
+
## ์ด๋ฒ ์ค๋
์ท์ ๊ฒฐ๋ก
|
| 20 |
+
|
| 21 |
+
- protected multi-stroke family selector ๊ธฐ์ค CROHME official test exact partition: **60.04%**
|
| 22 |
+
- boundary behavior guard ์ ์ฉ exact partition: **60.25%**
|
| 23 |
+
- pair-F1: **91.07% โ 91.26%**
|
| 24 |
+
- overmerge formula rate: **21.72% โ 20.49%**
|
| 25 |
+
- validation์์ ๊ณ ์ ํ boundary ์ค์ : probability threshold `0.5`, logit penalty weight `6.0`
|
| 26 |
+
- ์ ์ฒด ๋ก์ปฌ Python ํ๊ท: **271 passed**
|
| 27 |
+
|
| 28 |
+
Boundary head๋ grouping ์ ์ ๋คํ ํ๋ณด๊ฐ ๋ ์ด์์ ์ ๋ต ๊ธฐํธ ๊ฒฝ๊ณ๋ฅผ ์นจ๋ฒํ๋์ง ์์ธกํ๋ค. ๊ท์นํ component competition๊ณผ ์ด๋ฏธ grouping๋ partition ์ดํ์ local-baseline relation guard๋ ์ฑํ๋์ง ์์๋ค.
|
| 29 |
+
|
| 30 |
+
## ๋๋ ํฐ๋ฆฌ
|
| 31 |
+
|
| 32 |
+
- `artifacts/`: scikit-learn boundary ์ฐ๊ตฌ head
|
| 33 |
+
- `reports/`: validation/test ์งํ์ ์คํจ ๊ฐ์ฌ
|
| 34 |
+
- `scripts/`: ํ์ตยทsweepยท๊ฐ์ฌ entry point
|
| 35 |
+
- `src/`: ์ด๋ฒ ์คํ์ ์ง์ ๊ด๋ จ๋ selector/๊ด๊ณ ์ฝ๋ snapshot
|
| 36 |
+
|
| 37 |
+
## ์ฌ์ฉ ์ ํ
|
| 38 |
+
|
| 39 |
+
์ด ์ค๋
์ท์ **์ฐ๊ตฌ ์ ์ฉ**์ด๋ค.
|
| 40 |
+
|
| 41 |
+
- `boundary_behavior_guard.joblib`์ CROHME ์ ๋ต grouping์ผ๋ก ํ์ต๋ R-track ์ฐ์ถ๋ฌผ์ด๋ค.
|
| 42 |
+
- ์์ฉ ์ ํ, ๋ฐฐํฌ checkpoint, teacher ๋๋ distillation ์
๋ ฅ์ผ๋ก ์ฌ์ฉํ์ง ์๋๋ค.
|
| 43 |
+
- CROHME/HWRT ์๋ณธ ๋ฐ์ดํฐ, ํ๊ธฐ ํ๋ณธ, OCR cache๋ ์ด ์ ์ฅ์์ ํฌํจํ์ง ์๋๋ค.
|
| 44 |
+
- ์์ฉ ์ ํ์ ๋์ผ boundary target์ ์์ฉ ํ์ฉ P-track ์ฐ์์ ๋ฐ์ดํฐ๋ก ๋ค์ ํ์ตํด์ผ ํ๋ค.
|
| 45 |
+
- ๊ฐ๋ณ ์ธ๋ถ ๋ฐ์ดํฐ์
์ ์ ๋ผ์ด์ ์ค์ ์ฌ์ฉ ์กฐ๊ฑด์ ์ฌ์ฉ์๊ฐ ๋ณ๋๋ก ํ์ธํด์ผ ํ๋ค.
|
| 46 |
+
|
| 47 |
+
## ๋ณด์ ์ฃผ์
|
| 48 |
+
|
| 49 |
+
`joblib`/pickle ๊ณ์ด ํ์ผ์ ์์ ์ฝ๋๋ฅผ ์คํํ ์ ์๋ค. ์ด ์ ์ฅ์์ ํ์ผ checksum์ ์ ๋ขฐํ๋ ํ๊ฒฝ์์๋ง ๋ก๋ํ๋ค.
|
| 50 |
+
|
| 51 |
+
```python
|
| 52 |
+
import joblib
|
| 53 |
+
|
| 54 |
+
payload = joblib.load("artifacts/boundary_behavior_guard.joblib")
|
| 55 |
+
print(payload["schema"])
|
| 56 |
+
print(payload["feature_names"])
|
| 57 |
+
print(payload["threshold"], payload["weight"])
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
## ์ฌํ ๊ฒฝ๊ณ
|
| 61 |
+
|
| 62 |
+
ํฌํจ๋ script๋ ์ ์ฒด AIFlow ํ๋ก์ ํธ์ ์ฌ์ฉ์๊ฐ ๋ณ๋๋ก ํ๋ณดํ ๋ฐ์ดํฐ์
์ด ์์ด์ผ ์์ ์คํ๋๋ค. ๊ณต์ test๋ validation ์ ํ์ ์ฌ์ฉํ์ง ์์์ง๋ง, ๋ฐ๋ณต ์ฐ๊ตฌ ๊ณผ์ ์์ test ์คํจ๋ฅผ ๊ด์ฐฐํ์ผ๋ฏ๋ก ์ฌ๊ธฐ์ ๊ธฐ๋ก๋ ๊ฒฐ๊ณผ๋ ๋
๋ฆฝ ์ ํ ์ธ์ฆ ์์น๊ฐ ์๋๋ค.
|
| 63 |
+
|
| 64 |
+
์์ธํ ์ค๊ณยท์คํจ ๋ถ์ยทํ์ P-track ๊ณํ์ `reports/RESEARCH_REPORT.md`๋ฅผ ์ฐธ๊ณ ํ๋ค.
|
artifacts/boundary_behavior_guard.joblib
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:18aff472c1bdb25a519b1553364229c9869aed1f36a9d663cf88a1bc808fab8f
|
| 3 |
+
size 824191
|
pyproject.snapshot.toml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=68"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "math-grid-drawer-simulation"
|
| 7 |
+
version = "0.1.0"
|
| 8 |
+
description = "ํ๊ธฐ ์์ stroke ๊ทธ๋ฆฌ๋ฉ์ ์คํํ๋ ๋
๋ฆฝ Python ๋๋ก์"
|
| 9 |
+
requires-python = ">=3.11"
|
| 10 |
+
dependencies = ["Pillow>=10,<12"]
|
| 11 |
+
|
| 12 |
+
[project.optional-dependencies]
|
| 13 |
+
recognition = ["numpy>=1.26,<3", "torch>=2.2,<3", "transformers==4.47.0", "safetensors>=0.4,<1"]
|
| 14 |
+
research = [
|
| 15 |
+
"numpy>=1.26,<3",
|
| 16 |
+
"opencv-contrib-python-headless>=4.10,<5",
|
| 17 |
+
"scikit-learn>=1.5,<2",
|
| 18 |
+
"torch>=2.2,<3",
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
[project.scripts]
|
| 22 |
+
math-grid-drawer = "math_grid_drawer.app:main"
|
| 23 |
+
|
| 24 |
+
[tool.setuptools.packages.find]
|
| 25 |
+
where = ["src"]
|
reports/RESEARCH_REPORT.md
ADDED
|
@@ -0,0 +1,363 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# AIFlow Math Ink 0.6 โ ํ๋ ๋ฌธ๋งฅ ๋ชจ๋ธ ์ ํ ๋ณด๊ณ ์
|
| 2 |
+
|
| 3 |
+
## ๊ฒฐ๋ก
|
| 4 |
+
|
| 5 |
+
0.6์ ๋จ์ ์ ํ๋๋ ๊ณ ๋ฆฝ ๊ธฐํธ ๋ถ๋ฅ๊ธฐ๋ง ๋ ํ์ตํด์ ํด๊ฒฐ๋์ง ์๋๋ค. ๋์ผํ ๊ต์ฐจํ์ด `x`, `X`, `\times`๊ฐ ๋ ์ ์๊ณ , ๋์ผํ ๋๋ฌธ์๋ ๋ถ์ยท๋ถ๋ชจยท์ฒจ์์ฒ๋ผ ์ถ์๋ ์์ ์ฌ๋กฏ์์๋ ์ ์ ์ฒด ๋์ด์ ๋น๊ตํ ์ ์๊ธฐ ๋๋ฌธ์ด๋ค.
|
| 6 |
+
|
| 7 |
+
๋ฐ๋ผ์ ๋ค์ ์ฑ
์ ๋ถ๋ฆฌ๋ฅผ ์ฑํํ๋ค.
|
| 8 |
+
|
| 9 |
+
1. trajectory/raster ์ ๊ฒฝ๋ง์ ํํ ํ๋ณด top-k์ ํ๋ฅ ์ ๋ง๋ ๋ค.
|
| 10 |
+
2. Trayยท๊ด๊ณ ๋ ์ด์ด๋ ๋ถ์ยท๋ถ๋ชจยท์ฒจ์ยทํผ๊ฐํญ ๋ฑ ๋ก์ปฌ ์ขํ๊ณ๋ฅผ ๋ง๋ ๋ค.
|
| 11 |
+
3. ํ๋ ๋ฌธ๋งฅ ๋ ์ด์ด๋ ํ ํ๋, ๋ก์ปฌ ํฌ๊ธฐ, ์ด์ ํ๋ณด, ๊ฐ๊ฒฉ, Tray ์ญํ ์ ์ฌ์ฉํด ๊ฐ์ ํํ๊ตฐ ์์ ์๋ฏธ๋ง ์ฌ์ ์ํํ๋ค.
|
| 12 |
+
4. ์์ gate์ LaTeX ๋ฌธ๋ฒ์ ๊ทธ ๋ค์์ ์๋ํ๋ค.
|
| 13 |
+
5. ํ๋ ์ฆ๊ฑฐ๊ฐ ๋ถ์กฑํ๋ฉด ์๋ top-k๋ฅผ ๊ทธ๋๋ก ๋ฐํํ๋ค.
|
| 14 |
+
|
| 15 |
+
```text
|
| 16 |
+
touch / raster
|
| 17 |
+
โ 128ร128 ์ ๊ทํยทtrajectory encoder
|
| 18 |
+
โ ๊ธฐํธ top-k
|
| 19 |
+
โ segmentation latticeยทTrayยท2D relation
|
| 20 |
+
โ ํ๋ ๋ฌธ๋งฅ ๋ชจ๋ธ
|
| 21 |
+
โโ ๋ก์ปฌ ์ฌ๋กฏ๋ณ ๋์๋ฌธ์
|
| 22 |
+
โโ x / X / \times ์๋ฏธ ์ญํ
|
| 23 |
+
โโ O / o / 0 ๋ฑ hard-family ํ๋ณด ๋ณด์กด
|
| 24 |
+
โ math/non-math gate
|
| 25 |
+
โ Formula ExpertยทLaTeX
|
| 26 |
+
```
|
| 27 |
+
|
| 28 |
+
ํ๋ ๋ชจ๋ธ์ glyph ์ธ์๊ธฐ๋ฅผ ๋ฎ์ด์ฐ๋ ์ OCR์ด ์๋๋ค. ์ ๊ฒฝ๋ง์ด ๋ณด์กดํ ํ๋ณด๊ตฐ ์์์ ์ฌ์ฉ์๊ฐ ๊ทธ ๊ธฐํธ๋ฅผ ์ด๋ค ์ญํ ๋ก ๋ฐฐ์นํ๋์ง ์ถ์ ํ๋ ์์ ํ๋จ ๋ชจ๋ธ์ด๋ค.
|
| 29 |
+
|
| 30 |
+
## ์ง๊ธ๊น์ง ํ์ธ๋ ์ฐ๊ตฌ ๊ฒฐ๊ณผ
|
| 31 |
+
|
| 32 |
+
### Online/shared encoder
|
| 33 |
+
|
| 34 |
+
- ์น์ธ HWRTยทUJI Pen v1/v2ยทPendigits ๊ธฐ๋ฐ 128-stepยท19-channel ๊ฒฝ๋ก๋ฅผ ๊ตฌ์ถํ๋ค.
|
| 35 |
+
- seed 17ยท31ยท47 paired test ํ๊ท ์ exact top-1 `83.00%`, top-5 `97.78%`, family top-1 `91.37%`๋ค.
|
| 36 |
+
- logit ensemble์ exact `83.82%`, top-5 `98.04%`, casefold `90.93%`, shape-family `93.02%`๋ค.
|
| 37 |
+
- exact ์ค๋ฅ์ `43.95%`๊ฐ ๋์๋ฌธ์๋ง ๋ค๋ฅธ ์ค๋ฅ์ด๊ณ `56.86%`๊ฐ ๊ฐ์ shape family ์์ ์ค๋ฅ๋ค.
|
| 38 |
+
- ์ด๋ ํํ ํ๋ณด๋ ์๋น ๋ถ๋ถ ์ด์ ์์ผ๋ ์ต์ข
์๋ฏธ ๋ฐฐ์ ์ด ๋ณ๋ชฉ์ด๋ผ๋ ๊ทผ๊ฑฐ๋ค.
|
| 39 |
+
|
| 40 |
+
### Rasterโvirtual stroke
|
| 41 |
+
|
| 42 |
+
- top-4 virtual stroke์ shared encoder ์ฌ์
๋ ฅ ๊ฒฝ๋ก๋ฅผ ๊ตฌํํ๋ค.
|
| 43 |
+
- ํ๋ณด ๋ค์์ฑ์ ๊ฐ์ ๋์ง๋ง ์ต์ข
raster ์ ํ๋๋ release gate์ ํฌ๊ฒ ๋ฏธ๋ฌํ๋ค.
|
| 44 |
+
- 3-seed ์ฐจ์ด๊ฐ ์์ seed variance๋ณด๋ค vectorizer์ online feature ๋ถํฌ ์ ๋ ฌ์ด ์ฃผ ๋ณ๋ชฉ์ด๋ค.
|
| 45 |
+
- ๋ฐ๋ผ์ ํ๋ ๋ฌธ๋งฅ์ raster vectorizer ๋ณ๋ชฉ์ ๊ฐ์ถ๋ ๋์ฒด๋ฌผ์ด ์๋๋ฉฐ, ๋ ๋ฌธ์ ๋ฅผ ๋ณ๋ gate๋ก ์ธก์ ํด์ผ ํ๋ค.
|
| 46 |
+
|
| 47 |
+
### ๋์๋ฌธ์ ๋ฌธ๋งฅ
|
| 48 |
+
|
| 49 |
+
- `k/p/y`์ฒ๋ผ ํ ์์ฒด๊ฐ ๋ค๋ฅธ ์์ exact head๊ฐ ๋ด๋นํ๋ค.
|
| 50 |
+
- `c/o/s/u/v/w/x/z`์ฒ๋ผ ์ฃผ ์ฐจ์ด๊ฐ ์๋ ๋์ด์ธ ์๋ง context head๊ฐ ๋ด๋นํ๋ค.
|
| 51 |
+
- ๊ธฐ์กด ํ ์ ์ฒด ๊ธฐ์ค proxy๋ validation `70.31โ72.14%`, test `62.19โ67.34%`๋ก ๊ฐ์ ๋์ง๋ง, ์ฐ์ ์์ P ๋ฐ์ดํฐ์ ์ ํ ๊ทผ๊ฑฐ๋ ์๋๋ค.
|
| 52 |
+
- ํ ์ ์ฒด ๊ธฐ์ค์ ์ถ์๋ ๋ถ์ยท์ฒจ์ ์์ ๋๋ฌธ์๋ฅผ ์๋ฌธ์๋ก ๋ด๋ฆด ์ ์์ผ๋ฏ๋ก ์ด๋ฒ ๊ตฌํ์์ ๋ก์ปฌ ์ฌ๋กฏ ๊ธฐ์ค์ผ๋ก ๊ต์ฒดํ๋ค.
|
| 53 |
+
|
| 54 |
+
### ๊ตฌ์กฐยทTray
|
| 55 |
+
|
| 56 |
+
- ๋ถ์์ ์ ์์ฑ๋ ๋ถ์ยท๋ถ๋ชจ๊ฐ ์์ ๋๋ง `FRACTION` Tray๋ฅผ ๋ง๋ ๋ค.
|
| 57 |
+
- ๊ทผํธ๋ radicand์ ์ ํ์ degree, ์ ๋ถ์ integrand์ ์ ํ์ upper/lower/differential์ ๋ถ๋ฆฌํ๋ค.
|
| 58 |
+
- `=` ์ธ์์ ์ข์ฐ ์์ ์์ฑ ์ฌ๋ถ์ ๋
๋ฆฝ์ ์ผ๋ก ๋ณด์กดํ๋ค.
|
| 59 |
+
- Tray๋ ๊ธฐํธ ์ด๋ฆ์ ๊ฒฐ์ ํ์ง ์๊ณ ๊ธฐํธ๊ฐ ๋์ธ ์ํ์ ์ฌ๋กฏ๊ณผ ๋ก์ปฌ ๊ธฐ์ค์ ๋ง ์ ๊ณตํ๋ค.
|
| 60 |
+
|
| 61 |
+
## `x`, `X`, `\times` ํ๋จ
|
| 62 |
+
|
| 63 |
+
`2 ? 3`์ด๋ผ๋ ์ด์ ์ ๋ณด๋ง์ผ๋ก ๊ฐ์ด๋ฐ๋ฅผ ๊ณฑํ๊ธฐ๋ก ํ์ ํ ์ ์๋ค. ์ฌ์ฉ์๊ฐ ๋ณ์ `x` ๋๋ ๋ฌธ์ `X`๋ฅผ ์ซ์ ์ฌ์ด์ ์ธ ์๋ ์๊ธฐ ๋๋ฌธ์ด๋ค. ์ด ๋ชจํธ์ฑ์ ๋ฌธ๋ฒ๋ง์ผ๋ก ์์ ํ ์ ๊ฑฐํ ์ ์๋ค.
|
| 64 |
+
|
| 65 |
+
ํ์ฌ ํ๋ ๋ ์ด์ด๋ ๋ค์ ์์น์ ์ ์ฉํ๋ค.
|
| 66 |
+
|
| 67 |
+
- ์์ ์ซ์๋ ๋ณด์กฐ feature๋ก๋ง ์ฌ์ฉํ๋ค.
|
| 68 |
+
- ๊ต์ฐจํ ์๊ฐ ํ๋ฅ , infix ์ ์ , ์ฐ์ฐ์ํ ๊ฐ๊ฒฉ, ๋ช
์์ operator context ์ค ํ๋ ์ด์์ ๊ตฌ์กฐ ์ฆ๊ฑฐ๊ฐ ์์ผ๋ฉด no-op์ด๋ค.
|
| 69 |
+
- ์กฐ๊ฑด์ ํต๊ณผํด๋ `x/X/\times` family์ ์ด logit ์ง๋์ ๋ณด์กดํ ์ฑ `\times` prior๋ง ๋ํ๋ค.
|
| 70 |
+
- ์ ์ ๊ฒฝ๋ง์ `X` ์ฆ๊ฑฐ๊ฐ ์ถฉ๋ถํ ๊ฐํ๋ฉด ํ๋ prior๊ฐ ๋ค์ง์ง ๋ชปํ๋ค.
|
| 71 |
+
- ๋ณธ์ง์ ์ผ๋ก ๋ชจํธํ ์
๋ ฅ์ top-k์ ๋ฎ์ confidence๋ก ๋ฐํํ๋ฉฐ ๊ฑฐ์ง ํ์ ์ ํ์ง ์๋๋ค.
|
| 72 |
+
|
| 73 |
+
ํฅํ ํ์ต feature๋ ๋ค์๊ณผ ๊ฐ๋ค.
|
| 74 |
+
|
| 75 |
+
- ์ ์ ๊ฒฝ๋ง์ family logits์ entropy
|
| 76 |
+
- ์๋ค token์ digit/identifier/operator ํ๋ฅ ์ง๋
|
| 77 |
+
- ์ข์ฐ ๊ฐ๊ฒฉ, ์ค์ฌ์ ์ ๋ ฌ, ๋ก์ปฌ ๋์ด๋น
|
| 78 |
+
- ๊ต์ฐจํ์ ๋ฐฉํฅยทํ์์ยทpen-upยท์๋ยท์๊ฐ ๊ฐ๊ฒฉ
|
| 79 |
+
- segmentation lattice์ infix ์ ์
|
| 80 |
+
- Tray IDยทslotยท๊ด๊ณ edge
|
| 81 |
+
- observed/canonical timestamp์ missing mask
|
| 82 |
+
|
| 83 |
+
ํ์ต target์ ์ glyph ์ด๋ฆ ํ๋๊ฐ ์๋๋ผ `identifier_lower`, `identifier_upper`, `multiply_operator`, `other/abstain` ์๋ฏธ ์ญํ ๋ก ๋๋ค.
|
| 84 |
+
|
| 85 |
+
## ๋ถ์ยท์ฒจ์ ์์ ๋์๋ฌธ์
|
| 86 |
+
|
| 87 |
+
๋๋ฌธ์ ์ฌ๋ถ๋ ์ ์ ์ฒด canvas ๋์ด์ ๋น๊ตํ์ง ์๋๋ค.
|
| 88 |
+
|
| 89 |
+
- main row
|
| 90 |
+
- fraction numerator
|
| 91 |
+
- fraction denominator
|
| 92 |
+
- superscript/subscript
|
| 93 |
+
- root degree/radicand
|
| 94 |
+
- integral upper/lower/integrand
|
| 95 |
+
|
| 96 |
+
๊ฐ ์ฌ๋กฏ์ ๋ณ๋์ `local_context_id`๋ก ๋ฌถ๊ณ , ๊ฐ์ ์ฌ๋กฏ์ ์ ๋ขฐ ๊ฐ๋ฅํ ์ซ์ยทํํ ๋ฌธ์๋ง ๋์ด anchor๋ก ์ฌ์ฉํ๋ค. ์ฌ๋กฏ์ anchor๊ฐ ์์ผ๋ฉด case logits๋ฅผ ๋ณด์กดํ๋ค. ์ด ์์น์ ์์ ๋ถ์ ์์ `X`๋ฅผ main row๋ณด๋ค ์๋ค๋ ์ด์ ๋ก `x`๋ก ๊ฐ๋ฑํ๋ ์ค๋ฅ๋ฅผ ๋ง๋๋ค.
|
| 97 |
+
|
| 98 |
+
## ๊ตฌํ
|
| 99 |
+
|
| 100 |
+
`behavior_context06.py`๋ฅผ ์ถ๊ฐํ๊ณ `math_context_layer.py` ์๋จ์ ์ฐ๊ฒฐํ๋ค.
|
| 101 |
+
|
| 102 |
+
๊ณต๊ฐ ์
๋ ฅ ๊ณ์ฝ์ record๋ณ ๋ค์ ์ ํ feature๋ฅผ ์ฌ์ฉํ๋ค.
|
| 103 |
+
|
| 104 |
+
```json
|
| 105 |
+
{
|
| 106 |
+
"behavior_context": {
|
| 107 |
+
"local_context_id": "fraction:f1:numerator",
|
| 108 |
+
"cross_visual_probability": 0.91,
|
| 109 |
+
"infix_occupancy": 0.94,
|
| 110 |
+
"operator_spacing_score": 0.82,
|
| 111 |
+
"explicit_operator_context": 0.0
|
| 112 |
+
}
|
| 113 |
+
}
|
| 114 |
+
```
|
| 115 |
+
|
| 116 |
+
Tray๊ฐ ์์ผ๋ฉด `tray_id + tray_slot`, ์ผ๋ฐ ํ์ด๋ฉด `baseline_group`, ์๋ฌด ์ ๋ณด๋ ์์ผ๋ฉด `main`์ ์ฌ์ฉํ๋ค. `FRACTION.numerator/denominator`์ ๊ตฌ์กฐ ๊ด๊ณ์ `SUPERSCRIPT/SUBSCRIPT/ABOVE/BELOW/CONTAINS`๋ ๋ณ๋ ์์์
์์ด ๋ก์ปฌ context๋ก ์๋ ๋ณํํ๋ค.
|
| 117 |
+
|
| 118 |
+
ํ๊ท ๊ฒ์ฆ์ ๋ค์ ๋ค ํญ๋ชฉ์ ๊ณ ์ ํ๋ค.
|
| 119 |
+
|
| 120 |
+
1. ์ถ์๋ ๋ถ์ ์์ `X` ๋ณด์กด
|
| 121 |
+
2. ์ซ์ ์ด์๋ง์ผ๋ก `\times` ๊ฐ์ ๊ธ์ง
|
| 122 |
+
3. ํยทinfixยท๊ฐ๊ฒฉ ์ฆ๊ฑฐ๊ฐ ๊ฒน์น ์ ๋งคํ ๊ต์ฐจํ์ ์ํํธ `\times` ์ ํ๊ณผ family ์ง๋ ๋ณด์กด
|
| 123 |
+
4. ๊ฐํ ์๋ณธ `X` logit ๋ณด์กด
|
| 124 |
+
|
| 125 |
+
๊ด๋ จ context ํ๊ท ํ
์คํธ๋ `15/15`, ์ ์ฒด Python ํ๊ท ํ
์คํธ๋ `264/264` ํต๊ณผํ๋ค.
|
| 126 |
+
|
| 127 |
+
## ํ๋ ๋ชจ๋ธ ํ์ธํ๋ ๊ณํ
|
| 128 |
+
|
| 129 |
+
### ๋จ๊ณ A โ shadow feature ์์ง
|
| 130 |
+
|
| 131 |
+
๊ธฐ์กด ๋ชจ๋ธ weight๋ ๊ณ ์ ํ๊ณ ๊ฐ ๊ธฐํธ์ top-k, geometry, stroke behavior, Tray slot์ ๋ถ์ฌ ์ ์ฅํ๋ค. ์ด๋ฏธ์ง๋ ์ stroke๋ฅผ ์๋ฒ๋ก ๋ณด๋ด์ง ์๋ ์จ๋๋ฐ์ด์ค ์์น์ ์ ์งํ๋ค. ์ฐ๊ตฌ ๋ฐ์ดํฐ์๋ ์๋ณธ provenance์ writer/source/device split์ ๋ณด์กดํ๋ค.
|
| 132 |
+
|
| 133 |
+
### ๋จ๊ณ B โ ์์ role head
|
| 134 |
+
|
| 135 |
+
ํ๋ head๋ ๋ชจ๋ฐ์ผ student์ ๋ถ์ผ ์ ์๋ ์์ TCN/MLP๋ก ์์ํ๋ค. pixel encoder๋ฅผ ๋ค์ ํฌ๊ฒ ํ์ตํ์ง ์๊ณ ๋ค์ multi-task๋ฅผ ์ฌ์ฉํ๋ค.
|
| 136 |
+
|
| 137 |
+
- cross semantic role CE
|
| 138 |
+
- local case role CE
|
| 139 |
+
- Tray slot consistency loss
|
| 140 |
+
- ์ family logit ๋ณด์กด KL
|
| 141 |
+
- abstain/calibration loss
|
| 142 |
+
|
| 143 |
+
์ด๊ธฐ loss weight๋ validation sweep ๋์์ด๋ฉฐ ์์ง ๋ฐฐํฌ ์์๋ก ๊ณ ์ ํ์ง ์๋๋ค.
|
| 144 |
+
|
| 145 |
+
### ๋จ๊ณ C โ ์ค์ ์์ ๊ฒ์ฆ
|
| 146 |
+
|
| 147 |
+
ํฉ์ฑ ์กฐํฉ์ ๋ฐ์ดํฐ ์ฆ๊ฐ์ผ๋ก๋ง ์ฌ์ฉํ๋ค. ์ต์ข
์ ํ์ writer/device/source-disjoint ์ฐ์ ์์์์ ์ํํ๋ค.
|
| 148 |
+
|
| 149 |
+
- `x/X/\times` role top-1ยทtop-k์ confusion matrix
|
| 150 |
+
- local slot๋ณ case exact
|
| 151 |
+
- unchanged-safe rate: ํ๋ ์ฆ๊ฑฐ๊ฐ ์๋ ํ๋ณธ์ ์ top-1 ๋ณด์กด์จ
|
| 152 |
+
- family oracle ๋๋น ํ์์จ
|
| 153 |
+
- ECE์ abstain coverage/accuracy
|
| 154 |
+
- main/numerator/denominator/script slice ์ต์ ์ฑ๋ฅ
|
| 155 |
+
|
| 156 |
+
๋น์์ฉยท์ฉ๋ ์ ํ ๋ฐ์ดํฐ๋ ์ฐ๊ตฌ ํ๊ฐ์๋ง ์ฌ์ฉํ๊ณ ๋ฐฐํฌ checkpoint ํ์ต์๋ ํฌํจํ์ง ์๋๋ค.
|
| 157 |
+
|
| 158 |
+
## ํ์ฌ ํ๋จ
|
| 159 |
+
|
| 160 |
+
- ํ๋ ๋ชจ๋ธ ๋ฐฉํฅ ์ฑํ: **์**
|
| 161 |
+
- ๊ธฐ์กด case layer๋ฅผ ๋ก์ปฌ ์ฌ๋กฏ ํ๋ layer๋ก ๊ต์ฒด: **์**
|
| 162 |
+
- `x/X/\times` hard rewrite: **์๋์ค**
|
| 163 |
+
- CROHME R-track ์กฐ๊ฑด๋ถ ์ญํ ์ ํ๋ ์์น ํ์ธ: **์**
|
| 164 |
+
- ์ด๋ฅผ ์ ํ ์์น๋ก ์ฃผ์ฅ: **์๋์ค**
|
| 165 |
+
- 0.6 ์์ฉ ์นด๋๋ฆฌ: **์์ง ์๋์ค**
|
| 166 |
+
|
| 167 |
+
## ์ฒซ GPU ํ๋ํ์ต ๊ฒฐ๊ณผ
|
| 168 |
+
|
| 169 |
+
๊ธฐ์กด 3-seed glyph teacher๋ฅผ ๊ฐ๊ฐ ๋๊ฒฐํ๊ณ `128ร19 stroke TCN + 49์ฐจ ๋ฌธ๋งฅ MLP` ํ๋ head๋ฅผ GTX 1650 CUDA์์ ํ์ตํ๋ค. CROHME 2012 `trainData`๋ writer ๊ธฐ์ค fit 1,063์/validation 275์์ผ๋ก ๋๋ด๊ณ `testDataGT` 488์์ ๊ณต์ held-out์ผ๋ก๋ง ํ ๋ฒ ํ๊ฐํ๋ค.
|
| 170 |
+
|
| 171 |
+
์ญํ ๋ถ๋ชจ๋ ๋ค์๊ณผ ๊ฐ๋ค.
|
| 172 |
+
|
| 173 |
+
| split | lower identifier | upper identifier | multiply operator |
|
| 174 |
+
|---|---:|---:|---:|
|
| 175 |
+
| fit | 1,573 | 99 | 177 |
|
| 176 |
+
| validation | 401 | 22 | 44 |
|
| 177 |
+
| official test | 568 | 31 | 36 |
|
| 178 |
+
|
| 179 |
+
head ์
๋ ฅ์๋ ๋ค์์ ์ฌ์ฉํ๋ค.
|
| 180 |
+
|
| 181 |
+
- seed๋ณ frozen 0.6 teacher์ lower/upper/times logits
|
| 182 |
+
- target ์๋ณธ stroke์ ์์ยท๋ฐฉํฅยทpen-up์ ๋ณด์กดํ 128ร19 sequence
|
| 183 |
+
- ์ข์ฐ ์ด์์ ์ ๋ต label์ด ์๋ teacher ์์ธก ์ญํ ํ๋ฅ
|
| 184 |
+
- bbox, ๊ฐ๊ฒฉ, ์ค์ฌ์ , ๋ก์ปฌ ๋์ดยทํญ ๋น์จ
|
| 185 |
+
- 19๊ฐ cross visual feature์ stroke/point ์
|
| 186 |
+
|
| 187 |
+
| ์งํ | seed 17 | seed 31 | seed 47 | ํ๊ท ยฑ ํ์คํธ์ฐจ |
|
| 188 |
+
|---|---:|---:|---:|---:|
|
| 189 |
+
| teacher role accuracy | 74.33% | 60.79% | 60.94% | 65.35 ยฑ 6.35%p |
|
| 190 |
+
| behavior role accuracy | 93.07% | 93.86% | 93.07% | **93.33 ยฑ 0.37%p** |
|
| 191 |
+
| behavior macro-F1 | 76.64% | 77.17% | 75.42% | **76.41 ยฑ 0.73%p** |
|
| 192 |
+
| lower recall | 95.07% | 96.30% | 95.60% | 95.66 ยฑ 0.50%p |
|
| 193 |
+
| upper recall | 61.29% | 51.61% | 54.84% | **55.91 ยฑ 4.02%p** |
|
| 194 |
+
| multiply recall | 88.89% | 91.67% | 86.11% | **88.89 ยฑ 2.27%p** |
|
| 195 |
+
| ECE | 4.73% | 5.09% | 4.82% | 4.88 ยฑ 0.15%p |
|
| 196 |
+
|
| 197 |
+
ํ๊ท accuracy ์ด๋์ `+27.98%p`์ด๊ณ seed ๊ฐ ํธ์ฐจ๋ ์์ ํ๋ ๋ฌธ๋งฅ ๋ฐฉํฅ์ ์ ํจํ๋ค. ํนํ frozen teacher๊ฐ ์ธ seed ๋ชจ๋ `\times`๋ฅผ 0% ํ์ํ๋ ์กฐ๊ฑด์์ ํ๋ head๋ 86.11~91.67%๋ฅผ ํ์ํ๋ค.
|
| 198 |
+
|
| 199 |
+
๊ทธ๋ฌ๋ ๋ค์ ํ๊ณ ๋๋ฌธ์ ์ ํ ์น๊ฒฉ์ ๊ธ์งํ๋ค.
|
| 200 |
+
|
| 201 |
+
1. ์ ๋ต symbol grouping ์ดํ์ ์กฐ๏ฟฝ๏ฟฝ๏ฟฝ๋ถ ์ญํ ํ๊ฐ๋ค. segmentation ์ค๋ฅ๋ ๋ถ๋ชจ์ ์๋ค.
|
| 202 |
+
2. CROHME R-track weight๋ผ ์์ฉ checkpoint์ ๋ณํฉํ ์ ์๋ค.
|
| 203 |
+
3. official test์ ๋๋ฌธ์ ๋ถ๋ชจ๋ 31๊ฐ๋ฟ์ด๋ฉฐ recall ํ๊ท ๋ 55.91%๋ค.
|
| 204 |
+
4. ๋ถ์ยท์ฒจ์ ๋ก์ปฌ case์ ์ค์ device-disjoint ์ ํ ๋ถ๋ชจ๊ฐ ์๋๋ค.
|
| 205 |
+
5. ์ธ teacher๋ณ head๋ ์ฝ 75.8KB์ง๋ง distillation/์จ๋๋ฐ์ด์ค ๋ณํ ๋จ๊ณ๋ก ๋๊ธฐ์ง ์๋๋ค.
|
| 206 |
+
|
| 207 |
+
๋ฐ๋ผ์ ์ฒซ ํ์ต ๊ฒฐ๋ก ์ `x/X/\times` ์๋ฏธ ์ญํ head ์ฑํ, ๋๋ฌธ์ ๋ฐ์ดํฐ ํ์ฅ, predicted grouping end-to-end ์ฌํ๊ฐ๋ค. ์ ํ์ฉ ๋ค์ ํ์ต์ ์น์ธ๋ trajectory๋ฅผ ์ํ ๋ฐฐ์น๋ก ํฉ์ฑํ ์ฝํ P-compatible proxy์ ์ค์ ์์ฉ ํ์ฉ ์ฐ์์ ๋ฐ์ดํฐ๋ฅผ ๊ตฌ๋ถํด ์ฌ์ฉํด์ผ ํ๋ค.
|
| 208 |
+
|
| 209 |
+
## ํ์ ๋ฃจํ โ ํฉ์ฑ ๋๋ฌธ์์ predicted grouping
|
| 210 |
+
|
| 211 |
+
### ์น์ธ trajectory ๋ถ๋ชจ
|
| 212 |
+
|
| 213 |
+
HWRTยทUJI Pen v1/v2์ writer-disjoint training split์๋ ์๋ํฌ๊ธฐ๊ตฐ ๊ฐ ๋ยท์๋ฌธ์๊ฐ ์ฝ 90~111๊ฐ์ฉ ์๋ค. ์ด๋ฅผ ์ค์ stroke ๊ทธ๋๋ก ์ฌ์ฉํ๊ณ ์์ ํ์ ๋์ดยท์ข์ฐ ์ซ์ anchor๋ง ํฉ์ฑํ P-compatible proxy๋ฅผ ๋ง๋ค์๋ค.
|
| 214 |
+
|
| 215 |
+
์ฒซ ํ๋ณด๋ 8๊ฐ case family 1,590๊ฐ๋ฅผ loss weight 0.35๋ก ์ถ๊ฐํ๋ค. ๋ ๋ฒ์งธ ํ๋ณด๋ ์ค์ CROHME fit ์ ๊ทํ ํต๊ณ๋ฅผ ๊ณ ์ ํ๊ณ `c/x/z` family์ ๋๋ฌธ์ 284๊ฐ์ ์๋ฌธ์ 77๊ฐ๋ง weight 0.15๋ก ์ถ๊ฐํ๋ค.
|
| 216 |
+
|
| 217 |
+
| seed 17 official test | ์ behavior | ์ ์ฒด proxy | targeted proxy |
|
| 218 |
+
|---|---:|---:|---:|
|
| 219 |
+
| accuracy | 93.07% | 93.39% | 93.39% |
|
| 220 |
+
| macro-F1 | 76.64% | 75.62% | 76.45% |
|
| 221 |
+
| upper recall | **61.29%** | 51.61% | 54.84% |
|
| 222 |
+
| multiply recall | 88.89% | 86.11% | 88.89% |
|
| 223 |
+
|
| 224 |
+
๋ ํ๋ณด ๋ชจ๋ ๋ชฉํ์ธ upper recall์ ์
ํ์์ผฐ๋ค. ๊ณ ๋ฆฝ๊ธฐํธ๋ฅผ ์ธ์์ ์ผ๋ก ํค์ด ๋ฐ์ดํฐ๋ ์ค์ ์ฐ์์์ ๋๋ฌธ์ ํ๋ ๋ถํฌ๋ฅผ ๋์ฒดํ์ง ๋ชปํ๋ฏ๋ก ์กฐ๊ธฐ ๊ธฐ๊ฐํ๊ณ seed 31ยท47๋ก ํ์ฅํ์ง ์์๋ค. ์์ ์ ์ฒด accuracy ์์น์ ๊ทผ๊ฑฐ๋ก ์ฑํํ์ง ์๋๋ค.
|
| 225 |
+
|
| 226 |
+
๋ก์ปฌ MathWriting 2024 excerpt๋ train/valid/test ๊ฐ 100์๊ณผ symbols/synthetic ๊ฐ 100๊ฐ๊ฐ ์์ง๋ง formula InkML์ symbol-to-stroke group ์ ๋ต์ด ์๋ค. LaTeX ๋ฌธ์์ด๊ณผ ์ ์ฒด stroke๋ง์ผ๋ก ๋๋ฌธ์ ์ญํ target์ ๊ฐ์ ์ ๋ ฌํ๋ฉด labelยทํ์์ ๋ฐ๋ช
ํ๊ฒ ๋๋ฏ๋ก supervised behavior ํ์ต์ ๋ฃ์ง ์๋๋ค.
|
| 227 |
+
|
| 228 |
+
### Predicted grouping ๊ฐ์ฌ
|
| 229 |
+
|
| 230 |
+
๊ณต์ test 488์์ ํ๋ ๋์ 635๊ฐ๋ฅผ validation์์ ๊ณ ์ ํ lattice selector๋ก ๋ค์ ํ๊ฐํ๋ค.
|
| 231 |
+
|
| 232 |
+
| grouping ๊ฒฝ๋ก | ์ ์ฒด target | lower | upper | `\times` | `x` exact group |
|
| 233 |
+
|---|---:|---:|---:|---:|---:|
|
| 234 |
+
| base OCR fusion | 81.89% | 79.93% | 96.77% | 100% | 75.56% |
|
| 235 |
+
| Tray/infix joint | 81.42% | 79.58% | 93.55% | 100% | 75.11% |
|
| 236 |
+
| cross-visual + Tray/infix, gap 20% | 91.02% | 90.32% | 93.55% | 100% | 88.89% |
|
| 237 |
+
| full selector, gap 50% | 93.39% | 92.96% | 93.55% | 100% | **92.22%** |
|
| 238 |
+
| **protected family selector, gap 40%** | **92.91%** | **92.43%** | 93.55% | 100% | 91.56% |
|
| 239 |
+
|
| 240 |
+
base์์ ์คํจ 115๊ฑด ์ค `x`๊ฐ 110๊ฑด์ด๊ณ , ๊ทธ์ค 96๊ฑด์ด ๋ ํ oversplit์ด์๋ค. ์ต์ด cross-visual ๊ฒฐํฉ์ ์ ์ฒด oversplit์ 98โ42๊ฑด์ผ๋ก ์ค์ด๊ณ `x` exact grouping์ `+13.33%p` ์ฌ๋ ธ๋ค. ๋ฐ๋ฉด cross model ์๋ Tray/infix ๋จ๋
์ ์ํญ ์
ํ๋ผ ํ๋ grouping ๊ฒฝ๋ก๋ก ์ฑํํ์ง ์๋๋ค.
|
| 241 |
+
|
| 242 |
+
ํ์ฌ ๋ณ๋ชฉ์ ๋ค์์ฒ๋ผ ๋ถ๋ฆฌ๋๋ค.
|
| 243 |
+
|
| 244 |
+
1. `x`: ์ต์ข
๊ฒฐํฉ ํ grouping recall์ 91.56%๋ก 90%๋๋ฅผ ์ ์งํ๋ฉฐ ์์ฌ 38๊ฑด์ oversplit 26, overmerge/ํผํฉ 12๋ค.
|
| 245 |
+
2. `X/C`: upper ์ญํ grouping์ 93.55%์ด๋ฏ๋ก ์ฃผ ๋ณ๋ชฉ์ behavior role head์ ์ค์ ์ฐ์์ ๋๋ฌธ์ ๋ฐ์ดํฐ๋ค.
|
| 246 |
+
3. `\times`: grouping 100%, conditional role recall 88.89%์ด๋ฏ๋ก ์๋ฏธ head๊ฐ ๋ณ๋ชฉ์ด๋ค.
|
| 247 |
+
4. grouping๊ณผ role ์ค๋ฅ๊ฐ ๋
๋ฆฝ์ด๋ผ๊ณ ๊ฐ์ ํ ๊ทผ์ฌ end-to-end recall์ lower 88.42%, upper 52.31%, multiply 88.89%๋ค. ์ด๋ ์ค์ธก end-to-end ์์น๊ฐ ์๋๋ฏ๋ก ์ ํ ์งํ๋ก ์ฌ์ฉํ์ง ์๋๋ค.
|
| 248 |
+
5. ์ ์ฒด 488์ exact partition์ 60.04%์ด๋ฉฐ fractionยทsuperscriptยท๋จ์ผ๊ธฐํธ overmerge๊ฐ ๋ค์ segmentation ๋ณ๋ชฉ์ด๋ค.
|
| 249 |
+
|
| 250 |
+
๋ฐ๋ผ์ R-track grouping ๊ธฐ์ค์ ์ `equality + cross-visual + protected multistroke family + Tray/infix`, cross gap ratio `0.40`์ผ๋ก ๊ฐฑ์ ํ๋ค. ๋๋ฌธ์๋ ํฉ์ฑ isolated proxy๊ฐ ์๋๋ผ ์ค์ symbol-group annotation์ด ์๋ ์ฐ์์๋ง ๋ค์ supervised ํ์ต ํ๋ณด๋ก ํ์ฉํ๋ค.
|
| 251 |
+
|
| 252 |
+
### x prefilter ๋ณ๋ชฉ๊ณผ gap validation sweep
|
| 253 |
+
|
| 254 |
+
์คํจ๋ฅผ candidate ์์ฑโcross headโexact-cover selector๋ก ๋ถํดํ๋ค. ๋ ํ `x`์ lattice candidate recall์ validation/test ๋ชจ๋ 100%์ด๊ณ test cross threshold recall์ 98.37%์๋ค. ๊ทธ๋ฐ๋ฐ ์คํจ ํ๋ณด์ cross ํ๋ฅ ์ค์๊ฐ๋ 0.929์ธ๋ฐ `infix_signal=0`์ด์๋ค. ์์ธ์ cross head ์์ bbox gap 20% prefilter๊ฐ ์ข์ฐ ๊ณก์ `)(`ํ x๋ฅผ ๋ชจ๋ธ์ ์ ๋ฌํ์ง ์์ ๊ฒ์ด๋ค.
|
| 255 |
+
|
| 256 |
+
์ ์ญ threshold์ merge weight๋ฅผ ๊ณ ์ ํ๊ณ gap ratio `0.20~0.80`๋ง writer-validation์์ ์ ํํ๋ค. Validation winner `0.50`์ equality head๊น์ง ํฌํจํ full selector์ ๊ณต์ test์ ์ ์ฉํ๋ค.
|
| 257 |
+
|
| 258 |
+
| ์งํ | gap 20% | **gap 50%** | ๋ณํ |
|
| 259 |
+
|---|---:|---:|---:|
|
| 260 |
+
| validation `x` grouping | 82.31% | **88.45%** | +6.14%p |
|
| 261 |
+
| validation exact partition | 63.27% | **65.82%** | +2.55%p |
|
| 262 |
+
| validation pair-F1 | 91.00% | **91.69%** | +0.69%p |
|
| 263 |
+
| official test `x` grouping | 88.89% | **92.22%** | +3.33%p |
|
| 264 |
+
| official test exact partition | 53.69% | **54.51%** | +0.82%p |
|
| 265 |
+
| official test pair-F1 | 88.38% | **88.72%** | +0.35%p |
|
| 266 |
+
|
| 267 |
+
gap 50%๋ ์ด ๋จ๊ณ์์ ์ ์งํ๊ฐ ํจ๊ป ์ฌ๋ผ ์ฑํํ๋ค. ์ดํ ๊ฐ์ ๊ธฐ์ค์์ threshold `0.5`, weight `10`์ ๊ณต๊ฒฉ ํ๋ณด๋ test `x`๋ฅผ 94.44%๋ก ์ฌ๋ ธ์ง๋ง exact partition 54.51โ53.89%, pair-F1 88.72โ88.41%, overmerge 21.11โ23.57%๋ก ์
ํ๋ผ ๊ธฐ๊ฐํ๋ค. ๋คํ family boost์ ๊ฒฐํฉํ validation ์ฌ์ ํ์์๋ ๋์ผ test ์ฑ๋ฅ์ ๋ด๋ ๋ ์ข์ gap `0.40`์ด ์ ํ๋๋ค. ์ต์ข
cross ๊ฐ์ gap `0.40`, threshold `0.692841...`, infix weight `8`์ด๋ค.
|
| 268 |
+
|
| 269 |
+
### ๋คํ family must-link์ geometry ๋ณดํธ
|
| 270 |
+
|
| 271 |
+
`sum/pi/rightarrow/neq/pm`์ candidate recall์ด validation/test ๋ชจ๋ 100%์์ง๋ง ํฉ์น OCR alias๊ฐ ๊ธฐ์กด reliable ๋ชฉ๋ก๊ณผ ์ฐ๊ฒฐ๋์ง ์๊ฑฐ๋ 3๊ฐ์ singleton ์ ์์ ๋ฐ๋ ธ๋ค. `Sigma/pi-family/arrow-family/neq/pm` ํ๋ณด์๋ง ์ถ๊ฐ symbol signal์ ์ฃผ๊ณ boost `0~6`์ writer-validation์์ ์ ํํ๋ค.
|
| 272 |
+
|
| 273 |
+
| ๊ณต์ test grouping | ๊ธฐ์ค | boost 6 + ๋ณดํธ |
|
| 274 |
+
|---|---:|---:|
|
| 275 |
+
| ๋ค์ฏ family ์ ์ฒด | 45.19% | **82.96%** |
|
| 276 |
+
| `\sum` | 25.81% | **70.97%** |
|
| 277 |
+
| `\pi` | 40.00% | **90.00%** |
|
| 278 |
+
| `\rightarrow` | 71.43% | **89.80%** |
|
| 279 |
+
| `\neq` | 23.08% | **76.92%** |
|
| 280 |
+
| `\pm` | 25.00% | **75.00%** |
|
| 281 |
+
|
| 282 |
+
๋ฌด๋ณดํธ boost 6์ ์ ์ฒด์ exact 57.38%๊น์ง ์ฌ๋ ธ์ง๋ง `hookrightarrow`, ์ข์ `Sigma`, ์ธ๋ก๋ก ๊ธด `pm` ์ค์ธ์ด ๋จ์ผ๊ธฐํธ๋ฅผ ๋จน์๋ค. Training writer-validation์ ์ค์ geometry ๋ฒ์์์ arrow `pair_gap_maxโค0.50`, `Sigma width_refโฅ2.0`, `pm height_refโค2.0` guard๋ฅผ ๋ง๋ค๊ณ ontology ๋ฐ `hookrightarrow` alias๋ฅผ ์ ๊ฑฐํ๋ค. ๋ณดํธ ํ ๊ณต์ test exact๋ **60.04%**, pair-F1์ **91.07%**, overmerge formula rate๋ 21.72%๋ค. Gap-only ๋๋น ๊ฐ๊ฐ +5.53%p, +2.35%p์ด๋ฉฐ overmerge ์ฆ๊ฐ๋ +0.61%p๋ก ์ ํ๋๋ค.
|
| 283 |
+
|
| 284 |
+
### ์ต์ข
์์ฌ segmentation ๋ณ๋ชฉ
|
| 285 |
+
|
| 286 |
+
์ต์ข
full selector์ 488์ ์์ฌ ์คํจ๋ exact 293, oversplit 89, overmerge 72, ํผํฉ 34๋ค. ๊ตฌ์กฐ๋ณ exact๋ sqrt 68.00%, plain 63.95%, subscript 57.01%, superscript 52.75%, fraction 52.10%๋ค. 3ํ ๊ธฐํธ correct๋ 34โ63๊ฐ๋ก ๋์๊ณ oversplit์ 57โ32๊ฐ๋ก ์ค์๋ค.
|
| 287 |
+
|
| 288 |
+
| label | ์ค๋ฅ/์ ์ฒด | ์ฃผ ์คํจ |
|
| 289 |
+
|---|---:|---|
|
| 290 |
+
| `x` | 38/450 | oversplit 26, overmergeยทํผํฉ 12 |
|
| 291 |
+
| `1` | 36/482 | ์ฃผ๋ณ ํ๊ณผ overmerge |
|
| 292 |
+
| `-` | 36/584 | overmerge |
|
| 293 |
+
| `\sin` physical part | 35/297 | overmerge 15, oversplit 18 |
|
| 294 |
+
| `\sum` | 9/31 | oversplit 8 |
|
| 295 |
+
| `i` | 9/41 | oversplit |
|
| 296 |
+
| `F` | 7/10 | oversplit |
|
| 297 |
+
| `\div` | 7/11 | oversplit |
|
| 298 |
+
|
| 299 |
+
๋ฐ๋ผ์ ๋ค์ ์ฐ์ ์์๋ ์ ์ญ threshold๊ฐ ์๋๋ผ ๋ถ์ยท์์ฒจ์์์ `1/-/๊ดํธ`๊ฐ ์ด์ Tray๋ก ๋์ด๊ฐ์ง ์๊ฒ ํ๋ local-baseline must-not-link, ๊ทธ๋ฆฌ๊ณ ๋จ์ `F/div/i` ๋คํ family๋ค.
|
| 300 |
+
|
| 301 |
+
### Local-baseline ๊ฐ์ฌ์ ํ์ตํ boundary ํ๋ head
|
| 302 |
+
|
| 303 |
+
์ต์ข
selector์ overmerge๋ฅผ ์ ๋ต ๊ธฐํธ ๋จ์๋ก ๋ค์ ์ฐ๊ฒฐํ๋ค. ๊ณต์ test์๋ 299๊ฐ overmerge event์ 149๊ฐ ๊ณ ์ ์ ํ ํ๋ณด๊ฐ ์์๋ค. ์ค๋ฅ๊ฐ ์ง์ค๋ ์ ๋ต์ `1` 36๊ฑด, `-` 36๊ฑด, ๊ดํธ 53๊ฑด์ด๋ฉฐ ๊ตฌ์กฐ๋ณ๋ก superscript 150๊ฑด, fraction 112๊ฑด, subscript 96๊ฑด์ด ๊ฒน์ณ ์์๋ค. ๊ธฐ์กด fraction penalty๋ 299๊ฑด ์ค 4๊ฑด์๋ง ์ ํธ๋ฅผ ๋๋ค.
|
| 304 |
+
|
| 305 |
+
๋ ๊ท์นํ ๊ฐ๋๋ ์ฑํํ์ง ์์๋ค.
|
| 306 |
+
|
| 307 |
+
1. ํฉ์น OCR์ด ๊ตฌ์ฑํ๋ณด๋ค ์ฝํ ๋ ๊ฐ์ ํ๋ component competition์ validation exact๋ฅผ ๊ฐ์ ํ์ง ๋ชปํ๊ณ family grouping์ ์
ํ์์ผ weight 0์ ์ ํํ๋ค.
|
| 308 |
+
2. base partition์์ `SUPERSCRIPT/SUBSCRIPT/ABOVE/BELOW/CONTAINS` edge๋ฅผ ๋ง๋ local-baseline guard๋ validation 3,293๊ฐยทtest 5,061๊ฐ ํ๋ณด์ ์ ํธ๋ฅผ ๋ง๋ค์์ง๋ง ์ค์ 149๊ฐ ์ค๋ณํฉ ํ๋ณด์๋ ์ ํจ ์ ํธ๊ฐ 0๊ฐ์๋ค. ์ด๋ฏธ ์ค๋ณํฉ๋ partition ๋ค์์ ๊ด๊ณ๋ฅผ ๋ง๋ค์๊ธฐ ๋๋ฌธ์ ๋ด๋ถ ๊ฒฝ๊ณ๋ฅผ ๋ณต์ํ ์ ์์๋ค.
|
| 309 |
+
|
| 310 |
+
์ด์ grouping ์ด์ ์ ๋ชจ๋ ๋คํ ํ๋ณด๋ฅผ ๋์์ผ๋ก, ๋ ์ด์์ ์ ๋ต ๊ธฐํธ๋ฅผ ์นจ๋ฒํ๋์ง๋ฅผ ์์ธกํ๋ geometry boundary ํ๋ head๋ฅผ fit writer์ ํ์ตํ๋ค. 72,223๊ฐ ํ๋ณด ์ค 66,842๊ฐ๊ฐ boundary ์์ฑ์ด๋ฉฐ, validation์์ threshold 0.5ยทpenalty weight 6์ ์ ํํ๋ค.
|
| 311 |
+
|
| 312 |
+
| ์งํ | ๋ณดํธ selector | boundary ํ๋ head | ๋ณํ |
|
| 313 |
+
|---|---:|---:|---:|
|
| 314 |
+
| validation exact partition | 68.00% | **69.82%** | +1.82%p |
|
| 315 |
+
| validation pair-F1 | 92.98% | **93.64%** | +0.66%p |
|
| 316 |
+
| official exact partition | 60.04% | **60.25%** | +0.20%p |
|
| 317 |
+
| official pair-F1 | 91.07% | **91.26%** | +0.19%p |
|
| 318 |
+
| official overmerge formula | 21.72% | **20.49%** | -1.23%p |
|
| 319 |
+
| official `x` grouping | 91.56% | 91.33% | -0.22%p |
|
| 320 |
+
| official ๋คํ family grouping | 82.96% | 81.48% | -1.48%p |
|
| 321 |
+
|
| 322 |
+
์ฌ์ ์ ์ํ `x` -1%p, family -2%p, pair-F1 -0.25%p ๋ณดํธ gate ์์์ exact์ pair-F1์ด ํจ๊ป ์ฌ๋ผ R-track head๋ก ์ฑํํ๋ค. ๋ค๋ง CROHME ์ ๋ต ๊ฒฝ๊ณ๋ก ํ์ตํ ๋น์์
์ฐ๊ตฌ weight์ด๋ฏ๋ก ์ ํ checkpointยทteacherยทdistillation์๋ ์ฌ์ฉํ ์ ์๋ค. ์์ฉ ์ด์ ๊ฒฝ๋ก๋ ๋์ผ boundary target์ P-track ์ฐ์์์ผ๋ก ์ฌํ์ตํ๊ฑฐ๋ shared trajectory encoder์ auxiliary boundary loss๋ก ์ฎ๊ธฐ๋ ๊ฒ์ด๋ค. ์ ์ฒด Python ํ๊ท๋ 271๊ฐ๊ฐ ํต๊ณผํ๋ค.
|
| 323 |
+
|
| 324 |
+
## ์ฐ์ถ๋ฌผ
|
| 325 |
+
|
| 326 |
+
- `src/math_grid_drawer/research/behavior_context06.py`
|
| 327 |
+
- `src/math_grid_drawer/research/behavior_role_head06.py`
|
| 328 |
+
- `src/math_grid_drawer/research/math_context_layer.py`
|
| 329 |
+
- `scripts/train_math_ink_06_behavior_role.py`
|
| 330 |
+
- `scripts/summarize_math_ink_06_behavior_seeds.py`
|
| 331 |
+
- `scripts/audit_math_ink_06_behavior_grouping.py`
|
| 332 |
+
- `scripts/audit_math_ink_06_x_edge_bottleneck.py`
|
| 333 |
+
- `scripts/sweep_math_ink_06_x_grouping_guard.py`
|
| 334 |
+
- `scripts/sweep_math_ink_06_cross_gap_guard.py`
|
| 335 |
+
- `scripts/audit_math_ink_06_multistroke_families.py`
|
| 336 |
+
- `scripts/sweep_math_ink_06_multistroke_family_guard.py`
|
| 337 |
+
- `scripts/audit_math_ink_06_multistroke_guard_regressions.py`
|
| 338 |
+
- `scripts/audit_math_ink_06_local_baseline_overmerge.py`
|
| 339 |
+
- `scripts/sweep_math_ink_06_component_competition_guard.py`
|
| 340 |
+
- `scripts/sweep_math_ink_06_local_baseline_guard.py`
|
| 341 |
+
- `scripts/train_math_ink_06_boundary_behavior_guard.py`
|
| 342 |
+
- `scripts/analyze_crohme_lattice_failures.py`
|
| 343 |
+
- `tests/test_behavior_context06.py`
|
| 344 |
+
- `tests/test_behavior_role_head06.py`
|
| 345 |
+
- `research/AIFlow-MATH-INK-0.6-BEHAVIOR-CONTEXT-REPORT-20260724.md`
|
| 346 |
+
- `research/runs/math_ink_06_behavior_role_3seed_20260724/run_summary.json`
|
| 347 |
+
- `research/runs/math_ink_06_behavior_grouping_audit_20260724/report.json`
|
| 348 |
+
- `research/runs/math_ink_06_behavior_grouping_cross_joint_audit_20260724/report.json`
|
| 349 |
+
- `research/runs/math_ink_06_x_grouping_guard_20260724/report.json`
|
| 350 |
+
- `research/runs/math_ink_06_x_edge_bottleneck_20260724/report.json`
|
| 351 |
+
- `research/runs/math_ink_06_cross_gap_guard_full_selector_20260724/report.json`
|
| 352 |
+
- `research/runs/math_ink_06_behavior_grouping_full_selector_gap50_20260724/report.json`
|
| 353 |
+
- `research/runs/math_ink_06_x_grouping_guard_gap50_full_selector_20260724/report.json`
|
| 354 |
+
- `research/runs/math_ink_06_full_selector_gap50_failure_audit_20260724/report.json`
|
| 355 |
+
- `research/runs/math_ink_06_cross_gap_guard_family6_full_selector_20260724/report.json`
|
| 356 |
+
- `research/runs/math_ink_06_multistroke_family_guard_geometry_protected_20260724/report.json`
|
| 357 |
+
- `research/runs/math_ink_06_multistroke_guard_regression_audit_20260724/report.json`
|
| 358 |
+
- `research/runs/math_ink_06_full_selector_gap40_family6_protected_failure_audit_20260724/report.json`
|
| 359 |
+
- `research/runs/math_ink_06_behavior_grouping_full_selector_gap40_family6_protected_20260724/report.json`
|
| 360 |
+
- `research/runs/math_ink_06_component_competition_guard_20260724/report.json`
|
| 361 |
+
- `research/runs/math_ink_06_local_baseline_guard_20260724/report.json`
|
| 362 |
+
- `research/runs/math_ink_06_local_baseline_overmerge_audit_20260724/report.json`
|
| 363 |
+
- `research/runs/math_ink_06_boundary_behavior_guard_20260724/report.json`
|
reports/behavior_role_3seed_summary.json
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-BEHAVIOR-ROLE-3SEED-001",
|
| 3 |
+
"generated_at": "2026-07-23T16:44:02.899929+00:00",
|
| 4 |
+
"seeds": [
|
| 5 |
+
17,
|
| 6 |
+
31,
|
| 7 |
+
47
|
| 8 |
+
],
|
| 9 |
+
"reports": [
|
| 10 |
+
"research\\runs\\math_ink_06_behavior_role_seed17_20260724\\report.json",
|
| 11 |
+
"research\\runs\\math_ink_06_behavior_role_seed31_20260724\\report.json",
|
| 12 |
+
"research\\runs\\math_ink_06_behavior_role_seed47_20260724\\report.json"
|
| 13 |
+
],
|
| 14 |
+
"split_contract": "CROHME2012 trainData writer fit/validation; testDataGT official held-out",
|
| 15 |
+
"denominator": {
|
| 16 |
+
"formulas": {
|
| 17 |
+
"fit": 1063,
|
| 18 |
+
"validation": 275,
|
| 19 |
+
"official_test": 488
|
| 20 |
+
},
|
| 21 |
+
"role_samples": {
|
| 22 |
+
"fit": {
|
| 23 |
+
"identifier_lower": 1573,
|
| 24 |
+
"multiply_operator": 177,
|
| 25 |
+
"identifier_upper": 99
|
| 26 |
+
},
|
| 27 |
+
"validation": {
|
| 28 |
+
"identifier_lower": 401,
|
| 29 |
+
"multiply_operator": 44,
|
| 30 |
+
"identifier_upper": 22
|
| 31 |
+
},
|
| 32 |
+
"official_test": {
|
| 33 |
+
"identifier_lower": 568,
|
| 34 |
+
"identifier_upper": 31,
|
| 35 |
+
"multiply_operator": 36
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
},
|
| 39 |
+
"teacher_official_test": {
|
| 40 |
+
"accuracy": {
|
| 41 |
+
"values": [
|
| 42 |
+
0.7433071136474609,
|
| 43 |
+
0.6078740358352661,
|
| 44 |
+
0.6094487905502319
|
| 45 |
+
],
|
| 46 |
+
"mean": 0.6535433133443197,
|
| 47 |
+
"std": 0.06347584762070135,
|
| 48 |
+
"minimum": 0.6078740358352661
|
| 49 |
+
},
|
| 50 |
+
"macro_f1": {
|
| 51 |
+
"values": [
|
| 52 |
+
0.3644206619715246,
|
| 53 |
+
0.31082213130722774,
|
| 54 |
+
0.31121831754743146
|
| 55 |
+
],
|
| 56 |
+
"mean": 0.3288203702753946,
|
| 57 |
+
"std": 0.02517372727695006,
|
| 58 |
+
"minimum": 0.31082213130722774
|
| 59 |
+
}
|
| 60 |
+
},
|
| 61 |
+
"behavior_official_test": {
|
| 62 |
+
"accuracy": {
|
| 63 |
+
"values": [
|
| 64 |
+
0.930708646774292,
|
| 65 |
+
0.9385826587677002,
|
| 66 |
+
0.930708646774292
|
| 67 |
+
],
|
| 68 |
+
"mean": 0.9333333174387614,
|
| 69 |
+
"std": 0.00371184485045543,
|
| 70 |
+
"minimum": 0.930708646774292
|
| 71 |
+
},
|
| 72 |
+
"macro_f1": {
|
| 73 |
+
"values": [
|
| 74 |
+
0.7664031279554507,
|
| 75 |
+
0.7717349248806314,
|
| 76 |
+
0.7541782507773837
|
| 77 |
+
],
|
| 78 |
+
"mean": 0.764105434537822,
|
| 79 |
+
"std": 0.0073493195908634484,
|
| 80 |
+
"minimum": 0.7541782507773837
|
| 81 |
+
},
|
| 82 |
+
"identifier_lower_recall": {
|
| 83 |
+
"values": [
|
| 84 |
+
0.9507042253521126,
|
| 85 |
+
0.9630281690140845,
|
| 86 |
+
0.9559859154929577
|
| 87 |
+
],
|
| 88 |
+
"mean": 0.9565727699530516,
|
| 89 |
+
"std": 0.0050483129501424015,
|
| 90 |
+
"minimum": 0.9507042253521126
|
| 91 |
+
},
|
| 92 |
+
"identifier_upper_recall": {
|
| 93 |
+
"values": [
|
| 94 |
+
0.6129032258064516,
|
| 95 |
+
0.5161290322580645,
|
| 96 |
+
0.5483870967741935
|
| 97 |
+
],
|
| 98 |
+
"mean": 0.5591397849462365,
|
| 99 |
+
"std": 0.040232875126601536,
|
| 100 |
+
"minimum": 0.5161290322580645
|
| 101 |
+
},
|
| 102 |
+
"multiply_operator_recall": {
|
| 103 |
+
"values": [
|
| 104 |
+
0.8888888888888888,
|
| 105 |
+
0.9166666666666666,
|
| 106 |
+
0.8611111111111112
|
| 107 |
+
],
|
| 108 |
+
"mean": 0.8888888888888888,
|
| 109 |
+
"std": 0.02268046058132569,
|
| 110 |
+
"minimum": 0.8611111111111112
|
| 111 |
+
},
|
| 112 |
+
"ece": {
|
| 113 |
+
"values": [
|
| 114 |
+
0.04725504415735027,
|
| 115 |
+
0.050884793443143006,
|
| 116 |
+
0.048238128075605236
|
| 117 |
+
],
|
| 118 |
+
"mean": 0.04879265522536617,
|
| 119 |
+
"std": 0.0015328394652397476,
|
| 120 |
+
"minimum": 0.04725504415735027
|
| 121 |
+
}
|
| 122 |
+
},
|
| 123 |
+
"diagnosis": {
|
| 124 |
+
"behavior_accuracy_gain_pp": 27.979000409444165,
|
| 125 |
+
"stable_across_seeds": true,
|
| 126 |
+
"remaining_bottleneck": "identifier_upper recall; official test denominator 31",
|
| 127 |
+
"conditional_scope": "truth symbol grouping ์ดํ semantic role ๋ถ๋ฅ"
|
| 128 |
+
},
|
| 129 |
+
"track": "R_noncommercial_only",
|
| 130 |
+
"product_validation": false,
|
| 131 |
+
"distillation_allowed": false
|
| 132 |
+
}
|
reports/boundary_behavior_guard_report.json
ADDED
|
@@ -0,0 +1,2735 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-BOUNDARY-BEHAVIOR-GUARD-001",
|
| 3 |
+
"generated_at": "2026-07-23T18:44:45.356755+00:00",
|
| 4 |
+
"training": {
|
| 5 |
+
"candidate_rows": 72223,
|
| 6 |
+
"boundary_rows": 66842,
|
| 7 |
+
"boundary_rate": 0.9254946485191697
|
| 8 |
+
},
|
| 9 |
+
"feature_names": [
|
| 10 |
+
"stroke_count",
|
| 11 |
+
"point_count_log",
|
| 12 |
+
"width_ref",
|
| 13 |
+
"height_ref",
|
| 14 |
+
"aspect_log",
|
| 15 |
+
"temporal_span",
|
| 16 |
+
"temporal_contiguous",
|
| 17 |
+
"singleton",
|
| 18 |
+
"geometry_votes",
|
| 19 |
+
"has_temporal",
|
| 20 |
+
"has_spatial_pair",
|
| 21 |
+
"pair_gap_mean",
|
| 22 |
+
"pair_gap_max",
|
| 23 |
+
"stroke_width_mean",
|
| 24 |
+
"stroke_width_std",
|
| 25 |
+
"stroke_height_mean",
|
| 26 |
+
"stroke_height_std"
|
| 27 |
+
],
|
| 28 |
+
"reference_validation": {
|
| 29 |
+
"threshold": 0.3,
|
| 30 |
+
"weight": 0.0,
|
| 31 |
+
"global": {
|
| 32 |
+
"formulas": 275,
|
| 33 |
+
"exact_partition": 0.68,
|
| 34 |
+
"exact_group_recall": 0.9528054535920294,
|
| 35 |
+
"overmerge_rate": 0.14909090909090908,
|
| 36 |
+
"oversplit_rate": 0.21818181818181817,
|
| 37 |
+
"pair_precision": 0.9563838223632039,
|
| 38 |
+
"pair_recall": 0.9047261815453863,
|
| 39 |
+
"pair_f1": 0.9298380878951427
|
| 40 |
+
},
|
| 41 |
+
"behavior_targets": {
|
| 42 |
+
"x": {
|
| 43 |
+
"samples": 277,
|
| 44 |
+
"matched": 244,
|
| 45 |
+
"oversplit": 31,
|
| 46 |
+
"overmerge": 3,
|
| 47 |
+
"grouping_recall": 0.8808664259927798
|
| 48 |
+
},
|
| 49 |
+
"X": {
|
| 50 |
+
"samples": 7,
|
| 51 |
+
"matched": 7,
|
| 52 |
+
"oversplit": 0,
|
| 53 |
+
"overmerge": 0,
|
| 54 |
+
"grouping_recall": 1.0
|
| 55 |
+
},
|
| 56 |
+
"\\times": {
|
| 57 |
+
"samples": 44,
|
| 58 |
+
"matched": 44,
|
| 59 |
+
"oversplit": 0,
|
| 60 |
+
"overmerge": 0,
|
| 61 |
+
"grouping_recall": 1.0
|
| 62 |
+
}
|
| 63 |
+
},
|
| 64 |
+
"families": {
|
| 65 |
+
"total": 59,
|
| 66 |
+
"matched": 51,
|
| 67 |
+
"grouping_recall": 0.864406779661017,
|
| 68 |
+
"by_label": {
|
| 69 |
+
"\\sum": {
|
| 70 |
+
"samples": 11,
|
| 71 |
+
"matched": 8,
|
| 72 |
+
"grouping_recall": 0.7272727272727273
|
| 73 |
+
},
|
| 74 |
+
"\\pi": {
|
| 75 |
+
"samples": 17,
|
| 76 |
+
"matched": 15,
|
| 77 |
+
"grouping_recall": 0.8823529411764706
|
| 78 |
+
},
|
| 79 |
+
"\\rightarrow": {
|
| 80 |
+
"samples": 20,
|
| 81 |
+
"matched": 17,
|
| 82 |
+
"grouping_recall": 0.85
|
| 83 |
+
},
|
| 84 |
+
"\\neq": {
|
| 85 |
+
"samples": 2,
|
| 86 |
+
"matched": 2,
|
| 87 |
+
"grouping_recall": 1.0
|
| 88 |
+
},
|
| 89 |
+
"\\pm": {
|
| 90 |
+
"samples": 9,
|
| 91 |
+
"matched": 9,
|
| 92 |
+
"grouping_recall": 1.0
|
| 93 |
+
}
|
| 94 |
+
}
|
| 95 |
+
}
|
| 96 |
+
},
|
| 97 |
+
"winner_validation": {
|
| 98 |
+
"threshold": 0.5,
|
| 99 |
+
"weight": 6.0,
|
| 100 |
+
"global": {
|
| 101 |
+
"formulas": 275,
|
| 102 |
+
"exact_partition": 0.6981818181818182,
|
| 103 |
+
"exact_group_recall": 0.9580492920818039,
|
| 104 |
+
"overmerge_rate": 0.12,
|
| 105 |
+
"oversplit_rate": 0.21454545454545454,
|
| 106 |
+
"pair_precision": 0.9687249398556536,
|
| 107 |
+
"pair_recall": 0.9062265566391597,
|
| 108 |
+
"pair_f1": 0.9364341085271317
|
| 109 |
+
},
|
| 110 |
+
"behavior_targets": {
|
| 111 |
+
"x": {
|
| 112 |
+
"samples": 277,
|
| 113 |
+
"matched": 245,
|
| 114 |
+
"oversplit": 31,
|
| 115 |
+
"overmerge": 2,
|
| 116 |
+
"grouping_recall": 0.8844765342960289
|
| 117 |
+
},
|
| 118 |
+
"X": {
|
| 119 |
+
"samples": 7,
|
| 120 |
+
"matched": 7,
|
| 121 |
+
"oversplit": 0,
|
| 122 |
+
"overmerge": 0,
|
| 123 |
+
"grouping_recall": 1.0
|
| 124 |
+
},
|
| 125 |
+
"\\times": {
|
| 126 |
+
"samples": 44,
|
| 127 |
+
"matched": 44,
|
| 128 |
+
"oversplit": 0,
|
| 129 |
+
"overmerge": 0,
|
| 130 |
+
"grouping_recall": 1.0
|
| 131 |
+
}
|
| 132 |
+
},
|
| 133 |
+
"families": {
|
| 134 |
+
"total": 59,
|
| 135 |
+
"matched": 50,
|
| 136 |
+
"grouping_recall": 0.847457627118644,
|
| 137 |
+
"by_label": {
|
| 138 |
+
"\\sum": {
|
| 139 |
+
"samples": 11,
|
| 140 |
+
"matched": 8,
|
| 141 |
+
"grouping_recall": 0.7272727272727273
|
| 142 |
+
},
|
| 143 |
+
"\\pi": {
|
| 144 |
+
"samples": 17,
|
| 145 |
+
"matched": 15,
|
| 146 |
+
"grouping_recall": 0.8823529411764706
|
| 147 |
+
},
|
| 148 |
+
"\\rightarrow": {
|
| 149 |
+
"samples": 20,
|
| 150 |
+
"matched": 16,
|
| 151 |
+
"grouping_recall": 0.8
|
| 152 |
+
},
|
| 153 |
+
"\\neq": {
|
| 154 |
+
"samples": 2,
|
| 155 |
+
"matched": 2,
|
| 156 |
+
"grouping_recall": 1.0
|
| 157 |
+
},
|
| 158 |
+
"\\pm": {
|
| 159 |
+
"samples": 9,
|
| 160 |
+
"matched": 9,
|
| 161 |
+
"grouping_recall": 1.0
|
| 162 |
+
}
|
| 163 |
+
}
|
| 164 |
+
}
|
| 165 |
+
},
|
| 166 |
+
"trials": [
|
| 167 |
+
{
|
| 168 |
+
"threshold": 0.3,
|
| 169 |
+
"weight": 0.0,
|
| 170 |
+
"global": {
|
| 171 |
+
"formulas": 275,
|
| 172 |
+
"exact_partition": 0.68,
|
| 173 |
+
"exact_group_recall": 0.9528054535920294,
|
| 174 |
+
"overmerge_rate": 0.14909090909090908,
|
| 175 |
+
"oversplit_rate": 0.21818181818181817,
|
| 176 |
+
"pair_precision": 0.9563838223632039,
|
| 177 |
+
"pair_recall": 0.9047261815453863,
|
| 178 |
+
"pair_f1": 0.9298380878951427
|
| 179 |
+
},
|
| 180 |
+
"behavior_targets": {
|
| 181 |
+
"x": {
|
| 182 |
+
"samples": 277,
|
| 183 |
+
"matched": 244,
|
| 184 |
+
"oversplit": 31,
|
| 185 |
+
"overmerge": 3,
|
| 186 |
+
"grouping_recall": 0.8808664259927798
|
| 187 |
+
},
|
| 188 |
+
"X": {
|
| 189 |
+
"samples": 7,
|
| 190 |
+
"matched": 7,
|
| 191 |
+
"oversplit": 0,
|
| 192 |
+
"overmerge": 0,
|
| 193 |
+
"grouping_recall": 1.0
|
| 194 |
+
},
|
| 195 |
+
"\\times": {
|
| 196 |
+
"samples": 44,
|
| 197 |
+
"matched": 44,
|
| 198 |
+
"oversplit": 0,
|
| 199 |
+
"overmerge": 0,
|
| 200 |
+
"grouping_recall": 1.0
|
| 201 |
+
}
|
| 202 |
+
},
|
| 203 |
+
"families": {
|
| 204 |
+
"total": 59,
|
| 205 |
+
"matched": 51,
|
| 206 |
+
"grouping_recall": 0.864406779661017,
|
| 207 |
+
"by_label": {
|
| 208 |
+
"\\sum": {
|
| 209 |
+
"samples": 11,
|
| 210 |
+
"matched": 8,
|
| 211 |
+
"grouping_recall": 0.7272727272727273
|
| 212 |
+
},
|
| 213 |
+
"\\pi": {
|
| 214 |
+
"samples": 17,
|
| 215 |
+
"matched": 15,
|
| 216 |
+
"grouping_recall": 0.8823529411764706
|
| 217 |
+
},
|
| 218 |
+
"\\rightarrow": {
|
| 219 |
+
"samples": 20,
|
| 220 |
+
"matched": 17,
|
| 221 |
+
"grouping_recall": 0.85
|
| 222 |
+
},
|
| 223 |
+
"\\neq": {
|
| 224 |
+
"samples": 2,
|
| 225 |
+
"matched": 2,
|
| 226 |
+
"grouping_recall": 1.0
|
| 227 |
+
},
|
| 228 |
+
"\\pm": {
|
| 229 |
+
"samples": 9,
|
| 230 |
+
"matched": 9,
|
| 231 |
+
"grouping_recall": 1.0
|
| 232 |
+
}
|
| 233 |
+
}
|
| 234 |
+
}
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"threshold": 0.3,
|
| 238 |
+
"weight": 1.0,
|
| 239 |
+
"global": {
|
| 240 |
+
"formulas": 275,
|
| 241 |
+
"exact_partition": 0.6872727272727273,
|
| 242 |
+
"exact_group_recall": 0.955951756685894,
|
| 243 |
+
"overmerge_rate": 0.13818181818181818,
|
| 244 |
+
"oversplit_rate": 0.21454545454545454,
|
| 245 |
+
"pair_precision": 0.9618138424821002,
|
| 246 |
+
"pair_recall": 0.9069767441860465,
|
| 247 |
+
"pair_f1": 0.9335907335907335
|
| 248 |
+
},
|
| 249 |
+
"behavior_targets": {
|
| 250 |
+
"x": {
|
| 251 |
+
"samples": 277,
|
| 252 |
+
"matched": 244,
|
| 253 |
+
"oversplit": 31,
|
| 254 |
+
"overmerge": 3,
|
| 255 |
+
"grouping_recall": 0.8808664259927798
|
| 256 |
+
},
|
| 257 |
+
"X": {
|
| 258 |
+
"samples": 7,
|
| 259 |
+
"matched": 7,
|
| 260 |
+
"oversplit": 0,
|
| 261 |
+
"overmerge": 0,
|
| 262 |
+
"grouping_recall": 1.0
|
| 263 |
+
},
|
| 264 |
+
"\\times": {
|
| 265 |
+
"samples": 44,
|
| 266 |
+
"matched": 44,
|
| 267 |
+
"oversplit": 0,
|
| 268 |
+
"overmerge": 0,
|
| 269 |
+
"grouping_recall": 1.0
|
| 270 |
+
}
|
| 271 |
+
},
|
| 272 |
+
"families": {
|
| 273 |
+
"total": 59,
|
| 274 |
+
"matched": 51,
|
| 275 |
+
"grouping_recall": 0.864406779661017,
|
| 276 |
+
"by_label": {
|
| 277 |
+
"\\sum": {
|
| 278 |
+
"samples": 11,
|
| 279 |
+
"matched": 8,
|
| 280 |
+
"grouping_recall": 0.7272727272727273
|
| 281 |
+
},
|
| 282 |
+
"\\pi": {
|
| 283 |
+
"samples": 17,
|
| 284 |
+
"matched": 15,
|
| 285 |
+
"grouping_recall": 0.8823529411764706
|
| 286 |
+
},
|
| 287 |
+
"\\rightarrow": {
|
| 288 |
+
"samples": 20,
|
| 289 |
+
"matched": 17,
|
| 290 |
+
"grouping_recall": 0.85
|
| 291 |
+
},
|
| 292 |
+
"\\neq": {
|
| 293 |
+
"samples": 2,
|
| 294 |
+
"matched": 2,
|
| 295 |
+
"grouping_recall": 1.0
|
| 296 |
+
},
|
| 297 |
+
"\\pm": {
|
| 298 |
+
"samples": 9,
|
| 299 |
+
"matched": 9,
|
| 300 |
+
"grouping_recall": 1.0
|
| 301 |
+
}
|
| 302 |
+
}
|
| 303 |
+
}
|
| 304 |
+
},
|
| 305 |
+
{
|
| 306 |
+
"threshold": 0.3,
|
| 307 |
+
"weight": 2.0,
|
| 308 |
+
"global": {
|
| 309 |
+
"formulas": 275,
|
| 310 |
+
"exact_partition": 0.6872727272727273,
|
| 311 |
+
"exact_group_recall": 0.9567383324593602,
|
| 312 |
+
"overmerge_rate": 0.12727272727272726,
|
| 313 |
+
"oversplit_rate": 0.22181818181818183,
|
| 314 |
+
"pair_precision": 0.9662650602409638,
|
| 315 |
+
"pair_recall": 0.9024756189047262,
|
| 316 |
+
"pair_f1": 0.9332816136539953
|
| 317 |
+
},
|
| 318 |
+
"behavior_targets": {
|
| 319 |
+
"x": {
|
| 320 |
+
"samples": 277,
|
| 321 |
+
"matched": 244,
|
| 322 |
+
"oversplit": 31,
|
| 323 |
+
"overmerge": 3,
|
| 324 |
+
"grouping_recall": 0.8808664259927798
|
| 325 |
+
},
|
| 326 |
+
"X": {
|
| 327 |
+
"samples": 7,
|
| 328 |
+
"matched": 7,
|
| 329 |
+
"oversplit": 0,
|
| 330 |
+
"overmerge": 0,
|
| 331 |
+
"grouping_recall": 1.0
|
| 332 |
+
},
|
| 333 |
+
"\\times": {
|
| 334 |
+
"samples": 44,
|
| 335 |
+
"matched": 44,
|
| 336 |
+
"oversplit": 0,
|
| 337 |
+
"overmerge": 0,
|
| 338 |
+
"grouping_recall": 1.0
|
| 339 |
+
}
|
| 340 |
+
},
|
| 341 |
+
"families": {
|
| 342 |
+
"total": 59,
|
| 343 |
+
"matched": 50,
|
| 344 |
+
"grouping_recall": 0.847457627118644,
|
| 345 |
+
"by_label": {
|
| 346 |
+
"\\sum": {
|
| 347 |
+
"samples": 11,
|
| 348 |
+
"matched": 8,
|
| 349 |
+
"grouping_recall": 0.7272727272727273
|
| 350 |
+
},
|
| 351 |
+
"\\pi": {
|
| 352 |
+
"samples": 17,
|
| 353 |
+
"matched": 15,
|
| 354 |
+
"grouping_recall": 0.8823529411764706
|
| 355 |
+
},
|
| 356 |
+
"\\rightarrow": {
|
| 357 |
+
"samples": 20,
|
| 358 |
+
"matched": 16,
|
| 359 |
+
"grouping_recall": 0.8
|
| 360 |
+
},
|
| 361 |
+
"\\neq": {
|
| 362 |
+
"samples": 2,
|
| 363 |
+
"matched": 2,
|
| 364 |
+
"grouping_recall": 1.0
|
| 365 |
+
},
|
| 366 |
+
"\\pm": {
|
| 367 |
+
"samples": 9,
|
| 368 |
+
"matched": 9,
|
| 369 |
+
"grouping_recall": 1.0
|
| 370 |
+
}
|
| 371 |
+
}
|
| 372 |
+
}
|
| 373 |
+
},
|
| 374 |
+
{
|
| 375 |
+
"threshold": 0.3,
|
| 376 |
+
"weight": 4.0,
|
| 377 |
+
"global": {
|
| 378 |
+
"formulas": 275,
|
| 379 |
+
"exact_partition": 0.6909090909090909,
|
| 380 |
+
"exact_group_recall": 0.9572627163083377,
|
| 381 |
+
"overmerge_rate": 0.12363636363636364,
|
| 382 |
+
"oversplit_rate": 0.21818181818181817,
|
| 383 |
+
"pair_precision": 0.9670682730923694,
|
| 384 |
+
"pair_recall": 0.9032258064516129,
|
| 385 |
+
"pair_f1": 0.9340574088440652
|
| 386 |
+
},
|
| 387 |
+
"behavior_targets": {
|
| 388 |
+
"x": {
|
| 389 |
+
"samples": 277,
|
| 390 |
+
"matched": 244,
|
| 391 |
+
"oversplit": 31,
|
| 392 |
+
"overmerge": 3,
|
| 393 |
+
"grouping_recall": 0.8808664259927798
|
| 394 |
+
},
|
| 395 |
+
"X": {
|
| 396 |
+
"samples": 7,
|
| 397 |
+
"matched": 7,
|
| 398 |
+
"oversplit": 0,
|
| 399 |
+
"overmerge": 0,
|
| 400 |
+
"grouping_recall": 1.0
|
| 401 |
+
},
|
| 402 |
+
"\\times": {
|
| 403 |
+
"samples": 44,
|
| 404 |
+
"matched": 44,
|
| 405 |
+
"oversplit": 0,
|
| 406 |
+
"overmerge": 0,
|
| 407 |
+
"grouping_recall": 1.0
|
| 408 |
+
}
|
| 409 |
+
},
|
| 410 |
+
"families": {
|
| 411 |
+
"total": 59,
|
| 412 |
+
"matched": 50,
|
| 413 |
+
"grouping_recall": 0.847457627118644,
|
| 414 |
+
"by_label": {
|
| 415 |
+
"\\sum": {
|
| 416 |
+
"samples": 11,
|
| 417 |
+
"matched": 8,
|
| 418 |
+
"grouping_recall": 0.7272727272727273
|
| 419 |
+
},
|
| 420 |
+
"\\pi": {
|
| 421 |
+
"samples": 17,
|
| 422 |
+
"matched": 15,
|
| 423 |
+
"grouping_recall": 0.8823529411764706
|
| 424 |
+
},
|
| 425 |
+
"\\rightarrow": {
|
| 426 |
+
"samples": 20,
|
| 427 |
+
"matched": 16,
|
| 428 |
+
"grouping_recall": 0.8
|
| 429 |
+
},
|
| 430 |
+
"\\neq": {
|
| 431 |
+
"samples": 2,
|
| 432 |
+
"matched": 2,
|
| 433 |
+
"grouping_recall": 1.0
|
| 434 |
+
},
|
| 435 |
+
"\\pm": {
|
| 436 |
+
"samples": 9,
|
| 437 |
+
"matched": 9,
|
| 438 |
+
"grouping_recall": 1.0
|
| 439 |
+
}
|
| 440 |
+
}
|
| 441 |
+
}
|
| 442 |
+
},
|
| 443 |
+
{
|
| 444 |
+
"threshold": 0.3,
|
| 445 |
+
"weight": 6.0,
|
| 446 |
+
"global": {
|
| 447 |
+
"formulas": 275,
|
| 448 |
+
"exact_partition": 0.6872727272727273,
|
| 449 |
+
"exact_group_recall": 0.9575249082328264,
|
| 450 |
+
"overmerge_rate": 0.11636363636363636,
|
| 451 |
+
"oversplit_rate": 0.2290909090909091,
|
| 452 |
+
"pair_precision": 0.9692307692307692,
|
| 453 |
+
"pair_recall": 0.8979744936234059,
|
| 454 |
+
"pair_f1": 0.9322429906542056
|
| 455 |
+
},
|
| 456 |
+
"behavior_targets": {
|
| 457 |
+
"x": {
|
| 458 |
+
"samples": 277,
|
| 459 |
+
"matched": 243,
|
| 460 |
+
"oversplit": 33,
|
| 461 |
+
"overmerge": 2,
|
| 462 |
+
"grouping_recall": 0.8772563176895307
|
| 463 |
+
},
|
| 464 |
+
"X": {
|
| 465 |
+
"samples": 7,
|
| 466 |
+
"matched": 7,
|
| 467 |
+
"oversplit": 0,
|
| 468 |
+
"overmerge": 0,
|
| 469 |
+
"grouping_recall": 1.0
|
| 470 |
+
},
|
| 471 |
+
"\\times": {
|
| 472 |
+
"samples": 44,
|
| 473 |
+
"matched": 44,
|
| 474 |
+
"oversplit": 0,
|
| 475 |
+
"overmerge": 0,
|
| 476 |
+
"grouping_recall": 1.0
|
| 477 |
+
}
|
| 478 |
+
},
|
| 479 |
+
"families": {
|
| 480 |
+
"total": 59,
|
| 481 |
+
"matched": 50,
|
| 482 |
+
"grouping_recall": 0.847457627118644,
|
| 483 |
+
"by_label": {
|
| 484 |
+
"\\sum": {
|
| 485 |
+
"samples": 11,
|
| 486 |
+
"matched": 8,
|
| 487 |
+
"grouping_recall": 0.7272727272727273
|
| 488 |
+
},
|
| 489 |
+
"\\pi": {
|
| 490 |
+
"samples": 17,
|
| 491 |
+
"matched": 15,
|
| 492 |
+
"grouping_recall": 0.8823529411764706
|
| 493 |
+
},
|
| 494 |
+
"\\rightarrow": {
|
| 495 |
+
"samples": 20,
|
| 496 |
+
"matched": 16,
|
| 497 |
+
"grouping_recall": 0.8
|
| 498 |
+
},
|
| 499 |
+
"\\neq": {
|
| 500 |
+
"samples": 2,
|
| 501 |
+
"matched": 2,
|
| 502 |
+
"grouping_recall": 1.0
|
| 503 |
+
},
|
| 504 |
+
"\\pm": {
|
| 505 |
+
"samples": 9,
|
| 506 |
+
"matched": 9,
|
| 507 |
+
"grouping_recall": 1.0
|
| 508 |
+
}
|
| 509 |
+
}
|
| 510 |
+
}
|
| 511 |
+
},
|
| 512 |
+
{
|
| 513 |
+
"threshold": 0.3,
|
| 514 |
+
"weight": 8.0,
|
| 515 |
+
"global": {
|
| 516 |
+
"formulas": 275,
|
| 517 |
+
"exact_partition": 0.6872727272727273,
|
| 518 |
+
"exact_group_recall": 0.9575249082328264,
|
| 519 |
+
"overmerge_rate": 0.11636363636363636,
|
| 520 |
+
"oversplit_rate": 0.2290909090909091,
|
| 521 |
+
"pair_precision": 0.9692307692307692,
|
| 522 |
+
"pair_recall": 0.8979744936234059,
|
| 523 |
+
"pair_f1": 0.9322429906542056
|
| 524 |
+
},
|
| 525 |
+
"behavior_targets": {
|
| 526 |
+
"x": {
|
| 527 |
+
"samples": 277,
|
| 528 |
+
"matched": 243,
|
| 529 |
+
"oversplit": 33,
|
| 530 |
+
"overmerge": 2,
|
| 531 |
+
"grouping_recall": 0.8772563176895307
|
| 532 |
+
},
|
| 533 |
+
"X": {
|
| 534 |
+
"samples": 7,
|
| 535 |
+
"matched": 7,
|
| 536 |
+
"oversplit": 0,
|
| 537 |
+
"overmerge": 0,
|
| 538 |
+
"grouping_recall": 1.0
|
| 539 |
+
},
|
| 540 |
+
"\\times": {
|
| 541 |
+
"samples": 44,
|
| 542 |
+
"matched": 44,
|
| 543 |
+
"oversplit": 0,
|
| 544 |
+
"overmerge": 0,
|
| 545 |
+
"grouping_recall": 1.0
|
| 546 |
+
}
|
| 547 |
+
},
|
| 548 |
+
"families": {
|
| 549 |
+
"total": 59,
|
| 550 |
+
"matched": 50,
|
| 551 |
+
"grouping_recall": 0.847457627118644,
|
| 552 |
+
"by_label": {
|
| 553 |
+
"\\sum": {
|
| 554 |
+
"samples": 11,
|
| 555 |
+
"matched": 8,
|
| 556 |
+
"grouping_recall": 0.7272727272727273
|
| 557 |
+
},
|
| 558 |
+
"\\pi": {
|
| 559 |
+
"samples": 17,
|
| 560 |
+
"matched": 15,
|
| 561 |
+
"grouping_recall": 0.8823529411764706
|
| 562 |
+
},
|
| 563 |
+
"\\rightarrow": {
|
| 564 |
+
"samples": 20,
|
| 565 |
+
"matched": 16,
|
| 566 |
+
"grouping_recall": 0.8
|
| 567 |
+
},
|
| 568 |
+
"\\neq": {
|
| 569 |
+
"samples": 2,
|
| 570 |
+
"matched": 2,
|
| 571 |
+
"grouping_recall": 1.0
|
| 572 |
+
},
|
| 573 |
+
"\\pm": {
|
| 574 |
+
"samples": 9,
|
| 575 |
+
"matched": 9,
|
| 576 |
+
"grouping_recall": 1.0
|
| 577 |
+
}
|
| 578 |
+
}
|
| 579 |
+
}
|
| 580 |
+
},
|
| 581 |
+
{
|
| 582 |
+
"threshold": 0.3,
|
| 583 |
+
"weight": 12.0,
|
| 584 |
+
"global": {
|
| 585 |
+
"formulas": 275,
|
| 586 |
+
"exact_partition": 0.6836363636363636,
|
| 587 |
+
"exact_group_recall": 0.9572627163083377,
|
| 588 |
+
"overmerge_rate": 0.11636363636363636,
|
| 589 |
+
"oversplit_rate": 0.23272727272727273,
|
| 590 |
+
"pair_precision": 0.9691808596918086,
|
| 591 |
+
"pair_recall": 0.8964741185296324,
|
| 592 |
+
"pair_f1": 0.93141075604053
|
| 593 |
+
},
|
| 594 |
+
"behavior_targets": {
|
| 595 |
+
"x": {
|
| 596 |
+
"samples": 277,
|
| 597 |
+
"matched": 243,
|
| 598 |
+
"oversplit": 33,
|
| 599 |
+
"overmerge": 2,
|
| 600 |
+
"grouping_recall": 0.8772563176895307
|
| 601 |
+
},
|
| 602 |
+
"X": {
|
| 603 |
+
"samples": 7,
|
| 604 |
+
"matched": 7,
|
| 605 |
+
"oversplit": 0,
|
| 606 |
+
"overmerge": 0,
|
| 607 |
+
"grouping_recall": 1.0
|
| 608 |
+
},
|
| 609 |
+
"\\times": {
|
| 610 |
+
"samples": 44,
|
| 611 |
+
"matched": 44,
|
| 612 |
+
"oversplit": 0,
|
| 613 |
+
"overmerge": 0,
|
| 614 |
+
"grouping_recall": 1.0
|
| 615 |
+
}
|
| 616 |
+
},
|
| 617 |
+
"families": {
|
| 618 |
+
"total": 59,
|
| 619 |
+
"matched": 49,
|
| 620 |
+
"grouping_recall": 0.8305084745762712,
|
| 621 |
+
"by_label": {
|
| 622 |
+
"\\sum": {
|
| 623 |
+
"samples": 11,
|
| 624 |
+
"matched": 8,
|
| 625 |
+
"grouping_recall": 0.7272727272727273
|
| 626 |
+
},
|
| 627 |
+
"\\pi": {
|
| 628 |
+
"samples": 17,
|
| 629 |
+
"matched": 14,
|
| 630 |
+
"grouping_recall": 0.8235294117647058
|
| 631 |
+
},
|
| 632 |
+
"\\rightarrow": {
|
| 633 |
+
"samples": 20,
|
| 634 |
+
"matched": 16,
|
| 635 |
+
"grouping_recall": 0.8
|
| 636 |
+
},
|
| 637 |
+
"\\neq": {
|
| 638 |
+
"samples": 2,
|
| 639 |
+
"matched": 2,
|
| 640 |
+
"grouping_recall": 1.0
|
| 641 |
+
},
|
| 642 |
+
"\\pm": {
|
| 643 |
+
"samples": 9,
|
| 644 |
+
"matched": 9,
|
| 645 |
+
"grouping_recall": 1.0
|
| 646 |
+
}
|
| 647 |
+
}
|
| 648 |
+
}
|
| 649 |
+
},
|
| 650 |
+
{
|
| 651 |
+
"threshold": 0.5,
|
| 652 |
+
"weight": 0.0,
|
| 653 |
+
"global": {
|
| 654 |
+
"formulas": 275,
|
| 655 |
+
"exact_partition": 0.68,
|
| 656 |
+
"exact_group_recall": 0.9528054535920294,
|
| 657 |
+
"overmerge_rate": 0.14909090909090908,
|
| 658 |
+
"oversplit_rate": 0.21818181818181817,
|
| 659 |
+
"pair_precision": 0.9563838223632039,
|
| 660 |
+
"pair_recall": 0.9047261815453863,
|
| 661 |
+
"pair_f1": 0.9298380878951427
|
| 662 |
+
},
|
| 663 |
+
"behavior_targets": {
|
| 664 |
+
"x": {
|
| 665 |
+
"samples": 277,
|
| 666 |
+
"matched": 244,
|
| 667 |
+
"oversplit": 31,
|
| 668 |
+
"overmerge": 3,
|
| 669 |
+
"grouping_recall": 0.8808664259927798
|
| 670 |
+
},
|
| 671 |
+
"X": {
|
| 672 |
+
"samples": 7,
|
| 673 |
+
"matched": 7,
|
| 674 |
+
"oversplit": 0,
|
| 675 |
+
"overmerge": 0,
|
| 676 |
+
"grouping_recall": 1.0
|
| 677 |
+
},
|
| 678 |
+
"\\times": {
|
| 679 |
+
"samples": 44,
|
| 680 |
+
"matched": 44,
|
| 681 |
+
"oversplit": 0,
|
| 682 |
+
"overmerge": 0,
|
| 683 |
+
"grouping_recall": 1.0
|
| 684 |
+
}
|
| 685 |
+
},
|
| 686 |
+
"families": {
|
| 687 |
+
"total": 59,
|
| 688 |
+
"matched": 51,
|
| 689 |
+
"grouping_recall": 0.864406779661017,
|
| 690 |
+
"by_label": {
|
| 691 |
+
"\\sum": {
|
| 692 |
+
"samples": 11,
|
| 693 |
+
"matched": 8,
|
| 694 |
+
"grouping_recall": 0.7272727272727273
|
| 695 |
+
},
|
| 696 |
+
"\\pi": {
|
| 697 |
+
"samples": 17,
|
| 698 |
+
"matched": 15,
|
| 699 |
+
"grouping_recall": 0.8823529411764706
|
| 700 |
+
},
|
| 701 |
+
"\\rightarrow": {
|
| 702 |
+
"samples": 20,
|
| 703 |
+
"matched": 17,
|
| 704 |
+
"grouping_recall": 0.85
|
| 705 |
+
},
|
| 706 |
+
"\\neq": {
|
| 707 |
+
"samples": 2,
|
| 708 |
+
"matched": 2,
|
| 709 |
+
"grouping_recall": 1.0
|
| 710 |
+
},
|
| 711 |
+
"\\pm": {
|
| 712 |
+
"samples": 9,
|
| 713 |
+
"matched": 9,
|
| 714 |
+
"grouping_recall": 1.0
|
| 715 |
+
}
|
| 716 |
+
}
|
| 717 |
+
}
|
| 718 |
+
},
|
| 719 |
+
{
|
| 720 |
+
"threshold": 0.5,
|
| 721 |
+
"weight": 1.0,
|
| 722 |
+
"global": {
|
| 723 |
+
"formulas": 275,
|
| 724 |
+
"exact_partition": 0.6836363636363636,
|
| 725 |
+
"exact_group_recall": 0.9554273728369166,
|
| 726 |
+
"overmerge_rate": 0.14181818181818182,
|
| 727 |
+
"oversplit_rate": 0.21454545454545454,
|
| 728 |
+
"pair_precision": 0.9610492845786963,
|
| 729 |
+
"pair_recall": 0.9069767441860465,
|
| 730 |
+
"pair_f1": 0.933230412967966
|
| 731 |
+
},
|
| 732 |
+
"behavior_targets": {
|
| 733 |
+
"x": {
|
| 734 |
+
"samples": 277,
|
| 735 |
+
"matched": 244,
|
| 736 |
+
"oversplit": 31,
|
| 737 |
+
"overmerge": 3,
|
| 738 |
+
"grouping_recall": 0.8808664259927798
|
| 739 |
+
},
|
| 740 |
+
"X": {
|
| 741 |
+
"samples": 7,
|
| 742 |
+
"matched": 7,
|
| 743 |
+
"oversplit": 0,
|
| 744 |
+
"overmerge": 0,
|
| 745 |
+
"grouping_recall": 1.0
|
| 746 |
+
},
|
| 747 |
+
"\\times": {
|
| 748 |
+
"samples": 44,
|
| 749 |
+
"matched": 44,
|
| 750 |
+
"oversplit": 0,
|
| 751 |
+
"overmerge": 0,
|
| 752 |
+
"grouping_recall": 1.0
|
| 753 |
+
}
|
| 754 |
+
},
|
| 755 |
+
"families": {
|
| 756 |
+
"total": 59,
|
| 757 |
+
"matched": 51,
|
| 758 |
+
"grouping_recall": 0.864406779661017,
|
| 759 |
+
"by_label": {
|
| 760 |
+
"\\sum": {
|
| 761 |
+
"samples": 11,
|
| 762 |
+
"matched": 8,
|
| 763 |
+
"grouping_recall": 0.7272727272727273
|
| 764 |
+
},
|
| 765 |
+
"\\pi": {
|
| 766 |
+
"samples": 17,
|
| 767 |
+
"matched": 15,
|
| 768 |
+
"grouping_recall": 0.8823529411764706
|
| 769 |
+
},
|
| 770 |
+
"\\rightarrow": {
|
| 771 |
+
"samples": 20,
|
| 772 |
+
"matched": 17,
|
| 773 |
+
"grouping_recall": 0.85
|
| 774 |
+
},
|
| 775 |
+
"\\neq": {
|
| 776 |
+
"samples": 2,
|
| 777 |
+
"matched": 2,
|
| 778 |
+
"grouping_recall": 1.0
|
| 779 |
+
},
|
| 780 |
+
"\\pm": {
|
| 781 |
+
"samples": 9,
|
| 782 |
+
"matched": 9,
|
| 783 |
+
"grouping_recall": 1.0
|
| 784 |
+
}
|
| 785 |
+
}
|
| 786 |
+
}
|
| 787 |
+
},
|
| 788 |
+
{
|
| 789 |
+
"threshold": 0.5,
|
| 790 |
+
"weight": 2.0,
|
| 791 |
+
"global": {
|
| 792 |
+
"formulas": 275,
|
| 793 |
+
"exact_partition": 0.6872727272727273,
|
| 794 |
+
"exact_group_recall": 0.9562139486103828,
|
| 795 |
+
"overmerge_rate": 0.13454545454545455,
|
| 796 |
+
"oversplit_rate": 0.21454545454545454,
|
| 797 |
+
"pair_precision": 0.9648562300319489,
|
| 798 |
+
"pair_recall": 0.9062265566391597,
|
| 799 |
+
"pair_f1": 0.9346228239845261
|
| 800 |
+
},
|
| 801 |
+
"behavior_targets": {
|
| 802 |
+
"x": {
|
| 803 |
+
"samples": 277,
|
| 804 |
+
"matched": 244,
|
| 805 |
+
"oversplit": 31,
|
| 806 |
+
"overmerge": 3,
|
| 807 |
+
"grouping_recall": 0.8808664259927798
|
| 808 |
+
},
|
| 809 |
+
"X": {
|
| 810 |
+
"samples": 7,
|
| 811 |
+
"matched": 7,
|
| 812 |
+
"oversplit": 0,
|
| 813 |
+
"overmerge": 0,
|
| 814 |
+
"grouping_recall": 1.0
|
| 815 |
+
},
|
| 816 |
+
"\\times": {
|
| 817 |
+
"samples": 44,
|
| 818 |
+
"matched": 44,
|
| 819 |
+
"oversplit": 0,
|
| 820 |
+
"overmerge": 0,
|
| 821 |
+
"grouping_recall": 1.0
|
| 822 |
+
}
|
| 823 |
+
},
|
| 824 |
+
"families": {
|
| 825 |
+
"total": 59,
|
| 826 |
+
"matched": 51,
|
| 827 |
+
"grouping_recall": 0.864406779661017,
|
| 828 |
+
"by_label": {
|
| 829 |
+
"\\sum": {
|
| 830 |
+
"samples": 11,
|
| 831 |
+
"matched": 8,
|
| 832 |
+
"grouping_recall": 0.7272727272727273
|
| 833 |
+
},
|
| 834 |
+
"\\pi": {
|
| 835 |
+
"samples": 17,
|
| 836 |
+
"matched": 15,
|
| 837 |
+
"grouping_recall": 0.8823529411764706
|
| 838 |
+
},
|
| 839 |
+
"\\rightarrow": {
|
| 840 |
+
"samples": 20,
|
| 841 |
+
"matched": 17,
|
| 842 |
+
"grouping_recall": 0.85
|
| 843 |
+
},
|
| 844 |
+
"\\neq": {
|
| 845 |
+
"samples": 2,
|
| 846 |
+
"matched": 2,
|
| 847 |
+
"grouping_recall": 1.0
|
| 848 |
+
},
|
| 849 |
+
"\\pm": {
|
| 850 |
+
"samples": 9,
|
| 851 |
+
"matched": 9,
|
| 852 |
+
"grouping_recall": 1.0
|
| 853 |
+
}
|
| 854 |
+
}
|
| 855 |
+
}
|
| 856 |
+
},
|
| 857 |
+
{
|
| 858 |
+
"threshold": 0.5,
|
| 859 |
+
"weight": 4.0,
|
| 860 |
+
"global": {
|
| 861 |
+
"formulas": 275,
|
| 862 |
+
"exact_partition": 0.6909090909090909,
|
| 863 |
+
"exact_group_recall": 0.957000524383849,
|
| 864 |
+
"overmerge_rate": 0.12727272727272726,
|
| 865 |
+
"oversplit_rate": 0.21454545454545454,
|
| 866 |
+
"pair_precision": 0.9664,
|
| 867 |
+
"pair_recall": 0.9062265566391597,
|
| 868 |
+
"pair_f1": 0.9353464963221061
|
| 869 |
+
},
|
| 870 |
+
"behavior_targets": {
|
| 871 |
+
"x": {
|
| 872 |
+
"samples": 277,
|
| 873 |
+
"matched": 244,
|
| 874 |
+
"oversplit": 31,
|
| 875 |
+
"overmerge": 3,
|
| 876 |
+
"grouping_recall": 0.8808664259927798
|
| 877 |
+
},
|
| 878 |
+
"X": {
|
| 879 |
+
"samples": 7,
|
| 880 |
+
"matched": 7,
|
| 881 |
+
"oversplit": 0,
|
| 882 |
+
"overmerge": 0,
|
| 883 |
+
"grouping_recall": 1.0
|
| 884 |
+
},
|
| 885 |
+
"\\times": {
|
| 886 |
+
"samples": 44,
|
| 887 |
+
"matched": 44,
|
| 888 |
+
"oversplit": 0,
|
| 889 |
+
"overmerge": 0,
|
| 890 |
+
"grouping_recall": 1.0
|
| 891 |
+
}
|
| 892 |
+
},
|
| 893 |
+
"families": {
|
| 894 |
+
"total": 59,
|
| 895 |
+
"matched": 50,
|
| 896 |
+
"grouping_recall": 0.847457627118644,
|
| 897 |
+
"by_label": {
|
| 898 |
+
"\\sum": {
|
| 899 |
+
"samples": 11,
|
| 900 |
+
"matched": 8,
|
| 901 |
+
"grouping_recall": 0.7272727272727273
|
| 902 |
+
},
|
| 903 |
+
"\\pi": {
|
| 904 |
+
"samples": 17,
|
| 905 |
+
"matched": 15,
|
| 906 |
+
"grouping_recall": 0.8823529411764706
|
| 907 |
+
},
|
| 908 |
+
"\\rightarrow": {
|
| 909 |
+
"samples": 20,
|
| 910 |
+
"matched": 16,
|
| 911 |
+
"grouping_recall": 0.8
|
| 912 |
+
},
|
| 913 |
+
"\\neq": {
|
| 914 |
+
"samples": 2,
|
| 915 |
+
"matched": 2,
|
| 916 |
+
"grouping_recall": 1.0
|
| 917 |
+
},
|
| 918 |
+
"\\pm": {
|
| 919 |
+
"samples": 9,
|
| 920 |
+
"matched": 9,
|
| 921 |
+
"grouping_recall": 1.0
|
| 922 |
+
}
|
| 923 |
+
}
|
| 924 |
+
}
|
| 925 |
+
},
|
| 926 |
+
{
|
| 927 |
+
"threshold": 0.5,
|
| 928 |
+
"weight": 6.0,
|
| 929 |
+
"global": {
|
| 930 |
+
"formulas": 275,
|
| 931 |
+
"exact_partition": 0.6981818181818182,
|
| 932 |
+
"exact_group_recall": 0.9580492920818039,
|
| 933 |
+
"overmerge_rate": 0.12,
|
| 934 |
+
"oversplit_rate": 0.21454545454545454,
|
| 935 |
+
"pair_precision": 0.9687249398556536,
|
| 936 |
+
"pair_recall": 0.9062265566391597,
|
| 937 |
+
"pair_f1": 0.9364341085271317
|
| 938 |
+
},
|
| 939 |
+
"behavior_targets": {
|
| 940 |
+
"x": {
|
| 941 |
+
"samples": 277,
|
| 942 |
+
"matched": 245,
|
| 943 |
+
"oversplit": 31,
|
| 944 |
+
"overmerge": 2,
|
| 945 |
+
"grouping_recall": 0.8844765342960289
|
| 946 |
+
},
|
| 947 |
+
"X": {
|
| 948 |
+
"samples": 7,
|
| 949 |
+
"matched": 7,
|
| 950 |
+
"oversplit": 0,
|
| 951 |
+
"overmerge": 0,
|
| 952 |
+
"grouping_recall": 1.0
|
| 953 |
+
},
|
| 954 |
+
"\\times": {
|
| 955 |
+
"samples": 44,
|
| 956 |
+
"matched": 44,
|
| 957 |
+
"oversplit": 0,
|
| 958 |
+
"overmerge": 0,
|
| 959 |
+
"grouping_recall": 1.0
|
| 960 |
+
}
|
| 961 |
+
},
|
| 962 |
+
"families": {
|
| 963 |
+
"total": 59,
|
| 964 |
+
"matched": 50,
|
| 965 |
+
"grouping_recall": 0.847457627118644,
|
| 966 |
+
"by_label": {
|
| 967 |
+
"\\sum": {
|
| 968 |
+
"samples": 11,
|
| 969 |
+
"matched": 8,
|
| 970 |
+
"grouping_recall": 0.7272727272727273
|
| 971 |
+
},
|
| 972 |
+
"\\pi": {
|
| 973 |
+
"samples": 17,
|
| 974 |
+
"matched": 15,
|
| 975 |
+
"grouping_recall": 0.8823529411764706
|
| 976 |
+
},
|
| 977 |
+
"\\rightarrow": {
|
| 978 |
+
"samples": 20,
|
| 979 |
+
"matched": 16,
|
| 980 |
+
"grouping_recall": 0.8
|
| 981 |
+
},
|
| 982 |
+
"\\neq": {
|
| 983 |
+
"samples": 2,
|
| 984 |
+
"matched": 2,
|
| 985 |
+
"grouping_recall": 1.0
|
| 986 |
+
},
|
| 987 |
+
"\\pm": {
|
| 988 |
+
"samples": 9,
|
| 989 |
+
"matched": 9,
|
| 990 |
+
"grouping_recall": 1.0
|
| 991 |
+
}
|
| 992 |
+
}
|
| 993 |
+
}
|
| 994 |
+
},
|
| 995 |
+
{
|
| 996 |
+
"threshold": 0.5,
|
| 997 |
+
"weight": 8.0,
|
| 998 |
+
"global": {
|
| 999 |
+
"formulas": 275,
|
| 1000 |
+
"exact_partition": 0.6981818181818182,
|
| 1001 |
+
"exact_group_recall": 0.9580492920818039,
|
| 1002 |
+
"overmerge_rate": 0.12,
|
| 1003 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1004 |
+
"pair_precision": 0.9687249398556536,
|
| 1005 |
+
"pair_recall": 0.9062265566391597,
|
| 1006 |
+
"pair_f1": 0.9364341085271317
|
| 1007 |
+
},
|
| 1008 |
+
"behavior_targets": {
|
| 1009 |
+
"x": {
|
| 1010 |
+
"samples": 277,
|
| 1011 |
+
"matched": 245,
|
| 1012 |
+
"oversplit": 31,
|
| 1013 |
+
"overmerge": 2,
|
| 1014 |
+
"grouping_recall": 0.8844765342960289
|
| 1015 |
+
},
|
| 1016 |
+
"X": {
|
| 1017 |
+
"samples": 7,
|
| 1018 |
+
"matched": 7,
|
| 1019 |
+
"oversplit": 0,
|
| 1020 |
+
"overmerge": 0,
|
| 1021 |
+
"grouping_recall": 1.0
|
| 1022 |
+
},
|
| 1023 |
+
"\\times": {
|
| 1024 |
+
"samples": 44,
|
| 1025 |
+
"matched": 44,
|
| 1026 |
+
"oversplit": 0,
|
| 1027 |
+
"overmerge": 0,
|
| 1028 |
+
"grouping_recall": 1.0
|
| 1029 |
+
}
|
| 1030 |
+
},
|
| 1031 |
+
"families": {
|
| 1032 |
+
"total": 59,
|
| 1033 |
+
"matched": 50,
|
| 1034 |
+
"grouping_recall": 0.847457627118644,
|
| 1035 |
+
"by_label": {
|
| 1036 |
+
"\\sum": {
|
| 1037 |
+
"samples": 11,
|
| 1038 |
+
"matched": 8,
|
| 1039 |
+
"grouping_recall": 0.7272727272727273
|
| 1040 |
+
},
|
| 1041 |
+
"\\pi": {
|
| 1042 |
+
"samples": 17,
|
| 1043 |
+
"matched": 15,
|
| 1044 |
+
"grouping_recall": 0.8823529411764706
|
| 1045 |
+
},
|
| 1046 |
+
"\\rightarrow": {
|
| 1047 |
+
"samples": 20,
|
| 1048 |
+
"matched": 16,
|
| 1049 |
+
"grouping_recall": 0.8
|
| 1050 |
+
},
|
| 1051 |
+
"\\neq": {
|
| 1052 |
+
"samples": 2,
|
| 1053 |
+
"matched": 2,
|
| 1054 |
+
"grouping_recall": 1.0
|
| 1055 |
+
},
|
| 1056 |
+
"\\pm": {
|
| 1057 |
+
"samples": 9,
|
| 1058 |
+
"matched": 9,
|
| 1059 |
+
"grouping_recall": 1.0
|
| 1060 |
+
}
|
| 1061 |
+
}
|
| 1062 |
+
}
|
| 1063 |
+
},
|
| 1064 |
+
{
|
| 1065 |
+
"threshold": 0.5,
|
| 1066 |
+
"weight": 12.0,
|
| 1067 |
+
"global": {
|
| 1068 |
+
"formulas": 275,
|
| 1069 |
+
"exact_partition": 0.6909090909090909,
|
| 1070 |
+
"exact_group_recall": 0.9575249082328264,
|
| 1071 |
+
"overmerge_rate": 0.12,
|
| 1072 |
+
"oversplit_rate": 0.22181818181818183,
|
| 1073 |
+
"pair_precision": 0.9686495176848875,
|
| 1074 |
+
"pair_recall": 0.9039759939984996,
|
| 1075 |
+
"pair_f1": 0.9351959642995732
|
| 1076 |
+
},
|
| 1077 |
+
"behavior_targets": {
|
| 1078 |
+
"x": {
|
| 1079 |
+
"samples": 277,
|
| 1080 |
+
"matched": 244,
|
| 1081 |
+
"oversplit": 32,
|
| 1082 |
+
"overmerge": 2,
|
| 1083 |
+
"grouping_recall": 0.8808664259927798
|
| 1084 |
+
},
|
| 1085 |
+
"X": {
|
| 1086 |
+
"samples": 7,
|
| 1087 |
+
"matched": 7,
|
| 1088 |
+
"oversplit": 0,
|
| 1089 |
+
"overmerge": 0,
|
| 1090 |
+
"grouping_recall": 1.0
|
| 1091 |
+
},
|
| 1092 |
+
"\\times": {
|
| 1093 |
+
"samples": 44,
|
| 1094 |
+
"matched": 44,
|
| 1095 |
+
"oversplit": 0,
|
| 1096 |
+
"overmerge": 0,
|
| 1097 |
+
"grouping_recall": 1.0
|
| 1098 |
+
}
|
| 1099 |
+
},
|
| 1100 |
+
"families": {
|
| 1101 |
+
"total": 59,
|
| 1102 |
+
"matched": 49,
|
| 1103 |
+
"grouping_recall": 0.8305084745762712,
|
| 1104 |
+
"by_label": {
|
| 1105 |
+
"\\sum": {
|
| 1106 |
+
"samples": 11,
|
| 1107 |
+
"matched": 8,
|
| 1108 |
+
"grouping_recall": 0.7272727272727273
|
| 1109 |
+
},
|
| 1110 |
+
"\\pi": {
|
| 1111 |
+
"samples": 17,
|
| 1112 |
+
"matched": 14,
|
| 1113 |
+
"grouping_recall": 0.8235294117647058
|
| 1114 |
+
},
|
| 1115 |
+
"\\rightarrow": {
|
| 1116 |
+
"samples": 20,
|
| 1117 |
+
"matched": 16,
|
| 1118 |
+
"grouping_recall": 0.8
|
| 1119 |
+
},
|
| 1120 |
+
"\\neq": {
|
| 1121 |
+
"samples": 2,
|
| 1122 |
+
"matched": 2,
|
| 1123 |
+
"grouping_recall": 1.0
|
| 1124 |
+
},
|
| 1125 |
+
"\\pm": {
|
| 1126 |
+
"samples": 9,
|
| 1127 |
+
"matched": 9,
|
| 1128 |
+
"grouping_recall": 1.0
|
| 1129 |
+
}
|
| 1130 |
+
}
|
| 1131 |
+
}
|
| 1132 |
+
},
|
| 1133 |
+
{
|
| 1134 |
+
"threshold": 0.65,
|
| 1135 |
+
"weight": 0.0,
|
| 1136 |
+
"global": {
|
| 1137 |
+
"formulas": 275,
|
| 1138 |
+
"exact_partition": 0.68,
|
| 1139 |
+
"exact_group_recall": 0.9528054535920294,
|
| 1140 |
+
"overmerge_rate": 0.14909090909090908,
|
| 1141 |
+
"oversplit_rate": 0.21818181818181817,
|
| 1142 |
+
"pair_precision": 0.9563838223632039,
|
| 1143 |
+
"pair_recall": 0.9047261815453863,
|
| 1144 |
+
"pair_f1": 0.9298380878951427
|
| 1145 |
+
},
|
| 1146 |
+
"behavior_targets": {
|
| 1147 |
+
"x": {
|
| 1148 |
+
"samples": 277,
|
| 1149 |
+
"matched": 244,
|
| 1150 |
+
"oversplit": 31,
|
| 1151 |
+
"overmerge": 3,
|
| 1152 |
+
"grouping_recall": 0.8808664259927798
|
| 1153 |
+
},
|
| 1154 |
+
"X": {
|
| 1155 |
+
"samples": 7,
|
| 1156 |
+
"matched": 7,
|
| 1157 |
+
"oversplit": 0,
|
| 1158 |
+
"overmerge": 0,
|
| 1159 |
+
"grouping_recall": 1.0
|
| 1160 |
+
},
|
| 1161 |
+
"\\times": {
|
| 1162 |
+
"samples": 44,
|
| 1163 |
+
"matched": 44,
|
| 1164 |
+
"oversplit": 0,
|
| 1165 |
+
"overmerge": 0,
|
| 1166 |
+
"grouping_recall": 1.0
|
| 1167 |
+
}
|
| 1168 |
+
},
|
| 1169 |
+
"families": {
|
| 1170 |
+
"total": 59,
|
| 1171 |
+
"matched": 51,
|
| 1172 |
+
"grouping_recall": 0.864406779661017,
|
| 1173 |
+
"by_label": {
|
| 1174 |
+
"\\sum": {
|
| 1175 |
+
"samples": 11,
|
| 1176 |
+
"matched": 8,
|
| 1177 |
+
"grouping_recall": 0.7272727272727273
|
| 1178 |
+
},
|
| 1179 |
+
"\\pi": {
|
| 1180 |
+
"samples": 17,
|
| 1181 |
+
"matched": 15,
|
| 1182 |
+
"grouping_recall": 0.8823529411764706
|
| 1183 |
+
},
|
| 1184 |
+
"\\rightarrow": {
|
| 1185 |
+
"samples": 20,
|
| 1186 |
+
"matched": 17,
|
| 1187 |
+
"grouping_recall": 0.85
|
| 1188 |
+
},
|
| 1189 |
+
"\\neq": {
|
| 1190 |
+
"samples": 2,
|
| 1191 |
+
"matched": 2,
|
| 1192 |
+
"grouping_recall": 1.0
|
| 1193 |
+
},
|
| 1194 |
+
"\\pm": {
|
| 1195 |
+
"samples": 9,
|
| 1196 |
+
"matched": 9,
|
| 1197 |
+
"grouping_recall": 1.0
|
| 1198 |
+
}
|
| 1199 |
+
}
|
| 1200 |
+
}
|
| 1201 |
+
},
|
| 1202 |
+
{
|
| 1203 |
+
"threshold": 0.65,
|
| 1204 |
+
"weight": 1.0,
|
| 1205 |
+
"global": {
|
| 1206 |
+
"formulas": 275,
|
| 1207 |
+
"exact_partition": 0.68,
|
| 1208 |
+
"exact_group_recall": 0.9543786051389617,
|
| 1209 |
+
"overmerge_rate": 0.14545454545454545,
|
| 1210 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1211 |
+
"pair_precision": 0.9594916600476568,
|
| 1212 |
+
"pair_recall": 0.9062265566391597,
|
| 1213 |
+
"pair_f1": 0.9320987654320987
|
| 1214 |
+
},
|
| 1215 |
+
"behavior_targets": {
|
| 1216 |
+
"x": {
|
| 1217 |
+
"samples": 277,
|
| 1218 |
+
"matched": 244,
|
| 1219 |
+
"oversplit": 31,
|
| 1220 |
+
"overmerge": 3,
|
| 1221 |
+
"grouping_recall": 0.8808664259927798
|
| 1222 |
+
},
|
| 1223 |
+
"X": {
|
| 1224 |
+
"samples": 7,
|
| 1225 |
+
"matched": 7,
|
| 1226 |
+
"oversplit": 0,
|
| 1227 |
+
"overmerge": 0,
|
| 1228 |
+
"grouping_recall": 1.0
|
| 1229 |
+
},
|
| 1230 |
+
"\\times": {
|
| 1231 |
+
"samples": 44,
|
| 1232 |
+
"matched": 44,
|
| 1233 |
+
"oversplit": 0,
|
| 1234 |
+
"overmerge": 0,
|
| 1235 |
+
"grouping_recall": 1.0
|
| 1236 |
+
}
|
| 1237 |
+
},
|
| 1238 |
+
"families": {
|
| 1239 |
+
"total": 59,
|
| 1240 |
+
"matched": 51,
|
| 1241 |
+
"grouping_recall": 0.864406779661017,
|
| 1242 |
+
"by_label": {
|
| 1243 |
+
"\\sum": {
|
| 1244 |
+
"samples": 11,
|
| 1245 |
+
"matched": 8,
|
| 1246 |
+
"grouping_recall": 0.7272727272727273
|
| 1247 |
+
},
|
| 1248 |
+
"\\pi": {
|
| 1249 |
+
"samples": 17,
|
| 1250 |
+
"matched": 15,
|
| 1251 |
+
"grouping_recall": 0.8823529411764706
|
| 1252 |
+
},
|
| 1253 |
+
"\\rightarrow": {
|
| 1254 |
+
"samples": 20,
|
| 1255 |
+
"matched": 17,
|
| 1256 |
+
"grouping_recall": 0.85
|
| 1257 |
+
},
|
| 1258 |
+
"\\neq": {
|
| 1259 |
+
"samples": 2,
|
| 1260 |
+
"matched": 2,
|
| 1261 |
+
"grouping_recall": 1.0
|
| 1262 |
+
},
|
| 1263 |
+
"\\pm": {
|
| 1264 |
+
"samples": 9,
|
| 1265 |
+
"matched": 9,
|
| 1266 |
+
"grouping_recall": 1.0
|
| 1267 |
+
}
|
| 1268 |
+
}
|
| 1269 |
+
}
|
| 1270 |
+
},
|
| 1271 |
+
{
|
| 1272 |
+
"threshold": 0.65,
|
| 1273 |
+
"weight": 2.0,
|
| 1274 |
+
"global": {
|
| 1275 |
+
"formulas": 275,
|
| 1276 |
+
"exact_partition": 0.6836363636363636,
|
| 1277 |
+
"exact_group_recall": 0.9556895647614053,
|
| 1278 |
+
"overmerge_rate": 0.13818181818181818,
|
| 1279 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1280 |
+
"pair_precision": 0.9640861931364725,
|
| 1281 |
+
"pair_recall": 0.9062265566391597,
|
| 1282 |
+
"pair_f1": 0.934261407579273
|
| 1283 |
+
},
|
| 1284 |
+
"behavior_targets": {
|
| 1285 |
+
"x": {
|
| 1286 |
+
"samples": 277,
|
| 1287 |
+
"matched": 244,
|
| 1288 |
+
"oversplit": 31,
|
| 1289 |
+
"overmerge": 3,
|
| 1290 |
+
"grouping_recall": 0.8808664259927798
|
| 1291 |
+
},
|
| 1292 |
+
"X": {
|
| 1293 |
+
"samples": 7,
|
| 1294 |
+
"matched": 7,
|
| 1295 |
+
"oversplit": 0,
|
| 1296 |
+
"overmerge": 0,
|
| 1297 |
+
"grouping_recall": 1.0
|
| 1298 |
+
},
|
| 1299 |
+
"\\times": {
|
| 1300 |
+
"samples": 44,
|
| 1301 |
+
"matched": 44,
|
| 1302 |
+
"oversplit": 0,
|
| 1303 |
+
"overmerge": 0,
|
| 1304 |
+
"grouping_recall": 1.0
|
| 1305 |
+
}
|
| 1306 |
+
},
|
| 1307 |
+
"families": {
|
| 1308 |
+
"total": 59,
|
| 1309 |
+
"matched": 51,
|
| 1310 |
+
"grouping_recall": 0.864406779661017,
|
| 1311 |
+
"by_label": {
|
| 1312 |
+
"\\sum": {
|
| 1313 |
+
"samples": 11,
|
| 1314 |
+
"matched": 8,
|
| 1315 |
+
"grouping_recall": 0.7272727272727273
|
| 1316 |
+
},
|
| 1317 |
+
"\\pi": {
|
| 1318 |
+
"samples": 17,
|
| 1319 |
+
"matched": 15,
|
| 1320 |
+
"grouping_recall": 0.8823529411764706
|
| 1321 |
+
},
|
| 1322 |
+
"\\rightarrow": {
|
| 1323 |
+
"samples": 20,
|
| 1324 |
+
"matched": 17,
|
| 1325 |
+
"grouping_recall": 0.85
|
| 1326 |
+
},
|
| 1327 |
+
"\\neq": {
|
| 1328 |
+
"samples": 2,
|
| 1329 |
+
"matched": 2,
|
| 1330 |
+
"grouping_recall": 1.0
|
| 1331 |
+
},
|
| 1332 |
+
"\\pm": {
|
| 1333 |
+
"samples": 9,
|
| 1334 |
+
"matched": 9,
|
| 1335 |
+
"grouping_recall": 1.0
|
| 1336 |
+
}
|
| 1337 |
+
}
|
| 1338 |
+
}
|
| 1339 |
+
},
|
| 1340 |
+
{
|
| 1341 |
+
"threshold": 0.65,
|
| 1342 |
+
"weight": 4.0,
|
| 1343 |
+
"global": {
|
| 1344 |
+
"formulas": 275,
|
| 1345 |
+
"exact_partition": 0.6836363636363636,
|
| 1346 |
+
"exact_group_recall": 0.9556895647614053,
|
| 1347 |
+
"overmerge_rate": 0.13818181818181818,
|
| 1348 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1349 |
+
"pair_precision": 0.9640861931364725,
|
| 1350 |
+
"pair_recall": 0.9062265566391597,
|
| 1351 |
+
"pair_f1": 0.934261407579273
|
| 1352 |
+
},
|
| 1353 |
+
"behavior_targets": {
|
| 1354 |
+
"x": {
|
| 1355 |
+
"samples": 277,
|
| 1356 |
+
"matched": 244,
|
| 1357 |
+
"oversplit": 31,
|
| 1358 |
+
"overmerge": 3,
|
| 1359 |
+
"grouping_recall": 0.8808664259927798
|
| 1360 |
+
},
|
| 1361 |
+
"X": {
|
| 1362 |
+
"samples": 7,
|
| 1363 |
+
"matched": 7,
|
| 1364 |
+
"oversplit": 0,
|
| 1365 |
+
"overmerge": 0,
|
| 1366 |
+
"grouping_recall": 1.0
|
| 1367 |
+
},
|
| 1368 |
+
"\\times": {
|
| 1369 |
+
"samples": 44,
|
| 1370 |
+
"matched": 44,
|
| 1371 |
+
"oversplit": 0,
|
| 1372 |
+
"overmerge": 0,
|
| 1373 |
+
"grouping_recall": 1.0
|
| 1374 |
+
}
|
| 1375 |
+
},
|
| 1376 |
+
"families": {
|
| 1377 |
+
"total": 59,
|
| 1378 |
+
"matched": 51,
|
| 1379 |
+
"grouping_recall": 0.864406779661017,
|
| 1380 |
+
"by_label": {
|
| 1381 |
+
"\\sum": {
|
| 1382 |
+
"samples": 11,
|
| 1383 |
+
"matched": 8,
|
| 1384 |
+
"grouping_recall": 0.7272727272727273
|
| 1385 |
+
},
|
| 1386 |
+
"\\pi": {
|
| 1387 |
+
"samples": 17,
|
| 1388 |
+
"matched": 15,
|
| 1389 |
+
"grouping_recall": 0.8823529411764706
|
| 1390 |
+
},
|
| 1391 |
+
"\\rightarrow": {
|
| 1392 |
+
"samples": 20,
|
| 1393 |
+
"matched": 17,
|
| 1394 |
+
"grouping_recall": 0.85
|
| 1395 |
+
},
|
| 1396 |
+
"\\neq": {
|
| 1397 |
+
"samples": 2,
|
| 1398 |
+
"matched": 2,
|
| 1399 |
+
"grouping_recall": 1.0
|
| 1400 |
+
},
|
| 1401 |
+
"\\pm": {
|
| 1402 |
+
"samples": 9,
|
| 1403 |
+
"matched": 9,
|
| 1404 |
+
"grouping_recall": 1.0
|
| 1405 |
+
}
|
| 1406 |
+
}
|
| 1407 |
+
}
|
| 1408 |
+
},
|
| 1409 |
+
{
|
| 1410 |
+
"threshold": 0.65,
|
| 1411 |
+
"weight": 6.0,
|
| 1412 |
+
"global": {
|
| 1413 |
+
"formulas": 275,
|
| 1414 |
+
"exact_partition": 0.6909090909090909,
|
| 1415 |
+
"exact_group_recall": 0.9567383324593602,
|
| 1416 |
+
"overmerge_rate": 0.13090909090909092,
|
| 1417 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1418 |
+
"pair_precision": 0.9664,
|
| 1419 |
+
"pair_recall": 0.9062265566391597,
|
| 1420 |
+
"pair_f1": 0.9353464963221061
|
| 1421 |
+
},
|
| 1422 |
+
"behavior_targets": {
|
| 1423 |
+
"x": {
|
| 1424 |
+
"samples": 277,
|
| 1425 |
+
"matched": 245,
|
| 1426 |
+
"oversplit": 31,
|
| 1427 |
+
"overmerge": 2,
|
| 1428 |
+
"grouping_recall": 0.8844765342960289
|
| 1429 |
+
},
|
| 1430 |
+
"X": {
|
| 1431 |
+
"samples": 7,
|
| 1432 |
+
"matched": 7,
|
| 1433 |
+
"oversplit": 0,
|
| 1434 |
+
"overmerge": 0,
|
| 1435 |
+
"grouping_recall": 1.0
|
| 1436 |
+
},
|
| 1437 |
+
"\\times": {
|
| 1438 |
+
"samples": 44,
|
| 1439 |
+
"matched": 44,
|
| 1440 |
+
"oversplit": 0,
|
| 1441 |
+
"overmerge": 0,
|
| 1442 |
+
"grouping_recall": 1.0
|
| 1443 |
+
}
|
| 1444 |
+
},
|
| 1445 |
+
"families": {
|
| 1446 |
+
"total": 59,
|
| 1447 |
+
"matched": 51,
|
| 1448 |
+
"grouping_recall": 0.864406779661017,
|
| 1449 |
+
"by_label": {
|
| 1450 |
+
"\\sum": {
|
| 1451 |
+
"samples": 11,
|
| 1452 |
+
"matched": 8,
|
| 1453 |
+
"grouping_recall": 0.7272727272727273
|
| 1454 |
+
},
|
| 1455 |
+
"\\pi": {
|
| 1456 |
+
"samples": 17,
|
| 1457 |
+
"matched": 15,
|
| 1458 |
+
"grouping_recall": 0.8823529411764706
|
| 1459 |
+
},
|
| 1460 |
+
"\\rightarrow": {
|
| 1461 |
+
"samples": 20,
|
| 1462 |
+
"matched": 17,
|
| 1463 |
+
"grouping_recall": 0.85
|
| 1464 |
+
},
|
| 1465 |
+
"\\neq": {
|
| 1466 |
+
"samples": 2,
|
| 1467 |
+
"matched": 2,
|
| 1468 |
+
"grouping_recall": 1.0
|
| 1469 |
+
},
|
| 1470 |
+
"\\pm": {
|
| 1471 |
+
"samples": 9,
|
| 1472 |
+
"matched": 9,
|
| 1473 |
+
"grouping_recall": 1.0
|
| 1474 |
+
}
|
| 1475 |
+
}
|
| 1476 |
+
}
|
| 1477 |
+
},
|
| 1478 |
+
{
|
| 1479 |
+
"threshold": 0.65,
|
| 1480 |
+
"weight": 8.0,
|
| 1481 |
+
"global": {
|
| 1482 |
+
"formulas": 275,
|
| 1483 |
+
"exact_partition": 0.6909090909090909,
|
| 1484 |
+
"exact_group_recall": 0.9567383324593602,
|
| 1485 |
+
"overmerge_rate": 0.13090909090909092,
|
| 1486 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1487 |
+
"pair_precision": 0.9664,
|
| 1488 |
+
"pair_recall": 0.9062265566391597,
|
| 1489 |
+
"pair_f1": 0.9353464963221061
|
| 1490 |
+
},
|
| 1491 |
+
"behavior_targets": {
|
| 1492 |
+
"x": {
|
| 1493 |
+
"samples": 277,
|
| 1494 |
+
"matched": 245,
|
| 1495 |
+
"oversplit": 31,
|
| 1496 |
+
"overmerge": 2,
|
| 1497 |
+
"grouping_recall": 0.8844765342960289
|
| 1498 |
+
},
|
| 1499 |
+
"X": {
|
| 1500 |
+
"samples": 7,
|
| 1501 |
+
"matched": 7,
|
| 1502 |
+
"oversplit": 0,
|
| 1503 |
+
"overmerge": 0,
|
| 1504 |
+
"grouping_recall": 1.0
|
| 1505 |
+
},
|
| 1506 |
+
"\\times": {
|
| 1507 |
+
"samples": 44,
|
| 1508 |
+
"matched": 44,
|
| 1509 |
+
"oversplit": 0,
|
| 1510 |
+
"overmerge": 0,
|
| 1511 |
+
"grouping_recall": 1.0
|
| 1512 |
+
}
|
| 1513 |
+
},
|
| 1514 |
+
"families": {
|
| 1515 |
+
"total": 59,
|
| 1516 |
+
"matched": 51,
|
| 1517 |
+
"grouping_recall": 0.864406779661017,
|
| 1518 |
+
"by_label": {
|
| 1519 |
+
"\\sum": {
|
| 1520 |
+
"samples": 11,
|
| 1521 |
+
"matched": 8,
|
| 1522 |
+
"grouping_recall": 0.7272727272727273
|
| 1523 |
+
},
|
| 1524 |
+
"\\pi": {
|
| 1525 |
+
"samples": 17,
|
| 1526 |
+
"matched": 15,
|
| 1527 |
+
"grouping_recall": 0.8823529411764706
|
| 1528 |
+
},
|
| 1529 |
+
"\\rightarrow": {
|
| 1530 |
+
"samples": 20,
|
| 1531 |
+
"matched": 17,
|
| 1532 |
+
"grouping_recall": 0.85
|
| 1533 |
+
},
|
| 1534 |
+
"\\neq": {
|
| 1535 |
+
"samples": 2,
|
| 1536 |
+
"matched": 2,
|
| 1537 |
+
"grouping_recall": 1.0
|
| 1538 |
+
},
|
| 1539 |
+
"\\pm": {
|
| 1540 |
+
"samples": 9,
|
| 1541 |
+
"matched": 9,
|
| 1542 |
+
"grouping_recall": 1.0
|
| 1543 |
+
}
|
| 1544 |
+
}
|
| 1545 |
+
}
|
| 1546 |
+
},
|
| 1547 |
+
{
|
| 1548 |
+
"threshold": 0.65,
|
| 1549 |
+
"weight": 12.0,
|
| 1550 |
+
"global": {
|
| 1551 |
+
"formulas": 275,
|
| 1552 |
+
"exact_partition": 0.6945454545454546,
|
| 1553 |
+
"exact_group_recall": 0.9575249082328264,
|
| 1554 |
+
"overmerge_rate": 0.12363636363636364,
|
| 1555 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1556 |
+
"pair_precision": 0.9679230152365678,
|
| 1557 |
+
"pair_recall": 0.905476369092273,
|
| 1558 |
+
"pair_f1": 0.9356589147286822
|
| 1559 |
+
},
|
| 1560 |
+
"behavior_targets": {
|
| 1561 |
+
"x": {
|
| 1562 |
+
"samples": 277,
|
| 1563 |
+
"matched": 245,
|
| 1564 |
+
"oversplit": 31,
|
| 1565 |
+
"overmerge": 2,
|
| 1566 |
+
"grouping_recall": 0.8844765342960289
|
| 1567 |
+
},
|
| 1568 |
+
"X": {
|
| 1569 |
+
"samples": 7,
|
| 1570 |
+
"matched": 7,
|
| 1571 |
+
"oversplit": 0,
|
| 1572 |
+
"overmerge": 0,
|
| 1573 |
+
"grouping_recall": 1.0
|
| 1574 |
+
},
|
| 1575 |
+
"\\times": {
|
| 1576 |
+
"samples": 44,
|
| 1577 |
+
"matched": 44,
|
| 1578 |
+
"oversplit": 0,
|
| 1579 |
+
"overmerge": 0,
|
| 1580 |
+
"grouping_recall": 1.0
|
| 1581 |
+
}
|
| 1582 |
+
},
|
| 1583 |
+
"families": {
|
| 1584 |
+
"total": 59,
|
| 1585 |
+
"matched": 50,
|
| 1586 |
+
"grouping_recall": 0.847457627118644,
|
| 1587 |
+
"by_label": {
|
| 1588 |
+
"\\sum": {
|
| 1589 |
+
"samples": 11,
|
| 1590 |
+
"matched": 8,
|
| 1591 |
+
"grouping_recall": 0.7272727272727273
|
| 1592 |
+
},
|
| 1593 |
+
"\\pi": {
|
| 1594 |
+
"samples": 17,
|
| 1595 |
+
"matched": 14,
|
| 1596 |
+
"grouping_recall": 0.8235294117647058
|
| 1597 |
+
},
|
| 1598 |
+
"\\rightarrow": {
|
| 1599 |
+
"samples": 20,
|
| 1600 |
+
"matched": 17,
|
| 1601 |
+
"grouping_recall": 0.85
|
| 1602 |
+
},
|
| 1603 |
+
"\\neq": {
|
| 1604 |
+
"samples": 2,
|
| 1605 |
+
"matched": 2,
|
| 1606 |
+
"grouping_recall": 1.0
|
| 1607 |
+
},
|
| 1608 |
+
"\\pm": {
|
| 1609 |
+
"samples": 9,
|
| 1610 |
+
"matched": 9,
|
| 1611 |
+
"grouping_recall": 1.0
|
| 1612 |
+
}
|
| 1613 |
+
}
|
| 1614 |
+
}
|
| 1615 |
+
},
|
| 1616 |
+
{
|
| 1617 |
+
"threshold": 0.8,
|
| 1618 |
+
"weight": 0.0,
|
| 1619 |
+
"global": {
|
| 1620 |
+
"formulas": 275,
|
| 1621 |
+
"exact_partition": 0.68,
|
| 1622 |
+
"exact_group_recall": 0.9528054535920294,
|
| 1623 |
+
"overmerge_rate": 0.14909090909090908,
|
| 1624 |
+
"oversplit_rate": 0.21818181818181817,
|
| 1625 |
+
"pair_precision": 0.9563838223632039,
|
| 1626 |
+
"pair_recall": 0.9047261815453863,
|
| 1627 |
+
"pair_f1": 0.9298380878951427
|
| 1628 |
+
},
|
| 1629 |
+
"behavior_targets": {
|
| 1630 |
+
"x": {
|
| 1631 |
+
"samples": 277,
|
| 1632 |
+
"matched": 244,
|
| 1633 |
+
"oversplit": 31,
|
| 1634 |
+
"overmerge": 3,
|
| 1635 |
+
"grouping_recall": 0.8808664259927798
|
| 1636 |
+
},
|
| 1637 |
+
"X": {
|
| 1638 |
+
"samples": 7,
|
| 1639 |
+
"matched": 7,
|
| 1640 |
+
"oversplit": 0,
|
| 1641 |
+
"overmerge": 0,
|
| 1642 |
+
"grouping_recall": 1.0
|
| 1643 |
+
},
|
| 1644 |
+
"\\times": {
|
| 1645 |
+
"samples": 44,
|
| 1646 |
+
"matched": 44,
|
| 1647 |
+
"oversplit": 0,
|
| 1648 |
+
"overmerge": 0,
|
| 1649 |
+
"grouping_recall": 1.0
|
| 1650 |
+
}
|
| 1651 |
+
},
|
| 1652 |
+
"families": {
|
| 1653 |
+
"total": 59,
|
| 1654 |
+
"matched": 51,
|
| 1655 |
+
"grouping_recall": 0.864406779661017,
|
| 1656 |
+
"by_label": {
|
| 1657 |
+
"\\sum": {
|
| 1658 |
+
"samples": 11,
|
| 1659 |
+
"matched": 8,
|
| 1660 |
+
"grouping_recall": 0.7272727272727273
|
| 1661 |
+
},
|
| 1662 |
+
"\\pi": {
|
| 1663 |
+
"samples": 17,
|
| 1664 |
+
"matched": 15,
|
| 1665 |
+
"grouping_recall": 0.8823529411764706
|
| 1666 |
+
},
|
| 1667 |
+
"\\rightarrow": {
|
| 1668 |
+
"samples": 20,
|
| 1669 |
+
"matched": 17,
|
| 1670 |
+
"grouping_recall": 0.85
|
| 1671 |
+
},
|
| 1672 |
+
"\\neq": {
|
| 1673 |
+
"samples": 2,
|
| 1674 |
+
"matched": 2,
|
| 1675 |
+
"grouping_recall": 1.0
|
| 1676 |
+
},
|
| 1677 |
+
"\\pm": {
|
| 1678 |
+
"samples": 9,
|
| 1679 |
+
"matched": 9,
|
| 1680 |
+
"grouping_recall": 1.0
|
| 1681 |
+
}
|
| 1682 |
+
}
|
| 1683 |
+
}
|
| 1684 |
+
},
|
| 1685 |
+
{
|
| 1686 |
+
"threshold": 0.8,
|
| 1687 |
+
"weight": 1.0,
|
| 1688 |
+
"global": {
|
| 1689 |
+
"formulas": 275,
|
| 1690 |
+
"exact_partition": 0.68,
|
| 1691 |
+
"exact_group_recall": 0.9543786051389617,
|
| 1692 |
+
"overmerge_rate": 0.14545454545454545,
|
| 1693 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1694 |
+
"pair_precision": 0.9594916600476568,
|
| 1695 |
+
"pair_recall": 0.9062265566391597,
|
| 1696 |
+
"pair_f1": 0.9320987654320987
|
| 1697 |
+
},
|
| 1698 |
+
"behavior_targets": {
|
| 1699 |
+
"x": {
|
| 1700 |
+
"samples": 277,
|
| 1701 |
+
"matched": 244,
|
| 1702 |
+
"oversplit": 31,
|
| 1703 |
+
"overmerge": 3,
|
| 1704 |
+
"grouping_recall": 0.8808664259927798
|
| 1705 |
+
},
|
| 1706 |
+
"X": {
|
| 1707 |
+
"samples": 7,
|
| 1708 |
+
"matched": 7,
|
| 1709 |
+
"oversplit": 0,
|
| 1710 |
+
"overmerge": 0,
|
| 1711 |
+
"grouping_recall": 1.0
|
| 1712 |
+
},
|
| 1713 |
+
"\\times": {
|
| 1714 |
+
"samples": 44,
|
| 1715 |
+
"matched": 44,
|
| 1716 |
+
"oversplit": 0,
|
| 1717 |
+
"overmerge": 0,
|
| 1718 |
+
"grouping_recall": 1.0
|
| 1719 |
+
}
|
| 1720 |
+
},
|
| 1721 |
+
"families": {
|
| 1722 |
+
"total": 59,
|
| 1723 |
+
"matched": 51,
|
| 1724 |
+
"grouping_recall": 0.864406779661017,
|
| 1725 |
+
"by_label": {
|
| 1726 |
+
"\\sum": {
|
| 1727 |
+
"samples": 11,
|
| 1728 |
+
"matched": 8,
|
| 1729 |
+
"grouping_recall": 0.7272727272727273
|
| 1730 |
+
},
|
| 1731 |
+
"\\pi": {
|
| 1732 |
+
"samples": 17,
|
| 1733 |
+
"matched": 15,
|
| 1734 |
+
"grouping_recall": 0.8823529411764706
|
| 1735 |
+
},
|
| 1736 |
+
"\\rightarrow": {
|
| 1737 |
+
"samples": 20,
|
| 1738 |
+
"matched": 17,
|
| 1739 |
+
"grouping_recall": 0.85
|
| 1740 |
+
},
|
| 1741 |
+
"\\neq": {
|
| 1742 |
+
"samples": 2,
|
| 1743 |
+
"matched": 2,
|
| 1744 |
+
"grouping_recall": 1.0
|
| 1745 |
+
},
|
| 1746 |
+
"\\pm": {
|
| 1747 |
+
"samples": 9,
|
| 1748 |
+
"matched": 9,
|
| 1749 |
+
"grouping_recall": 1.0
|
| 1750 |
+
}
|
| 1751 |
+
}
|
| 1752 |
+
}
|
| 1753 |
+
},
|
| 1754 |
+
{
|
| 1755 |
+
"threshold": 0.8,
|
| 1756 |
+
"weight": 2.0,
|
| 1757 |
+
"global": {
|
| 1758 |
+
"formulas": 275,
|
| 1759 |
+
"exact_partition": 0.6836363636363636,
|
| 1760 |
+
"exact_group_recall": 0.9551651809124279,
|
| 1761 |
+
"overmerge_rate": 0.13818181818181818,
|
| 1762 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1763 |
+
"pair_precision": 0.9632881085395052,
|
| 1764 |
+
"pair_recall": 0.905476369092273,
|
| 1765 |
+
"pair_f1": 0.9334880123743232
|
| 1766 |
+
},
|
| 1767 |
+
"behavior_targets": {
|
| 1768 |
+
"x": {
|
| 1769 |
+
"samples": 277,
|
| 1770 |
+
"matched": 244,
|
| 1771 |
+
"oversplit": 31,
|
| 1772 |
+
"overmerge": 3,
|
| 1773 |
+
"grouping_recall": 0.8808664259927798
|
| 1774 |
+
},
|
| 1775 |
+
"X": {
|
| 1776 |
+
"samples": 7,
|
| 1777 |
+
"matched": 7,
|
| 1778 |
+
"oversplit": 0,
|
| 1779 |
+
"overmerge": 0,
|
| 1780 |
+
"grouping_recall": 1.0
|
| 1781 |
+
},
|
| 1782 |
+
"\\times": {
|
| 1783 |
+
"samples": 44,
|
| 1784 |
+
"matched": 44,
|
| 1785 |
+
"oversplit": 0,
|
| 1786 |
+
"overmerge": 0,
|
| 1787 |
+
"grouping_recall": 1.0
|
| 1788 |
+
}
|
| 1789 |
+
},
|
| 1790 |
+
"families": {
|
| 1791 |
+
"total": 59,
|
| 1792 |
+
"matched": 51,
|
| 1793 |
+
"grouping_recall": 0.864406779661017,
|
| 1794 |
+
"by_label": {
|
| 1795 |
+
"\\sum": {
|
| 1796 |
+
"samples": 11,
|
| 1797 |
+
"matched": 8,
|
| 1798 |
+
"grouping_recall": 0.7272727272727273
|
| 1799 |
+
},
|
| 1800 |
+
"\\pi": {
|
| 1801 |
+
"samples": 17,
|
| 1802 |
+
"matched": 15,
|
| 1803 |
+
"grouping_recall": 0.8823529411764706
|
| 1804 |
+
},
|
| 1805 |
+
"\\rightarrow": {
|
| 1806 |
+
"samples": 20,
|
| 1807 |
+
"matched": 17,
|
| 1808 |
+
"grouping_recall": 0.85
|
| 1809 |
+
},
|
| 1810 |
+
"\\neq": {
|
| 1811 |
+
"samples": 2,
|
| 1812 |
+
"matched": 2,
|
| 1813 |
+
"grouping_recall": 1.0
|
| 1814 |
+
},
|
| 1815 |
+
"\\pm": {
|
| 1816 |
+
"samples": 9,
|
| 1817 |
+
"matched": 9,
|
| 1818 |
+
"grouping_recall": 1.0
|
| 1819 |
+
}
|
| 1820 |
+
}
|
| 1821 |
+
}
|
| 1822 |
+
},
|
| 1823 |
+
{
|
| 1824 |
+
"threshold": 0.8,
|
| 1825 |
+
"weight": 4.0,
|
| 1826 |
+
"global": {
|
| 1827 |
+
"formulas": 275,
|
| 1828 |
+
"exact_partition": 0.6836363636363636,
|
| 1829 |
+
"exact_group_recall": 0.9551651809124279,
|
| 1830 |
+
"overmerge_rate": 0.13818181818181818,
|
| 1831 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1832 |
+
"pair_precision": 0.9632881085395052,
|
| 1833 |
+
"pair_recall": 0.905476369092273,
|
| 1834 |
+
"pair_f1": 0.9334880123743232
|
| 1835 |
+
},
|
| 1836 |
+
"behavior_targets": {
|
| 1837 |
+
"x": {
|
| 1838 |
+
"samples": 277,
|
| 1839 |
+
"matched": 244,
|
| 1840 |
+
"oversplit": 31,
|
| 1841 |
+
"overmerge": 3,
|
| 1842 |
+
"grouping_recall": 0.8808664259927798
|
| 1843 |
+
},
|
| 1844 |
+
"X": {
|
| 1845 |
+
"samples": 7,
|
| 1846 |
+
"matched": 7,
|
| 1847 |
+
"oversplit": 0,
|
| 1848 |
+
"overmerge": 0,
|
| 1849 |
+
"grouping_recall": 1.0
|
| 1850 |
+
},
|
| 1851 |
+
"\\times": {
|
| 1852 |
+
"samples": 44,
|
| 1853 |
+
"matched": 44,
|
| 1854 |
+
"oversplit": 0,
|
| 1855 |
+
"overmerge": 0,
|
| 1856 |
+
"grouping_recall": 1.0
|
| 1857 |
+
}
|
| 1858 |
+
},
|
| 1859 |
+
"families": {
|
| 1860 |
+
"total": 59,
|
| 1861 |
+
"matched": 51,
|
| 1862 |
+
"grouping_recall": 0.864406779661017,
|
| 1863 |
+
"by_label": {
|
| 1864 |
+
"\\sum": {
|
| 1865 |
+
"samples": 11,
|
| 1866 |
+
"matched": 8,
|
| 1867 |
+
"grouping_recall": 0.7272727272727273
|
| 1868 |
+
},
|
| 1869 |
+
"\\pi": {
|
| 1870 |
+
"samples": 17,
|
| 1871 |
+
"matched": 15,
|
| 1872 |
+
"grouping_recall": 0.8823529411764706
|
| 1873 |
+
},
|
| 1874 |
+
"\\rightarrow": {
|
| 1875 |
+
"samples": 20,
|
| 1876 |
+
"matched": 17,
|
| 1877 |
+
"grouping_recall": 0.85
|
| 1878 |
+
},
|
| 1879 |
+
"\\neq": {
|
| 1880 |
+
"samples": 2,
|
| 1881 |
+
"matched": 2,
|
| 1882 |
+
"grouping_recall": 1.0
|
| 1883 |
+
},
|
| 1884 |
+
"\\pm": {
|
| 1885 |
+
"samples": 9,
|
| 1886 |
+
"matched": 9,
|
| 1887 |
+
"grouping_recall": 1.0
|
| 1888 |
+
}
|
| 1889 |
+
}
|
| 1890 |
+
}
|
| 1891 |
+
},
|
| 1892 |
+
{
|
| 1893 |
+
"threshold": 0.8,
|
| 1894 |
+
"weight": 6.0,
|
| 1895 |
+
"global": {
|
| 1896 |
+
"formulas": 275,
|
| 1897 |
+
"exact_partition": 0.6872727272727273,
|
| 1898 |
+
"exact_group_recall": 0.9556895647614053,
|
| 1899 |
+
"overmerge_rate": 0.13454545454545455,
|
| 1900 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1901 |
+
"pair_precision": 0.964828137490008,
|
| 1902 |
+
"pair_recall": 0.905476369092273,
|
| 1903 |
+
"pair_f1": 0.9342105263157895
|
| 1904 |
+
},
|
| 1905 |
+
"behavior_targets": {
|
| 1906 |
+
"x": {
|
| 1907 |
+
"samples": 277,
|
| 1908 |
+
"matched": 245,
|
| 1909 |
+
"oversplit": 31,
|
| 1910 |
+
"overmerge": 2,
|
| 1911 |
+
"grouping_recall": 0.8844765342960289
|
| 1912 |
+
},
|
| 1913 |
+
"X": {
|
| 1914 |
+
"samples": 7,
|
| 1915 |
+
"matched": 7,
|
| 1916 |
+
"oversplit": 0,
|
| 1917 |
+
"overmerge": 0,
|
| 1918 |
+
"grouping_recall": 1.0
|
| 1919 |
+
},
|
| 1920 |
+
"\\times": {
|
| 1921 |
+
"samples": 44,
|
| 1922 |
+
"matched": 44,
|
| 1923 |
+
"oversplit": 0,
|
| 1924 |
+
"overmerge": 0,
|
| 1925 |
+
"grouping_recall": 1.0
|
| 1926 |
+
}
|
| 1927 |
+
},
|
| 1928 |
+
"families": {
|
| 1929 |
+
"total": 59,
|
| 1930 |
+
"matched": 51,
|
| 1931 |
+
"grouping_recall": 0.864406779661017,
|
| 1932 |
+
"by_label": {
|
| 1933 |
+
"\\sum": {
|
| 1934 |
+
"samples": 11,
|
| 1935 |
+
"matched": 8,
|
| 1936 |
+
"grouping_recall": 0.7272727272727273
|
| 1937 |
+
},
|
| 1938 |
+
"\\pi": {
|
| 1939 |
+
"samples": 17,
|
| 1940 |
+
"matched": 15,
|
| 1941 |
+
"grouping_recall": 0.8823529411764706
|
| 1942 |
+
},
|
| 1943 |
+
"\\rightarrow": {
|
| 1944 |
+
"samples": 20,
|
| 1945 |
+
"matched": 17,
|
| 1946 |
+
"grouping_recall": 0.85
|
| 1947 |
+
},
|
| 1948 |
+
"\\neq": {
|
| 1949 |
+
"samples": 2,
|
| 1950 |
+
"matched": 2,
|
| 1951 |
+
"grouping_recall": 1.0
|
| 1952 |
+
},
|
| 1953 |
+
"\\pm": {
|
| 1954 |
+
"samples": 9,
|
| 1955 |
+
"matched": 9,
|
| 1956 |
+
"grouping_recall": 1.0
|
| 1957 |
+
}
|
| 1958 |
+
}
|
| 1959 |
+
}
|
| 1960 |
+
},
|
| 1961 |
+
{
|
| 1962 |
+
"threshold": 0.8,
|
| 1963 |
+
"weight": 8.0,
|
| 1964 |
+
"global": {
|
| 1965 |
+
"formulas": 275,
|
| 1966 |
+
"exact_partition": 0.6872727272727273,
|
| 1967 |
+
"exact_group_recall": 0.9556895647614053,
|
| 1968 |
+
"overmerge_rate": 0.13454545454545455,
|
| 1969 |
+
"oversplit_rate": 0.21454545454545454,
|
| 1970 |
+
"pair_precision": 0.964828137490008,
|
| 1971 |
+
"pair_recall": 0.905476369092273,
|
| 1972 |
+
"pair_f1": 0.9342105263157895
|
| 1973 |
+
},
|
| 1974 |
+
"behavior_targets": {
|
| 1975 |
+
"x": {
|
| 1976 |
+
"samples": 277,
|
| 1977 |
+
"matched": 245,
|
| 1978 |
+
"oversplit": 31,
|
| 1979 |
+
"overmerge": 2,
|
| 1980 |
+
"grouping_recall": 0.8844765342960289
|
| 1981 |
+
},
|
| 1982 |
+
"X": {
|
| 1983 |
+
"samples": 7,
|
| 1984 |
+
"matched": 7,
|
| 1985 |
+
"oversplit": 0,
|
| 1986 |
+
"overmerge": 0,
|
| 1987 |
+
"grouping_recall": 1.0
|
| 1988 |
+
},
|
| 1989 |
+
"\\times": {
|
| 1990 |
+
"samples": 44,
|
| 1991 |
+
"matched": 44,
|
| 1992 |
+
"oversplit": 0,
|
| 1993 |
+
"overmerge": 0,
|
| 1994 |
+
"grouping_recall": 1.0
|
| 1995 |
+
}
|
| 1996 |
+
},
|
| 1997 |
+
"families": {
|
| 1998 |
+
"total": 59,
|
| 1999 |
+
"matched": 51,
|
| 2000 |
+
"grouping_recall": 0.864406779661017,
|
| 2001 |
+
"by_label": {
|
| 2002 |
+
"\\sum": {
|
| 2003 |
+
"samples": 11,
|
| 2004 |
+
"matched": 8,
|
| 2005 |
+
"grouping_recall": 0.7272727272727273
|
| 2006 |
+
},
|
| 2007 |
+
"\\pi": {
|
| 2008 |
+
"samples": 17,
|
| 2009 |
+
"matched": 15,
|
| 2010 |
+
"grouping_recall": 0.8823529411764706
|
| 2011 |
+
},
|
| 2012 |
+
"\\rightarrow": {
|
| 2013 |
+
"samples": 20,
|
| 2014 |
+
"matched": 17,
|
| 2015 |
+
"grouping_recall": 0.85
|
| 2016 |
+
},
|
| 2017 |
+
"\\neq": {
|
| 2018 |
+
"samples": 2,
|
| 2019 |
+
"matched": 2,
|
| 2020 |
+
"grouping_recall": 1.0
|
| 2021 |
+
},
|
| 2022 |
+
"\\pm": {
|
| 2023 |
+
"samples": 9,
|
| 2024 |
+
"matched": 9,
|
| 2025 |
+
"grouping_recall": 1.0
|
| 2026 |
+
}
|
| 2027 |
+
}
|
| 2028 |
+
}
|
| 2029 |
+
},
|
| 2030 |
+
{
|
| 2031 |
+
"threshold": 0.8,
|
| 2032 |
+
"weight": 12.0,
|
| 2033 |
+
"global": {
|
| 2034 |
+
"formulas": 275,
|
| 2035 |
+
"exact_partition": 0.6872727272727273,
|
| 2036 |
+
"exact_group_recall": 0.9556895647614053,
|
| 2037 |
+
"overmerge_rate": 0.13454545454545455,
|
| 2038 |
+
"oversplit_rate": 0.21454545454545454,
|
| 2039 |
+
"pair_precision": 0.964828137490008,
|
| 2040 |
+
"pair_recall": 0.905476369092273,
|
| 2041 |
+
"pair_f1": 0.9342105263157895
|
| 2042 |
+
},
|
| 2043 |
+
"behavior_targets": {
|
| 2044 |
+
"x": {
|
| 2045 |
+
"samples": 277,
|
| 2046 |
+
"matched": 245,
|
| 2047 |
+
"oversplit": 31,
|
| 2048 |
+
"overmerge": 2,
|
| 2049 |
+
"grouping_recall": 0.8844765342960289
|
| 2050 |
+
},
|
| 2051 |
+
"X": {
|
| 2052 |
+
"samples": 7,
|
| 2053 |
+
"matched": 7,
|
| 2054 |
+
"oversplit": 0,
|
| 2055 |
+
"overmerge": 0,
|
| 2056 |
+
"grouping_recall": 1.0
|
| 2057 |
+
},
|
| 2058 |
+
"\\times": {
|
| 2059 |
+
"samples": 44,
|
| 2060 |
+
"matched": 44,
|
| 2061 |
+
"oversplit": 0,
|
| 2062 |
+
"overmerge": 0,
|
| 2063 |
+
"grouping_recall": 1.0
|
| 2064 |
+
}
|
| 2065 |
+
},
|
| 2066 |
+
"families": {
|
| 2067 |
+
"total": 59,
|
| 2068 |
+
"matched": 51,
|
| 2069 |
+
"grouping_recall": 0.864406779661017,
|
| 2070 |
+
"by_label": {
|
| 2071 |
+
"\\sum": {
|
| 2072 |
+
"samples": 11,
|
| 2073 |
+
"matched": 8,
|
| 2074 |
+
"grouping_recall": 0.7272727272727273
|
| 2075 |
+
},
|
| 2076 |
+
"\\pi": {
|
| 2077 |
+
"samples": 17,
|
| 2078 |
+
"matched": 15,
|
| 2079 |
+
"grouping_recall": 0.8823529411764706
|
| 2080 |
+
},
|
| 2081 |
+
"\\rightarrow": {
|
| 2082 |
+
"samples": 20,
|
| 2083 |
+
"matched": 17,
|
| 2084 |
+
"grouping_recall": 0.85
|
| 2085 |
+
},
|
| 2086 |
+
"\\neq": {
|
| 2087 |
+
"samples": 2,
|
| 2088 |
+
"matched": 2,
|
| 2089 |
+
"grouping_recall": 1.0
|
| 2090 |
+
},
|
| 2091 |
+
"\\pm": {
|
| 2092 |
+
"samples": 9,
|
| 2093 |
+
"matched": 9,
|
| 2094 |
+
"grouping_recall": 1.0
|
| 2095 |
+
}
|
| 2096 |
+
}
|
| 2097 |
+
}
|
| 2098 |
+
},
|
| 2099 |
+
{
|
| 2100 |
+
"threshold": 0.9,
|
| 2101 |
+
"weight": 0.0,
|
| 2102 |
+
"global": {
|
| 2103 |
+
"formulas": 275,
|
| 2104 |
+
"exact_partition": 0.68,
|
| 2105 |
+
"exact_group_recall": 0.9528054535920294,
|
| 2106 |
+
"overmerge_rate": 0.14909090909090908,
|
| 2107 |
+
"oversplit_rate": 0.21818181818181817,
|
| 2108 |
+
"pair_precision": 0.9563838223632039,
|
| 2109 |
+
"pair_recall": 0.9047261815453863,
|
| 2110 |
+
"pair_f1": 0.9298380878951427
|
| 2111 |
+
},
|
| 2112 |
+
"behavior_targets": {
|
| 2113 |
+
"x": {
|
| 2114 |
+
"samples": 277,
|
| 2115 |
+
"matched": 244,
|
| 2116 |
+
"oversplit": 31,
|
| 2117 |
+
"overmerge": 3,
|
| 2118 |
+
"grouping_recall": 0.8808664259927798
|
| 2119 |
+
},
|
| 2120 |
+
"X": {
|
| 2121 |
+
"samples": 7,
|
| 2122 |
+
"matched": 7,
|
| 2123 |
+
"oversplit": 0,
|
| 2124 |
+
"overmerge": 0,
|
| 2125 |
+
"grouping_recall": 1.0
|
| 2126 |
+
},
|
| 2127 |
+
"\\times": {
|
| 2128 |
+
"samples": 44,
|
| 2129 |
+
"matched": 44,
|
| 2130 |
+
"oversplit": 0,
|
| 2131 |
+
"overmerge": 0,
|
| 2132 |
+
"grouping_recall": 1.0
|
| 2133 |
+
}
|
| 2134 |
+
},
|
| 2135 |
+
"families": {
|
| 2136 |
+
"total": 59,
|
| 2137 |
+
"matched": 51,
|
| 2138 |
+
"grouping_recall": 0.864406779661017,
|
| 2139 |
+
"by_label": {
|
| 2140 |
+
"\\sum": {
|
| 2141 |
+
"samples": 11,
|
| 2142 |
+
"matched": 8,
|
| 2143 |
+
"grouping_recall": 0.7272727272727273
|
| 2144 |
+
},
|
| 2145 |
+
"\\pi": {
|
| 2146 |
+
"samples": 17,
|
| 2147 |
+
"matched": 15,
|
| 2148 |
+
"grouping_recall": 0.8823529411764706
|
| 2149 |
+
},
|
| 2150 |
+
"\\rightarrow": {
|
| 2151 |
+
"samples": 20,
|
| 2152 |
+
"matched": 17,
|
| 2153 |
+
"grouping_recall": 0.85
|
| 2154 |
+
},
|
| 2155 |
+
"\\neq": {
|
| 2156 |
+
"samples": 2,
|
| 2157 |
+
"matched": 2,
|
| 2158 |
+
"grouping_recall": 1.0
|
| 2159 |
+
},
|
| 2160 |
+
"\\pm": {
|
| 2161 |
+
"samples": 9,
|
| 2162 |
+
"matched": 9,
|
| 2163 |
+
"grouping_recall": 1.0
|
| 2164 |
+
}
|
| 2165 |
+
}
|
| 2166 |
+
}
|
| 2167 |
+
},
|
| 2168 |
+
{
|
| 2169 |
+
"threshold": 0.9,
|
| 2170 |
+
"weight": 1.0,
|
| 2171 |
+
"global": {
|
| 2172 |
+
"formulas": 275,
|
| 2173 |
+
"exact_partition": 0.68,
|
| 2174 |
+
"exact_group_recall": 0.9543786051389617,
|
| 2175 |
+
"overmerge_rate": 0.14545454545454545,
|
| 2176 |
+
"oversplit_rate": 0.21454545454545454,
|
| 2177 |
+
"pair_precision": 0.9594916600476568,
|
| 2178 |
+
"pair_recall": 0.9062265566391597,
|
| 2179 |
+
"pair_f1": 0.9320987654320987
|
| 2180 |
+
},
|
| 2181 |
+
"behavior_targets": {
|
| 2182 |
+
"x": {
|
| 2183 |
+
"samples": 277,
|
| 2184 |
+
"matched": 244,
|
| 2185 |
+
"oversplit": 31,
|
| 2186 |
+
"overmerge": 3,
|
| 2187 |
+
"grouping_recall": 0.8808664259927798
|
| 2188 |
+
},
|
| 2189 |
+
"X": {
|
| 2190 |
+
"samples": 7,
|
| 2191 |
+
"matched": 7,
|
| 2192 |
+
"oversplit": 0,
|
| 2193 |
+
"overmerge": 0,
|
| 2194 |
+
"grouping_recall": 1.0
|
| 2195 |
+
},
|
| 2196 |
+
"\\times": {
|
| 2197 |
+
"samples": 44,
|
| 2198 |
+
"matched": 44,
|
| 2199 |
+
"oversplit": 0,
|
| 2200 |
+
"overmerge": 0,
|
| 2201 |
+
"grouping_recall": 1.0
|
| 2202 |
+
}
|
| 2203 |
+
},
|
| 2204 |
+
"families": {
|
| 2205 |
+
"total": 59,
|
| 2206 |
+
"matched": 51,
|
| 2207 |
+
"grouping_recall": 0.864406779661017,
|
| 2208 |
+
"by_label": {
|
| 2209 |
+
"\\sum": {
|
| 2210 |
+
"samples": 11,
|
| 2211 |
+
"matched": 8,
|
| 2212 |
+
"grouping_recall": 0.7272727272727273
|
| 2213 |
+
},
|
| 2214 |
+
"\\pi": {
|
| 2215 |
+
"samples": 17,
|
| 2216 |
+
"matched": 15,
|
| 2217 |
+
"grouping_recall": 0.8823529411764706
|
| 2218 |
+
},
|
| 2219 |
+
"\\rightarrow": {
|
| 2220 |
+
"samples": 20,
|
| 2221 |
+
"matched": 17,
|
| 2222 |
+
"grouping_recall": 0.85
|
| 2223 |
+
},
|
| 2224 |
+
"\\neq": {
|
| 2225 |
+
"samples": 2,
|
| 2226 |
+
"matched": 2,
|
| 2227 |
+
"grouping_recall": 1.0
|
| 2228 |
+
},
|
| 2229 |
+
"\\pm": {
|
| 2230 |
+
"samples": 9,
|
| 2231 |
+
"matched": 9,
|
| 2232 |
+
"grouping_recall": 1.0
|
| 2233 |
+
}
|
| 2234 |
+
}
|
| 2235 |
+
}
|
| 2236 |
+
},
|
| 2237 |
+
{
|
| 2238 |
+
"threshold": 0.9,
|
| 2239 |
+
"weight": 2.0,
|
| 2240 |
+
"global": {
|
| 2241 |
+
"formulas": 275,
|
| 2242 |
+
"exact_partition": 0.6836363636363636,
|
| 2243 |
+
"exact_group_recall": 0.9551651809124279,
|
| 2244 |
+
"overmerge_rate": 0.13818181818181818,
|
| 2245 |
+
"oversplit_rate": 0.21454545454545454,
|
| 2246 |
+
"pair_precision": 0.9632881085395052,
|
| 2247 |
+
"pair_recall": 0.905476369092273,
|
| 2248 |
+
"pair_f1": 0.9334880123743232
|
| 2249 |
+
},
|
| 2250 |
+
"behavior_targets": {
|
| 2251 |
+
"x": {
|
| 2252 |
+
"samples": 277,
|
| 2253 |
+
"matched": 244,
|
| 2254 |
+
"oversplit": 31,
|
| 2255 |
+
"overmerge": 3,
|
| 2256 |
+
"grouping_recall": 0.8808664259927798
|
| 2257 |
+
},
|
| 2258 |
+
"X": {
|
| 2259 |
+
"samples": 7,
|
| 2260 |
+
"matched": 7,
|
| 2261 |
+
"oversplit": 0,
|
| 2262 |
+
"overmerge": 0,
|
| 2263 |
+
"grouping_recall": 1.0
|
| 2264 |
+
},
|
| 2265 |
+
"\\times": {
|
| 2266 |
+
"samples": 44,
|
| 2267 |
+
"matched": 44,
|
| 2268 |
+
"oversplit": 0,
|
| 2269 |
+
"overmerge": 0,
|
| 2270 |
+
"grouping_recall": 1.0
|
| 2271 |
+
}
|
| 2272 |
+
},
|
| 2273 |
+
"families": {
|
| 2274 |
+
"total": 59,
|
| 2275 |
+
"matched": 51,
|
| 2276 |
+
"grouping_recall": 0.864406779661017,
|
| 2277 |
+
"by_label": {
|
| 2278 |
+
"\\sum": {
|
| 2279 |
+
"samples": 11,
|
| 2280 |
+
"matched": 8,
|
| 2281 |
+
"grouping_recall": 0.7272727272727273
|
| 2282 |
+
},
|
| 2283 |
+
"\\pi": {
|
| 2284 |
+
"samples": 17,
|
| 2285 |
+
"matched": 15,
|
| 2286 |
+
"grouping_recall": 0.8823529411764706
|
| 2287 |
+
},
|
| 2288 |
+
"\\rightarrow": {
|
| 2289 |
+
"samples": 20,
|
| 2290 |
+
"matched": 17,
|
| 2291 |
+
"grouping_recall": 0.85
|
| 2292 |
+
},
|
| 2293 |
+
"\\neq": {
|
| 2294 |
+
"samples": 2,
|
| 2295 |
+
"matched": 2,
|
| 2296 |
+
"grouping_recall": 1.0
|
| 2297 |
+
},
|
| 2298 |
+
"\\pm": {
|
| 2299 |
+
"samples": 9,
|
| 2300 |
+
"matched": 9,
|
| 2301 |
+
"grouping_recall": 1.0
|
| 2302 |
+
}
|
| 2303 |
+
}
|
| 2304 |
+
}
|
| 2305 |
+
},
|
| 2306 |
+
{
|
| 2307 |
+
"threshold": 0.9,
|
| 2308 |
+
"weight": 4.0,
|
| 2309 |
+
"global": {
|
| 2310 |
+
"formulas": 275,
|
| 2311 |
+
"exact_partition": 0.6836363636363636,
|
| 2312 |
+
"exact_group_recall": 0.9551651809124279,
|
| 2313 |
+
"overmerge_rate": 0.13818181818181818,
|
| 2314 |
+
"oversplit_rate": 0.21454545454545454,
|
| 2315 |
+
"pair_precision": 0.9632881085395052,
|
| 2316 |
+
"pair_recall": 0.905476369092273,
|
| 2317 |
+
"pair_f1": 0.9334880123743232
|
| 2318 |
+
},
|
| 2319 |
+
"behavior_targets": {
|
| 2320 |
+
"x": {
|
| 2321 |
+
"samples": 277,
|
| 2322 |
+
"matched": 244,
|
| 2323 |
+
"oversplit": 31,
|
| 2324 |
+
"overmerge": 3,
|
| 2325 |
+
"grouping_recall": 0.8808664259927798
|
| 2326 |
+
},
|
| 2327 |
+
"X": {
|
| 2328 |
+
"samples": 7,
|
| 2329 |
+
"matched": 7,
|
| 2330 |
+
"oversplit": 0,
|
| 2331 |
+
"overmerge": 0,
|
| 2332 |
+
"grouping_recall": 1.0
|
| 2333 |
+
},
|
| 2334 |
+
"\\times": {
|
| 2335 |
+
"samples": 44,
|
| 2336 |
+
"matched": 44,
|
| 2337 |
+
"oversplit": 0,
|
| 2338 |
+
"overmerge": 0,
|
| 2339 |
+
"grouping_recall": 1.0
|
| 2340 |
+
}
|
| 2341 |
+
},
|
| 2342 |
+
"families": {
|
| 2343 |
+
"total": 59,
|
| 2344 |
+
"matched": 51,
|
| 2345 |
+
"grouping_recall": 0.864406779661017,
|
| 2346 |
+
"by_label": {
|
| 2347 |
+
"\\sum": {
|
| 2348 |
+
"samples": 11,
|
| 2349 |
+
"matched": 8,
|
| 2350 |
+
"grouping_recall": 0.7272727272727273
|
| 2351 |
+
},
|
| 2352 |
+
"\\pi": {
|
| 2353 |
+
"samples": 17,
|
| 2354 |
+
"matched": 15,
|
| 2355 |
+
"grouping_recall": 0.8823529411764706
|
| 2356 |
+
},
|
| 2357 |
+
"\\rightarrow": {
|
| 2358 |
+
"samples": 20,
|
| 2359 |
+
"matched": 17,
|
| 2360 |
+
"grouping_recall": 0.85
|
| 2361 |
+
},
|
| 2362 |
+
"\\neq": {
|
| 2363 |
+
"samples": 2,
|
| 2364 |
+
"matched": 2,
|
| 2365 |
+
"grouping_recall": 1.0
|
| 2366 |
+
},
|
| 2367 |
+
"\\pm": {
|
| 2368 |
+
"samples": 9,
|
| 2369 |
+
"matched": 9,
|
| 2370 |
+
"grouping_recall": 1.0
|
| 2371 |
+
}
|
| 2372 |
+
}
|
| 2373 |
+
}
|
| 2374 |
+
},
|
| 2375 |
+
{
|
| 2376 |
+
"threshold": 0.9,
|
| 2377 |
+
"weight": 6.0,
|
| 2378 |
+
"global": {
|
| 2379 |
+
"formulas": 275,
|
| 2380 |
+
"exact_partition": 0.6872727272727273,
|
| 2381 |
+
"exact_group_recall": 0.9556895647614053,
|
| 2382 |
+
"overmerge_rate": 0.13454545454545455,
|
| 2383 |
+
"oversplit_rate": 0.21454545454545454,
|
| 2384 |
+
"pair_precision": 0.964828137490008,
|
| 2385 |
+
"pair_recall": 0.905476369092273,
|
| 2386 |
+
"pair_f1": 0.9342105263157895
|
| 2387 |
+
},
|
| 2388 |
+
"behavior_targets": {
|
| 2389 |
+
"x": {
|
| 2390 |
+
"samples": 277,
|
| 2391 |
+
"matched": 245,
|
| 2392 |
+
"oversplit": 31,
|
| 2393 |
+
"overmerge": 2,
|
| 2394 |
+
"grouping_recall": 0.8844765342960289
|
| 2395 |
+
},
|
| 2396 |
+
"X": {
|
| 2397 |
+
"samples": 7,
|
| 2398 |
+
"matched": 7,
|
| 2399 |
+
"oversplit": 0,
|
| 2400 |
+
"overmerge": 0,
|
| 2401 |
+
"grouping_recall": 1.0
|
| 2402 |
+
},
|
| 2403 |
+
"\\times": {
|
| 2404 |
+
"samples": 44,
|
| 2405 |
+
"matched": 44,
|
| 2406 |
+
"oversplit": 0,
|
| 2407 |
+
"overmerge": 0,
|
| 2408 |
+
"grouping_recall": 1.0
|
| 2409 |
+
}
|
| 2410 |
+
},
|
| 2411 |
+
"families": {
|
| 2412 |
+
"total": 59,
|
| 2413 |
+
"matched": 51,
|
| 2414 |
+
"grouping_recall": 0.864406779661017,
|
| 2415 |
+
"by_label": {
|
| 2416 |
+
"\\sum": {
|
| 2417 |
+
"samples": 11,
|
| 2418 |
+
"matched": 8,
|
| 2419 |
+
"grouping_recall": 0.7272727272727273
|
| 2420 |
+
},
|
| 2421 |
+
"\\pi": {
|
| 2422 |
+
"samples": 17,
|
| 2423 |
+
"matched": 15,
|
| 2424 |
+
"grouping_recall": 0.8823529411764706
|
| 2425 |
+
},
|
| 2426 |
+
"\\rightarrow": {
|
| 2427 |
+
"samples": 20,
|
| 2428 |
+
"matched": 17,
|
| 2429 |
+
"grouping_recall": 0.85
|
| 2430 |
+
},
|
| 2431 |
+
"\\neq": {
|
| 2432 |
+
"samples": 2,
|
| 2433 |
+
"matched": 2,
|
| 2434 |
+
"grouping_recall": 1.0
|
| 2435 |
+
},
|
| 2436 |
+
"\\pm": {
|
| 2437 |
+
"samples": 9,
|
| 2438 |
+
"matched": 9,
|
| 2439 |
+
"grouping_recall": 1.0
|
| 2440 |
+
}
|
| 2441 |
+
}
|
| 2442 |
+
}
|
| 2443 |
+
},
|
| 2444 |
+
{
|
| 2445 |
+
"threshold": 0.9,
|
| 2446 |
+
"weight": 8.0,
|
| 2447 |
+
"global": {
|
| 2448 |
+
"formulas": 275,
|
| 2449 |
+
"exact_partition": 0.6872727272727273,
|
| 2450 |
+
"exact_group_recall": 0.9556895647614053,
|
| 2451 |
+
"overmerge_rate": 0.13454545454545455,
|
| 2452 |
+
"oversplit_rate": 0.21454545454545454,
|
| 2453 |
+
"pair_precision": 0.964828137490008,
|
| 2454 |
+
"pair_recall": 0.905476369092273,
|
| 2455 |
+
"pair_f1": 0.9342105263157895
|
| 2456 |
+
},
|
| 2457 |
+
"behavior_targets": {
|
| 2458 |
+
"x": {
|
| 2459 |
+
"samples": 277,
|
| 2460 |
+
"matched": 245,
|
| 2461 |
+
"oversplit": 31,
|
| 2462 |
+
"overmerge": 2,
|
| 2463 |
+
"grouping_recall": 0.8844765342960289
|
| 2464 |
+
},
|
| 2465 |
+
"X": {
|
| 2466 |
+
"samples": 7,
|
| 2467 |
+
"matched": 7,
|
| 2468 |
+
"oversplit": 0,
|
| 2469 |
+
"overmerge": 0,
|
| 2470 |
+
"grouping_recall": 1.0
|
| 2471 |
+
},
|
| 2472 |
+
"\\times": {
|
| 2473 |
+
"samples": 44,
|
| 2474 |
+
"matched": 44,
|
| 2475 |
+
"oversplit": 0,
|
| 2476 |
+
"overmerge": 0,
|
| 2477 |
+
"grouping_recall": 1.0
|
| 2478 |
+
}
|
| 2479 |
+
},
|
| 2480 |
+
"families": {
|
| 2481 |
+
"total": 59,
|
| 2482 |
+
"matched": 51,
|
| 2483 |
+
"grouping_recall": 0.864406779661017,
|
| 2484 |
+
"by_label": {
|
| 2485 |
+
"\\sum": {
|
| 2486 |
+
"samples": 11,
|
| 2487 |
+
"matched": 8,
|
| 2488 |
+
"grouping_recall": 0.7272727272727273
|
| 2489 |
+
},
|
| 2490 |
+
"\\pi": {
|
| 2491 |
+
"samples": 17,
|
| 2492 |
+
"matched": 15,
|
| 2493 |
+
"grouping_recall": 0.8823529411764706
|
| 2494 |
+
},
|
| 2495 |
+
"\\rightarrow": {
|
| 2496 |
+
"samples": 20,
|
| 2497 |
+
"matched": 17,
|
| 2498 |
+
"grouping_recall": 0.85
|
| 2499 |
+
},
|
| 2500 |
+
"\\neq": {
|
| 2501 |
+
"samples": 2,
|
| 2502 |
+
"matched": 2,
|
| 2503 |
+
"grouping_recall": 1.0
|
| 2504 |
+
},
|
| 2505 |
+
"\\pm": {
|
| 2506 |
+
"samples": 9,
|
| 2507 |
+
"matched": 9,
|
| 2508 |
+
"grouping_recall": 1.0
|
| 2509 |
+
}
|
| 2510 |
+
}
|
| 2511 |
+
}
|
| 2512 |
+
},
|
| 2513 |
+
{
|
| 2514 |
+
"threshold": 0.9,
|
| 2515 |
+
"weight": 12.0,
|
| 2516 |
+
"global": {
|
| 2517 |
+
"formulas": 275,
|
| 2518 |
+
"exact_partition": 0.6872727272727273,
|
| 2519 |
+
"exact_group_recall": 0.9556895647614053,
|
| 2520 |
+
"overmerge_rate": 0.13454545454545455,
|
| 2521 |
+
"oversplit_rate": 0.21454545454545454,
|
| 2522 |
+
"pair_precision": 0.964828137490008,
|
| 2523 |
+
"pair_recall": 0.905476369092273,
|
| 2524 |
+
"pair_f1": 0.9342105263157895
|
| 2525 |
+
},
|
| 2526 |
+
"behavior_targets": {
|
| 2527 |
+
"x": {
|
| 2528 |
+
"samples": 277,
|
| 2529 |
+
"matched": 245,
|
| 2530 |
+
"oversplit": 31,
|
| 2531 |
+
"overmerge": 2,
|
| 2532 |
+
"grouping_recall": 0.8844765342960289
|
| 2533 |
+
},
|
| 2534 |
+
"X": {
|
| 2535 |
+
"samples": 7,
|
| 2536 |
+
"matched": 7,
|
| 2537 |
+
"oversplit": 0,
|
| 2538 |
+
"overmerge": 0,
|
| 2539 |
+
"grouping_recall": 1.0
|
| 2540 |
+
},
|
| 2541 |
+
"\\times": {
|
| 2542 |
+
"samples": 44,
|
| 2543 |
+
"matched": 44,
|
| 2544 |
+
"oversplit": 0,
|
| 2545 |
+
"overmerge": 0,
|
| 2546 |
+
"grouping_recall": 1.0
|
| 2547 |
+
}
|
| 2548 |
+
},
|
| 2549 |
+
"families": {
|
| 2550 |
+
"total": 59,
|
| 2551 |
+
"matched": 51,
|
| 2552 |
+
"grouping_recall": 0.864406779661017,
|
| 2553 |
+
"by_label": {
|
| 2554 |
+
"\\sum": {
|
| 2555 |
+
"samples": 11,
|
| 2556 |
+
"matched": 8,
|
| 2557 |
+
"grouping_recall": 0.7272727272727273
|
| 2558 |
+
},
|
| 2559 |
+
"\\pi": {
|
| 2560 |
+
"samples": 17,
|
| 2561 |
+
"matched": 15,
|
| 2562 |
+
"grouping_recall": 0.8823529411764706
|
| 2563 |
+
},
|
| 2564 |
+
"\\rightarrow": {
|
| 2565 |
+
"samples": 20,
|
| 2566 |
+
"matched": 17,
|
| 2567 |
+
"grouping_recall": 0.85
|
| 2568 |
+
},
|
| 2569 |
+
"\\neq": {
|
| 2570 |
+
"samples": 2,
|
| 2571 |
+
"matched": 2,
|
| 2572 |
+
"grouping_recall": 1.0
|
| 2573 |
+
},
|
| 2574 |
+
"\\pm": {
|
| 2575 |
+
"samples": 9,
|
| 2576 |
+
"matched": 9,
|
| 2577 |
+
"grouping_recall": 1.0
|
| 2578 |
+
}
|
| 2579 |
+
}
|
| 2580 |
+
}
|
| 2581 |
+
}
|
| 2582 |
+
],
|
| 2583 |
+
"official_test_reference": {
|
| 2584 |
+
"threshold": 0.3,
|
| 2585 |
+
"weight": 0.0,
|
| 2586 |
+
"global": {
|
| 2587 |
+
"formulas": 488,
|
| 2588 |
+
"exact_partition": 0.6004098360655737,
|
| 2589 |
+
"exact_group_recall": 0.9370456528060482,
|
| 2590 |
+
"overmerge_rate": 0.21721311475409835,
|
| 2591 |
+
"oversplit_rate": 0.2520491803278688,
|
| 2592 |
+
"pair_precision": 0.9231143552311436,
|
| 2593 |
+
"pair_recall": 0.8986262434864993,
|
| 2594 |
+
"pair_f1": 0.9107057129140663
|
| 2595 |
+
},
|
| 2596 |
+
"behavior_targets": {
|
| 2597 |
+
"x": {
|
| 2598 |
+
"samples": 450,
|
| 2599 |
+
"matched": 412,
|
| 2600 |
+
"oversplit": 30,
|
| 2601 |
+
"overmerge": 12,
|
| 2602 |
+
"grouping_recall": 0.9155555555555556
|
| 2603 |
+
},
|
| 2604 |
+
"X": {
|
| 2605 |
+
"samples": 14,
|
| 2606 |
+
"matched": 12,
|
| 2607 |
+
"oversplit": 2,
|
| 2608 |
+
"overmerge": 1,
|
| 2609 |
+
"grouping_recall": 0.8571428571428571
|
| 2610 |
+
},
|
| 2611 |
+
"\\times": {
|
| 2612 |
+
"samples": 36,
|
| 2613 |
+
"matched": 36,
|
| 2614 |
+
"oversplit": 0,
|
| 2615 |
+
"overmerge": 0,
|
| 2616 |
+
"grouping_recall": 1.0
|
| 2617 |
+
}
|
| 2618 |
+
},
|
| 2619 |
+
"families": {
|
| 2620 |
+
"total": 135,
|
| 2621 |
+
"matched": 112,
|
| 2622 |
+
"grouping_recall": 0.8296296296296296,
|
| 2623 |
+
"by_label": {
|
| 2624 |
+
"\\sum": {
|
| 2625 |
+
"samples": 31,
|
| 2626 |
+
"matched": 22,
|
| 2627 |
+
"grouping_recall": 0.7096774193548387
|
| 2628 |
+
},
|
| 2629 |
+
"\\pi": {
|
| 2630 |
+
"samples": 30,
|
| 2631 |
+
"matched": 27,
|
| 2632 |
+
"grouping_recall": 0.9
|
| 2633 |
+
},
|
| 2634 |
+
"\\rightarrow": {
|
| 2635 |
+
"samples": 49,
|
| 2636 |
+
"matched": 44,
|
| 2637 |
+
"grouping_recall": 0.8979591836734694
|
| 2638 |
+
},
|
| 2639 |
+
"\\neq": {
|
| 2640 |
+
"samples": 13,
|
| 2641 |
+
"matched": 10,
|
| 2642 |
+
"grouping_recall": 0.7692307692307693
|
| 2643 |
+
},
|
| 2644 |
+
"\\pm": {
|
| 2645 |
+
"samples": 12,
|
| 2646 |
+
"matched": 9,
|
| 2647 |
+
"grouping_recall": 0.75
|
| 2648 |
+
}
|
| 2649 |
+
}
|
| 2650 |
+
}
|
| 2651 |
+
},
|
| 2652 |
+
"official_test_winner": {
|
| 2653 |
+
"threshold": 0.5,
|
| 2654 |
+
"weight": 6.0,
|
| 2655 |
+
"global": {
|
| 2656 |
+
"formulas": 488,
|
| 2657 |
+
"exact_partition": 0.6024590163934426,
|
| 2658 |
+
"exact_group_recall": 0.9396626926432102,
|
| 2659 |
+
"overmerge_rate": 0.20491803278688525,
|
| 2660 |
+
"oversplit_rate": 0.2581967213114754,
|
| 2661 |
+
"pair_precision": 0.9301524840137727,
|
| 2662 |
+
"pair_recall": 0.8957839886309806,
|
| 2663 |
+
"pair_f1": 0.9126447876447876
|
| 2664 |
+
},
|
| 2665 |
+
"behavior_targets": {
|
| 2666 |
+
"x": {
|
| 2667 |
+
"samples": 450,
|
| 2668 |
+
"matched": 411,
|
| 2669 |
+
"oversplit": 32,
|
| 2670 |
+
"overmerge": 11,
|
| 2671 |
+
"grouping_recall": 0.9133333333333333
|
| 2672 |
+
},
|
| 2673 |
+
"X": {
|
| 2674 |
+
"samples": 14,
|
| 2675 |
+
"matched": 12,
|
| 2676 |
+
"oversplit": 2,
|
| 2677 |
+
"overmerge": 1,
|
| 2678 |
+
"grouping_recall": 0.8571428571428571
|
| 2679 |
+
},
|
| 2680 |
+
"\\times": {
|
| 2681 |
+
"samples": 36,
|
| 2682 |
+
"matched": 36,
|
| 2683 |
+
"oversplit": 0,
|
| 2684 |
+
"overmerge": 0,
|
| 2685 |
+
"grouping_recall": 1.0
|
| 2686 |
+
}
|
| 2687 |
+
},
|
| 2688 |
+
"families": {
|
| 2689 |
+
"total": 135,
|
| 2690 |
+
"matched": 110,
|
| 2691 |
+
"grouping_recall": 0.8148148148148148,
|
| 2692 |
+
"by_label": {
|
| 2693 |
+
"\\sum": {
|
| 2694 |
+
"samples": 31,
|
| 2695 |
+
"matched": 22,
|
| 2696 |
+
"grouping_recall": 0.7096774193548387
|
| 2697 |
+
},
|
| 2698 |
+
"\\pi": {
|
| 2699 |
+
"samples": 30,
|
| 2700 |
+
"matched": 26,
|
| 2701 |
+
"grouping_recall": 0.8666666666666667
|
| 2702 |
+
},
|
| 2703 |
+
"\\rightarrow": {
|
| 2704 |
+
"samples": 49,
|
| 2705 |
+
"matched": 43,
|
| 2706 |
+
"grouping_recall": 0.8775510204081632
|
| 2707 |
+
},
|
| 2708 |
+
"\\neq": {
|
| 2709 |
+
"samples": 13,
|
| 2710 |
+
"matched": 10,
|
| 2711 |
+
"grouping_recall": 0.7692307692307693
|
| 2712 |
+
},
|
| 2713 |
+
"\\pm": {
|
| 2714 |
+
"samples": 12,
|
| 2715 |
+
"matched": 9,
|
| 2716 |
+
"grouping_recall": 0.75
|
| 2717 |
+
}
|
| 2718 |
+
}
|
| 2719 |
+
}
|
| 2720 |
+
},
|
| 2721 |
+
"official_test_deltas": {
|
| 2722 |
+
"exact_partition_pp": 0.2049180327868827,
|
| 2723 |
+
"pair_f1_pp": 0.1939074730721302,
|
| 2724 |
+
"x_grouping_pp": -0.22222222222222365,
|
| 2725 |
+
"family_grouping_pp": -1.4814814814814836
|
| 2726 |
+
},
|
| 2727 |
+
"decision": {
|
| 2728 |
+
"adopted": true,
|
| 2729 |
+
"selected_threshold": 0.5,
|
| 2730 |
+
"selected_weight": 6.0
|
| 2731 |
+
},
|
| 2732 |
+
"artifact": "research\\runs\\math_ink_06_boundary_behavior_guard_20260724\\boundary_behavior_guard.joblib",
|
| 2733 |
+
"track": "R_noncommercial_only",
|
| 2734 |
+
"product_validation": false
|
| 2735 |
+
}
|
reports/component_competition_report.json
ADDED
|
@@ -0,0 +1,843 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-COMPONENT-COMPETITION-GUARD-001",
|
| 3 |
+
"generated_at": "2026-07-23T18:30:20.711476+00:00",
|
| 4 |
+
"selection_contract": {
|
| 5 |
+
"split": "CROHME trainData writer-validation only",
|
| 6 |
+
"cross_gap_ratio": 0.4,
|
| 7 |
+
"multistroke_family_boost": 6.0,
|
| 8 |
+
"maximum_x_regression_pp": 1.0,
|
| 9 |
+
"maximum_family_regression_pp": 2.0,
|
| 10 |
+
"maximum_pair_f1_regression_pp": 0.25
|
| 11 |
+
},
|
| 12 |
+
"reference_validation": {
|
| 13 |
+
"component_competition_weight": 0.0,
|
| 14 |
+
"global": {
|
| 15 |
+
"formulas": 275,
|
| 16 |
+
"exact_partition": 0.68,
|
| 17 |
+
"exact_group_recall": 0.9528054535920294,
|
| 18 |
+
"overmerge_rate": 0.14909090909090908,
|
| 19 |
+
"oversplit_rate": 0.21818181818181817,
|
| 20 |
+
"pair_precision": 0.9563838223632039,
|
| 21 |
+
"pair_recall": 0.9047261815453863,
|
| 22 |
+
"pair_f1": 0.9298380878951427
|
| 23 |
+
},
|
| 24 |
+
"behavior_targets": {
|
| 25 |
+
"x": {
|
| 26 |
+
"samples": 277,
|
| 27 |
+
"matched": 244,
|
| 28 |
+
"oversplit": 31,
|
| 29 |
+
"overmerge": 3,
|
| 30 |
+
"grouping_recall": 0.8808664259927798
|
| 31 |
+
},
|
| 32 |
+
"X": {
|
| 33 |
+
"samples": 7,
|
| 34 |
+
"matched": 7,
|
| 35 |
+
"oversplit": 0,
|
| 36 |
+
"overmerge": 0,
|
| 37 |
+
"grouping_recall": 1.0
|
| 38 |
+
},
|
| 39 |
+
"\\times": {
|
| 40 |
+
"samples": 44,
|
| 41 |
+
"matched": 44,
|
| 42 |
+
"oversplit": 0,
|
| 43 |
+
"overmerge": 0,
|
| 44 |
+
"grouping_recall": 1.0
|
| 45 |
+
}
|
| 46 |
+
},
|
| 47 |
+
"families": {
|
| 48 |
+
"total": 59,
|
| 49 |
+
"matched": 51,
|
| 50 |
+
"grouping_recall": 0.864406779661017,
|
| 51 |
+
"by_label": {
|
| 52 |
+
"\\sum": {
|
| 53 |
+
"samples": 11,
|
| 54 |
+
"matched": 8,
|
| 55 |
+
"grouping_recall": 0.7272727272727273
|
| 56 |
+
},
|
| 57 |
+
"\\pi": {
|
| 58 |
+
"samples": 17,
|
| 59 |
+
"matched": 15,
|
| 60 |
+
"grouping_recall": 0.8823529411764706
|
| 61 |
+
},
|
| 62 |
+
"\\rightarrow": {
|
| 63 |
+
"samples": 20,
|
| 64 |
+
"matched": 17,
|
| 65 |
+
"grouping_recall": 0.85
|
| 66 |
+
},
|
| 67 |
+
"\\neq": {
|
| 68 |
+
"samples": 2,
|
| 69 |
+
"matched": 2,
|
| 70 |
+
"grouping_recall": 1.0
|
| 71 |
+
},
|
| 72 |
+
"\\pm": {
|
| 73 |
+
"samples": 9,
|
| 74 |
+
"matched": 9,
|
| 75 |
+
"grouping_recall": 1.0
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
}
|
| 79 |
+
},
|
| 80 |
+
"winner_validation": {
|
| 81 |
+
"component_competition_weight": 0.0,
|
| 82 |
+
"global": {
|
| 83 |
+
"formulas": 275,
|
| 84 |
+
"exact_partition": 0.68,
|
| 85 |
+
"exact_group_recall": 0.9528054535920294,
|
| 86 |
+
"overmerge_rate": 0.14909090909090908,
|
| 87 |
+
"oversplit_rate": 0.21818181818181817,
|
| 88 |
+
"pair_precision": 0.9563838223632039,
|
| 89 |
+
"pair_recall": 0.9047261815453863,
|
| 90 |
+
"pair_f1": 0.9298380878951427
|
| 91 |
+
},
|
| 92 |
+
"behavior_targets": {
|
| 93 |
+
"x": {
|
| 94 |
+
"samples": 277,
|
| 95 |
+
"matched": 244,
|
| 96 |
+
"oversplit": 31,
|
| 97 |
+
"overmerge": 3,
|
| 98 |
+
"grouping_recall": 0.8808664259927798
|
| 99 |
+
},
|
| 100 |
+
"X": {
|
| 101 |
+
"samples": 7,
|
| 102 |
+
"matched": 7,
|
| 103 |
+
"oversplit": 0,
|
| 104 |
+
"overmerge": 0,
|
| 105 |
+
"grouping_recall": 1.0
|
| 106 |
+
},
|
| 107 |
+
"\\times": {
|
| 108 |
+
"samples": 44,
|
| 109 |
+
"matched": 44,
|
| 110 |
+
"oversplit": 0,
|
| 111 |
+
"overmerge": 0,
|
| 112 |
+
"grouping_recall": 1.0
|
| 113 |
+
}
|
| 114 |
+
},
|
| 115 |
+
"families": {
|
| 116 |
+
"total": 59,
|
| 117 |
+
"matched": 51,
|
| 118 |
+
"grouping_recall": 0.864406779661017,
|
| 119 |
+
"by_label": {
|
| 120 |
+
"\\sum": {
|
| 121 |
+
"samples": 11,
|
| 122 |
+
"matched": 8,
|
| 123 |
+
"grouping_recall": 0.7272727272727273
|
| 124 |
+
},
|
| 125 |
+
"\\pi": {
|
| 126 |
+
"samples": 17,
|
| 127 |
+
"matched": 15,
|
| 128 |
+
"grouping_recall": 0.8823529411764706
|
| 129 |
+
},
|
| 130 |
+
"\\rightarrow": {
|
| 131 |
+
"samples": 20,
|
| 132 |
+
"matched": 17,
|
| 133 |
+
"grouping_recall": 0.85
|
| 134 |
+
},
|
| 135 |
+
"\\neq": {
|
| 136 |
+
"samples": 2,
|
| 137 |
+
"matched": 2,
|
| 138 |
+
"grouping_recall": 1.0
|
| 139 |
+
},
|
| 140 |
+
"\\pm": {
|
| 141 |
+
"samples": 9,
|
| 142 |
+
"matched": 9,
|
| 143 |
+
"grouping_recall": 1.0
|
| 144 |
+
}
|
| 145 |
+
}
|
| 146 |
+
}
|
| 147 |
+
},
|
| 148 |
+
"trials": [
|
| 149 |
+
{
|
| 150 |
+
"component_competition_weight": 0.0,
|
| 151 |
+
"global": {
|
| 152 |
+
"formulas": 275,
|
| 153 |
+
"exact_partition": 0.68,
|
| 154 |
+
"exact_group_recall": 0.9528054535920294,
|
| 155 |
+
"overmerge_rate": 0.14909090909090908,
|
| 156 |
+
"oversplit_rate": 0.21818181818181817,
|
| 157 |
+
"pair_precision": 0.9563838223632039,
|
| 158 |
+
"pair_recall": 0.9047261815453863,
|
| 159 |
+
"pair_f1": 0.9298380878951427
|
| 160 |
+
},
|
| 161 |
+
"behavior_targets": {
|
| 162 |
+
"x": {
|
| 163 |
+
"samples": 277,
|
| 164 |
+
"matched": 244,
|
| 165 |
+
"oversplit": 31,
|
| 166 |
+
"overmerge": 3,
|
| 167 |
+
"grouping_recall": 0.8808664259927798
|
| 168 |
+
},
|
| 169 |
+
"X": {
|
| 170 |
+
"samples": 7,
|
| 171 |
+
"matched": 7,
|
| 172 |
+
"oversplit": 0,
|
| 173 |
+
"overmerge": 0,
|
| 174 |
+
"grouping_recall": 1.0
|
| 175 |
+
},
|
| 176 |
+
"\\times": {
|
| 177 |
+
"samples": 44,
|
| 178 |
+
"matched": 44,
|
| 179 |
+
"oversplit": 0,
|
| 180 |
+
"overmerge": 0,
|
| 181 |
+
"grouping_recall": 1.0
|
| 182 |
+
}
|
| 183 |
+
},
|
| 184 |
+
"families": {
|
| 185 |
+
"total": 59,
|
| 186 |
+
"matched": 51,
|
| 187 |
+
"grouping_recall": 0.864406779661017,
|
| 188 |
+
"by_label": {
|
| 189 |
+
"\\sum": {
|
| 190 |
+
"samples": 11,
|
| 191 |
+
"matched": 8,
|
| 192 |
+
"grouping_recall": 0.7272727272727273
|
| 193 |
+
},
|
| 194 |
+
"\\pi": {
|
| 195 |
+
"samples": 17,
|
| 196 |
+
"matched": 15,
|
| 197 |
+
"grouping_recall": 0.8823529411764706
|
| 198 |
+
},
|
| 199 |
+
"\\rightarrow": {
|
| 200 |
+
"samples": 20,
|
| 201 |
+
"matched": 17,
|
| 202 |
+
"grouping_recall": 0.85
|
| 203 |
+
},
|
| 204 |
+
"\\neq": {
|
| 205 |
+
"samples": 2,
|
| 206 |
+
"matched": 2,
|
| 207 |
+
"grouping_recall": 1.0
|
| 208 |
+
},
|
| 209 |
+
"\\pm": {
|
| 210 |
+
"samples": 9,
|
| 211 |
+
"matched": 9,
|
| 212 |
+
"grouping_recall": 1.0
|
| 213 |
+
}
|
| 214 |
+
}
|
| 215 |
+
}
|
| 216 |
+
},
|
| 217 |
+
{
|
| 218 |
+
"component_competition_weight": 1.0,
|
| 219 |
+
"global": {
|
| 220 |
+
"formulas": 275,
|
| 221 |
+
"exact_partition": 0.68,
|
| 222 |
+
"exact_group_recall": 0.9538542212899843,
|
| 223 |
+
"overmerge_rate": 0.14181818181818182,
|
| 224 |
+
"oversplit_rate": 0.22545454545454546,
|
| 225 |
+
"pair_precision": 0.9585326953748007,
|
| 226 |
+
"pair_recall": 0.9017254313578394,
|
| 227 |
+
"pair_f1": 0.9292616930807887
|
| 228 |
+
},
|
| 229 |
+
"behavior_targets": {
|
| 230 |
+
"x": {
|
| 231 |
+
"samples": 277,
|
| 232 |
+
"matched": 244,
|
| 233 |
+
"oversplit": 31,
|
| 234 |
+
"overmerge": 3,
|
| 235 |
+
"grouping_recall": 0.8808664259927798
|
| 236 |
+
},
|
| 237 |
+
"X": {
|
| 238 |
+
"samples": 7,
|
| 239 |
+
"matched": 7,
|
| 240 |
+
"oversplit": 0,
|
| 241 |
+
"overmerge": 0,
|
| 242 |
+
"grouping_recall": 1.0
|
| 243 |
+
},
|
| 244 |
+
"\\times": {
|
| 245 |
+
"samples": 44,
|
| 246 |
+
"matched": 44,
|
| 247 |
+
"oversplit": 0,
|
| 248 |
+
"overmerge": 0,
|
| 249 |
+
"grouping_recall": 1.0
|
| 250 |
+
}
|
| 251 |
+
},
|
| 252 |
+
"families": {
|
| 253 |
+
"total": 59,
|
| 254 |
+
"matched": 50,
|
| 255 |
+
"grouping_recall": 0.847457627118644,
|
| 256 |
+
"by_label": {
|
| 257 |
+
"\\sum": {
|
| 258 |
+
"samples": 11,
|
| 259 |
+
"matched": 8,
|
| 260 |
+
"grouping_recall": 0.7272727272727273
|
| 261 |
+
},
|
| 262 |
+
"\\pi": {
|
| 263 |
+
"samples": 17,
|
| 264 |
+
"matched": 15,
|
| 265 |
+
"grouping_recall": 0.8823529411764706
|
| 266 |
+
},
|
| 267 |
+
"\\rightarrow": {
|
| 268 |
+
"samples": 20,
|
| 269 |
+
"matched": 16,
|
| 270 |
+
"grouping_recall": 0.8
|
| 271 |
+
},
|
| 272 |
+
"\\neq": {
|
| 273 |
+
"samples": 2,
|
| 274 |
+
"matched": 2,
|
| 275 |
+
"grouping_recall": 1.0
|
| 276 |
+
},
|
| 277 |
+
"\\pm": {
|
| 278 |
+
"samples": 9,
|
| 279 |
+
"matched": 9,
|
| 280 |
+
"grouping_recall": 1.0
|
| 281 |
+
}
|
| 282 |
+
}
|
| 283 |
+
}
|
| 284 |
+
},
|
| 285 |
+
{
|
| 286 |
+
"component_competition_weight": 2.0,
|
| 287 |
+
"global": {
|
| 288 |
+
"formulas": 275,
|
| 289 |
+
"exact_partition": 0.6727272727272727,
|
| 290 |
+
"exact_group_recall": 0.9538542212899843,
|
| 291 |
+
"overmerge_rate": 0.13090909090909092,
|
| 292 |
+
"oversplit_rate": 0.23272727272727273,
|
| 293 |
+
"pair_precision": 0.9599037690457097,
|
| 294 |
+
"pair_recall": 0.8979744936234059,
|
| 295 |
+
"pair_f1": 0.9279069767441861
|
| 296 |
+
},
|
| 297 |
+
"behavior_targets": {
|
| 298 |
+
"x": {
|
| 299 |
+
"samples": 277,
|
| 300 |
+
"matched": 242,
|
| 301 |
+
"oversplit": 33,
|
| 302 |
+
"overmerge": 3,
|
| 303 |
+
"grouping_recall": 0.8736462093862816
|
| 304 |
+
},
|
| 305 |
+
"X": {
|
| 306 |
+
"samples": 7,
|
| 307 |
+
"matched": 7,
|
| 308 |
+
"oversplit": 0,
|
| 309 |
+
"overmerge": 0,
|
| 310 |
+
"grouping_recall": 1.0
|
| 311 |
+
},
|
| 312 |
+
"\\times": {
|
| 313 |
+
"samples": 44,
|
| 314 |
+
"matched": 44,
|
| 315 |
+
"oversplit": 0,
|
| 316 |
+
"overmerge": 0,
|
| 317 |
+
"grouping_recall": 1.0
|
| 318 |
+
}
|
| 319 |
+
},
|
| 320 |
+
"families": {
|
| 321 |
+
"total": 59,
|
| 322 |
+
"matched": 50,
|
| 323 |
+
"grouping_recall": 0.847457627118644,
|
| 324 |
+
"by_label": {
|
| 325 |
+
"\\sum": {
|
| 326 |
+
"samples": 11,
|
| 327 |
+
"matched": 8,
|
| 328 |
+
"grouping_recall": 0.7272727272727273
|
| 329 |
+
},
|
| 330 |
+
"\\pi": {
|
| 331 |
+
"samples": 17,
|
| 332 |
+
"matched": 15,
|
| 333 |
+
"grouping_recall": 0.8823529411764706
|
| 334 |
+
},
|
| 335 |
+
"\\rightarrow": {
|
| 336 |
+
"samples": 20,
|
| 337 |
+
"matched": 16,
|
| 338 |
+
"grouping_recall": 0.8
|
| 339 |
+
},
|
| 340 |
+
"\\neq": {
|
| 341 |
+
"samples": 2,
|
| 342 |
+
"matched": 2,
|
| 343 |
+
"grouping_recall": 1.0
|
| 344 |
+
},
|
| 345 |
+
"\\pm": {
|
| 346 |
+
"samples": 9,
|
| 347 |
+
"matched": 9,
|
| 348 |
+
"grouping_recall": 1.0
|
| 349 |
+
}
|
| 350 |
+
}
|
| 351 |
+
}
|
| 352 |
+
},
|
| 353 |
+
{
|
| 354 |
+
"component_competition_weight": 3.0,
|
| 355 |
+
"global": {
|
| 356 |
+
"formulas": 275,
|
| 357 |
+
"exact_partition": 0.6436363636363637,
|
| 358 |
+
"exact_group_recall": 0.9509701101206083,
|
| 359 |
+
"overmerge_rate": 0.13090909090909092,
|
| 360 |
+
"oversplit_rate": 0.26545454545454544,
|
| 361 |
+
"pair_precision": 0.9586038961038961,
|
| 362 |
+
"pair_recall": 0.8859714928732183,
|
| 363 |
+
"pair_f1": 0.9208576998050683
|
| 364 |
+
},
|
| 365 |
+
"behavior_targets": {
|
| 366 |
+
"x": {
|
| 367 |
+
"samples": 277,
|
| 368 |
+
"matched": 240,
|
| 369 |
+
"oversplit": 35,
|
| 370 |
+
"overmerge": 3,
|
| 371 |
+
"grouping_recall": 0.8664259927797834
|
| 372 |
+
},
|
| 373 |
+
"X": {
|
| 374 |
+
"samples": 7,
|
| 375 |
+
"matched": 7,
|
| 376 |
+
"oversplit": 0,
|
| 377 |
+
"overmerge": 0,
|
| 378 |
+
"grouping_recall": 1.0
|
| 379 |
+
},
|
| 380 |
+
"\\times": {
|
| 381 |
+
"samples": 44,
|
| 382 |
+
"matched": 44,
|
| 383 |
+
"oversplit": 0,
|
| 384 |
+
"overmerge": 0,
|
| 385 |
+
"grouping_recall": 1.0
|
| 386 |
+
}
|
| 387 |
+
},
|
| 388 |
+
"families": {
|
| 389 |
+
"total": 59,
|
| 390 |
+
"matched": 50,
|
| 391 |
+
"grouping_recall": 0.847457627118644,
|
| 392 |
+
"by_label": {
|
| 393 |
+
"\\sum": {
|
| 394 |
+
"samples": 11,
|
| 395 |
+
"matched": 8,
|
| 396 |
+
"grouping_recall": 0.7272727272727273
|
| 397 |
+
},
|
| 398 |
+
"\\pi": {
|
| 399 |
+
"samples": 17,
|
| 400 |
+
"matched": 15,
|
| 401 |
+
"grouping_recall": 0.8823529411764706
|
| 402 |
+
},
|
| 403 |
+
"\\rightarrow": {
|
| 404 |
+
"samples": 20,
|
| 405 |
+
"matched": 16,
|
| 406 |
+
"grouping_recall": 0.8
|
| 407 |
+
},
|
| 408 |
+
"\\neq": {
|
| 409 |
+
"samples": 2,
|
| 410 |
+
"matched": 2,
|
| 411 |
+
"grouping_recall": 1.0
|
| 412 |
+
},
|
| 413 |
+
"\\pm": {
|
| 414 |
+
"samples": 9,
|
| 415 |
+
"matched": 9,
|
| 416 |
+
"grouping_recall": 1.0
|
| 417 |
+
}
|
| 418 |
+
}
|
| 419 |
+
}
|
| 420 |
+
},
|
| 421 |
+
{
|
| 422 |
+
"component_competition_weight": 4.0,
|
| 423 |
+
"global": {
|
| 424 |
+
"formulas": 275,
|
| 425 |
+
"exact_partition": 0.64,
|
| 426 |
+
"exact_group_recall": 0.9507079181961196,
|
| 427 |
+
"overmerge_rate": 0.12363636363636364,
|
| 428 |
+
"oversplit_rate": 0.28,
|
| 429 |
+
"pair_precision": 0.9607843137254902,
|
| 430 |
+
"pair_recall": 0.8822205551387847,
|
| 431 |
+
"pair_f1": 0.9198279233476732
|
| 432 |
+
},
|
| 433 |
+
"behavior_targets": {
|
| 434 |
+
"x": {
|
| 435 |
+
"samples": 277,
|
| 436 |
+
"matched": 240,
|
| 437 |
+
"oversplit": 35,
|
| 438 |
+
"overmerge": 3,
|
| 439 |
+
"grouping_recall": 0.8664259927797834
|
| 440 |
+
},
|
| 441 |
+
"X": {
|
| 442 |
+
"samples": 7,
|
| 443 |
+
"matched": 7,
|
| 444 |
+
"oversplit": 0,
|
| 445 |
+
"overmerge": 0,
|
| 446 |
+
"grouping_recall": 1.0
|
| 447 |
+
},
|
| 448 |
+
"\\times": {
|
| 449 |
+
"samples": 44,
|
| 450 |
+
"matched": 44,
|
| 451 |
+
"oversplit": 0,
|
| 452 |
+
"overmerge": 0,
|
| 453 |
+
"grouping_recall": 1.0
|
| 454 |
+
}
|
| 455 |
+
},
|
| 456 |
+
"families": {
|
| 457 |
+
"total": 59,
|
| 458 |
+
"matched": 50,
|
| 459 |
+
"grouping_recall": 0.847457627118644,
|
| 460 |
+
"by_label": {
|
| 461 |
+
"\\sum": {
|
| 462 |
+
"samples": 11,
|
| 463 |
+
"matched": 8,
|
| 464 |
+
"grouping_recall": 0.7272727272727273
|
| 465 |
+
},
|
| 466 |
+
"\\pi": {
|
| 467 |
+
"samples": 17,
|
| 468 |
+
"matched": 15,
|
| 469 |
+
"grouping_recall": 0.8823529411764706
|
| 470 |
+
},
|
| 471 |
+
"\\rightarrow": {
|
| 472 |
+
"samples": 20,
|
| 473 |
+
"matched": 16,
|
| 474 |
+
"grouping_recall": 0.8
|
| 475 |
+
},
|
| 476 |
+
"\\neq": {
|
| 477 |
+
"samples": 2,
|
| 478 |
+
"matched": 2,
|
| 479 |
+
"grouping_recall": 1.0
|
| 480 |
+
},
|
| 481 |
+
"\\pm": {
|
| 482 |
+
"samples": 9,
|
| 483 |
+
"matched": 9,
|
| 484 |
+
"grouping_recall": 1.0
|
| 485 |
+
}
|
| 486 |
+
}
|
| 487 |
+
}
|
| 488 |
+
},
|
| 489 |
+
{
|
| 490 |
+
"component_competition_weight": 6.0,
|
| 491 |
+
"global": {
|
| 492 |
+
"formulas": 275,
|
| 493 |
+
"exact_partition": 0.6290909090909091,
|
| 494 |
+
"exact_group_recall": 0.9499213424226534,
|
| 495 |
+
"overmerge_rate": 0.11636363636363636,
|
| 496 |
+
"oversplit_rate": 0.29818181818181816,
|
| 497 |
+
"pair_precision": 0.9628406275805119,
|
| 498 |
+
"pair_recall": 0.8747186796699175,
|
| 499 |
+
"pair_f1": 0.9166666666666666
|
| 500 |
+
},
|
| 501 |
+
"behavior_targets": {
|
| 502 |
+
"x": {
|
| 503 |
+
"samples": 277,
|
| 504 |
+
"matched": 239,
|
| 505 |
+
"oversplit": 36,
|
| 506 |
+
"overmerge": 3,
|
| 507 |
+
"grouping_recall": 0.8628158844765343
|
| 508 |
+
},
|
| 509 |
+
"X": {
|
| 510 |
+
"samples": 7,
|
| 511 |
+
"matched": 7,
|
| 512 |
+
"oversplit": 0,
|
| 513 |
+
"overmerge": 0,
|
| 514 |
+
"grouping_recall": 1.0
|
| 515 |
+
},
|
| 516 |
+
"\\times": {
|
| 517 |
+
"samples": 44,
|
| 518 |
+
"matched": 44,
|
| 519 |
+
"oversplit": 0,
|
| 520 |
+
"overmerge": 0,
|
| 521 |
+
"grouping_recall": 1.0
|
| 522 |
+
}
|
| 523 |
+
},
|
| 524 |
+
"families": {
|
| 525 |
+
"total": 59,
|
| 526 |
+
"matched": 50,
|
| 527 |
+
"grouping_recall": 0.847457627118644,
|
| 528 |
+
"by_label": {
|
| 529 |
+
"\\sum": {
|
| 530 |
+
"samples": 11,
|
| 531 |
+
"matched": 8,
|
| 532 |
+
"grouping_recall": 0.7272727272727273
|
| 533 |
+
},
|
| 534 |
+
"\\pi": {
|
| 535 |
+
"samples": 17,
|
| 536 |
+
"matched": 15,
|
| 537 |
+
"grouping_recall": 0.8823529411764706
|
| 538 |
+
},
|
| 539 |
+
"\\rightarrow": {
|
| 540 |
+
"samples": 20,
|
| 541 |
+
"matched": 16,
|
| 542 |
+
"grouping_recall": 0.8
|
| 543 |
+
},
|
| 544 |
+
"\\neq": {
|
| 545 |
+
"samples": 2,
|
| 546 |
+
"matched": 2,
|
| 547 |
+
"grouping_recall": 1.0
|
| 548 |
+
},
|
| 549 |
+
"\\pm": {
|
| 550 |
+
"samples": 9,
|
| 551 |
+
"matched": 9,
|
| 552 |
+
"grouping_recall": 1.0
|
| 553 |
+
}
|
| 554 |
+
}
|
| 555 |
+
}
|
| 556 |
+
},
|
| 557 |
+
{
|
| 558 |
+
"component_competition_weight": 8.0,
|
| 559 |
+
"global": {
|
| 560 |
+
"formulas": 275,
|
| 561 |
+
"exact_partition": 0.6145454545454545,
|
| 562 |
+
"exact_group_recall": 0.948348190875721,
|
| 563 |
+
"overmerge_rate": 0.11272727272727273,
|
| 564 |
+
"oversplit_rate": 0.31636363636363635,
|
| 565 |
+
"pair_precision": 0.9633943427620633,
|
| 566 |
+
"pair_recall": 0.8687171792948237,
|
| 567 |
+
"pair_f1": 0.9136094674556214
|
| 568 |
+
},
|
| 569 |
+
"behavior_targets": {
|
| 570 |
+
"x": {
|
| 571 |
+
"samples": 277,
|
| 572 |
+
"matched": 237,
|
| 573 |
+
"oversplit": 38,
|
| 574 |
+
"overmerge": 3,
|
| 575 |
+
"grouping_recall": 0.855595667870036
|
| 576 |
+
},
|
| 577 |
+
"X": {
|
| 578 |
+
"samples": 7,
|
| 579 |
+
"matched": 7,
|
| 580 |
+
"oversplit": 0,
|
| 581 |
+
"overmerge": 0,
|
| 582 |
+
"grouping_recall": 1.0
|
| 583 |
+
},
|
| 584 |
+
"\\times": {
|
| 585 |
+
"samples": 44,
|
| 586 |
+
"matched": 44,
|
| 587 |
+
"oversplit": 0,
|
| 588 |
+
"overmerge": 0,
|
| 589 |
+
"grouping_recall": 1.0
|
| 590 |
+
}
|
| 591 |
+
},
|
| 592 |
+
"families": {
|
| 593 |
+
"total": 59,
|
| 594 |
+
"matched": 50,
|
| 595 |
+
"grouping_recall": 0.847457627118644,
|
| 596 |
+
"by_label": {
|
| 597 |
+
"\\sum": {
|
| 598 |
+
"samples": 11,
|
| 599 |
+
"matched": 8,
|
| 600 |
+
"grouping_recall": 0.7272727272727273
|
| 601 |
+
},
|
| 602 |
+
"\\pi": {
|
| 603 |
+
"samples": 17,
|
| 604 |
+
"matched": 15,
|
| 605 |
+
"grouping_recall": 0.8823529411764706
|
| 606 |
+
},
|
| 607 |
+
"\\rightarrow": {
|
| 608 |
+
"samples": 20,
|
| 609 |
+
"matched": 16,
|
| 610 |
+
"grouping_recall": 0.8
|
| 611 |
+
},
|
| 612 |
+
"\\neq": {
|
| 613 |
+
"samples": 2,
|
| 614 |
+
"matched": 2,
|
| 615 |
+
"grouping_recall": 1.0
|
| 616 |
+
},
|
| 617 |
+
"\\pm": {
|
| 618 |
+
"samples": 9,
|
| 619 |
+
"matched": 9,
|
| 620 |
+
"grouping_recall": 1.0
|
| 621 |
+
}
|
| 622 |
+
}
|
| 623 |
+
}
|
| 624 |
+
},
|
| 625 |
+
{
|
| 626 |
+
"component_competition_weight": 12.0,
|
| 627 |
+
"global": {
|
| 628 |
+
"formulas": 275,
|
| 629 |
+
"exact_partition": 0.5527272727272727,
|
| 630 |
+
"exact_group_recall": 0.9396958573675931,
|
| 631 |
+
"overmerge_rate": 0.11272727272727273,
|
| 632 |
+
"oversplit_rate": 0.3927272727272727,
|
| 633 |
+
"pair_precision": 0.9621993127147767,
|
| 634 |
+
"pair_recall": 0.8402100525131283,
|
| 635 |
+
"pair_f1": 0.8970764917901483
|
| 636 |
+
},
|
| 637 |
+
"behavior_targets": {
|
| 638 |
+
"x": {
|
| 639 |
+
"samples": 277,
|
| 640 |
+
"matched": 229,
|
| 641 |
+
"oversplit": 46,
|
| 642 |
+
"overmerge": 3,
|
| 643 |
+
"grouping_recall": 0.8267148014440433
|
| 644 |
+
},
|
| 645 |
+
"X": {
|
| 646 |
+
"samples": 7,
|
| 647 |
+
"matched": 7,
|
| 648 |
+
"oversplit": 0,
|
| 649 |
+
"overmerge": 0,
|
| 650 |
+
"grouping_recall": 1.0
|
| 651 |
+
},
|
| 652 |
+
"\\times": {
|
| 653 |
+
"samples": 44,
|
| 654 |
+
"matched": 44,
|
| 655 |
+
"oversplit": 0,
|
| 656 |
+
"overmerge": 0,
|
| 657 |
+
"grouping_recall": 1.0
|
| 658 |
+
}
|
| 659 |
+
},
|
| 660 |
+
"families": {
|
| 661 |
+
"total": 59,
|
| 662 |
+
"matched": 48,
|
| 663 |
+
"grouping_recall": 0.8135593220338984,
|
| 664 |
+
"by_label": {
|
| 665 |
+
"\\sum": {
|
| 666 |
+
"samples": 11,
|
| 667 |
+
"matched": 8,
|
| 668 |
+
"grouping_recall": 0.7272727272727273
|
| 669 |
+
},
|
| 670 |
+
"\\pi": {
|
| 671 |
+
"samples": 17,
|
| 672 |
+
"matched": 14,
|
| 673 |
+
"grouping_recall": 0.8235294117647058
|
| 674 |
+
},
|
| 675 |
+
"\\rightarrow": {
|
| 676 |
+
"samples": 20,
|
| 677 |
+
"matched": 15,
|
| 678 |
+
"grouping_recall": 0.75
|
| 679 |
+
},
|
| 680 |
+
"\\neq": {
|
| 681 |
+
"samples": 2,
|
| 682 |
+
"matched": 2,
|
| 683 |
+
"grouping_recall": 1.0
|
| 684 |
+
},
|
| 685 |
+
"\\pm": {
|
| 686 |
+
"samples": 9,
|
| 687 |
+
"matched": 9,
|
| 688 |
+
"grouping_recall": 1.0
|
| 689 |
+
}
|
| 690 |
+
}
|
| 691 |
+
}
|
| 692 |
+
}
|
| 693 |
+
],
|
| 694 |
+
"official_test_reference": {
|
| 695 |
+
"component_competition_weight": 0.0,
|
| 696 |
+
"global": {
|
| 697 |
+
"formulas": 488,
|
| 698 |
+
"exact_partition": 0.6004098360655737,
|
| 699 |
+
"exact_group_recall": 0.9370456528060482,
|
| 700 |
+
"overmerge_rate": 0.21721311475409835,
|
| 701 |
+
"oversplit_rate": 0.2520491803278688,
|
| 702 |
+
"pair_precision": 0.9231143552311436,
|
| 703 |
+
"pair_recall": 0.8986262434864993,
|
| 704 |
+
"pair_f1": 0.9107057129140663
|
| 705 |
+
},
|
| 706 |
+
"behavior_targets": {
|
| 707 |
+
"x": {
|
| 708 |
+
"samples": 450,
|
| 709 |
+
"matched": 412,
|
| 710 |
+
"oversplit": 30,
|
| 711 |
+
"overmerge": 12,
|
| 712 |
+
"grouping_recall": 0.9155555555555556
|
| 713 |
+
},
|
| 714 |
+
"X": {
|
| 715 |
+
"samples": 14,
|
| 716 |
+
"matched": 12,
|
| 717 |
+
"oversplit": 2,
|
| 718 |
+
"overmerge": 1,
|
| 719 |
+
"grouping_recall": 0.8571428571428571
|
| 720 |
+
},
|
| 721 |
+
"\\times": {
|
| 722 |
+
"samples": 36,
|
| 723 |
+
"matched": 36,
|
| 724 |
+
"oversplit": 0,
|
| 725 |
+
"overmerge": 0,
|
| 726 |
+
"grouping_recall": 1.0
|
| 727 |
+
}
|
| 728 |
+
},
|
| 729 |
+
"families": {
|
| 730 |
+
"total": 135,
|
| 731 |
+
"matched": 112,
|
| 732 |
+
"grouping_recall": 0.8296296296296296,
|
| 733 |
+
"by_label": {
|
| 734 |
+
"\\sum": {
|
| 735 |
+
"samples": 31,
|
| 736 |
+
"matched": 22,
|
| 737 |
+
"grouping_recall": 0.7096774193548387
|
| 738 |
+
},
|
| 739 |
+
"\\pi": {
|
| 740 |
+
"samples": 30,
|
| 741 |
+
"matched": 27,
|
| 742 |
+
"grouping_recall": 0.9
|
| 743 |
+
},
|
| 744 |
+
"\\rightarrow": {
|
| 745 |
+
"samples": 49,
|
| 746 |
+
"matched": 44,
|
| 747 |
+
"grouping_recall": 0.8979591836734694
|
| 748 |
+
},
|
| 749 |
+
"\\neq": {
|
| 750 |
+
"samples": 13,
|
| 751 |
+
"matched": 10,
|
| 752 |
+
"grouping_recall": 0.7692307692307693
|
| 753 |
+
},
|
| 754 |
+
"\\pm": {
|
| 755 |
+
"samples": 12,
|
| 756 |
+
"matched": 9,
|
| 757 |
+
"grouping_recall": 0.75
|
| 758 |
+
}
|
| 759 |
+
}
|
| 760 |
+
}
|
| 761 |
+
},
|
| 762 |
+
"official_test_winner": {
|
| 763 |
+
"component_competition_weight": 0.0,
|
| 764 |
+
"global": {
|
| 765 |
+
"formulas": 488,
|
| 766 |
+
"exact_partition": 0.6004098360655737,
|
| 767 |
+
"exact_group_recall": 0.9370456528060482,
|
| 768 |
+
"overmerge_rate": 0.21721311475409835,
|
| 769 |
+
"oversplit_rate": 0.2520491803278688,
|
| 770 |
+
"pair_precision": 0.9231143552311436,
|
| 771 |
+
"pair_recall": 0.8986262434864993,
|
| 772 |
+
"pair_f1": 0.9107057129140663
|
| 773 |
+
},
|
| 774 |
+
"behavior_targets": {
|
| 775 |
+
"x": {
|
| 776 |
+
"samples": 450,
|
| 777 |
+
"matched": 412,
|
| 778 |
+
"oversplit": 30,
|
| 779 |
+
"overmerge": 12,
|
| 780 |
+
"grouping_recall": 0.9155555555555556
|
| 781 |
+
},
|
| 782 |
+
"X": {
|
| 783 |
+
"samples": 14,
|
| 784 |
+
"matched": 12,
|
| 785 |
+
"oversplit": 2,
|
| 786 |
+
"overmerge": 1,
|
| 787 |
+
"grouping_recall": 0.8571428571428571
|
| 788 |
+
},
|
| 789 |
+
"\\times": {
|
| 790 |
+
"samples": 36,
|
| 791 |
+
"matched": 36,
|
| 792 |
+
"oversplit": 0,
|
| 793 |
+
"overmerge": 0,
|
| 794 |
+
"grouping_recall": 1.0
|
| 795 |
+
}
|
| 796 |
+
},
|
| 797 |
+
"families": {
|
| 798 |
+
"total": 135,
|
| 799 |
+
"matched": 112,
|
| 800 |
+
"grouping_recall": 0.8296296296296296,
|
| 801 |
+
"by_label": {
|
| 802 |
+
"\\sum": {
|
| 803 |
+
"samples": 31,
|
| 804 |
+
"matched": 22,
|
| 805 |
+
"grouping_recall": 0.7096774193548387
|
| 806 |
+
},
|
| 807 |
+
"\\pi": {
|
| 808 |
+
"samples": 30,
|
| 809 |
+
"matched": 27,
|
| 810 |
+
"grouping_recall": 0.9
|
| 811 |
+
},
|
| 812 |
+
"\\rightarrow": {
|
| 813 |
+
"samples": 49,
|
| 814 |
+
"matched": 44,
|
| 815 |
+
"grouping_recall": 0.8979591836734694
|
| 816 |
+
},
|
| 817 |
+
"\\neq": {
|
| 818 |
+
"samples": 13,
|
| 819 |
+
"matched": 10,
|
| 820 |
+
"grouping_recall": 0.7692307692307693
|
| 821 |
+
},
|
| 822 |
+
"\\pm": {
|
| 823 |
+
"samples": 12,
|
| 824 |
+
"matched": 9,
|
| 825 |
+
"grouping_recall": 0.75
|
| 826 |
+
}
|
| 827 |
+
}
|
| 828 |
+
}
|
| 829 |
+
},
|
| 830 |
+
"official_test_deltas": {
|
| 831 |
+
"exact_partition_pp": 0.0,
|
| 832 |
+
"pair_f1_pp": 0.0,
|
| 833 |
+
"x_grouping_pp": 0.0,
|
| 834 |
+
"family_grouping_pp": 0.0
|
| 835 |
+
},
|
| 836 |
+
"decision": {
|
| 837 |
+
"adopted": false,
|
| 838 |
+
"selected_component_competition_weight": 0.0,
|
| 839 |
+
"reason": "official adoption gate failed"
|
| 840 |
+
},
|
| 841 |
+
"track": "R_noncommercial_only",
|
| 842 |
+
"product_validation": false
|
| 843 |
+
}
|
reports/local_baseline_guard_report.json
ADDED
|
@@ -0,0 +1,915 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-LOCAL-BASELINE-GUARD-001",
|
| 3 |
+
"generated_at": "2026-07-23T18:36:25.447949+00:00",
|
| 4 |
+
"selection_contract": {
|
| 5 |
+
"split": "CROHME trainData writer-validation only",
|
| 6 |
+
"cross_gap_ratio": 0.4,
|
| 7 |
+
"multistroke_family_boost": 6.0,
|
| 8 |
+
"maximum_x_regression_pp": 1.0,
|
| 9 |
+
"maximum_family_regression_pp": 2.0,
|
| 10 |
+
"maximum_pair_f1_regression_pp": 0.25
|
| 11 |
+
},
|
| 12 |
+
"signal_coverage": {
|
| 13 |
+
"validation_candidates": 3293,
|
| 14 |
+
"official_test_candidates": 5061
|
| 15 |
+
},
|
| 16 |
+
"reference_validation": {
|
| 17 |
+
"local_baseline_weight": 0.0,
|
| 18 |
+
"global": {
|
| 19 |
+
"formulas": 275,
|
| 20 |
+
"exact_partition": 0.68,
|
| 21 |
+
"exact_group_recall": 0.9528054535920294,
|
| 22 |
+
"overmerge_rate": 0.14909090909090908,
|
| 23 |
+
"oversplit_rate": 0.21818181818181817,
|
| 24 |
+
"pair_precision": 0.9563838223632039,
|
| 25 |
+
"pair_recall": 0.9047261815453863,
|
| 26 |
+
"pair_f1": 0.9298380878951427
|
| 27 |
+
},
|
| 28 |
+
"behavior_targets": {
|
| 29 |
+
"x": {
|
| 30 |
+
"samples": 277,
|
| 31 |
+
"matched": 244,
|
| 32 |
+
"oversplit": 31,
|
| 33 |
+
"overmerge": 3,
|
| 34 |
+
"grouping_recall": 0.8808664259927798
|
| 35 |
+
},
|
| 36 |
+
"X": {
|
| 37 |
+
"samples": 7,
|
| 38 |
+
"matched": 7,
|
| 39 |
+
"oversplit": 0,
|
| 40 |
+
"overmerge": 0,
|
| 41 |
+
"grouping_recall": 1.0
|
| 42 |
+
},
|
| 43 |
+
"\\times": {
|
| 44 |
+
"samples": 44,
|
| 45 |
+
"matched": 44,
|
| 46 |
+
"oversplit": 0,
|
| 47 |
+
"overmerge": 0,
|
| 48 |
+
"grouping_recall": 1.0
|
| 49 |
+
}
|
| 50 |
+
},
|
| 51 |
+
"families": {
|
| 52 |
+
"total": 59,
|
| 53 |
+
"matched": 51,
|
| 54 |
+
"grouping_recall": 0.864406779661017,
|
| 55 |
+
"by_label": {
|
| 56 |
+
"\\sum": {
|
| 57 |
+
"samples": 11,
|
| 58 |
+
"matched": 8,
|
| 59 |
+
"grouping_recall": 0.7272727272727273
|
| 60 |
+
},
|
| 61 |
+
"\\pi": {
|
| 62 |
+
"samples": 17,
|
| 63 |
+
"matched": 15,
|
| 64 |
+
"grouping_recall": 0.8823529411764706
|
| 65 |
+
},
|
| 66 |
+
"\\rightarrow": {
|
| 67 |
+
"samples": 20,
|
| 68 |
+
"matched": 17,
|
| 69 |
+
"grouping_recall": 0.85
|
| 70 |
+
},
|
| 71 |
+
"\\neq": {
|
| 72 |
+
"samples": 2,
|
| 73 |
+
"matched": 2,
|
| 74 |
+
"grouping_recall": 1.0
|
| 75 |
+
},
|
| 76 |
+
"\\pm": {
|
| 77 |
+
"samples": 9,
|
| 78 |
+
"matched": 9,
|
| 79 |
+
"grouping_recall": 1.0
|
| 80 |
+
}
|
| 81 |
+
}
|
| 82 |
+
}
|
| 83 |
+
},
|
| 84 |
+
"winner_validation": {
|
| 85 |
+
"local_baseline_weight": 0.0,
|
| 86 |
+
"global": {
|
| 87 |
+
"formulas": 275,
|
| 88 |
+
"exact_partition": 0.68,
|
| 89 |
+
"exact_group_recall": 0.9528054535920294,
|
| 90 |
+
"overmerge_rate": 0.14909090909090908,
|
| 91 |
+
"oversplit_rate": 0.21818181818181817,
|
| 92 |
+
"pair_precision": 0.9563838223632039,
|
| 93 |
+
"pair_recall": 0.9047261815453863,
|
| 94 |
+
"pair_f1": 0.9298380878951427
|
| 95 |
+
},
|
| 96 |
+
"behavior_targets": {
|
| 97 |
+
"x": {
|
| 98 |
+
"samples": 277,
|
| 99 |
+
"matched": 244,
|
| 100 |
+
"oversplit": 31,
|
| 101 |
+
"overmerge": 3,
|
| 102 |
+
"grouping_recall": 0.8808664259927798
|
| 103 |
+
},
|
| 104 |
+
"X": {
|
| 105 |
+
"samples": 7,
|
| 106 |
+
"matched": 7,
|
| 107 |
+
"oversplit": 0,
|
| 108 |
+
"overmerge": 0,
|
| 109 |
+
"grouping_recall": 1.0
|
| 110 |
+
},
|
| 111 |
+
"\\times": {
|
| 112 |
+
"samples": 44,
|
| 113 |
+
"matched": 44,
|
| 114 |
+
"oversplit": 0,
|
| 115 |
+
"overmerge": 0,
|
| 116 |
+
"grouping_recall": 1.0
|
| 117 |
+
}
|
| 118 |
+
},
|
| 119 |
+
"families": {
|
| 120 |
+
"total": 59,
|
| 121 |
+
"matched": 51,
|
| 122 |
+
"grouping_recall": 0.864406779661017,
|
| 123 |
+
"by_label": {
|
| 124 |
+
"\\sum": {
|
| 125 |
+
"samples": 11,
|
| 126 |
+
"matched": 8,
|
| 127 |
+
"grouping_recall": 0.7272727272727273
|
| 128 |
+
},
|
| 129 |
+
"\\pi": {
|
| 130 |
+
"samples": 17,
|
| 131 |
+
"matched": 15,
|
| 132 |
+
"grouping_recall": 0.8823529411764706
|
| 133 |
+
},
|
| 134 |
+
"\\rightarrow": {
|
| 135 |
+
"samples": 20,
|
| 136 |
+
"matched": 17,
|
| 137 |
+
"grouping_recall": 0.85
|
| 138 |
+
},
|
| 139 |
+
"\\neq": {
|
| 140 |
+
"samples": 2,
|
| 141 |
+
"matched": 2,
|
| 142 |
+
"grouping_recall": 1.0
|
| 143 |
+
},
|
| 144 |
+
"\\pm": {
|
| 145 |
+
"samples": 9,
|
| 146 |
+
"matched": 9,
|
| 147 |
+
"grouping_recall": 1.0
|
| 148 |
+
}
|
| 149 |
+
}
|
| 150 |
+
}
|
| 151 |
+
},
|
| 152 |
+
"trials": [
|
| 153 |
+
{
|
| 154 |
+
"local_baseline_weight": 0.0,
|
| 155 |
+
"global": {
|
| 156 |
+
"formulas": 275,
|
| 157 |
+
"exact_partition": 0.68,
|
| 158 |
+
"exact_group_recall": 0.9528054535920294,
|
| 159 |
+
"overmerge_rate": 0.14909090909090908,
|
| 160 |
+
"oversplit_rate": 0.21818181818181817,
|
| 161 |
+
"pair_precision": 0.9563838223632039,
|
| 162 |
+
"pair_recall": 0.9047261815453863,
|
| 163 |
+
"pair_f1": 0.9298380878951427
|
| 164 |
+
},
|
| 165 |
+
"behavior_targets": {
|
| 166 |
+
"x": {
|
| 167 |
+
"samples": 277,
|
| 168 |
+
"matched": 244,
|
| 169 |
+
"oversplit": 31,
|
| 170 |
+
"overmerge": 3,
|
| 171 |
+
"grouping_recall": 0.8808664259927798
|
| 172 |
+
},
|
| 173 |
+
"X": {
|
| 174 |
+
"samples": 7,
|
| 175 |
+
"matched": 7,
|
| 176 |
+
"oversplit": 0,
|
| 177 |
+
"overmerge": 0,
|
| 178 |
+
"grouping_recall": 1.0
|
| 179 |
+
},
|
| 180 |
+
"\\times": {
|
| 181 |
+
"samples": 44,
|
| 182 |
+
"matched": 44,
|
| 183 |
+
"oversplit": 0,
|
| 184 |
+
"overmerge": 0,
|
| 185 |
+
"grouping_recall": 1.0
|
| 186 |
+
}
|
| 187 |
+
},
|
| 188 |
+
"families": {
|
| 189 |
+
"total": 59,
|
| 190 |
+
"matched": 51,
|
| 191 |
+
"grouping_recall": 0.864406779661017,
|
| 192 |
+
"by_label": {
|
| 193 |
+
"\\sum": {
|
| 194 |
+
"samples": 11,
|
| 195 |
+
"matched": 8,
|
| 196 |
+
"grouping_recall": 0.7272727272727273
|
| 197 |
+
},
|
| 198 |
+
"\\pi": {
|
| 199 |
+
"samples": 17,
|
| 200 |
+
"matched": 15,
|
| 201 |
+
"grouping_recall": 0.8823529411764706
|
| 202 |
+
},
|
| 203 |
+
"\\rightarrow": {
|
| 204 |
+
"samples": 20,
|
| 205 |
+
"matched": 17,
|
| 206 |
+
"grouping_recall": 0.85
|
| 207 |
+
},
|
| 208 |
+
"\\neq": {
|
| 209 |
+
"samples": 2,
|
| 210 |
+
"matched": 2,
|
| 211 |
+
"grouping_recall": 1.0
|
| 212 |
+
},
|
| 213 |
+
"\\pm": {
|
| 214 |
+
"samples": 9,
|
| 215 |
+
"matched": 9,
|
| 216 |
+
"grouping_recall": 1.0
|
| 217 |
+
}
|
| 218 |
+
}
|
| 219 |
+
}
|
| 220 |
+
},
|
| 221 |
+
{
|
| 222 |
+
"local_baseline_weight": 0.5,
|
| 223 |
+
"global": {
|
| 224 |
+
"formulas": 275,
|
| 225 |
+
"exact_partition": 0.68,
|
| 226 |
+
"exact_group_recall": 0.9528054535920294,
|
| 227 |
+
"overmerge_rate": 0.14909090909090908,
|
| 228 |
+
"oversplit_rate": 0.21818181818181817,
|
| 229 |
+
"pair_precision": 0.9563838223632039,
|
| 230 |
+
"pair_recall": 0.9047261815453863,
|
| 231 |
+
"pair_f1": 0.9298380878951427
|
| 232 |
+
},
|
| 233 |
+
"behavior_targets": {
|
| 234 |
+
"x": {
|
| 235 |
+
"samples": 277,
|
| 236 |
+
"matched": 244,
|
| 237 |
+
"oversplit": 31,
|
| 238 |
+
"overmerge": 3,
|
| 239 |
+
"grouping_recall": 0.8808664259927798
|
| 240 |
+
},
|
| 241 |
+
"X": {
|
| 242 |
+
"samples": 7,
|
| 243 |
+
"matched": 7,
|
| 244 |
+
"oversplit": 0,
|
| 245 |
+
"overmerge": 0,
|
| 246 |
+
"grouping_recall": 1.0
|
| 247 |
+
},
|
| 248 |
+
"\\times": {
|
| 249 |
+
"samples": 44,
|
| 250 |
+
"matched": 44,
|
| 251 |
+
"oversplit": 0,
|
| 252 |
+
"overmerge": 0,
|
| 253 |
+
"grouping_recall": 1.0
|
| 254 |
+
}
|
| 255 |
+
},
|
| 256 |
+
"families": {
|
| 257 |
+
"total": 59,
|
| 258 |
+
"matched": 51,
|
| 259 |
+
"grouping_recall": 0.864406779661017,
|
| 260 |
+
"by_label": {
|
| 261 |
+
"\\sum": {
|
| 262 |
+
"samples": 11,
|
| 263 |
+
"matched": 8,
|
| 264 |
+
"grouping_recall": 0.7272727272727273
|
| 265 |
+
},
|
| 266 |
+
"\\pi": {
|
| 267 |
+
"samples": 17,
|
| 268 |
+
"matched": 15,
|
| 269 |
+
"grouping_recall": 0.8823529411764706
|
| 270 |
+
},
|
| 271 |
+
"\\rightarrow": {
|
| 272 |
+
"samples": 20,
|
| 273 |
+
"matched": 17,
|
| 274 |
+
"grouping_recall": 0.85
|
| 275 |
+
},
|
| 276 |
+
"\\neq": {
|
| 277 |
+
"samples": 2,
|
| 278 |
+
"matched": 2,
|
| 279 |
+
"grouping_recall": 1.0
|
| 280 |
+
},
|
| 281 |
+
"\\pm": {
|
| 282 |
+
"samples": 9,
|
| 283 |
+
"matched": 9,
|
| 284 |
+
"grouping_recall": 1.0
|
| 285 |
+
}
|
| 286 |
+
}
|
| 287 |
+
}
|
| 288 |
+
},
|
| 289 |
+
{
|
| 290 |
+
"local_baseline_weight": 1.0,
|
| 291 |
+
"global": {
|
| 292 |
+
"formulas": 275,
|
| 293 |
+
"exact_partition": 0.68,
|
| 294 |
+
"exact_group_recall": 0.9528054535920294,
|
| 295 |
+
"overmerge_rate": 0.14909090909090908,
|
| 296 |
+
"oversplit_rate": 0.21818181818181817,
|
| 297 |
+
"pair_precision": 0.9563838223632039,
|
| 298 |
+
"pair_recall": 0.9047261815453863,
|
| 299 |
+
"pair_f1": 0.9298380878951427
|
| 300 |
+
},
|
| 301 |
+
"behavior_targets": {
|
| 302 |
+
"x": {
|
| 303 |
+
"samples": 277,
|
| 304 |
+
"matched": 244,
|
| 305 |
+
"oversplit": 31,
|
| 306 |
+
"overmerge": 3,
|
| 307 |
+
"grouping_recall": 0.8808664259927798
|
| 308 |
+
},
|
| 309 |
+
"X": {
|
| 310 |
+
"samples": 7,
|
| 311 |
+
"matched": 7,
|
| 312 |
+
"oversplit": 0,
|
| 313 |
+
"overmerge": 0,
|
| 314 |
+
"grouping_recall": 1.0
|
| 315 |
+
},
|
| 316 |
+
"\\times": {
|
| 317 |
+
"samples": 44,
|
| 318 |
+
"matched": 44,
|
| 319 |
+
"oversplit": 0,
|
| 320 |
+
"overmerge": 0,
|
| 321 |
+
"grouping_recall": 1.0
|
| 322 |
+
}
|
| 323 |
+
},
|
| 324 |
+
"families": {
|
| 325 |
+
"total": 59,
|
| 326 |
+
"matched": 51,
|
| 327 |
+
"grouping_recall": 0.864406779661017,
|
| 328 |
+
"by_label": {
|
| 329 |
+
"\\sum": {
|
| 330 |
+
"samples": 11,
|
| 331 |
+
"matched": 8,
|
| 332 |
+
"grouping_recall": 0.7272727272727273
|
| 333 |
+
},
|
| 334 |
+
"\\pi": {
|
| 335 |
+
"samples": 17,
|
| 336 |
+
"matched": 15,
|
| 337 |
+
"grouping_recall": 0.8823529411764706
|
| 338 |
+
},
|
| 339 |
+
"\\rightarrow": {
|
| 340 |
+
"samples": 20,
|
| 341 |
+
"matched": 17,
|
| 342 |
+
"grouping_recall": 0.85
|
| 343 |
+
},
|
| 344 |
+
"\\neq": {
|
| 345 |
+
"samples": 2,
|
| 346 |
+
"matched": 2,
|
| 347 |
+
"grouping_recall": 1.0
|
| 348 |
+
},
|
| 349 |
+
"\\pm": {
|
| 350 |
+
"samples": 9,
|
| 351 |
+
"matched": 9,
|
| 352 |
+
"grouping_recall": 1.0
|
| 353 |
+
}
|
| 354 |
+
}
|
| 355 |
+
}
|
| 356 |
+
},
|
| 357 |
+
{
|
| 358 |
+
"local_baseline_weight": 2.0,
|
| 359 |
+
"global": {
|
| 360 |
+
"formulas": 275,
|
| 361 |
+
"exact_partition": 0.68,
|
| 362 |
+
"exact_group_recall": 0.9528054535920294,
|
| 363 |
+
"overmerge_rate": 0.14909090909090908,
|
| 364 |
+
"oversplit_rate": 0.21818181818181817,
|
| 365 |
+
"pair_precision": 0.9563838223632039,
|
| 366 |
+
"pair_recall": 0.9047261815453863,
|
| 367 |
+
"pair_f1": 0.9298380878951427
|
| 368 |
+
},
|
| 369 |
+
"behavior_targets": {
|
| 370 |
+
"x": {
|
| 371 |
+
"samples": 277,
|
| 372 |
+
"matched": 244,
|
| 373 |
+
"oversplit": 31,
|
| 374 |
+
"overmerge": 3,
|
| 375 |
+
"grouping_recall": 0.8808664259927798
|
| 376 |
+
},
|
| 377 |
+
"X": {
|
| 378 |
+
"samples": 7,
|
| 379 |
+
"matched": 7,
|
| 380 |
+
"oversplit": 0,
|
| 381 |
+
"overmerge": 0,
|
| 382 |
+
"grouping_recall": 1.0
|
| 383 |
+
},
|
| 384 |
+
"\\times": {
|
| 385 |
+
"samples": 44,
|
| 386 |
+
"matched": 44,
|
| 387 |
+
"oversplit": 0,
|
| 388 |
+
"overmerge": 0,
|
| 389 |
+
"grouping_recall": 1.0
|
| 390 |
+
}
|
| 391 |
+
},
|
| 392 |
+
"families": {
|
| 393 |
+
"total": 59,
|
| 394 |
+
"matched": 51,
|
| 395 |
+
"grouping_recall": 0.864406779661017,
|
| 396 |
+
"by_label": {
|
| 397 |
+
"\\sum": {
|
| 398 |
+
"samples": 11,
|
| 399 |
+
"matched": 8,
|
| 400 |
+
"grouping_recall": 0.7272727272727273
|
| 401 |
+
},
|
| 402 |
+
"\\pi": {
|
| 403 |
+
"samples": 17,
|
| 404 |
+
"matched": 15,
|
| 405 |
+
"grouping_recall": 0.8823529411764706
|
| 406 |
+
},
|
| 407 |
+
"\\rightarrow": {
|
| 408 |
+
"samples": 20,
|
| 409 |
+
"matched": 17,
|
| 410 |
+
"grouping_recall": 0.85
|
| 411 |
+
},
|
| 412 |
+
"\\neq": {
|
| 413 |
+
"samples": 2,
|
| 414 |
+
"matched": 2,
|
| 415 |
+
"grouping_recall": 1.0
|
| 416 |
+
},
|
| 417 |
+
"\\pm": {
|
| 418 |
+
"samples": 9,
|
| 419 |
+
"matched": 9,
|
| 420 |
+
"grouping_recall": 1.0
|
| 421 |
+
}
|
| 422 |
+
}
|
| 423 |
+
}
|
| 424 |
+
},
|
| 425 |
+
{
|
| 426 |
+
"local_baseline_weight": 3.0,
|
| 427 |
+
"global": {
|
| 428 |
+
"formulas": 275,
|
| 429 |
+
"exact_partition": 0.68,
|
| 430 |
+
"exact_group_recall": 0.9528054535920294,
|
| 431 |
+
"overmerge_rate": 0.14909090909090908,
|
| 432 |
+
"oversplit_rate": 0.21818181818181817,
|
| 433 |
+
"pair_precision": 0.9563838223632039,
|
| 434 |
+
"pair_recall": 0.9047261815453863,
|
| 435 |
+
"pair_f1": 0.9298380878951427
|
| 436 |
+
},
|
| 437 |
+
"behavior_targets": {
|
| 438 |
+
"x": {
|
| 439 |
+
"samples": 277,
|
| 440 |
+
"matched": 244,
|
| 441 |
+
"oversplit": 31,
|
| 442 |
+
"overmerge": 3,
|
| 443 |
+
"grouping_recall": 0.8808664259927798
|
| 444 |
+
},
|
| 445 |
+
"X": {
|
| 446 |
+
"samples": 7,
|
| 447 |
+
"matched": 7,
|
| 448 |
+
"oversplit": 0,
|
| 449 |
+
"overmerge": 0,
|
| 450 |
+
"grouping_recall": 1.0
|
| 451 |
+
},
|
| 452 |
+
"\\times": {
|
| 453 |
+
"samples": 44,
|
| 454 |
+
"matched": 44,
|
| 455 |
+
"oversplit": 0,
|
| 456 |
+
"overmerge": 0,
|
| 457 |
+
"grouping_recall": 1.0
|
| 458 |
+
}
|
| 459 |
+
},
|
| 460 |
+
"families": {
|
| 461 |
+
"total": 59,
|
| 462 |
+
"matched": 51,
|
| 463 |
+
"grouping_recall": 0.864406779661017,
|
| 464 |
+
"by_label": {
|
| 465 |
+
"\\sum": {
|
| 466 |
+
"samples": 11,
|
| 467 |
+
"matched": 8,
|
| 468 |
+
"grouping_recall": 0.7272727272727273
|
| 469 |
+
},
|
| 470 |
+
"\\pi": {
|
| 471 |
+
"samples": 17,
|
| 472 |
+
"matched": 15,
|
| 473 |
+
"grouping_recall": 0.8823529411764706
|
| 474 |
+
},
|
| 475 |
+
"\\rightarrow": {
|
| 476 |
+
"samples": 20,
|
| 477 |
+
"matched": 17,
|
| 478 |
+
"grouping_recall": 0.85
|
| 479 |
+
},
|
| 480 |
+
"\\neq": {
|
| 481 |
+
"samples": 2,
|
| 482 |
+
"matched": 2,
|
| 483 |
+
"grouping_recall": 1.0
|
| 484 |
+
},
|
| 485 |
+
"\\pm": {
|
| 486 |
+
"samples": 9,
|
| 487 |
+
"matched": 9,
|
| 488 |
+
"grouping_recall": 1.0
|
| 489 |
+
}
|
| 490 |
+
}
|
| 491 |
+
}
|
| 492 |
+
},
|
| 493 |
+
{
|
| 494 |
+
"local_baseline_weight": 4.0,
|
| 495 |
+
"global": {
|
| 496 |
+
"formulas": 275,
|
| 497 |
+
"exact_partition": 0.68,
|
| 498 |
+
"exact_group_recall": 0.9528054535920294,
|
| 499 |
+
"overmerge_rate": 0.14909090909090908,
|
| 500 |
+
"oversplit_rate": 0.21818181818181817,
|
| 501 |
+
"pair_precision": 0.9563838223632039,
|
| 502 |
+
"pair_recall": 0.9047261815453863,
|
| 503 |
+
"pair_f1": 0.9298380878951427
|
| 504 |
+
},
|
| 505 |
+
"behavior_targets": {
|
| 506 |
+
"x": {
|
| 507 |
+
"samples": 277,
|
| 508 |
+
"matched": 244,
|
| 509 |
+
"oversplit": 31,
|
| 510 |
+
"overmerge": 3,
|
| 511 |
+
"grouping_recall": 0.8808664259927798
|
| 512 |
+
},
|
| 513 |
+
"X": {
|
| 514 |
+
"samples": 7,
|
| 515 |
+
"matched": 7,
|
| 516 |
+
"oversplit": 0,
|
| 517 |
+
"overmerge": 0,
|
| 518 |
+
"grouping_recall": 1.0
|
| 519 |
+
},
|
| 520 |
+
"\\times": {
|
| 521 |
+
"samples": 44,
|
| 522 |
+
"matched": 44,
|
| 523 |
+
"oversplit": 0,
|
| 524 |
+
"overmerge": 0,
|
| 525 |
+
"grouping_recall": 1.0
|
| 526 |
+
}
|
| 527 |
+
},
|
| 528 |
+
"families": {
|
| 529 |
+
"total": 59,
|
| 530 |
+
"matched": 51,
|
| 531 |
+
"grouping_recall": 0.864406779661017,
|
| 532 |
+
"by_label": {
|
| 533 |
+
"\\sum": {
|
| 534 |
+
"samples": 11,
|
| 535 |
+
"matched": 8,
|
| 536 |
+
"grouping_recall": 0.7272727272727273
|
| 537 |
+
},
|
| 538 |
+
"\\pi": {
|
| 539 |
+
"samples": 17,
|
| 540 |
+
"matched": 15,
|
| 541 |
+
"grouping_recall": 0.8823529411764706
|
| 542 |
+
},
|
| 543 |
+
"\\rightarrow": {
|
| 544 |
+
"samples": 20,
|
| 545 |
+
"matched": 17,
|
| 546 |
+
"grouping_recall": 0.85
|
| 547 |
+
},
|
| 548 |
+
"\\neq": {
|
| 549 |
+
"samples": 2,
|
| 550 |
+
"matched": 2,
|
| 551 |
+
"grouping_recall": 1.0
|
| 552 |
+
},
|
| 553 |
+
"\\pm": {
|
| 554 |
+
"samples": 9,
|
| 555 |
+
"matched": 9,
|
| 556 |
+
"grouping_recall": 1.0
|
| 557 |
+
}
|
| 558 |
+
}
|
| 559 |
+
}
|
| 560 |
+
},
|
| 561 |
+
{
|
| 562 |
+
"local_baseline_weight": 6.0,
|
| 563 |
+
"global": {
|
| 564 |
+
"formulas": 275,
|
| 565 |
+
"exact_partition": 0.68,
|
| 566 |
+
"exact_group_recall": 0.9528054535920294,
|
| 567 |
+
"overmerge_rate": 0.14909090909090908,
|
| 568 |
+
"oversplit_rate": 0.21818181818181817,
|
| 569 |
+
"pair_precision": 0.9563838223632039,
|
| 570 |
+
"pair_recall": 0.9047261815453863,
|
| 571 |
+
"pair_f1": 0.9298380878951427
|
| 572 |
+
},
|
| 573 |
+
"behavior_targets": {
|
| 574 |
+
"x": {
|
| 575 |
+
"samples": 277,
|
| 576 |
+
"matched": 244,
|
| 577 |
+
"oversplit": 31,
|
| 578 |
+
"overmerge": 3,
|
| 579 |
+
"grouping_recall": 0.8808664259927798
|
| 580 |
+
},
|
| 581 |
+
"X": {
|
| 582 |
+
"samples": 7,
|
| 583 |
+
"matched": 7,
|
| 584 |
+
"oversplit": 0,
|
| 585 |
+
"overmerge": 0,
|
| 586 |
+
"grouping_recall": 1.0
|
| 587 |
+
},
|
| 588 |
+
"\\times": {
|
| 589 |
+
"samples": 44,
|
| 590 |
+
"matched": 44,
|
| 591 |
+
"oversplit": 0,
|
| 592 |
+
"overmerge": 0,
|
| 593 |
+
"grouping_recall": 1.0
|
| 594 |
+
}
|
| 595 |
+
},
|
| 596 |
+
"families": {
|
| 597 |
+
"total": 59,
|
| 598 |
+
"matched": 51,
|
| 599 |
+
"grouping_recall": 0.864406779661017,
|
| 600 |
+
"by_label": {
|
| 601 |
+
"\\sum": {
|
| 602 |
+
"samples": 11,
|
| 603 |
+
"matched": 8,
|
| 604 |
+
"grouping_recall": 0.7272727272727273
|
| 605 |
+
},
|
| 606 |
+
"\\pi": {
|
| 607 |
+
"samples": 17,
|
| 608 |
+
"matched": 15,
|
| 609 |
+
"grouping_recall": 0.8823529411764706
|
| 610 |
+
},
|
| 611 |
+
"\\rightarrow": {
|
| 612 |
+
"samples": 20,
|
| 613 |
+
"matched": 17,
|
| 614 |
+
"grouping_recall": 0.85
|
| 615 |
+
},
|
| 616 |
+
"\\neq": {
|
| 617 |
+
"samples": 2,
|
| 618 |
+
"matched": 2,
|
| 619 |
+
"grouping_recall": 1.0
|
| 620 |
+
},
|
| 621 |
+
"\\pm": {
|
| 622 |
+
"samples": 9,
|
| 623 |
+
"matched": 9,
|
| 624 |
+
"grouping_recall": 1.0
|
| 625 |
+
}
|
| 626 |
+
}
|
| 627 |
+
}
|
| 628 |
+
},
|
| 629 |
+
{
|
| 630 |
+
"local_baseline_weight": 8.0,
|
| 631 |
+
"global": {
|
| 632 |
+
"formulas": 275,
|
| 633 |
+
"exact_partition": 0.68,
|
| 634 |
+
"exact_group_recall": 0.9528054535920294,
|
| 635 |
+
"overmerge_rate": 0.14909090909090908,
|
| 636 |
+
"oversplit_rate": 0.21818181818181817,
|
| 637 |
+
"pair_precision": 0.9563838223632039,
|
| 638 |
+
"pair_recall": 0.9047261815453863,
|
| 639 |
+
"pair_f1": 0.9298380878951427
|
| 640 |
+
},
|
| 641 |
+
"behavior_targets": {
|
| 642 |
+
"x": {
|
| 643 |
+
"samples": 277,
|
| 644 |
+
"matched": 244,
|
| 645 |
+
"oversplit": 31,
|
| 646 |
+
"overmerge": 3,
|
| 647 |
+
"grouping_recall": 0.8808664259927798
|
| 648 |
+
},
|
| 649 |
+
"X": {
|
| 650 |
+
"samples": 7,
|
| 651 |
+
"matched": 7,
|
| 652 |
+
"oversplit": 0,
|
| 653 |
+
"overmerge": 0,
|
| 654 |
+
"grouping_recall": 1.0
|
| 655 |
+
},
|
| 656 |
+
"\\times": {
|
| 657 |
+
"samples": 44,
|
| 658 |
+
"matched": 44,
|
| 659 |
+
"oversplit": 0,
|
| 660 |
+
"overmerge": 0,
|
| 661 |
+
"grouping_recall": 1.0
|
| 662 |
+
}
|
| 663 |
+
},
|
| 664 |
+
"families": {
|
| 665 |
+
"total": 59,
|
| 666 |
+
"matched": 51,
|
| 667 |
+
"grouping_recall": 0.864406779661017,
|
| 668 |
+
"by_label": {
|
| 669 |
+
"\\sum": {
|
| 670 |
+
"samples": 11,
|
| 671 |
+
"matched": 8,
|
| 672 |
+
"grouping_recall": 0.7272727272727273
|
| 673 |
+
},
|
| 674 |
+
"\\pi": {
|
| 675 |
+
"samples": 17,
|
| 676 |
+
"matched": 15,
|
| 677 |
+
"grouping_recall": 0.8823529411764706
|
| 678 |
+
},
|
| 679 |
+
"\\rightarrow": {
|
| 680 |
+
"samples": 20,
|
| 681 |
+
"matched": 17,
|
| 682 |
+
"grouping_recall": 0.85
|
| 683 |
+
},
|
| 684 |
+
"\\neq": {
|
| 685 |
+
"samples": 2,
|
| 686 |
+
"matched": 2,
|
| 687 |
+
"grouping_recall": 1.0
|
| 688 |
+
},
|
| 689 |
+
"\\pm": {
|
| 690 |
+
"samples": 9,
|
| 691 |
+
"matched": 9,
|
| 692 |
+
"grouping_recall": 1.0
|
| 693 |
+
}
|
| 694 |
+
}
|
| 695 |
+
}
|
| 696 |
+
},
|
| 697 |
+
{
|
| 698 |
+
"local_baseline_weight": 12.0,
|
| 699 |
+
"global": {
|
| 700 |
+
"formulas": 275,
|
| 701 |
+
"exact_partition": 0.68,
|
| 702 |
+
"exact_group_recall": 0.9528054535920294,
|
| 703 |
+
"overmerge_rate": 0.14909090909090908,
|
| 704 |
+
"oversplit_rate": 0.21818181818181817,
|
| 705 |
+
"pair_precision": 0.9563838223632039,
|
| 706 |
+
"pair_recall": 0.9047261815453863,
|
| 707 |
+
"pair_f1": 0.9298380878951427
|
| 708 |
+
},
|
| 709 |
+
"behavior_targets": {
|
| 710 |
+
"x": {
|
| 711 |
+
"samples": 277,
|
| 712 |
+
"matched": 244,
|
| 713 |
+
"oversplit": 31,
|
| 714 |
+
"overmerge": 3,
|
| 715 |
+
"grouping_recall": 0.8808664259927798
|
| 716 |
+
},
|
| 717 |
+
"X": {
|
| 718 |
+
"samples": 7,
|
| 719 |
+
"matched": 7,
|
| 720 |
+
"oversplit": 0,
|
| 721 |
+
"overmerge": 0,
|
| 722 |
+
"grouping_recall": 1.0
|
| 723 |
+
},
|
| 724 |
+
"\\times": {
|
| 725 |
+
"samples": 44,
|
| 726 |
+
"matched": 44,
|
| 727 |
+
"oversplit": 0,
|
| 728 |
+
"overmerge": 0,
|
| 729 |
+
"grouping_recall": 1.0
|
| 730 |
+
}
|
| 731 |
+
},
|
| 732 |
+
"families": {
|
| 733 |
+
"total": 59,
|
| 734 |
+
"matched": 51,
|
| 735 |
+
"grouping_recall": 0.864406779661017,
|
| 736 |
+
"by_label": {
|
| 737 |
+
"\\sum": {
|
| 738 |
+
"samples": 11,
|
| 739 |
+
"matched": 8,
|
| 740 |
+
"grouping_recall": 0.7272727272727273
|
| 741 |
+
},
|
| 742 |
+
"\\pi": {
|
| 743 |
+
"samples": 17,
|
| 744 |
+
"matched": 15,
|
| 745 |
+
"grouping_recall": 0.8823529411764706
|
| 746 |
+
},
|
| 747 |
+
"\\rightarrow": {
|
| 748 |
+
"samples": 20,
|
| 749 |
+
"matched": 17,
|
| 750 |
+
"grouping_recall": 0.85
|
| 751 |
+
},
|
| 752 |
+
"\\neq": {
|
| 753 |
+
"samples": 2,
|
| 754 |
+
"matched": 2,
|
| 755 |
+
"grouping_recall": 1.0
|
| 756 |
+
},
|
| 757 |
+
"\\pm": {
|
| 758 |
+
"samples": 9,
|
| 759 |
+
"matched": 9,
|
| 760 |
+
"grouping_recall": 1.0
|
| 761 |
+
}
|
| 762 |
+
}
|
| 763 |
+
}
|
| 764 |
+
}
|
| 765 |
+
],
|
| 766 |
+
"official_test_reference": {
|
| 767 |
+
"local_baseline_weight": 0.0,
|
| 768 |
+
"global": {
|
| 769 |
+
"formulas": 488,
|
| 770 |
+
"exact_partition": 0.6004098360655737,
|
| 771 |
+
"exact_group_recall": 0.9370456528060482,
|
| 772 |
+
"overmerge_rate": 0.21721311475409835,
|
| 773 |
+
"oversplit_rate": 0.2520491803278688,
|
| 774 |
+
"pair_precision": 0.9231143552311436,
|
| 775 |
+
"pair_recall": 0.8986262434864993,
|
| 776 |
+
"pair_f1": 0.9107057129140663
|
| 777 |
+
},
|
| 778 |
+
"behavior_targets": {
|
| 779 |
+
"x": {
|
| 780 |
+
"samples": 450,
|
| 781 |
+
"matched": 412,
|
| 782 |
+
"oversplit": 30,
|
| 783 |
+
"overmerge": 12,
|
| 784 |
+
"grouping_recall": 0.9155555555555556
|
| 785 |
+
},
|
| 786 |
+
"X": {
|
| 787 |
+
"samples": 14,
|
| 788 |
+
"matched": 12,
|
| 789 |
+
"oversplit": 2,
|
| 790 |
+
"overmerge": 1,
|
| 791 |
+
"grouping_recall": 0.8571428571428571
|
| 792 |
+
},
|
| 793 |
+
"\\times": {
|
| 794 |
+
"samples": 36,
|
| 795 |
+
"matched": 36,
|
| 796 |
+
"oversplit": 0,
|
| 797 |
+
"overmerge": 0,
|
| 798 |
+
"grouping_recall": 1.0
|
| 799 |
+
}
|
| 800 |
+
},
|
| 801 |
+
"families": {
|
| 802 |
+
"total": 135,
|
| 803 |
+
"matched": 112,
|
| 804 |
+
"grouping_recall": 0.8296296296296296,
|
| 805 |
+
"by_label": {
|
| 806 |
+
"\\sum": {
|
| 807 |
+
"samples": 31,
|
| 808 |
+
"matched": 22,
|
| 809 |
+
"grouping_recall": 0.7096774193548387
|
| 810 |
+
},
|
| 811 |
+
"\\pi": {
|
| 812 |
+
"samples": 30,
|
| 813 |
+
"matched": 27,
|
| 814 |
+
"grouping_recall": 0.9
|
| 815 |
+
},
|
| 816 |
+
"\\rightarrow": {
|
| 817 |
+
"samples": 49,
|
| 818 |
+
"matched": 44,
|
| 819 |
+
"grouping_recall": 0.8979591836734694
|
| 820 |
+
},
|
| 821 |
+
"\\neq": {
|
| 822 |
+
"samples": 13,
|
| 823 |
+
"matched": 10,
|
| 824 |
+
"grouping_recall": 0.7692307692307693
|
| 825 |
+
},
|
| 826 |
+
"\\pm": {
|
| 827 |
+
"samples": 12,
|
| 828 |
+
"matched": 9,
|
| 829 |
+
"grouping_recall": 0.75
|
| 830 |
+
}
|
| 831 |
+
}
|
| 832 |
+
}
|
| 833 |
+
},
|
| 834 |
+
"official_test_winner": {
|
| 835 |
+
"local_baseline_weight": 0.0,
|
| 836 |
+
"global": {
|
| 837 |
+
"formulas": 488,
|
| 838 |
+
"exact_partition": 0.6004098360655737,
|
| 839 |
+
"exact_group_recall": 0.9370456528060482,
|
| 840 |
+
"overmerge_rate": 0.21721311475409835,
|
| 841 |
+
"oversplit_rate": 0.2520491803278688,
|
| 842 |
+
"pair_precision": 0.9231143552311436,
|
| 843 |
+
"pair_recall": 0.8986262434864993,
|
| 844 |
+
"pair_f1": 0.9107057129140663
|
| 845 |
+
},
|
| 846 |
+
"behavior_targets": {
|
| 847 |
+
"x": {
|
| 848 |
+
"samples": 450,
|
| 849 |
+
"matched": 412,
|
| 850 |
+
"oversplit": 30,
|
| 851 |
+
"overmerge": 12,
|
| 852 |
+
"grouping_recall": 0.9155555555555556
|
| 853 |
+
},
|
| 854 |
+
"X": {
|
| 855 |
+
"samples": 14,
|
| 856 |
+
"matched": 12,
|
| 857 |
+
"oversplit": 2,
|
| 858 |
+
"overmerge": 1,
|
| 859 |
+
"grouping_recall": 0.8571428571428571
|
| 860 |
+
},
|
| 861 |
+
"\\times": {
|
| 862 |
+
"samples": 36,
|
| 863 |
+
"matched": 36,
|
| 864 |
+
"oversplit": 0,
|
| 865 |
+
"overmerge": 0,
|
| 866 |
+
"grouping_recall": 1.0
|
| 867 |
+
}
|
| 868 |
+
},
|
| 869 |
+
"families": {
|
| 870 |
+
"total": 135,
|
| 871 |
+
"matched": 112,
|
| 872 |
+
"grouping_recall": 0.8296296296296296,
|
| 873 |
+
"by_label": {
|
| 874 |
+
"\\sum": {
|
| 875 |
+
"samples": 31,
|
| 876 |
+
"matched": 22,
|
| 877 |
+
"grouping_recall": 0.7096774193548387
|
| 878 |
+
},
|
| 879 |
+
"\\pi": {
|
| 880 |
+
"samples": 30,
|
| 881 |
+
"matched": 27,
|
| 882 |
+
"grouping_recall": 0.9
|
| 883 |
+
},
|
| 884 |
+
"\\rightarrow": {
|
| 885 |
+
"samples": 49,
|
| 886 |
+
"matched": 44,
|
| 887 |
+
"grouping_recall": 0.8979591836734694
|
| 888 |
+
},
|
| 889 |
+
"\\neq": {
|
| 890 |
+
"samples": 13,
|
| 891 |
+
"matched": 10,
|
| 892 |
+
"grouping_recall": 0.7692307692307693
|
| 893 |
+
},
|
| 894 |
+
"\\pm": {
|
| 895 |
+
"samples": 12,
|
| 896 |
+
"matched": 9,
|
| 897 |
+
"grouping_recall": 0.75
|
| 898 |
+
}
|
| 899 |
+
}
|
| 900 |
+
}
|
| 901 |
+
},
|
| 902 |
+
"official_test_deltas": {
|
| 903 |
+
"exact_partition_pp": 0.0,
|
| 904 |
+
"pair_f1_pp": 0.0,
|
| 905 |
+
"x_grouping_pp": 0.0,
|
| 906 |
+
"family_grouping_pp": 0.0
|
| 907 |
+
},
|
| 908 |
+
"decision": {
|
| 909 |
+
"adopted": false,
|
| 910 |
+
"selected_local_baseline_weight": 0.0,
|
| 911 |
+
"reason": "official adoption gate failed"
|
| 912 |
+
},
|
| 913 |
+
"track": "R_noncommercial_only",
|
| 914 |
+
"product_validation": false
|
| 915 |
+
}
|
reports/local_baseline_overmerge_audit.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
reports/multistroke_family_guard_report.json
ADDED
|
@@ -0,0 +1,565 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"experiment": "R-MATH-INK-06-MULTISTROKE-FAMILY-GUARD-001",
|
| 3 |
+
"generated_at": "2026-07-23T18:17:32.159942+00:00",
|
| 4 |
+
"selection_contract": {
|
| 5 |
+
"split": "CROHME trainData writer-validation only",
|
| 6 |
+
"cross_gap_ratio": 0.4,
|
| 7 |
+
"symbol_weight": 4.0,
|
| 8 |
+
"maximum_exact_regression_pp": 0.5,
|
| 9 |
+
"maximum_pair_f1_regression_pp": 0.25
|
| 10 |
+
},
|
| 11 |
+
"reference_validation": {
|
| 12 |
+
"multistroke_family_boost": 0.0,
|
| 13 |
+
"global": {
|
| 14 |
+
"formulas": 275,
|
| 15 |
+
"exact_partition": 0.6581818181818182,
|
| 16 |
+
"exact_group_recall": 0.9480859989512324,
|
| 17 |
+
"overmerge_rate": 0.14181818181818182,
|
| 18 |
+
"oversplit_rate": 0.24727272727272728,
|
| 19 |
+
"pair_precision": 0.9605587510271159,
|
| 20 |
+
"pair_recall": 0.8769692423105776,
|
| 21 |
+
"pair_f1": 0.9168627450980391
|
| 22 |
+
},
|
| 23 |
+
"families": {
|
| 24 |
+
"total": 59,
|
| 25 |
+
"matched": 28,
|
| 26 |
+
"grouping_recall": 0.4745762711864407,
|
| 27 |
+
"by_label": {
|
| 28 |
+
"\\sum": {
|
| 29 |
+
"samples": 11,
|
| 30 |
+
"matched": 2,
|
| 31 |
+
"grouping_recall": 0.18181818181818182
|
| 32 |
+
},
|
| 33 |
+
"\\pi": {
|
| 34 |
+
"samples": 17,
|
| 35 |
+
"matched": 10,
|
| 36 |
+
"grouping_recall": 0.5882352941176471
|
| 37 |
+
},
|
| 38 |
+
"\\rightarrow": {
|
| 39 |
+
"samples": 20,
|
| 40 |
+
"matched": 14,
|
| 41 |
+
"grouping_recall": 0.7
|
| 42 |
+
},
|
| 43 |
+
"\\neq": {
|
| 44 |
+
"samples": 2,
|
| 45 |
+
"matched": 1,
|
| 46 |
+
"grouping_recall": 0.5
|
| 47 |
+
},
|
| 48 |
+
"\\pm": {
|
| 49 |
+
"samples": 9,
|
| 50 |
+
"matched": 1,
|
| 51 |
+
"grouping_recall": 0.1111111111111111
|
| 52 |
+
}
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
},
|
| 56 |
+
"winner_validation": {
|
| 57 |
+
"multistroke_family_boost": 6.0,
|
| 58 |
+
"global": {
|
| 59 |
+
"formulas": 275,
|
| 60 |
+
"exact_partition": 0.68,
|
| 61 |
+
"exact_group_recall": 0.9528054535920294,
|
| 62 |
+
"overmerge_rate": 0.14909090909090908,
|
| 63 |
+
"oversplit_rate": 0.21818181818181817,
|
| 64 |
+
"pair_precision": 0.9563838223632039,
|
| 65 |
+
"pair_recall": 0.9047261815453863,
|
| 66 |
+
"pair_f1": 0.9298380878951427
|
| 67 |
+
},
|
| 68 |
+
"families": {
|
| 69 |
+
"total": 59,
|
| 70 |
+
"matched": 51,
|
| 71 |
+
"grouping_recall": 0.864406779661017,
|
| 72 |
+
"by_label": {
|
| 73 |
+
"\\sum": {
|
| 74 |
+
"samples": 11,
|
| 75 |
+
"matched": 8,
|
| 76 |
+
"grouping_recall": 0.7272727272727273
|
| 77 |
+
},
|
| 78 |
+
"\\pi": {
|
| 79 |
+
"samples": 17,
|
| 80 |
+
"matched": 15,
|
| 81 |
+
"grouping_recall": 0.8823529411764706
|
| 82 |
+
},
|
| 83 |
+
"\\rightarrow": {
|
| 84 |
+
"samples": 20,
|
| 85 |
+
"matched": 17,
|
| 86 |
+
"grouping_recall": 0.85
|
| 87 |
+
},
|
| 88 |
+
"\\neq": {
|
| 89 |
+
"samples": 2,
|
| 90 |
+
"matched": 2,
|
| 91 |
+
"grouping_recall": 1.0
|
| 92 |
+
},
|
| 93 |
+
"\\pm": {
|
| 94 |
+
"samples": 9,
|
| 95 |
+
"matched": 9,
|
| 96 |
+
"grouping_recall": 1.0
|
| 97 |
+
}
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
},
|
| 101 |
+
"trials": [
|
| 102 |
+
{
|
| 103 |
+
"multistroke_family_boost": 0.0,
|
| 104 |
+
"global": {
|
| 105 |
+
"formulas": 275,
|
| 106 |
+
"exact_partition": 0.6581818181818182,
|
| 107 |
+
"exact_group_recall": 0.9480859989512324,
|
| 108 |
+
"overmerge_rate": 0.14181818181818182,
|
| 109 |
+
"oversplit_rate": 0.24727272727272728,
|
| 110 |
+
"pair_precision": 0.9605587510271159,
|
| 111 |
+
"pair_recall": 0.8769692423105776,
|
| 112 |
+
"pair_f1": 0.9168627450980391
|
| 113 |
+
},
|
| 114 |
+
"families": {
|
| 115 |
+
"total": 59,
|
| 116 |
+
"matched": 28,
|
| 117 |
+
"grouping_recall": 0.4745762711864407,
|
| 118 |
+
"by_label": {
|
| 119 |
+
"\\sum": {
|
| 120 |
+
"samples": 11,
|
| 121 |
+
"matched": 2,
|
| 122 |
+
"grouping_recall": 0.18181818181818182
|
| 123 |
+
},
|
| 124 |
+
"\\pi": {
|
| 125 |
+
"samples": 17,
|
| 126 |
+
"matched": 10,
|
| 127 |
+
"grouping_recall": 0.5882352941176471
|
| 128 |
+
},
|
| 129 |
+
"\\rightarrow": {
|
| 130 |
+
"samples": 20,
|
| 131 |
+
"matched": 14,
|
| 132 |
+
"grouping_recall": 0.7
|
| 133 |
+
},
|
| 134 |
+
"\\neq": {
|
| 135 |
+
"samples": 2,
|
| 136 |
+
"matched": 1,
|
| 137 |
+
"grouping_recall": 0.5
|
| 138 |
+
},
|
| 139 |
+
"\\pm": {
|
| 140 |
+
"samples": 9,
|
| 141 |
+
"matched": 1,
|
| 142 |
+
"grouping_recall": 0.1111111111111111
|
| 143 |
+
}
|
| 144 |
+
}
|
| 145 |
+
}
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"multistroke_family_boost": 0.5,
|
| 149 |
+
"global": {
|
| 150 |
+
"formulas": 275,
|
| 151 |
+
"exact_partition": 0.6654545454545454,
|
| 152 |
+
"exact_group_recall": 0.9488725747246984,
|
| 153 |
+
"overmerge_rate": 0.14181818181818182,
|
| 154 |
+
"oversplit_rate": 0.23636363636363636,
|
| 155 |
+
"pair_precision": 0.9607522485690924,
|
| 156 |
+
"pair_recall": 0.881470367591898,
|
| 157 |
+
"pair_f1": 0.9194053208137716
|
| 158 |
+
},
|
| 159 |
+
"families": {
|
| 160 |
+
"total": 59,
|
| 161 |
+
"matched": 31,
|
| 162 |
+
"grouping_recall": 0.5254237288135594,
|
| 163 |
+
"by_label": {
|
| 164 |
+
"\\sum": {
|
| 165 |
+
"samples": 11,
|
| 166 |
+
"matched": 2,
|
| 167 |
+
"grouping_recall": 0.18181818181818182
|
| 168 |
+
},
|
| 169 |
+
"\\pi": {
|
| 170 |
+
"samples": 17,
|
| 171 |
+
"matched": 11,
|
| 172 |
+
"grouping_recall": 0.6470588235294118
|
| 173 |
+
},
|
| 174 |
+
"\\rightarrow": {
|
| 175 |
+
"samples": 20,
|
| 176 |
+
"matched": 14,
|
| 177 |
+
"grouping_recall": 0.7
|
| 178 |
+
},
|
| 179 |
+
"\\neq": {
|
| 180 |
+
"samples": 2,
|
| 181 |
+
"matched": 1,
|
| 182 |
+
"grouping_recall": 0.5
|
| 183 |
+
},
|
| 184 |
+
"\\pm": {
|
| 185 |
+
"samples": 9,
|
| 186 |
+
"matched": 3,
|
| 187 |
+
"grouping_recall": 0.3333333333333333
|
| 188 |
+
}
|
| 189 |
+
}
|
| 190 |
+
}
|
| 191 |
+
},
|
| 192 |
+
{
|
| 193 |
+
"multistroke_family_boost": 1.0,
|
| 194 |
+
"global": {
|
| 195 |
+
"formulas": 275,
|
| 196 |
+
"exact_partition": 0.6654545454545454,
|
| 197 |
+
"exact_group_recall": 0.9499213424226534,
|
| 198 |
+
"overmerge_rate": 0.14181818181818182,
|
| 199 |
+
"oversplit_rate": 0.23272727272727273,
|
| 200 |
+
"pair_precision": 0.9609756097560975,
|
| 201 |
+
"pair_recall": 0.886721680420105,
|
| 202 |
+
"pair_f1": 0.9223566133437378
|
| 203 |
+
},
|
| 204 |
+
"families": {
|
| 205 |
+
"total": 59,
|
| 206 |
+
"matched": 35,
|
| 207 |
+
"grouping_recall": 0.5932203389830508,
|
| 208 |
+
"by_label": {
|
| 209 |
+
"\\sum": {
|
| 210 |
+
"samples": 11,
|
| 211 |
+
"matched": 3,
|
| 212 |
+
"grouping_recall": 0.2727272727272727
|
| 213 |
+
},
|
| 214 |
+
"\\pi": {
|
| 215 |
+
"samples": 17,
|
| 216 |
+
"matched": 11,
|
| 217 |
+
"grouping_recall": 0.6470588235294118
|
| 218 |
+
},
|
| 219 |
+
"\\rightarrow": {
|
| 220 |
+
"samples": 20,
|
| 221 |
+
"matched": 14,
|
| 222 |
+
"grouping_recall": 0.7
|
| 223 |
+
},
|
| 224 |
+
"\\neq": {
|
| 225 |
+
"samples": 2,
|
| 226 |
+
"matched": 2,
|
| 227 |
+
"grouping_recall": 1.0
|
| 228 |
+
},
|
| 229 |
+
"\\pm": {
|
| 230 |
+
"samples": 9,
|
| 231 |
+
"matched": 5,
|
| 232 |
+
"grouping_recall": 0.5555555555555556
|
| 233 |
+
}
|
| 234 |
+
}
|
| 235 |
+
}
|
| 236 |
+
},
|
| 237 |
+
{
|
| 238 |
+
"multistroke_family_boost": 1.5,
|
| 239 |
+
"global": {
|
| 240 |
+
"formulas": 275,
|
| 241 |
+
"exact_partition": 0.6690909090909091,
|
| 242 |
+
"exact_group_recall": 0.951232302045097,
|
| 243 |
+
"overmerge_rate": 0.13818181818181818,
|
| 244 |
+
"oversplit_rate": 0.2290909090909091,
|
| 245 |
+
"pair_precision": 0.9627831715210357,
|
| 246 |
+
"pair_recall": 0.8927231807951987,
|
| 247 |
+
"pair_f1": 0.9264305177111717
|
| 248 |
+
},
|
| 249 |
+
"families": {
|
| 250 |
+
"total": 59,
|
| 251 |
+
"matched": 39,
|
| 252 |
+
"grouping_recall": 0.6610169491525424,
|
| 253 |
+
"by_label": {
|
| 254 |
+
"\\sum": {
|
| 255 |
+
"samples": 11,
|
| 256 |
+
"matched": 3,
|
| 257 |
+
"grouping_recall": 0.2727272727272727
|
| 258 |
+
},
|
| 259 |
+
"\\pi": {
|
| 260 |
+
"samples": 17,
|
| 261 |
+
"matched": 12,
|
| 262 |
+
"grouping_recall": 0.7058823529411765
|
| 263 |
+
},
|
| 264 |
+
"\\rightarrow": {
|
| 265 |
+
"samples": 20,
|
| 266 |
+
"matched": 14,
|
| 267 |
+
"grouping_recall": 0.7
|
| 268 |
+
},
|
| 269 |
+
"\\neq": {
|
| 270 |
+
"samples": 2,
|
| 271 |
+
"matched": 2,
|
| 272 |
+
"grouping_recall": 1.0
|
| 273 |
+
},
|
| 274 |
+
"\\pm": {
|
| 275 |
+
"samples": 9,
|
| 276 |
+
"matched": 8,
|
| 277 |
+
"grouping_recall": 0.8888888888888888
|
| 278 |
+
}
|
| 279 |
+
}
|
| 280 |
+
}
|
| 281 |
+
},
|
| 282 |
+
{
|
| 283 |
+
"multistroke_family_boost": 2.0,
|
| 284 |
+
"global": {
|
| 285 |
+
"formulas": 275,
|
| 286 |
+
"exact_partition": 0.6727272727272727,
|
| 287 |
+
"exact_group_recall": 0.9520188778185632,
|
| 288 |
+
"overmerge_rate": 0.13818181818181818,
|
| 289 |
+
"oversplit_rate": 0.22545454545454546,
|
| 290 |
+
"pair_precision": 0.9629032258064516,
|
| 291 |
+
"pair_recall": 0.8957239309827457,
|
| 292 |
+
"pair_f1": 0.9280994947532063
|
| 293 |
+
},
|
| 294 |
+
"families": {
|
| 295 |
+
"total": 59,
|
| 296 |
+
"matched": 42,
|
| 297 |
+
"grouping_recall": 0.711864406779661,
|
| 298 |
+
"by_label": {
|
| 299 |
+
"\\sum": {
|
| 300 |
+
"samples": 11,
|
| 301 |
+
"matched": 5,
|
| 302 |
+
"grouping_recall": 0.45454545454545453
|
| 303 |
+
},
|
| 304 |
+
"\\pi": {
|
| 305 |
+
"samples": 17,
|
| 306 |
+
"matched": 12,
|
| 307 |
+
"grouping_recall": 0.7058823529411765
|
| 308 |
+
},
|
| 309 |
+
"\\rightarrow": {
|
| 310 |
+
"samples": 20,
|
| 311 |
+
"matched": 14,
|
| 312 |
+
"grouping_recall": 0.7
|
| 313 |
+
},
|
| 314 |
+
"\\neq": {
|
| 315 |
+
"samples": 2,
|
| 316 |
+
"matched": 2,
|
| 317 |
+
"grouping_recall": 1.0
|
| 318 |
+
},
|
| 319 |
+
"\\pm": {
|
| 320 |
+
"samples": 9,
|
| 321 |
+
"matched": 9,
|
| 322 |
+
"grouping_recall": 1.0
|
| 323 |
+
}
|
| 324 |
+
}
|
| 325 |
+
}
|
| 326 |
+
},
|
| 327 |
+
{
|
| 328 |
+
"multistroke_family_boost": 3.0,
|
| 329 |
+
"global": {
|
| 330 |
+
"formulas": 275,
|
| 331 |
+
"exact_partition": 0.6763636363636364,
|
| 332 |
+
"exact_group_recall": 0.9530676455165181,
|
| 333 |
+
"overmerge_rate": 0.13818181818181818,
|
| 334 |
+
"oversplit_rate": 0.22181818181818183,
|
| 335 |
+
"pair_precision": 0.9630522088353414,
|
| 336 |
+
"pair_recall": 0.8994748687171793,
|
| 337 |
+
"pair_f1": 0.930178432893716
|
| 338 |
+
},
|
| 339 |
+
"families": {
|
| 340 |
+
"total": 59,
|
| 341 |
+
"matched": 45,
|
| 342 |
+
"grouping_recall": 0.7627118644067796,
|
| 343 |
+
"by_label": {
|
| 344 |
+
"\\sum": {
|
| 345 |
+
"samples": 11,
|
| 346 |
+
"matched": 5,
|
| 347 |
+
"grouping_recall": 0.45454545454545453
|
| 348 |
+
},
|
| 349 |
+
"\\pi": {
|
| 350 |
+
"samples": 17,
|
| 351 |
+
"matched": 13,
|
| 352 |
+
"grouping_recall": 0.7647058823529411
|
| 353 |
+
},
|
| 354 |
+
"\\rightarrow": {
|
| 355 |
+
"samples": 20,
|
| 356 |
+
"matched": 16,
|
| 357 |
+
"grouping_recall": 0.8
|
| 358 |
+
},
|
| 359 |
+
"\\neq": {
|
| 360 |
+
"samples": 2,
|
| 361 |
+
"matched": 2,
|
| 362 |
+
"grouping_recall": 1.0
|
| 363 |
+
},
|
| 364 |
+
"\\pm": {
|
| 365 |
+
"samples": 9,
|
| 366 |
+
"matched": 9,
|
| 367 |
+
"grouping_recall": 1.0
|
| 368 |
+
}
|
| 369 |
+
}
|
| 370 |
+
}
|
| 371 |
+
},
|
| 372 |
+
{
|
| 373 |
+
"multistroke_family_boost": 4.0,
|
| 374 |
+
"global": {
|
| 375 |
+
"formulas": 275,
|
| 376 |
+
"exact_partition": 0.6763636363636364,
|
| 377 |
+
"exact_group_recall": 0.9533298374410069,
|
| 378 |
+
"overmerge_rate": 0.14181818181818182,
|
| 379 |
+
"oversplit_rate": 0.21818181818181817,
|
| 380 |
+
"pair_precision": 0.9616613418530351,
|
| 381 |
+
"pair_recall": 0.9032258064516129,
|
| 382 |
+
"pair_f1": 0.9315280464216633
|
| 383 |
+
},
|
| 384 |
+
"families": {
|
| 385 |
+
"total": 59,
|
| 386 |
+
"matched": 48,
|
| 387 |
+
"grouping_recall": 0.8135593220338984,
|
| 388 |
+
"by_label": {
|
| 389 |
+
"\\sum": {
|
| 390 |
+
"samples": 11,
|
| 391 |
+
"matched": 6,
|
| 392 |
+
"grouping_recall": 0.5454545454545454
|
| 393 |
+
},
|
| 394 |
+
"\\pi": {
|
| 395 |
+
"samples": 17,
|
| 396 |
+
"matched": 15,
|
| 397 |
+
"grouping_recall": 0.8823529411764706
|
| 398 |
+
},
|
| 399 |
+
"\\rightarrow": {
|
| 400 |
+
"samples": 20,
|
| 401 |
+
"matched": 16,
|
| 402 |
+
"grouping_recall": 0.8
|
| 403 |
+
},
|
| 404 |
+
"\\neq": {
|
| 405 |
+
"samples": 2,
|
| 406 |
+
"matched": 2,
|
| 407 |
+
"grouping_recall": 1.0
|
| 408 |
+
},
|
| 409 |
+
"\\pm": {
|
| 410 |
+
"samples": 9,
|
| 411 |
+
"matched": 9,
|
| 412 |
+
"grouping_recall": 1.0
|
| 413 |
+
}
|
| 414 |
+
}
|
| 415 |
+
}
|
| 416 |
+
},
|
| 417 |
+
{
|
| 418 |
+
"multistroke_family_boost": 6.0,
|
| 419 |
+
"global": {
|
| 420 |
+
"formulas": 275,
|
| 421 |
+
"exact_partition": 0.68,
|
| 422 |
+
"exact_group_recall": 0.9528054535920294,
|
| 423 |
+
"overmerge_rate": 0.14909090909090908,
|
| 424 |
+
"oversplit_rate": 0.21818181818181817,
|
| 425 |
+
"pair_precision": 0.9563838223632039,
|
| 426 |
+
"pair_recall": 0.9047261815453863,
|
| 427 |
+
"pair_f1": 0.9298380878951427
|
| 428 |
+
},
|
| 429 |
+
"families": {
|
| 430 |
+
"total": 59,
|
| 431 |
+
"matched": 51,
|
| 432 |
+
"grouping_recall": 0.864406779661017,
|
| 433 |
+
"by_label": {
|
| 434 |
+
"\\sum": {
|
| 435 |
+
"samples": 11,
|
| 436 |
+
"matched": 8,
|
| 437 |
+
"grouping_recall": 0.7272727272727273
|
| 438 |
+
},
|
| 439 |
+
"\\pi": {
|
| 440 |
+
"samples": 17,
|
| 441 |
+
"matched": 15,
|
| 442 |
+
"grouping_recall": 0.8823529411764706
|
| 443 |
+
},
|
| 444 |
+
"\\rightarrow": {
|
| 445 |
+
"samples": 20,
|
| 446 |
+
"matched": 17,
|
| 447 |
+
"grouping_recall": 0.85
|
| 448 |
+
},
|
| 449 |
+
"\\neq": {
|
| 450 |
+
"samples": 2,
|
| 451 |
+
"matched": 2,
|
| 452 |
+
"grouping_recall": 1.0
|
| 453 |
+
},
|
| 454 |
+
"\\pm": {
|
| 455 |
+
"samples": 9,
|
| 456 |
+
"matched": 9,
|
| 457 |
+
"grouping_recall": 1.0
|
| 458 |
+
}
|
| 459 |
+
}
|
| 460 |
+
}
|
| 461 |
+
}
|
| 462 |
+
],
|
| 463 |
+
"official_test_reference": {
|
| 464 |
+
"multistroke_family_boost": 0.0,
|
| 465 |
+
"global": {
|
| 466 |
+
"formulas": 488,
|
| 467 |
+
"exact_partition": 0.5450819672131147,
|
| 468 |
+
"exact_group_recall": 0.9310846176214016,
|
| 469 |
+
"overmerge_rate": 0.21106557377049182,
|
| 470 |
+
"oversplit_rate": 0.32786885245901637,
|
| 471 |
+
"pair_precision": 0.9236289082521784,
|
| 472 |
+
"pair_recall": 0.8536238749407864,
|
| 473 |
+
"pair_f1": 0.8872476612506155
|
| 474 |
+
},
|
| 475 |
+
"families": {
|
| 476 |
+
"total": 135,
|
| 477 |
+
"matched": 61,
|
| 478 |
+
"grouping_recall": 0.45185185185185184,
|
| 479 |
+
"by_label": {
|
| 480 |
+
"\\sum": {
|
| 481 |
+
"samples": 31,
|
| 482 |
+
"matched": 8,
|
| 483 |
+
"grouping_recall": 0.25806451612903225
|
| 484 |
+
},
|
| 485 |
+
"\\pi": {
|
| 486 |
+
"samples": 30,
|
| 487 |
+
"matched": 12,
|
| 488 |
+
"grouping_recall": 0.4
|
| 489 |
+
},
|
| 490 |
+
"\\rightarrow": {
|
| 491 |
+
"samples": 49,
|
| 492 |
+
"matched": 35,
|
| 493 |
+
"grouping_recall": 0.7142857142857143
|
| 494 |
+
},
|
| 495 |
+
"\\neq": {
|
| 496 |
+
"samples": 13,
|
| 497 |
+
"matched": 3,
|
| 498 |
+
"grouping_recall": 0.23076923076923078
|
| 499 |
+
},
|
| 500 |
+
"\\pm": {
|
| 501 |
+
"samples": 12,
|
| 502 |
+
"matched": 3,
|
| 503 |
+
"grouping_recall": 0.25
|
| 504 |
+
}
|
| 505 |
+
}
|
| 506 |
+
}
|
| 507 |
+
},
|
| 508 |
+
"official_test_winner": {
|
| 509 |
+
"multistroke_family_boost": 6.0,
|
| 510 |
+
"global": {
|
| 511 |
+
"formulas": 488,
|
| 512 |
+
"exact_partition": 0.6004098360655737,
|
| 513 |
+
"exact_group_recall": 0.9370456528060482,
|
| 514 |
+
"overmerge_rate": 0.21721311475409835,
|
| 515 |
+
"oversplit_rate": 0.2520491803278688,
|
| 516 |
+
"pair_precision": 0.9231143552311436,
|
| 517 |
+
"pair_recall": 0.8986262434864993,
|
| 518 |
+
"pair_f1": 0.9107057129140663
|
| 519 |
+
},
|
| 520 |
+
"families": {
|
| 521 |
+
"total": 135,
|
| 522 |
+
"matched": 112,
|
| 523 |
+
"grouping_recall": 0.8296296296296296,
|
| 524 |
+
"by_label": {
|
| 525 |
+
"\\sum": {
|
| 526 |
+
"samples": 31,
|
| 527 |
+
"matched": 22,
|
| 528 |
+
"grouping_recall": 0.7096774193548387
|
| 529 |
+
},
|
| 530 |
+
"\\pi": {
|
| 531 |
+
"samples": 30,
|
| 532 |
+
"matched": 27,
|
| 533 |
+
"grouping_recall": 0.9
|
| 534 |
+
},
|
| 535 |
+
"\\rightarrow": {
|
| 536 |
+
"samples": 49,
|
| 537 |
+
"matched": 44,
|
| 538 |
+
"grouping_recall": 0.8979591836734694
|
| 539 |
+
},
|
| 540 |
+
"\\neq": {
|
| 541 |
+
"samples": 13,
|
| 542 |
+
"matched": 10,
|
| 543 |
+
"grouping_recall": 0.7692307692307693
|
| 544 |
+
},
|
| 545 |
+
"\\pm": {
|
| 546 |
+
"samples": 12,
|
| 547 |
+
"matched": 9,
|
| 548 |
+
"grouping_recall": 0.75
|
| 549 |
+
}
|
| 550 |
+
}
|
| 551 |
+
}
|
| 552 |
+
},
|
| 553 |
+
"official_test_deltas": {
|
| 554 |
+
"family_grouping_pp": 37.77777777777778,
|
| 555 |
+
"exact_partition_pp": 5.5327868852458995,
|
| 556 |
+
"pair_f1_pp": 2.345805166345083
|
| 557 |
+
},
|
| 558 |
+
"decision": {
|
| 559 |
+
"adopted": true,
|
| 560 |
+
"selected_multistroke_family_boost": 6.0,
|
| 561 |
+
"reason": "family grouping improves within global partition guards"
|
| 562 |
+
},
|
| 563 |
+
"track": "R_noncommercial_only",
|
| 564 |
+
"product_validation": false
|
| 565 |
+
}
|
scripts/audit_math_ink_06_local_baseline_overmerge.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""์ต์ข
selector์ overmerge๋ฅผ ์ด์ truthยทOCR familyยทTray penalty ๊ธฐ์ค์ผ๋ก ๋ถํดํ๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from collections import Counter
|
| 7 |
+
from datetime import datetime, timezone
|
| 8 |
+
import json
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
import sys
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 14 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 15 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 16 |
+
if str(path) not in sys.path:
|
| 17 |
+
sys.path.insert(0, str(path))
|
| 18 |
+
|
| 19 |
+
from math_grid_drawer.research.cross_visual import CrossVisualModel
|
| 20 |
+
from math_grid_drawer.research.equality_visual import EqualityVisualModel
|
| 21 |
+
from math_grid_drawer.research.segmentation_lattice import (
|
| 22 |
+
LATTICE_FEATURE_NAMES,
|
| 23 |
+
select_lattice_partition,
|
| 24 |
+
)
|
| 25 |
+
from scripts.crohme_lattice_common import load_cached_split, writer_fit_validation
|
| 26 |
+
from scripts.evaluate_crohme_gt_free_grouping import _truth_partition
|
| 27 |
+
from scripts.evaluate_crohme_lattice_ocr_fusion import _fit_geometry
|
| 28 |
+
from scripts.evaluate_crohme_structure_presence import _truth_structures
|
| 29 |
+
from scripts.evaluate_crohme_tray_joint_selector import _prepared_signals, _weighted
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def _parse_args() -> argparse.Namespace:
|
| 33 |
+
"""ํ์ ๋ณ์: ๊ณต์ testยทcacheยทfull selector head. ์๋ ์๋ฆฌ: ์ต์ข
overmerge ๊ฐ์ฌ CLI๋ฅผ ๋ง๋ ๋ค."""
|
| 34 |
+
|
| 35 |
+
parser = argparse.ArgumentParser(description="Audit Math Ink 0.6 local-baseline overmerge")
|
| 36 |
+
parser.add_argument(
|
| 37 |
+
"--train-root", type=Path,
|
| 38 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/trainData",
|
| 39 |
+
)
|
| 40 |
+
parser.add_argument(
|
| 41 |
+
"--test-root", type=Path,
|
| 42 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/testDataGT",
|
| 43 |
+
)
|
| 44 |
+
parser.add_argument(
|
| 45 |
+
"--cache-dir", type=Path,
|
| 46 |
+
default=PROJECT_ROOT / "research/runs/crohme_lattice_ocr_cache_v2_20260722",
|
| 47 |
+
)
|
| 48 |
+
parser.add_argument(
|
| 49 |
+
"--bundle", type=Path,
|
| 50 |
+
default=Path(r"research\runs\aiflow_ocr_05_dual_trajectory_3seed_20260720\bundle.manifest.json"),
|
| 51 |
+
)
|
| 52 |
+
parser.add_argument(
|
| 53 |
+
"--cross-model", type=Path,
|
| 54 |
+
default=PROJECT_ROOT / "research/runs/crohme_cross_visual_loop3_polyline_20260722/cross_visual.json",
|
| 55 |
+
)
|
| 56 |
+
parser.add_argument(
|
| 57 |
+
"--equality-model", type=Path,
|
| 58 |
+
default=PROJECT_ROOT / "research/runs/crohme_equality_visual_loop1_20260722/equality_visual.json",
|
| 59 |
+
)
|
| 60 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 61 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 62 |
+
return parser.parse_args()
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def main() -> None:
|
| 66 |
+
"""ํ์ ๋ณ์: gap40ยทfamily6 ๋ณดํธ selector. ์๋ ์๋ฆฌ: overmerge candidate์ ์นจ๋ฒ truth๋ฅผ 1:1 ์ฐ๊ฒฐํ๋ค."""
|
| 67 |
+
|
| 68 |
+
args = _parse_args()
|
| 69 |
+
fit, _validation = writer_fit_validation(args.train_root, args.profile)
|
| 70 |
+
geometry_model = _fit_geometry(fit)
|
| 71 |
+
equality_model = EqualityVisualModel.load(args.equality_model)
|
| 72 |
+
cross_model = CrossVisualModel.load(args.cross_model)
|
| 73 |
+
samples, cached = load_cached_split(
|
| 74 |
+
args.test_root,
|
| 75 |
+
args.cache_dir,
|
| 76 |
+
split="official_test",
|
| 77 |
+
profile=args.profile,
|
| 78 |
+
bundle=args.bundle,
|
| 79 |
+
version=2,
|
| 80 |
+
)
|
| 81 |
+
prepared = _prepared_signals(
|
| 82 |
+
samples,
|
| 83 |
+
cached,
|
| 84 |
+
geometry_model,
|
| 85 |
+
equality_model=equality_model,
|
| 86 |
+
cross_model=cross_model,
|
| 87 |
+
cross_gap_ratio=0.40,
|
| 88 |
+
multistroke_family_boost=6.0,
|
| 89 |
+
)
|
| 90 |
+
weighted = _weighted(
|
| 91 |
+
prepared,
|
| 92 |
+
tray_weight=4.0,
|
| 93 |
+
symbol_weight=4.0,
|
| 94 |
+
fraction_weight=8.0,
|
| 95 |
+
infix_weight=8.0,
|
| 96 |
+
)
|
| 97 |
+
path_by_id = {path.stem: path for path in sorted(args.test_root.rglob("*.inkml"))}
|
| 98 |
+
truth_labels: Counter[str] = Counter()
|
| 99 |
+
candidate_labels: Counter[str] = Counter()
|
| 100 |
+
candidate_families: Counter[str] = Counter()
|
| 101 |
+
invaded_pairs: Counter[str] = Counter()
|
| 102 |
+
structure_counts: Counter[str] = Counter()
|
| 103 |
+
fraction_penalty_counts: Counter[str] = Counter()
|
| 104 |
+
local_penalty_counts: Counter[str] = Counter()
|
| 105 |
+
rows: list[dict[str, Any]] = []
|
| 106 |
+
unique_bad_candidates: dict[tuple[str, tuple[int, ...]], dict[str, Any]] = {}
|
| 107 |
+
correct_multistroke_rows: list[dict[str, Any]] = []
|
| 108 |
+
for sample, row in zip(samples, weighted, strict=True):
|
| 109 |
+
truth_groups, labels = _truth_partition(sample, "aiflow_geometry")
|
| 110 |
+
label_by_group = dict(zip(truth_groups, (str(value) for value in labels), strict=True))
|
| 111 |
+
predicted = set(select_lattice_partition(
|
| 112 |
+
row["candidates"], row["logits"], row["stroke_count"], group_bias=-2.0,
|
| 113 |
+
))
|
| 114 |
+
candidate_index = {
|
| 115 |
+
frozenset(int(value) for value in candidate["source_indices"]): index
|
| 116 |
+
for index, candidate in enumerate(row["candidates"])
|
| 117 |
+
}
|
| 118 |
+
families = row.get("ocr_families") or [""] * len(row["candidates"])
|
| 119 |
+
structures = _truth_structures(path_by_id[sample["sample_id"]])
|
| 120 |
+
for truth, truth_label in label_by_group.items():
|
| 121 |
+
if truth in predicted:
|
| 122 |
+
if len(truth) > 1:
|
| 123 |
+
index = candidate_index[truth]
|
| 124 |
+
correct_multistroke_rows.append({
|
| 125 |
+
"sample_id": sample["sample_id"],
|
| 126 |
+
"truth_label": truth_label,
|
| 127 |
+
"truth_group": sorted(truth),
|
| 128 |
+
"candidate_label": str(row["ocr_labels"][index]),
|
| 129 |
+
"candidate_family": str(families[index]),
|
| 130 |
+
"features": {
|
| 131 |
+
"ocr_top1": float(
|
| 132 |
+
row["features"][index][len(LATTICE_FEATURE_NAMES)]
|
| 133 |
+
),
|
| 134 |
+
"merge_top1_gain": float(
|
| 135 |
+
row["features"][index][len(LATTICE_FEATURE_NAMES) + 6]
|
| 136 |
+
),
|
| 137 |
+
"merge_entropy_gain": float(
|
| 138 |
+
row["features"][index][len(LATTICE_FEATURE_NAMES) + 7]
|
| 139 |
+
),
|
| 140 |
+
"pair_gap_max": float(row["features"][index][12]),
|
| 141 |
+
},
|
| 142 |
+
})
|
| 143 |
+
continue
|
| 144 |
+
overmerged = [
|
| 145 |
+
group for group in predicted
|
| 146 |
+
if group & truth and bool(group - truth)
|
| 147 |
+
]
|
| 148 |
+
for group in overmerged:
|
| 149 |
+
index = candidate_index[group]
|
| 150 |
+
candidate_label = str(row["ocr_labels"][index])
|
| 151 |
+
candidate_family = str(families[index])
|
| 152 |
+
invaded = [
|
| 153 |
+
other_label
|
| 154 |
+
for other_group, other_label in label_by_group.items()
|
| 155 |
+
if other_group != truth and other_group & group
|
| 156 |
+
]
|
| 157 |
+
truth_labels[truth_label] += 1
|
| 158 |
+
candidate_labels[candidate_label] += 1
|
| 159 |
+
candidate_families[candidate_family] += 1
|
| 160 |
+
for other_label in invaded:
|
| 161 |
+
invaded_pairs[f"{truth_label} -> {other_label}"] += 1
|
| 162 |
+
for structure in structures or {"plain"}:
|
| 163 |
+
structure_counts[structure] += 1
|
| 164 |
+
fraction_penalty = float(row["fraction_penalty"][index])
|
| 165 |
+
fraction_penalty_counts[
|
| 166 |
+
"nonzero" if fraction_penalty > 0.0 else "zero"
|
| 167 |
+
] += 1
|
| 168 |
+
raw_local_penalty = float(row["raw_local_baseline_penalty"][index])
|
| 169 |
+
local_penalty = float(row["local_baseline_penalty"][index])
|
| 170 |
+
if local_penalty > 0.0:
|
| 171 |
+
local_penalty_counts["effective_nonzero"] += 1
|
| 172 |
+
elif raw_local_penalty > 0.0:
|
| 173 |
+
local_penalty_counts["protected_by_positive_signal"] += 1
|
| 174 |
+
else:
|
| 175 |
+
local_penalty_counts["not_detected"] += 1
|
| 176 |
+
covered_truth = [
|
| 177 |
+
other_group for other_group in truth_groups if other_group & group
|
| 178 |
+
]
|
| 179 |
+
replacement_scores = [
|
| 180 |
+
float(row["logits"][candidate_index[other_group]])
|
| 181 |
+
for other_group in covered_truth if other_group in candidate_index
|
| 182 |
+
]
|
| 183 |
+
oracle_margin = (
|
| 184 |
+
float(row["logits"][index]) - sum(replacement_scores)
|
| 185 |
+
+ 2.0 * (len(replacement_scores) - 1)
|
| 186 |
+
if len(replacement_scores) == len(covered_truth) else None
|
| 187 |
+
)
|
| 188 |
+
detail = {
|
| 189 |
+
"sample_id": sample["sample_id"],
|
| 190 |
+
"structures": sorted(structures),
|
| 191 |
+
"truth_label": truth_label,
|
| 192 |
+
"truth_group": sorted(truth),
|
| 193 |
+
"candidate_group": sorted(group),
|
| 194 |
+
"candidate_label": candidate_label,
|
| 195 |
+
"candidate_family": candidate_family,
|
| 196 |
+
"invaded_truth_labels": invaded,
|
| 197 |
+
"fraction_penalty": fraction_penalty,
|
| 198 |
+
"raw_local_baseline_penalty": raw_local_penalty,
|
| 199 |
+
"local_baseline_penalty": local_penalty,
|
| 200 |
+
"oracle_truth_partition_margin": oracle_margin,
|
| 201 |
+
"tray_signal": float(row["tray_signal"][index]),
|
| 202 |
+
"symbol_signal": float(row["symbol_signal"][index]),
|
| 203 |
+
"infix_signal": float(row["infix_signal"][index]),
|
| 204 |
+
"score": float(row["logits"][index]),
|
| 205 |
+
"geometry": {
|
| 206 |
+
"width_ref": float(row["features"][index][2]),
|
| 207 |
+
"height_ref": float(row["features"][index][3]),
|
| 208 |
+
"aspect_log": float(row["features"][index][4]),
|
| 209 |
+
"temporal_span": float(row["features"][index][5]),
|
| 210 |
+
"pair_gap_max": float(row["features"][index][12]),
|
| 211 |
+
},
|
| 212 |
+
"ocr_features": {
|
| 213 |
+
"ocr_top1": float(
|
| 214 |
+
row["features"][index][len(LATTICE_FEATURE_NAMES)]
|
| 215 |
+
),
|
| 216 |
+
"merge_top1_gain": float(
|
| 217 |
+
row["features"][index][len(LATTICE_FEATURE_NAMES) + 6]
|
| 218 |
+
),
|
| 219 |
+
"merge_entropy_gain": float(
|
| 220 |
+
row["features"][index][len(LATTICE_FEATURE_NAMES) + 7]
|
| 221 |
+
),
|
| 222 |
+
},
|
| 223 |
+
}
|
| 224 |
+
rows.append(detail)
|
| 225 |
+
unique_bad_candidates[(sample["sample_id"], tuple(sorted(group)))] = detail
|
| 226 |
+
report = {
|
| 227 |
+
"experiment": "R-MATH-INK-06-LOCAL-BASELINE-OVERMERGE-AUDIT-001",
|
| 228 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 229 |
+
"configuration": {
|
| 230 |
+
"cross_gap_ratio": 0.40,
|
| 231 |
+
"multistroke_family_boost": 6.0,
|
| 232 |
+
},
|
| 233 |
+
"overmerge_events": len(rows),
|
| 234 |
+
"truth_labels": truth_labels.most_common(),
|
| 235 |
+
"candidate_labels": candidate_labels.most_common(),
|
| 236 |
+
"candidate_families": candidate_families.most_common(),
|
| 237 |
+
"invaded_pairs": invaded_pairs.most_common(),
|
| 238 |
+
"structures": structure_counts.most_common(),
|
| 239 |
+
"fraction_penalty": dict(fraction_penalty_counts),
|
| 240 |
+
"local_baseline_penalty": dict(local_penalty_counts),
|
| 241 |
+
"unique_bad_candidate_count": len(unique_bad_candidates),
|
| 242 |
+
"unique_bad_candidates": list(unique_bad_candidates.values()),
|
| 243 |
+
"correct_multistroke_rows": correct_multistroke_rows,
|
| 244 |
+
"rows": rows,
|
| 245 |
+
"track": "R_noncommercial_only",
|
| 246 |
+
"product_validation": False,
|
| 247 |
+
}
|
| 248 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 249 |
+
args.output.write_text(
|
| 250 |
+
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
| 251 |
+
encoding="utf-8",
|
| 252 |
+
)
|
| 253 |
+
print(json.dumps({
|
| 254 |
+
key: value for key, value in report.items()
|
| 255 |
+
if key not in {"rows", "unique_bad_candidates", "correct_multistroke_rows"}
|
| 256 |
+
}, ensure_ascii=False, indent=2))
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
if __name__ == "__main__":
|
| 260 |
+
main()
|
scripts/evaluate_crohme_tray_joint_selector.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exact OCR anchor์ ์ํ Tray ๊ณ์ฝ์ lattice positive/negative score๋ก ๊ณต๋ ๊ฒ์ฆํ๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from datetime import datetime, timezone
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import numpy as np
|
| 11 |
+
|
| 12 |
+
from math_grid_drawer.research.cross_visual import CrossVisualModel
|
| 13 |
+
from math_grid_drawer.research.equality_visual import EqualityVisualModel
|
| 14 |
+
from math_grid_drawer.research.math_tray import fraction_tray_boundary_penalties
|
| 15 |
+
from math_grid_drawer.research.segmentation_lattice import select_lattice_partition
|
| 16 |
+
from math_grid_drawer.research.tray_joint import (
|
| 17 |
+
TrayJointWeights,
|
| 18 |
+
adjusted_logits,
|
| 19 |
+
candidate_signals,
|
| 20 |
+
component_competition_penalties,
|
| 21 |
+
local_baseline_boundary_penalties,
|
| 22 |
+
)
|
| 23 |
+
from scripts.crohme_lattice_common import load_cache_for_samples, load_cached_split, writer_fit_validation
|
| 24 |
+
from scripts.evaluate_crohme_lattice_ocr_fusion import _fit_geometry, _score
|
| 25 |
+
from scripts.train_crohme_segmentation_lattice_joint_selector import _metrics
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _prepared_signals(
|
| 29 |
+
samples: list[dict], cached: list[dict], model, *, strict_equality: bool = False,
|
| 30 |
+
equality_model: EqualityVisualModel | None = None, cross_model: CrossVisualModel | None = None,
|
| 31 |
+
cross_gap_ratio: float = 0.40,
|
| 32 |
+
multistroke_family_boost: float = 6.0,
|
| 33 |
+
) -> list[dict]:
|
| 34 |
+
"""ํ์ ๋ณ์: ์๋ณธยทv2 cacheยทgeometry modelยทcross gap. ์๋ ์๋ฆฌ: base fusion๊ณผ ๊ตฌ์กฐ signal์ ํ ๋ฒ ๊ณ์ฐํ๋ค."""
|
| 35 |
+
|
| 36 |
+
output = []
|
| 37 |
+
for sample, scored in zip(samples, _score(cached, model, ocr_weight=0.5), strict=True):
|
| 38 |
+
partition = select_lattice_partition(scored["candidates"], scored["logits"], scored["stroke_count"], group_bias=-2.0)
|
| 39 |
+
tray_signal, symbol_signal, infix_signal = candidate_signals(
|
| 40 |
+
sample["profiled_strokes"], scored["candidates"], scored["ocr_labels"],
|
| 41 |
+
scored["features"], partition, ocr_families=scored.get("ocr_families"),
|
| 42 |
+
strict_equality=strict_equality, equality_model=equality_model,
|
| 43 |
+
cross_model=cross_model, cross_gap_ratio=cross_gap_ratio,
|
| 44 |
+
multistroke_family_boost=multistroke_family_boost,
|
| 45 |
+
)
|
| 46 |
+
fraction_penalty = fraction_tray_boundary_penalties(scored["candidates"], sample["profiled_strokes"])
|
| 47 |
+
competition_penalty = component_competition_penalties(
|
| 48 |
+
scored["candidates"], scored["features"], mode="joint",
|
| 49 |
+
)
|
| 50 |
+
raw_local_baseline_penalty = local_baseline_boundary_penalties(
|
| 51 |
+
sample["profiled_strokes"], scored["candidates"], scored["ocr_labels"],
|
| 52 |
+
scored["features"], partition, ocr_families=scored.get("ocr_families"),
|
| 53 |
+
)
|
| 54 |
+
local_baseline_penalty = raw_local_baseline_penalty.copy()
|
| 55 |
+
# ์ ๋ขฐํ ์ ์๋ ์์ฑ ๋คํ ๊ธฐํธ๋ ๊ตฌ์กฐ ๊ฒฝ๊ณ์ ๊ฒน์ณ๋ ๊ธฐ์กด positive evidence๋ฅผ ์ฐ์ ํ๋ค.
|
| 56 |
+
protected = (tray_signal > 0.0) | (symbol_signal > 0.0) | (infix_signal > 0.0)
|
| 57 |
+
local_baseline_penalty[protected] = 0.0
|
| 58 |
+
output.append({
|
| 59 |
+
**scored, "tray_signal": tray_signal, "symbol_signal": symbol_signal,
|
| 60 |
+
"infix_signal": infix_signal, "fraction_penalty": fraction_penalty,
|
| 61 |
+
"competition_penalty": competition_penalty,
|
| 62 |
+
"raw_local_baseline_penalty": raw_local_baseline_penalty,
|
| 63 |
+
"local_baseline_penalty": local_baseline_penalty,
|
| 64 |
+
})
|
| 65 |
+
return output
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _weighted(
|
| 69 |
+
rows: list[dict], *, tray_weight: float, symbol_weight: float,
|
| 70 |
+
fraction_weight: float, infix_weight: float = 0.0, competition_weight: float = 0.0,
|
| 71 |
+
local_baseline_weight: float = 0.0,
|
| 72 |
+
) -> list[dict]:
|
| 73 |
+
"""ํ์ ๋ณ์: ์ฌ์ ๊ณ์ฐ signalยท๊ฐ์ค์น. ์๋ ์๋ฆฌ: positive ๊ตฌ์กฐ์ ์ธ negative guard๋ฅผ ๊ฒฐํฉํ๋ค."""
|
| 74 |
+
|
| 75 |
+
weights = TrayJointWeights(
|
| 76 |
+
tray=tray_weight, symbol=symbol_weight, fraction=fraction_weight,
|
| 77 |
+
infix=infix_weight, competition=competition_weight,
|
| 78 |
+
local_baseline=local_baseline_weight,
|
| 79 |
+
)
|
| 80 |
+
return [{**row, "logits": adjusted_logits(
|
| 81 |
+
row["logits"], row["tray_signal"], row["symbol_signal"], row["fraction_penalty"], weights,
|
| 82 |
+
row["infix_signal"], row["competition_penalty"], row["local_baseline_penalty"],
|
| 83 |
+
)} for row in rows]
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def main() -> None:
|
| 87 |
+
"""ํ์ ๋ณ์: CROHME train/testยทv2 cache. ์๋ ์๋ฆฌ: writer-validation์์ joint ๊ฐ์ค์น๋ฅผ ๊ณ ์ ํ๊ณ official test์ ํ ๋ฒ ์ ์ฉํ๋ค."""
|
| 88 |
+
|
| 89 |
+
parser = argparse.ArgumentParser(description="Evaluate CROHME Tray joint selector")
|
| 90 |
+
parser.add_argument("--train-root", type=Path, required=True)
|
| 91 |
+
parser.add_argument("--test-root", type=Path, required=True)
|
| 92 |
+
parser.add_argument("--cache-dir", type=Path, required=True)
|
| 93 |
+
parser.add_argument("--bundle", type=Path, required=True)
|
| 94 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 95 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 96 |
+
parser.add_argument("--equality-model", type=Path, help="์ ํ์ equality visual JSON head")
|
| 97 |
+
parser.add_argument("--cross-model", type=Path, help="์ ํ์ cross visual JSON head")
|
| 98 |
+
parser.add_argument("--cross-gap-ratio", type=float, default=0.40, help="cross head ์ฌ์ bbox gap/์์ ๋์ด ๋น์จ")
|
| 99 |
+
parser.add_argument("--multistroke-family-boost", type=float, default=6.0, help="OCR family ํธํ ๋คํ ํ๋ณด์ ์ถ๊ฐ symbol signal")
|
| 100 |
+
parser.add_argument("--fixed-selected", action="store_true", help="๊ธฐ์กด validation ์ ํ๊ฐ 4/4/8/-2๋ฅผ ์ฌ๊ฒ์ฆํ๊ณ sweep์ ์๋ต")
|
| 101 |
+
parser.add_argument("--sweep-infix", action="store_true", help="๊ธฐ์กด ๊ฐ์ค์น๋ ๊ณ ์ ํ๊ณ x/= ๊ตฌ์กฐ ๊ฐ์ค์น๋ง validation ์ ํ")
|
| 102 |
+
equality_mode = parser.add_mutually_exclusive_group()
|
| 103 |
+
equality_mode.add_argument("--strict-equality", dest="strict_equality", action="store_true", help="= ํ๋ณด์ ๋ถ๋ถ์ parser ๊ณ์ฝ ์ ์ฉ")
|
| 104 |
+
equality_mode.add_argument("--relaxed-equality", dest="strict_equality", action="store_false", help="=๋ฅผ ์์ ์๊ฒฐ์ฑ๊ณผ ๋
๋ฆฝ๋ shape๋ก ์ธ์")
|
| 105 |
+
parser.set_defaults(strict_equality=False)
|
| 106 |
+
args = parser.parse_args()
|
| 107 |
+
equality_model = EqualityVisualModel.load(args.equality_model) if args.equality_model else None
|
| 108 |
+
cross_model = CrossVisualModel.load(args.cross_model) if args.cross_model else None
|
| 109 |
+
fit, validation = writer_fit_validation(args.train_root, args.profile)
|
| 110 |
+
model = _fit_geometry(fit)
|
| 111 |
+
validation_cache = load_cache_for_samples(
|
| 112 |
+
validation, args.cache_dir, split="validation", profile=args.profile, bundle=args.bundle, version=2,
|
| 113 |
+
)
|
| 114 |
+
validation_rows = _prepared_signals(
|
| 115 |
+
validation, validation_cache, model, strict_equality=args.strict_equality,
|
| 116 |
+
equality_model=equality_model,
|
| 117 |
+
cross_model=cross_model, cross_gap_ratio=args.cross_gap_ratio,
|
| 118 |
+
multistroke_family_boost=args.multistroke_family_boost,
|
| 119 |
+
)
|
| 120 |
+
if args.sweep_infix:
|
| 121 |
+
fixed = TrayJointWeights()
|
| 122 |
+
trials = []
|
| 123 |
+
for infix_weight in (0.0, 0.5, 1.0, 2.0, 4.0, 6.0, 8.0, 12.0, 16.0):
|
| 124 |
+
trials.append({
|
| 125 |
+
"tray_weight": fixed.tray, "symbol_weight": fixed.symbol,
|
| 126 |
+
"fraction_weight": fixed.fraction, "infix_weight": infix_weight,
|
| 127 |
+
"group_bias": fixed.group_bias,
|
| 128 |
+
"metrics": _metrics(_weighted(
|
| 129 |
+
validation_rows, tray_weight=fixed.tray, symbol_weight=fixed.symbol,
|
| 130 |
+
fraction_weight=fixed.fraction, infix_weight=infix_weight,
|
| 131 |
+
), fixed.group_bias),
|
| 132 |
+
})
|
| 133 |
+
winner = max(trials, key=lambda row: (row["metrics"]["exact_partition"], row["metrics"]["pair_f1"]))
|
| 134 |
+
elif args.fixed_selected:
|
| 135 |
+
fixed = TrayJointWeights()
|
| 136 |
+
winner = {
|
| 137 |
+
"tray_weight": fixed.tray, "symbol_weight": fixed.symbol,
|
| 138 |
+
"fraction_weight": fixed.fraction, "infix_weight": fixed.infix, "group_bias": fixed.group_bias,
|
| 139 |
+
"metrics": _metrics(_weighted(
|
| 140 |
+
validation_rows, tray_weight=fixed.tray, symbol_weight=fixed.symbol,
|
| 141 |
+
fraction_weight=fixed.fraction, infix_weight=fixed.infix,
|
| 142 |
+
), fixed.group_bias),
|
| 143 |
+
}
|
| 144 |
+
else:
|
| 145 |
+
trials = []
|
| 146 |
+
for tray_weight in (0.0, 0.5, 1.0, 2.0, 4.0, 8.0):
|
| 147 |
+
for symbol_weight in (0.0, 0.5, 1.0, 2.0, 4.0):
|
| 148 |
+
for fraction_weight in (0.0, 8.0):
|
| 149 |
+
weighted = _weighted(
|
| 150 |
+
validation_rows, tray_weight=tray_weight,
|
| 151 |
+
symbol_weight=symbol_weight, fraction_weight=fraction_weight,
|
| 152 |
+
)
|
| 153 |
+
for bias in (-2.5, -2.0, -1.5, -1.0):
|
| 154 |
+
trials.append({
|
| 155 |
+
"tray_weight": tray_weight, "symbol_weight": symbol_weight,
|
| 156 |
+
"fraction_weight": fraction_weight, "group_bias": bias,
|
| 157 |
+
"infix_weight": 0.0,
|
| 158 |
+
"metrics": _metrics(weighted, bias),
|
| 159 |
+
})
|
| 160 |
+
winner = max(trials, key=lambda row: (row["metrics"]["exact_partition"], row["metrics"]["pair_f1"]))
|
| 161 |
+
test, test_cache = load_cached_split(
|
| 162 |
+
args.test_root, args.cache_dir, split="official_test", profile=args.profile,
|
| 163 |
+
bundle=args.bundle, version=2,
|
| 164 |
+
)
|
| 165 |
+
test_rows = _prepared_signals(
|
| 166 |
+
test, test_cache, model, strict_equality=args.strict_equality,
|
| 167 |
+
equality_model=equality_model,
|
| 168 |
+
cross_model=cross_model, cross_gap_ratio=args.cross_gap_ratio,
|
| 169 |
+
multistroke_family_boost=args.multistroke_family_boost,
|
| 170 |
+
)
|
| 171 |
+
weighted_test = _weighted(
|
| 172 |
+
test_rows, tray_weight=float(winner["tray_weight"]), symbol_weight=float(winner["symbol_weight"]),
|
| 173 |
+
fraction_weight=float(winner["fraction_weight"]),
|
| 174 |
+
infix_weight=float(winner.get("infix_weight", 0.0)),
|
| 175 |
+
)
|
| 176 |
+
report = {
|
| 177 |
+
"experiment": "R-CROHME-TRAY-JOINT-SELECTOR-001",
|
| 178 |
+
"generated_at": datetime.now(timezone.utc).isoformat(), "track": "R_noncommercial_only",
|
| 179 |
+
"selection_mode": "infix_validation_sweep" if args.sweep_infix else ("fixed_refactor_verification" if args.fixed_selected else "writer_validation_sweep"),
|
| 180 |
+
"strict_equality": args.strict_equality,
|
| 181 |
+
"equality_model": str(args.equality_model) if args.equality_model else None,
|
| 182 |
+
"cross_model": str(args.cross_model) if args.cross_model else None,
|
| 183 |
+
"selected": winner, "official_test": _metrics(weighted_test, float(winner["group_bias"])),
|
| 184 |
+
"validation_trials": trials if args.sweep_infix else None,
|
| 185 |
+
"signal_coverage": {
|
| 186 |
+
"validation_tray_candidates": int(sum(np.count_nonzero(row["tray_signal"]) for row in validation_rows)),
|
| 187 |
+
"validation_symbol_candidates": int(sum(np.count_nonzero(row["symbol_signal"]) for row in validation_rows)),
|
| 188 |
+
"test_tray_candidates": int(sum(np.count_nonzero(row["tray_signal"]) for row in test_rows)),
|
| 189 |
+
"test_symbol_candidates": int(sum(np.count_nonzero(row["symbol_signal"]) for row in test_rows)),
|
| 190 |
+
"validation_infix_candidates": int(sum(np.count_nonzero(row["infix_signal"]) for row in validation_rows)),
|
| 191 |
+
"test_infix_candidates": int(sum(np.count_nonzero(row["infix_signal"]) for row in test_rows)),
|
| 192 |
+
},
|
| 193 |
+
"reference_test": {"exact_partition": 0.4836065574, "pair_f1": 0.86628},
|
| 194 |
+
"product_validation": False,
|
| 195 |
+
"interpretation_limit": "HWRT expanded top-label + CROHME R-track Tray joint selector",
|
| 196 |
+
}
|
| 197 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 198 |
+
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
| 199 |
+
print(json.dumps(report, ensure_ascii=False, indent=2))
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
if __name__ == "__main__":
|
| 203 |
+
main()
|
scripts/sweep_math_ink_06_component_competition_guard.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""์ต์ข
selector์์ ๊ตฌ์ฑํ๋ณด๋ค ์ฝํ merge ํ๋ณด์ must-not-link weight๋ฅผ validation ์ ํํ๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from datetime import datetime, timezone
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import sys
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 13 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 14 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 15 |
+
if str(path) not in sys.path:
|
| 16 |
+
sys.path.insert(0, str(path))
|
| 17 |
+
|
| 18 |
+
from math_grid_drawer.research.cross_visual import CrossVisualModel
|
| 19 |
+
from math_grid_drawer.research.equality_visual import EqualityVisualModel
|
| 20 |
+
from scripts.crohme_lattice_common import load_cache_for_samples, load_cached_split, writer_fit_validation
|
| 21 |
+
from scripts.evaluate_crohme_lattice_ocr_fusion import _fit_geometry
|
| 22 |
+
from scripts.evaluate_crohme_tray_joint_selector import _prepared_signals, _weighted
|
| 23 |
+
from scripts.sweep_math_ink_06_multistroke_family_guard import _family_metrics06
|
| 24 |
+
from scripts.sweep_math_ink_06_x_grouping_guard import _target_metrics06
|
| 25 |
+
from scripts.train_crohme_segmentation_lattice_joint_selector import _metrics
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _parse_args() -> argparse.Namespace:
|
| 29 |
+
"""ํ์ ๋ณ์: CROHME splitยทcacheยทfull selector head. ์๋ ์๋ฆฌ: test ๋น๊ฐ์
competition sweep CLI๋ฅผ ๋ง๋ ๋ค."""
|
| 30 |
+
|
| 31 |
+
parser = argparse.ArgumentParser(description="Sweep Math Ink 0.6 component competition guard")
|
| 32 |
+
parser.add_argument(
|
| 33 |
+
"--train-root", type=Path,
|
| 34 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/trainData",
|
| 35 |
+
)
|
| 36 |
+
parser.add_argument(
|
| 37 |
+
"--test-root", type=Path,
|
| 38 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/testDataGT",
|
| 39 |
+
)
|
| 40 |
+
parser.add_argument(
|
| 41 |
+
"--cache-dir", type=Path,
|
| 42 |
+
default=PROJECT_ROOT / "research/runs/crohme_lattice_ocr_cache_v2_20260722",
|
| 43 |
+
)
|
| 44 |
+
parser.add_argument(
|
| 45 |
+
"--bundle", type=Path,
|
| 46 |
+
default=Path(r"research\runs\aiflow_ocr_05_dual_trajectory_3seed_20260720\bundle.manifest.json"),
|
| 47 |
+
)
|
| 48 |
+
parser.add_argument(
|
| 49 |
+
"--cross-model", type=Path,
|
| 50 |
+
default=PROJECT_ROOT / "research/runs/crohme_cross_visual_loop3_polyline_20260722/cross_visual.json",
|
| 51 |
+
)
|
| 52 |
+
parser.add_argument(
|
| 53 |
+
"--equality-model", type=Path,
|
| 54 |
+
default=PROJECT_ROOT / "research/runs/crohme_equality_visual_loop1_20260722/equality_visual.json",
|
| 55 |
+
)
|
| 56 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 57 |
+
parser.add_argument("--maximum-x-regression-pp", type=float, default=1.0)
|
| 58 |
+
parser.add_argument("--maximum-family-regression-pp", type=float, default=2.0)
|
| 59 |
+
parser.add_argument("--maximum-pair-f1-regression-pp", type=float, default=0.25)
|
| 60 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 61 |
+
return parser.parse_args()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _evaluate_weight06(
|
| 65 |
+
samples: list[dict[str, Any]],
|
| 66 |
+
prepared: list[dict[str, Any]],
|
| 67 |
+
weight: float,
|
| 68 |
+
) -> dict[str, Any]:
|
| 69 |
+
"""ํ์ ๋ณ์: ํ split์ cached signalยทcompetition weight. ์๋ ์๋ฆฌ: ์ ์ญ/x/family ์งํ๋ฅผ ๊ฐ์ partition์์ ๊ณ์ฐํ๋ค."""
|
| 70 |
+
|
| 71 |
+
weighted = _weighted(
|
| 72 |
+
prepared,
|
| 73 |
+
tray_weight=4.0,
|
| 74 |
+
symbol_weight=4.0,
|
| 75 |
+
fraction_weight=8.0,
|
| 76 |
+
infix_weight=8.0,
|
| 77 |
+
competition_weight=weight,
|
| 78 |
+
)
|
| 79 |
+
return {
|
| 80 |
+
"component_competition_weight": weight,
|
| 81 |
+
"global": _metrics(weighted, -2.0),
|
| 82 |
+
"behavior_targets": _target_metrics06(
|
| 83 |
+
samples, weighted, group_bias=-2.0,
|
| 84 |
+
),
|
| 85 |
+
"families": _family_metrics06(samples, weighted),
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _delta_pp06(candidate: float, reference: float) -> float:
|
| 90 |
+
"""ํ์ ๋ณ์: ํ๋ณดยท๊ธฐ์ค ๋น์จ. ์๋ ์๋ฆฌ: ์ฑํ ํ๋จ์ฉ percentage-point ์ฐจ์ด๋ฅผ ๋ฐํํ๋ค."""
|
| 91 |
+
|
| 92 |
+
return (candidate - reference) * 100.0
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def main() -> None:
|
| 96 |
+
"""ํ์ ๋ณ์: writer-validationยทofficial test. ์๋ ์๋ฆฌ: ๋ณดํธ gate ์ exact winner์ ๊ธฐ์ค๋ง test์์ ๋น๊ตํ๋ค."""
|
| 97 |
+
|
| 98 |
+
args = _parse_args()
|
| 99 |
+
fit, validation = writer_fit_validation(args.train_root, args.profile)
|
| 100 |
+
geometry_model = _fit_geometry(fit)
|
| 101 |
+
equality_model = EqualityVisualModel.load(args.equality_model)
|
| 102 |
+
cross_model = CrossVisualModel.load(args.cross_model)
|
| 103 |
+
validation_cache = load_cache_for_samples(
|
| 104 |
+
validation,
|
| 105 |
+
args.cache_dir,
|
| 106 |
+
split="validation",
|
| 107 |
+
profile=args.profile,
|
| 108 |
+
bundle=args.bundle,
|
| 109 |
+
version=2,
|
| 110 |
+
)
|
| 111 |
+
validation_prepared = _prepared_signals(
|
| 112 |
+
validation,
|
| 113 |
+
validation_cache,
|
| 114 |
+
geometry_model,
|
| 115 |
+
equality_model=equality_model,
|
| 116 |
+
cross_model=cross_model,
|
| 117 |
+
cross_gap_ratio=0.40,
|
| 118 |
+
multistroke_family_boost=6.0,
|
| 119 |
+
)
|
| 120 |
+
trials = [
|
| 121 |
+
_evaluate_weight06(validation, validation_prepared, weight)
|
| 122 |
+
for weight in (0.0, 1.0, 2.0, 3.0, 4.0, 6.0, 8.0, 12.0)
|
| 123 |
+
]
|
| 124 |
+
reference = trials[0]
|
| 125 |
+
minimum_x = (
|
| 126 |
+
reference["behavior_targets"]["x"]["grouping_recall"]
|
| 127 |
+
- args.maximum_x_regression_pp / 100.0
|
| 128 |
+
)
|
| 129 |
+
minimum_family = (
|
| 130 |
+
reference["families"]["grouping_recall"]
|
| 131 |
+
- args.maximum_family_regression_pp / 100.0
|
| 132 |
+
)
|
| 133 |
+
minimum_pair_f1 = (
|
| 134 |
+
reference["global"]["pair_f1"]
|
| 135 |
+
- args.maximum_pair_f1_regression_pp / 100.0
|
| 136 |
+
)
|
| 137 |
+
eligible = [
|
| 138 |
+
row for row in trials
|
| 139 |
+
if (
|
| 140 |
+
row["behavior_targets"]["x"]["grouping_recall"] >= minimum_x
|
| 141 |
+
and row["families"]["grouping_recall"] >= minimum_family
|
| 142 |
+
and row["global"]["pair_f1"] >= minimum_pair_f1
|
| 143 |
+
)
|
| 144 |
+
]
|
| 145 |
+
winner = max(eligible, key=lambda row: (
|
| 146 |
+
row["global"]["exact_partition"],
|
| 147 |
+
row["global"]["pair_f1"],
|
| 148 |
+
row["global"]["exact_group_recall"],
|
| 149 |
+
-row["component_competition_weight"],
|
| 150 |
+
))
|
| 151 |
+
test, test_cache = load_cached_split(
|
| 152 |
+
args.test_root,
|
| 153 |
+
args.cache_dir,
|
| 154 |
+
split="official_test",
|
| 155 |
+
profile=args.profile,
|
| 156 |
+
bundle=args.bundle,
|
| 157 |
+
version=2,
|
| 158 |
+
)
|
| 159 |
+
test_prepared = _prepared_signals(
|
| 160 |
+
test,
|
| 161 |
+
test_cache,
|
| 162 |
+
geometry_model,
|
| 163 |
+
equality_model=equality_model,
|
| 164 |
+
cross_model=cross_model,
|
| 165 |
+
cross_gap_ratio=0.40,
|
| 166 |
+
multistroke_family_boost=6.0,
|
| 167 |
+
)
|
| 168 |
+
official_reference = _evaluate_weight06(test, test_prepared, 0.0)
|
| 169 |
+
official_winner = _evaluate_weight06(
|
| 170 |
+
test, test_prepared, float(winner["component_competition_weight"]),
|
| 171 |
+
)
|
| 172 |
+
deltas = {
|
| 173 |
+
"exact_partition_pp": _delta_pp06(
|
| 174 |
+
official_winner["global"]["exact_partition"],
|
| 175 |
+
official_reference["global"]["exact_partition"],
|
| 176 |
+
),
|
| 177 |
+
"pair_f1_pp": _delta_pp06(
|
| 178 |
+
official_winner["global"]["pair_f1"],
|
| 179 |
+
official_reference["global"]["pair_f1"],
|
| 180 |
+
),
|
| 181 |
+
"x_grouping_pp": _delta_pp06(
|
| 182 |
+
official_winner["behavior_targets"]["x"]["grouping_recall"],
|
| 183 |
+
official_reference["behavior_targets"]["x"]["grouping_recall"],
|
| 184 |
+
),
|
| 185 |
+
"family_grouping_pp": _delta_pp06(
|
| 186 |
+
official_winner["families"]["grouping_recall"],
|
| 187 |
+
official_reference["families"]["grouping_recall"],
|
| 188 |
+
),
|
| 189 |
+
}
|
| 190 |
+
adopted = bool(
|
| 191 |
+
float(winner["component_competition_weight"]) > 0.0
|
| 192 |
+
and deltas["exact_partition_pp"] > 0.0
|
| 193 |
+
and deltas["pair_f1_pp"] >= -args.maximum_pair_f1_regression_pp
|
| 194 |
+
and deltas["x_grouping_pp"] >= -args.maximum_x_regression_pp
|
| 195 |
+
and deltas["family_grouping_pp"] >= -args.maximum_family_regression_pp
|
| 196 |
+
)
|
| 197 |
+
report = {
|
| 198 |
+
"experiment": "R-MATH-INK-06-COMPONENT-COMPETITION-GUARD-001",
|
| 199 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 200 |
+
"selection_contract": {
|
| 201 |
+
"split": "CROHME trainData writer-validation only",
|
| 202 |
+
"cross_gap_ratio": 0.40,
|
| 203 |
+
"multistroke_family_boost": 6.0,
|
| 204 |
+
"maximum_x_regression_pp": args.maximum_x_regression_pp,
|
| 205 |
+
"maximum_family_regression_pp": args.maximum_family_regression_pp,
|
| 206 |
+
"maximum_pair_f1_regression_pp": args.maximum_pair_f1_regression_pp,
|
| 207 |
+
},
|
| 208 |
+
"reference_validation": reference,
|
| 209 |
+
"winner_validation": winner,
|
| 210 |
+
"trials": trials,
|
| 211 |
+
"official_test_reference": official_reference,
|
| 212 |
+
"official_test_winner": official_winner,
|
| 213 |
+
"official_test_deltas": deltas,
|
| 214 |
+
"decision": {
|
| 215 |
+
"adopted": adopted,
|
| 216 |
+
"selected_component_competition_weight": (
|
| 217 |
+
float(winner["component_competition_weight"]) if adopted else 0.0
|
| 218 |
+
),
|
| 219 |
+
"reason": (
|
| 220 |
+
"exact partition improves within x/family/pair-F1 guards"
|
| 221 |
+
if adopted else "official adoption gate failed"
|
| 222 |
+
),
|
| 223 |
+
},
|
| 224 |
+
"track": "R_noncommercial_only",
|
| 225 |
+
"product_validation": False,
|
| 226 |
+
}
|
| 227 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 228 |
+
args.output.write_text(
|
| 229 |
+
json.dumps(report, ensure_ascii=False, indent=2) + "\n",
|
| 230 |
+
encoding="utf-8",
|
| 231 |
+
)
|
| 232 |
+
print(json.dumps({key: value for key, value in report.items() if key != "trials"}, ensure_ascii=False, indent=2))
|
| 233 |
+
|
| 234 |
+
|
| 235 |
+
if __name__ == "__main__":
|
| 236 |
+
main()
|
scripts/sweep_math_ink_06_local_baseline_guard.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""์ต์ข
selector์์ ์์ ๊ณต๊ฐ ๊ด๊ณ ๊ฒฝ๊ณ๋ฅผ ์นจ๋ฒํ๋ merge์ ๊ฐ์ weight๋ฅผ validation ์ ํํ๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from datetime import datetime, timezone
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import sys
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 13 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 14 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 15 |
+
if str(path) not in sys.path:
|
| 16 |
+
sys.path.insert(0, str(path))
|
| 17 |
+
|
| 18 |
+
from math_grid_drawer.research.cross_visual import CrossVisualModel
|
| 19 |
+
from math_grid_drawer.research.equality_visual import EqualityVisualModel
|
| 20 |
+
from scripts.crohme_lattice_common import load_cache_for_samples, load_cached_split, writer_fit_validation
|
| 21 |
+
from scripts.evaluate_crohme_lattice_ocr_fusion import _fit_geometry
|
| 22 |
+
from scripts.evaluate_crohme_tray_joint_selector import _prepared_signals, _weighted
|
| 23 |
+
from scripts.sweep_math_ink_06_multistroke_family_guard import _family_metrics06
|
| 24 |
+
from scripts.sweep_math_ink_06_x_grouping_guard import _target_metrics06
|
| 25 |
+
from scripts.train_crohme_segmentation_lattice_joint_selector import _metrics
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _parse_args() -> argparse.Namespace:
|
| 29 |
+
"""ํ์ ๋ณ์: CROHME splitยทcacheยทfull selector head. ์๋ ์๋ฆฌ: test ๋น๊ฐ์
local-baseline sweep CLI๋ฅผ ๋ง๋ ๋ค."""
|
| 30 |
+
|
| 31 |
+
parser = argparse.ArgumentParser(description="Sweep Math Ink 0.6 local baseline guard")
|
| 32 |
+
parser.add_argument(
|
| 33 |
+
"--train-root", type=Path,
|
| 34 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/trainData",
|
| 35 |
+
)
|
| 36 |
+
parser.add_argument(
|
| 37 |
+
"--test-root", type=Path,
|
| 38 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/testDataGT",
|
| 39 |
+
)
|
| 40 |
+
parser.add_argument(
|
| 41 |
+
"--cache-dir", type=Path,
|
| 42 |
+
default=PROJECT_ROOT / "research/runs/crohme_lattice_ocr_cache_v2_20260722",
|
| 43 |
+
)
|
| 44 |
+
parser.add_argument(
|
| 45 |
+
"--bundle", type=Path,
|
| 46 |
+
default=Path(r"research\runs\aiflow_ocr_05_dual_trajectory_3seed_20260720\bundle.manifest.json"),
|
| 47 |
+
)
|
| 48 |
+
parser.add_argument(
|
| 49 |
+
"--cross-model", type=Path,
|
| 50 |
+
default=PROJECT_ROOT / "research/runs/crohme_cross_visual_loop3_polyline_20260722/cross_visual.json",
|
| 51 |
+
)
|
| 52 |
+
parser.add_argument(
|
| 53 |
+
"--equality-model", type=Path,
|
| 54 |
+
default=PROJECT_ROOT / "research/runs/crohme_equality_visual_loop1_20260722/equality_visual.json",
|
| 55 |
+
)
|
| 56 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 57 |
+
parser.add_argument("--maximum-x-regression-pp", type=float, default=1.0)
|
| 58 |
+
parser.add_argument("--maximum-family-regression-pp", type=float, default=2.0)
|
| 59 |
+
parser.add_argument("--maximum-pair-f1-regression-pp", type=float, default=0.25)
|
| 60 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 61 |
+
return parser.parse_args()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _evaluate_weight06(
|
| 65 |
+
samples: list[dict[str, Any]],
|
| 66 |
+
prepared: list[dict[str, Any]],
|
| 67 |
+
weight: float,
|
| 68 |
+
) -> dict[str, Any]:
|
| 69 |
+
"""ํ์ ๋ณ์: ํ split์ cached signalยทlocal-baseline weight. ์๋ ์๋ฆฌ: ์ ์ญ/x/family ์งํ๋ฅผ ๊ฐ์ partition์์ ๊ณ์ฐํ๋ค."""
|
| 70 |
+
|
| 71 |
+
weighted = _weighted(
|
| 72 |
+
prepared,
|
| 73 |
+
tray_weight=4.0,
|
| 74 |
+
symbol_weight=4.0,
|
| 75 |
+
fraction_weight=8.0,
|
| 76 |
+
infix_weight=8.0,
|
| 77 |
+
local_baseline_weight=weight,
|
| 78 |
+
)
|
| 79 |
+
return {
|
| 80 |
+
"local_baseline_weight": weight,
|
| 81 |
+
"global": _metrics(weighted, -2.0),
|
| 82 |
+
"behavior_targets": _target_metrics06(samples, weighted, group_bias=-2.0),
|
| 83 |
+
"families": _family_metrics06(samples, weighted),
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _delta_pp06(candidate: float, reference: float) -> float:
|
| 88 |
+
"""ํ์ ๋ณ์: ํ๋ณดยท๊ธฐ์ค ๋น์จ. ์๋ ์๋ฆฌ: ์ฑํ ํ๋จ์ฉ percentage-point ์ฐจ์ด๋ฅผ ๋ฐํํ๋ค."""
|
| 89 |
+
|
| 90 |
+
return (candidate - reference) * 100.0
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def main() -> None:
|
| 94 |
+
"""ํ์ ๋ณ์: writer-validationยทofficial test. ์๋ ์๋ฆฌ: ๋ณดํธ gate ์ exact winner์ ๊ธฐ์ค๋ง test์์ ๋น๊ตํ๋ค."""
|
| 95 |
+
|
| 96 |
+
args = _parse_args()
|
| 97 |
+
fit, validation = writer_fit_validation(args.train_root, args.profile)
|
| 98 |
+
geometry_model = _fit_geometry(fit)
|
| 99 |
+
equality_model = EqualityVisualModel.load(args.equality_model)
|
| 100 |
+
cross_model = CrossVisualModel.load(args.cross_model)
|
| 101 |
+
validation_cache = load_cache_for_samples(
|
| 102 |
+
validation, args.cache_dir, split="validation", profile=args.profile,
|
| 103 |
+
bundle=args.bundle, version=2,
|
| 104 |
+
)
|
| 105 |
+
validation_prepared = _prepared_signals(
|
| 106 |
+
validation, validation_cache, geometry_model,
|
| 107 |
+
equality_model=equality_model, cross_model=cross_model,
|
| 108 |
+
cross_gap_ratio=0.40, multistroke_family_boost=6.0,
|
| 109 |
+
)
|
| 110 |
+
trials = [
|
| 111 |
+
_evaluate_weight06(validation, validation_prepared, weight)
|
| 112 |
+
for weight in (0.0, 0.5, 1.0, 2.0, 3.0, 4.0, 6.0, 8.0, 12.0)
|
| 113 |
+
]
|
| 114 |
+
reference = trials[0]
|
| 115 |
+
minimum_x = reference["behavior_targets"]["x"]["grouping_recall"] - args.maximum_x_regression_pp / 100.0
|
| 116 |
+
minimum_family = reference["families"]["grouping_recall"] - args.maximum_family_regression_pp / 100.0
|
| 117 |
+
minimum_pair_f1 = reference["global"]["pair_f1"] - args.maximum_pair_f1_regression_pp / 100.0
|
| 118 |
+
eligible = [
|
| 119 |
+
row for row in trials
|
| 120 |
+
if (
|
| 121 |
+
row["behavior_targets"]["x"]["grouping_recall"] >= minimum_x
|
| 122 |
+
and row["families"]["grouping_recall"] >= minimum_family
|
| 123 |
+
and row["global"]["pair_f1"] >= minimum_pair_f1
|
| 124 |
+
)
|
| 125 |
+
]
|
| 126 |
+
winner = max(eligible, key=lambda row: (
|
| 127 |
+
row["global"]["exact_partition"],
|
| 128 |
+
row["global"]["pair_f1"],
|
| 129 |
+
row["global"]["exact_group_recall"],
|
| 130 |
+
-row["local_baseline_weight"],
|
| 131 |
+
))
|
| 132 |
+
test, test_cache = load_cached_split(
|
| 133 |
+
args.test_root, args.cache_dir, split="official_test", profile=args.profile,
|
| 134 |
+
bundle=args.bundle, version=2,
|
| 135 |
+
)
|
| 136 |
+
test_prepared = _prepared_signals(
|
| 137 |
+
test, test_cache, geometry_model,
|
| 138 |
+
equality_model=equality_model, cross_model=cross_model,
|
| 139 |
+
cross_gap_ratio=0.40, multistroke_family_boost=6.0,
|
| 140 |
+
)
|
| 141 |
+
official_reference = _evaluate_weight06(test, test_prepared, 0.0)
|
| 142 |
+
official_winner = _evaluate_weight06(test, test_prepared, float(winner["local_baseline_weight"]))
|
| 143 |
+
deltas = {
|
| 144 |
+
"exact_partition_pp": _delta_pp06(
|
| 145 |
+
official_winner["global"]["exact_partition"], official_reference["global"]["exact_partition"],
|
| 146 |
+
),
|
| 147 |
+
"pair_f1_pp": _delta_pp06(
|
| 148 |
+
official_winner["global"]["pair_f1"], official_reference["global"]["pair_f1"],
|
| 149 |
+
),
|
| 150 |
+
"x_grouping_pp": _delta_pp06(
|
| 151 |
+
official_winner["behavior_targets"]["x"]["grouping_recall"],
|
| 152 |
+
official_reference["behavior_targets"]["x"]["grouping_recall"],
|
| 153 |
+
),
|
| 154 |
+
"family_grouping_pp": _delta_pp06(
|
| 155 |
+
official_winner["families"]["grouping_recall"], official_reference["families"]["grouping_recall"],
|
| 156 |
+
),
|
| 157 |
+
}
|
| 158 |
+
adopted = bool(
|
| 159 |
+
float(winner["local_baseline_weight"]) > 0.0
|
| 160 |
+
and deltas["exact_partition_pp"] > 0.0
|
| 161 |
+
and deltas["pair_f1_pp"] >= -args.maximum_pair_f1_regression_pp
|
| 162 |
+
and deltas["x_grouping_pp"] >= -args.maximum_x_regression_pp
|
| 163 |
+
and deltas["family_grouping_pp"] >= -args.maximum_family_regression_pp
|
| 164 |
+
)
|
| 165 |
+
report = {
|
| 166 |
+
"experiment": "R-MATH-INK-06-LOCAL-BASELINE-GUARD-001",
|
| 167 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 168 |
+
"selection_contract": {
|
| 169 |
+
"split": "CROHME trainData writer-validation only",
|
| 170 |
+
"cross_gap_ratio": 0.40,
|
| 171 |
+
"multistroke_family_boost": 6.0,
|
| 172 |
+
"maximum_x_regression_pp": args.maximum_x_regression_pp,
|
| 173 |
+
"maximum_family_regression_pp": args.maximum_family_regression_pp,
|
| 174 |
+
"maximum_pair_f1_regression_pp": args.maximum_pair_f1_regression_pp,
|
| 175 |
+
},
|
| 176 |
+
"signal_coverage": {
|
| 177 |
+
"validation_candidates": sum(
|
| 178 |
+
int((row["local_baseline_penalty"] > 0.0).sum()) for row in validation_prepared
|
| 179 |
+
),
|
| 180 |
+
"official_test_candidates": sum(
|
| 181 |
+
int((row["local_baseline_penalty"] > 0.0).sum()) for row in test_prepared
|
| 182 |
+
),
|
| 183 |
+
},
|
| 184 |
+
"reference_validation": reference,
|
| 185 |
+
"winner_validation": winner,
|
| 186 |
+
"trials": trials,
|
| 187 |
+
"official_test_reference": official_reference,
|
| 188 |
+
"official_test_winner": official_winner,
|
| 189 |
+
"official_test_deltas": deltas,
|
| 190 |
+
"decision": {
|
| 191 |
+
"adopted": adopted,
|
| 192 |
+
"selected_local_baseline_weight": float(winner["local_baseline_weight"]) if adopted else 0.0,
|
| 193 |
+
"reason": (
|
| 194 |
+
"exact partition improves within x/family/pair-F1 guards"
|
| 195 |
+
if adopted else "official adoption gate failed"
|
| 196 |
+
),
|
| 197 |
+
},
|
| 198 |
+
"track": "R_noncommercial_only",
|
| 199 |
+
"product_validation": False,
|
| 200 |
+
}
|
| 201 |
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
| 202 |
+
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
| 203 |
+
print(json.dumps({key: value for key, value in report.items() if key != "trials"}, ensure_ascii=False, indent=2))
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
if __name__ == "__main__":
|
| 207 |
+
main()
|
scripts/train_math_ink_06_boundary_behavior_guard.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Grouping ์ด์ geometry๋ก ๊ธฐํธ ๊ฒฝ๊ณ ์นจ๋ฒ ํ๋ฅ ์ ํ์ตํ๊ณ ์ต์ข
selector์์ ๊ฒ์ฆํ๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
from datetime import datetime, timezone
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import sys
|
| 10 |
+
from typing import Any
|
| 11 |
+
|
| 12 |
+
import joblib
|
| 13 |
+
import numpy as np
|
| 14 |
+
from sklearn.ensemble import HistGradientBoostingClassifier
|
| 15 |
+
|
| 16 |
+
PROJECT_ROOT = Path(__file__).parents[1]
|
| 17 |
+
SOURCE_ROOT = PROJECT_ROOT / "src"
|
| 18 |
+
for path in (PROJECT_ROOT, SOURCE_ROOT):
|
| 19 |
+
if str(path) not in sys.path:
|
| 20 |
+
sys.path.insert(0, str(path))
|
| 21 |
+
|
| 22 |
+
from math_grid_drawer.aiflow_ocr05 import AIFlowOCR05
|
| 23 |
+
from math_grid_drawer.research.cross_visual import CrossVisualModel
|
| 24 |
+
from math_grid_drawer.research.equality_visual import EqualityVisualModel
|
| 25 |
+
from math_grid_drawer.research.segmentation_lattice import (
|
| 26 |
+
LATTICE_FEATURE_NAMES,
|
| 27 |
+
lattice_candidate_features,
|
| 28 |
+
)
|
| 29 |
+
from scripts.crohme_lattice_common import load_cache_for_samples, load_cached_split, writer_fit_validation
|
| 30 |
+
from scripts.evaluate_crohme_gt_free_grouping import _truth_partition
|
| 31 |
+
from scripts.evaluate_crohme_lattice_ocr_fusion import _fit_geometry
|
| 32 |
+
from scripts.evaluate_crohme_tray_joint_selector import _prepared_signals, _weighted
|
| 33 |
+
from scripts.sweep_math_ink_06_multistroke_family_guard import _family_metrics06
|
| 34 |
+
from scripts.sweep_math_ink_06_x_grouping_guard import _target_metrics06
|
| 35 |
+
from scripts.train_crohme_segmentation_lattice_joint_selector import _metrics
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _parse_args() -> argparse.Namespace:
|
| 39 |
+
"""ํ์ ๋ณ์: CROHME splitยทcacheยทhead ๊ฒฝ๋ก. ์๋ ์๋ฆฌ: writer-disjoint boundary ํ์ต/๊ฒ์ฆ CLI๋ฅผ ๋ง๋ ๋ค."""
|
| 40 |
+
|
| 41 |
+
parser = argparse.ArgumentParser(description="Train Math Ink 0.6 boundary behavior guard")
|
| 42 |
+
parser.add_argument(
|
| 43 |
+
"--train-root", type=Path,
|
| 44 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/trainData",
|
| 45 |
+
)
|
| 46 |
+
parser.add_argument(
|
| 47 |
+
"--test-root", type=Path,
|
| 48 |
+
default=PROJECT_ROOT / "research/data/R_noncommercial/ICFHR_package/CROHME2012_data/testDataGT",
|
| 49 |
+
)
|
| 50 |
+
parser.add_argument(
|
| 51 |
+
"--cache-dir", type=Path,
|
| 52 |
+
default=PROJECT_ROOT / "research/runs/crohme_lattice_ocr_cache_v2_20260722",
|
| 53 |
+
)
|
| 54 |
+
parser.add_argument(
|
| 55 |
+
"--bundle", type=Path,
|
| 56 |
+
default=Path(r"research\runs\aiflow_ocr_05_dual_trajectory_3seed_20260720\bundle.manifest.json"),
|
| 57 |
+
)
|
| 58 |
+
parser.add_argument(
|
| 59 |
+
"--cross-model", type=Path,
|
| 60 |
+
default=PROJECT_ROOT / "research/runs/crohme_cross_visual_loop3_polyline_20260722/cross_visual.json",
|
| 61 |
+
)
|
| 62 |
+
parser.add_argument(
|
| 63 |
+
"--equality-model", type=Path,
|
| 64 |
+
default=PROJECT_ROOT / "research/runs/crohme_equality_visual_loop1_20260722/equality_visual.json",
|
| 65 |
+
)
|
| 66 |
+
parser.add_argument("--profile", default="median_height_32")
|
| 67 |
+
parser.add_argument("--maximum-x-regression-pp", type=float, default=1.0)
|
| 68 |
+
parser.add_argument("--maximum-family-regression-pp", type=float, default=2.0)
|
| 69 |
+
parser.add_argument("--maximum-pair-f1-regression-pp", type=float, default=0.25)
|
| 70 |
+
parser.add_argument("--output", type=Path, required=True)
|
| 71 |
+
return parser.parse_args()
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _boundary_training_rows(samples: list[dict[str, Any]]) -> tuple[np.ndarray, np.ndarray]:
|
| 75 |
+
"""ํ์ ๋ณ์: fit writer ์์ยทtruth partition. ์๋ ์๋ฆฌ: ๋ ์ด์์ ์ ๋ต ๊ธฐํธ๋ฅผ ์นจ๋ฒํ ๋คํ ํ๋ณด๋ฅผ boundary=1๋ก ํ์ตํ๋ค."""
|
| 76 |
+
|
| 77 |
+
feature_rows: list[np.ndarray] = []
|
| 78 |
+
targets: list[bool] = []
|
| 79 |
+
for sample in samples:
|
| 80 |
+
strokes = sample["profiled_strokes"]
|
| 81 |
+
candidates = AIFlowOCR05.build_segmentation_lattice(strokes)
|
| 82 |
+
features = lattice_candidate_features(candidates, strokes)
|
| 83 |
+
truth_groups, _labels = _truth_partition(sample, "aiflow_geometry")
|
| 84 |
+
for candidate, feature in zip(candidates, features, strict=True):
|
| 85 |
+
group = frozenset(int(value) for value in candidate["source_indices"])
|
| 86 |
+
if len(group) <= 1:
|
| 87 |
+
continue
|
| 88 |
+
feature_rows.append(feature)
|
| 89 |
+
targets.append(sum(bool(group & truth) for truth in truth_groups) > 1)
|
| 90 |
+
return np.asarray(feature_rows, dtype=np.float32), np.asarray(targets, dtype=np.int64)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _fit_boundary_model(samples: list[dict[str, Any]]) -> tuple[HistGradientBoostingClassifier, dict[str, Any]]:
|
| 94 |
+
"""ํ์ ๋ณ์: fit writer ํ๋ณด. ์๋ ์๋ฆฌ: class-balanced gradient boosting์ผ๋ก ๊ธฐํธ ๊ฒฝ๊ณ ์นจ๋ฒ ํ๋ฅ ์ ํ์ตํ๋ค."""
|
| 95 |
+
|
| 96 |
+
features, targets = _boundary_training_rows(samples)
|
| 97 |
+
positives = max(int(targets.sum()), 1)
|
| 98 |
+
negatives = max(len(targets) - positives, 1)
|
| 99 |
+
sample_weight = np.where(
|
| 100 |
+
targets == 1, len(targets) / (2 * positives), len(targets) / (2 * negatives),
|
| 101 |
+
)
|
| 102 |
+
model = HistGradientBoostingClassifier(
|
| 103 |
+
learning_rate=0.06,
|
| 104 |
+
max_iter=220,
|
| 105 |
+
max_leaf_nodes=31,
|
| 106 |
+
l2_regularization=2.0,
|
| 107 |
+
min_samples_leaf=40,
|
| 108 |
+
random_state=17,
|
| 109 |
+
).fit(features, targets, sample_weight=sample_weight)
|
| 110 |
+
return model, {
|
| 111 |
+
"candidate_rows": len(targets),
|
| 112 |
+
"boundary_rows": int(targets.sum()),
|
| 113 |
+
"boundary_rate": float(targets.mean()),
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _boundary_probabilities(
|
| 118 |
+
prepared: list[dict[str, Any]], model: HistGradientBoostingClassifier,
|
| 119 |
+
) -> list[np.ndarray]:
|
| 120 |
+
"""ํ์ ๋ณ์: ํ๊ฐ ํ๋ณด geometryยทํ์ต ๋ชจ๋ธ. ์๋ ์๋ฆฌ: singleton์ 0, ๋คํ ํ๋ณด๋ง ๊ฒฝ๊ณ ์นจ๋ฒ ํ๋ฅ ์ ๋ฐํํ๋ค."""
|
| 121 |
+
|
| 122 |
+
output = []
|
| 123 |
+
for row in prepared:
|
| 124 |
+
probabilities = model.predict_proba(row["features"][:, :len(LATTICE_FEATURE_NAMES)])[:, 1]
|
| 125 |
+
multistroke = np.asarray(
|
| 126 |
+
[len(candidate["source_indices"]) > 1 for candidate in row["candidates"]],
|
| 127 |
+
dtype=bool,
|
| 128 |
+
)
|
| 129 |
+
output.append(np.where(multistroke, probabilities, 0.0).astype(np.float32))
|
| 130 |
+
return output
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _evaluate06(
|
| 134 |
+
samples: list[dict[str, Any]],
|
| 135 |
+
prepared: list[dict[str, Any]],
|
| 136 |
+
probabilities: list[np.ndarray],
|
| 137 |
+
*,
|
| 138 |
+
threshold: float,
|
| 139 |
+
weight: float,
|
| 140 |
+
) -> dict[str, Any]:
|
| 141 |
+
"""ํ์ ๋ณ์: selector rowยทboundary ํ๋ฅ ยทthreshold/weight. ์๋ ์๋ฆฌ: ํ๋ฅ ์ด๊ณผ๋ถ๋ง logit์์ ๋นผ๊ณ ๋ณดํธ ์งํ๋ฅผ ๊ณ์ฐํ๋ค."""
|
| 142 |
+
|
| 143 |
+
weighted = _weighted(
|
| 144 |
+
prepared, tray_weight=4.0, symbol_weight=4.0,
|
| 145 |
+
fraction_weight=8.0, infix_weight=8.0,
|
| 146 |
+
)
|
| 147 |
+
guarded = []
|
| 148 |
+
for row, probability in zip(weighted, probabilities, strict=True):
|
| 149 |
+
penalty = np.maximum(probability - threshold, 0.0) / max(1.0 - threshold, 1e-6)
|
| 150 |
+
guarded.append({**row, "logits": row["logits"] - weight * penalty})
|
| 151 |
+
return {
|
| 152 |
+
"threshold": threshold,
|
| 153 |
+
"weight": weight,
|
| 154 |
+
"global": _metrics(guarded, -2.0),
|
| 155 |
+
"behavior_targets": _target_metrics06(samples, guarded, group_bias=-2.0),
|
| 156 |
+
"families": _family_metrics06(samples, guarded),
|
| 157 |
+
}
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _delta_pp06(candidate: float, reference: float) -> float:
|
| 161 |
+
"""ํ์ ๋ณ์: ํ๋ณดยท๊ธฐ์ค ๋น์จ. ์๋ ์๋ฆฌ: ์ฑํ ํ๋จ์ฉ percentage-point ์ฐจ์ด๋ฅผ ๋ฐํํ๋ค."""
|
| 162 |
+
|
| 163 |
+
return (candidate - reference) * 100.0
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def main() -> None:
|
| 167 |
+
"""ํ์ ๋ณ์: fit/validation/test writer split. ์๋ ์๋ฆฌ: fit ํ์ตยทvalidation ์ ํ ํ ๊ณ ์ winner๋ง official test์ ์ ์ฉํ๋ค."""
|
| 168 |
+
|
| 169 |
+
args = _parse_args()
|
| 170 |
+
fit, validation = writer_fit_validation(args.train_root, args.profile)
|
| 171 |
+
boundary_model, training = _fit_boundary_model(fit)
|
| 172 |
+
geometry_model = _fit_geometry(fit)
|
| 173 |
+
equality_model = EqualityVisualModel.load(args.equality_model)
|
| 174 |
+
cross_model = CrossVisualModel.load(args.cross_model)
|
| 175 |
+
validation_cache = load_cache_for_samples(
|
| 176 |
+
validation, args.cache_dir, split="validation", profile=args.profile,
|
| 177 |
+
bundle=args.bundle, version=2,
|
| 178 |
+
)
|
| 179 |
+
validation_prepared = _prepared_signals(
|
| 180 |
+
validation, validation_cache, geometry_model,
|
| 181 |
+
equality_model=equality_model, cross_model=cross_model,
|
| 182 |
+
cross_gap_ratio=0.40, multistroke_family_boost=6.0,
|
| 183 |
+
)
|
| 184 |
+
validation_probability = _boundary_probabilities(validation_prepared, boundary_model)
|
| 185 |
+
trials = [
|
| 186 |
+
_evaluate06(
|
| 187 |
+
validation, validation_prepared, validation_probability,
|
| 188 |
+
threshold=threshold, weight=weight,
|
| 189 |
+
)
|
| 190 |
+
for threshold in (0.30, 0.50, 0.65, 0.80, 0.90)
|
| 191 |
+
for weight in (0.0, 1.0, 2.0, 4.0, 6.0, 8.0, 12.0)
|
| 192 |
+
]
|
| 193 |
+
reference = trials[0]
|
| 194 |
+
minimum_x = reference["behavior_targets"]["x"]["grouping_recall"] - args.maximum_x_regression_pp / 100.0
|
| 195 |
+
minimum_family = reference["families"]["grouping_recall"] - args.maximum_family_regression_pp / 100.0
|
| 196 |
+
minimum_pair_f1 = reference["global"]["pair_f1"] - args.maximum_pair_f1_regression_pp / 100.0
|
| 197 |
+
eligible = [
|
| 198 |
+
row for row in trials
|
| 199 |
+
if (
|
| 200 |
+
row["behavior_targets"]["x"]["grouping_recall"] >= minimum_x
|
| 201 |
+
and row["families"]["grouping_recall"] >= minimum_family
|
| 202 |
+
and row["global"]["pair_f1"] >= minimum_pair_f1
|
| 203 |
+
)
|
| 204 |
+
]
|
| 205 |
+
winner = max(eligible, key=lambda row: (
|
| 206 |
+
row["global"]["exact_partition"], row["global"]["pair_f1"],
|
| 207 |
+
row["global"]["exact_group_recall"], -row["weight"], row["threshold"],
|
| 208 |
+
))
|
| 209 |
+
test, test_cache = load_cached_split(
|
| 210 |
+
args.test_root, args.cache_dir, split="official_test", profile=args.profile,
|
| 211 |
+
bundle=args.bundle, version=2,
|
| 212 |
+
)
|
| 213 |
+
test_prepared = _prepared_signals(
|
| 214 |
+
test, test_cache, geometry_model,
|
| 215 |
+
equality_model=equality_model, cross_model=cross_model,
|
| 216 |
+
cross_gap_ratio=0.40, multistroke_family_boost=6.0,
|
| 217 |
+
)
|
| 218 |
+
test_probability = _boundary_probabilities(test_prepared, boundary_model)
|
| 219 |
+
official_reference = _evaluate06(
|
| 220 |
+
test, test_prepared, test_probability, threshold=0.30, weight=0.0,
|
| 221 |
+
)
|
| 222 |
+
official_winner = _evaluate06(
|
| 223 |
+
test, test_prepared, test_probability,
|
| 224 |
+
threshold=float(winner["threshold"]), weight=float(winner["weight"]),
|
| 225 |
+
)
|
| 226 |
+
deltas = {
|
| 227 |
+
"exact_partition_pp": _delta_pp06(
|
| 228 |
+
official_winner["global"]["exact_partition"], official_reference["global"]["exact_partition"],
|
| 229 |
+
),
|
| 230 |
+
"pair_f1_pp": _delta_pp06(
|
| 231 |
+
official_winner["global"]["pair_f1"], official_reference["global"]["pair_f1"],
|
| 232 |
+
),
|
| 233 |
+
"x_grouping_pp": _delta_pp06(
|
| 234 |
+
official_winner["behavior_targets"]["x"]["grouping_recall"],
|
| 235 |
+
official_reference["behavior_targets"]["x"]["grouping_recall"],
|
| 236 |
+
),
|
| 237 |
+
"family_grouping_pp": _delta_pp06(
|
| 238 |
+
official_winner["families"]["grouping_recall"], official_reference["families"]["grouping_recall"],
|
| 239 |
+
),
|
| 240 |
+
}
|
| 241 |
+
adopted = bool(
|
| 242 |
+
float(winner["weight"]) > 0.0
|
| 243 |
+
and deltas["exact_partition_pp"] > 0.0
|
| 244 |
+
and deltas["pair_f1_pp"] >= -args.maximum_pair_f1_regression_pp
|
| 245 |
+
and deltas["x_grouping_pp"] >= -args.maximum_x_regression_pp
|
| 246 |
+
and deltas["family_grouping_pp"] >= -args.maximum_family_regression_pp
|
| 247 |
+
)
|
| 248 |
+
artifact_path = args.output.parent / "boundary_behavior_guard.joblib"
|
| 249 |
+
artifact_path.parent.mkdir(parents=True, exist_ok=True)
|
| 250 |
+
joblib.dump({
|
| 251 |
+
"schema": "aiflow-math-ink-06-boundary-behavior-v1",
|
| 252 |
+
"feature_names": list(LATTICE_FEATURE_NAMES),
|
| 253 |
+
"model": boundary_model,
|
| 254 |
+
"threshold": float(winner["threshold"]) if adopted else None,
|
| 255 |
+
"weight": float(winner["weight"]) if adopted else 0.0,
|
| 256 |
+
"track": "R_noncommercial_only",
|
| 257 |
+
}, artifact_path)
|
| 258 |
+
report = {
|
| 259 |
+
"experiment": "R-MATH-INK-06-BOUNDARY-BEHAVIOR-GUARD-001",
|
| 260 |
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
| 261 |
+
"training": training,
|
| 262 |
+
"feature_names": list(LATTICE_FEATURE_NAMES),
|
| 263 |
+
"reference_validation": reference,
|
| 264 |
+
"winner_validation": winner,
|
| 265 |
+
"trials": trials,
|
| 266 |
+
"official_test_reference": official_reference,
|
| 267 |
+
"official_test_winner": official_winner,
|
| 268 |
+
"official_test_deltas": deltas,
|
| 269 |
+
"decision": {
|
| 270 |
+
"adopted": adopted,
|
| 271 |
+
"selected_threshold": float(winner["threshold"]) if adopted else None,
|
| 272 |
+
"selected_weight": float(winner["weight"]) if adopted else 0.0,
|
| 273 |
+
},
|
| 274 |
+
"artifact": str(artifact_path),
|
| 275 |
+
"track": "R_noncommercial_only",
|
| 276 |
+
"product_validation": False,
|
| 277 |
+
}
|
| 278 |
+
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
| 279 |
+
print(json.dumps({key: value for key, value in report.items() if key != "trials"}, ensure_ascii=False, indent=2))
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
if __name__ == "__main__":
|
| 283 |
+
main()
|
src/segmentation_lattice.py
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ํ grouping์ ์กฐ๊ธฐ ํ์ ํ์ง ์๊ณ ๋ณต์ symbol ํ๋ณด๋ก ๋ณด์กดํ๋ ์ฐ๊ตฌ์ฉ lattice๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from collections import defaultdict
|
| 6 |
+
from typing import Any, Sequence
|
| 7 |
+
import math
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def _stroke_box(stroke: dict[str, Any]) -> tuple[float, float, float, float]:
|
| 13 |
+
"""ํ์ ๋ณ์: point๊ฐ ์๋ stroke. ์๋ ์๋ฆฌ: ํ๋ณด ๊ฑฐ๋ฆฌยท์ ๋ ฌ์ ์ฌ์ฉํ bbox๋ฅผ ๊ณ์ฐํ๋ค."""
|
| 14 |
+
|
| 15 |
+
points = stroke.get("points") or []
|
| 16 |
+
xs = [float(point.get("x", 0.0) if isinstance(point, dict) else point[0]) for point in points]
|
| 17 |
+
ys = [float(point.get("y", 0.0) if isinstance(point, dict) else point[1]) for point in points]
|
| 18 |
+
if not xs:
|
| 19 |
+
raise ValueError("๋น stroke๋ segmentation lattice์ ๋ฃ์ ์ ์์ต๋๋ค.")
|
| 20 |
+
return min(xs), min(ys), max(xs), max(ys)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _group_box(indices: frozenset[int], boxes: Sequence[tuple[float, float, float, float]]) -> dict[str, float]:
|
| 24 |
+
"""ํ์ ๋ณ์: stroke index ์งํฉยท๊ฐ๋ณ bbox. ์๋ ์๋ฆฌ: group ์ ์ฒด bbox๋ฅผ ์ง๋ ฌํ ๊ฐ๋ฅํ ๊ฐ์ผ๋ก ํฉ์น๋ค."""
|
| 25 |
+
|
| 26 |
+
return {
|
| 27 |
+
"left": min(boxes[index][0] for index in indices),
|
| 28 |
+
"top": min(boxes[index][1] for index in indices),
|
| 29 |
+
"right": max(boxes[index][2] for index in indices),
|
| 30 |
+
"bottom": max(boxes[index][3] for index in indices),
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _box_gap(first: tuple[float, float, float, float], second: tuple[float, float, float, float]) -> float:
|
| 35 |
+
"""ํ์ ๋ณ์: ๋ bbox. ์๋ ์๋ฆฌ: ๊ฒน์น๋ฉด 0, ์๋๋ฉด ์ ํด๋ฆฌ๋ ์ธ๊ณฝ ๊ฐ๊ฒฉ์ ๋ฐํํ๋ค."""
|
| 36 |
+
|
| 37 |
+
dx = max(first[0] - second[2], second[0] - first[2], 0.0)
|
| 38 |
+
dy = max(first[1] - second[3], second[1] - first[3], 0.0)
|
| 39 |
+
return math.hypot(dx, dy)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def build_group_candidate_lattice(
|
| 43 |
+
strokes: Sequence[dict[str, Any]],
|
| 44 |
+
seed_partitions: Sequence[Sequence[dict[str, Any]]],
|
| 45 |
+
*,
|
| 46 |
+
temporal_window: int = 4,
|
| 47 |
+
spatial_neighbors: int = 3,
|
| 48 |
+
) -> list[dict[str, Any]]:
|
| 49 |
+
"""ํ์ ๋ณ์: ์ strokeยท์ฌ๋ฌ geometry partition. ์๋ ์๋ฆฌ: singleton/์๊ฐ์ฐฝ/๊ณต๊ฐ์ด์/๊ธฐ์กด group์ ์ค๋ณต ์ ๊ฑฐํด ๋ณด์กดํ๋ค."""
|
| 50 |
+
|
| 51 |
+
usable = [(index, stroke) for index, stroke in enumerate(strokes) if stroke.get("points")]
|
| 52 |
+
if not usable:
|
| 53 |
+
return []
|
| 54 |
+
source_indices = [index for index, _stroke in usable]
|
| 55 |
+
stroke_by_index = {index: stroke for index, stroke in usable}
|
| 56 |
+
boxes_by_source = {index: _stroke_box(stroke) for index, stroke in usable}
|
| 57 |
+
dense_boxes = [boxes_by_source[index] for index in source_indices]
|
| 58 |
+
evidence: dict[frozenset[int], set[str]] = defaultdict(set)
|
| 59 |
+
|
| 60 |
+
def add(indices: frozenset[int], reason: str) -> None:
|
| 61 |
+
if indices and indices.issubset(stroke_by_index):
|
| 62 |
+
evidence[indices].add(reason)
|
| 63 |
+
|
| 64 |
+
for index in source_indices:
|
| 65 |
+
add(frozenset({index}), "singleton")
|
| 66 |
+
for partition_index, partition in enumerate(seed_partitions):
|
| 67 |
+
for segment in partition:
|
| 68 |
+
add(frozenset(int(index) for index in segment["source_indices"]), f"geometry:{partition_index}")
|
| 69 |
+
|
| 70 |
+
# ํ๊ธฐ ์์๊ฐ ์ธ์ ํ ๋คํ ๊ธฐํธ๋ฅผ ๋ณต๊ตฌํ๋ ์ง์ ํญ๋ฐ์ ๋ง๊ธฐ ์ํด ์งง์ ์ฐ์ ์ฐฝ๋ง ๋ง๋ ๋ค.
|
| 71 |
+
for start in range(len(source_indices)):
|
| 72 |
+
for length in range(2, min(temporal_window, len(source_indices) - start) + 1):
|
| 73 |
+
add(frozenset(source_indices[start:start + length]), f"temporal:{length}")
|
| 74 |
+
|
| 75 |
+
# ์ค์ ๊ธฐํธ ํ์ด ์๊ฐ์ ๋ผ์ด ์ฐ์์ ๋๋ฅผ ์ํด ๊ฐ ํ์ ๊ฐ๊น์ด ๊ณต๊ฐ ์ด์ pair๋ฅผ ์ถ๊ฐํ๋ค.
|
| 76 |
+
for dense_index, source_index in enumerate(source_indices):
|
| 77 |
+
neighbor_order = sorted(
|
| 78 |
+
(other for other in range(len(source_indices)) if other != dense_index),
|
| 79 |
+
key=lambda other: (_box_gap(dense_boxes[dense_index], dense_boxes[other]), other),
|
| 80 |
+
)[:spatial_neighbors]
|
| 81 |
+
for other in neighbor_order:
|
| 82 |
+
add(frozenset({source_index, source_indices[other]}), "spatial_pair")
|
| 83 |
+
|
| 84 |
+
candidates = []
|
| 85 |
+
for indices, reasons in evidence.items():
|
| 86 |
+
box = _group_box(indices, boxes_by_source)
|
| 87 |
+
candidates.append({
|
| 88 |
+
"source_indices": sorted(indices),
|
| 89 |
+
"strokes": [stroke_by_index[index] for index in sorted(indices)],
|
| 90 |
+
"box": box,
|
| 91 |
+
"evidence": sorted(reasons),
|
| 92 |
+
"candidate_id": "g:" + ",".join(str(index) for index in sorted(indices)),
|
| 93 |
+
})
|
| 94 |
+
candidates.sort(key=lambda row: (row["box"]["left"], len(row["source_indices"]), row["source_indices"]))
|
| 95 |
+
return candidates
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def lattice_partition_oracle(
|
| 99 |
+
candidates: Sequence[dict[str, Any]], truth_groups: Sequence[frozenset[int]],
|
| 100 |
+
) -> dict[str, Any]:
|
| 101 |
+
"""ํ์ ๋ณ์: ํ๋ณด latticeยท์ ๋ต partition. ์๋ ์๋ฆฌ: ์ ์์ ๋ฌด๊ดํ ํ๋ณด coverage ์ํ์ ๊ณ์ฐํ๋ค."""
|
| 102 |
+
|
| 103 |
+
candidate_groups = {frozenset(int(index) for index in row["source_indices"]) for row in candidates}
|
| 104 |
+
matched = [group for group in truth_groups if group in candidate_groups]
|
| 105 |
+
missing = [sorted(group) for group in truth_groups if group not in candidate_groups]
|
| 106 |
+
return {
|
| 107 |
+
"truth_groups": len(truth_groups),
|
| 108 |
+
"matched_groups": len(matched),
|
| 109 |
+
"group_recall": len(matched) / max(len(truth_groups), 1),
|
| 110 |
+
"exact_partition_recoverable": not missing,
|
| 111 |
+
"missing_groups": missing,
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
LATTICE_FEATURE_NAMES = (
|
| 116 |
+
"stroke_count", "point_count_log", "width_ref", "height_ref", "aspect_log",
|
| 117 |
+
"temporal_span", "temporal_contiguous", "singleton", "geometry_votes",
|
| 118 |
+
"has_temporal", "has_spatial_pair", "pair_gap_mean", "pair_gap_max",
|
| 119 |
+
"stroke_width_mean", "stroke_width_std", "stroke_height_mean", "stroke_height_std",
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
LATTICE_OCR_FEATURE_NAMES = (
|
| 123 |
+
"ocr_top1", "ocr_margin", "ocr_top5_mass", "ocr_entropy", "family_top1",
|
| 124 |
+
"family_margin", "merge_top1_gain", "merge_entropy_gain", "label_alnum",
|
| 125 |
+
"label_operator", "label_container", "label_delimiter", "label_dot_like",
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def lattice_candidate_features(
|
| 130 |
+
candidates: Sequence[dict[str, Any]], strokes: Sequence[dict[str, Any]],
|
| 131 |
+
) -> np.ndarray:
|
| 132 |
+
"""ํ์ ๋ณ์: lattice ํ๋ณดยท์ stroke. ์๋ ์๋ฆฌ: label ์์ด ๊ณต๋ selector๊ฐ ์ฌ์ฉํ scale-normalized geometry feature๋ฅผ ๋ง๋ ๋ค."""
|
| 133 |
+
|
| 134 |
+
boxes = [_stroke_box(stroke) for stroke in strokes]
|
| 135 |
+
nonzero_heights = [box[3] - box[1] for box in boxes if box[3] - box[1] > 1e-6]
|
| 136 |
+
reference_height = float(np.median(nonzero_heights)) if nonzero_heights else 1.0
|
| 137 |
+
rows = []
|
| 138 |
+
for candidate in candidates:
|
| 139 |
+
indices = [int(index) for index in candidate["source_indices"]]
|
| 140 |
+
selected = [boxes[index] for index in indices]
|
| 141 |
+
box = candidate["box"]
|
| 142 |
+
width = float(box["right"] - box["left"])
|
| 143 |
+
height = float(box["bottom"] - box["top"])
|
| 144 |
+
gaps = [_box_gap(selected[first], selected[second]) / reference_height
|
| 145 |
+
for first in range(len(selected)) for second in range(first + 1, len(selected))]
|
| 146 |
+
widths = [(value[2] - value[0]) / reference_height for value in selected]
|
| 147 |
+
heights = [(value[3] - value[1]) / reference_height for value in selected]
|
| 148 |
+
evidence = set(candidate.get("evidence") or [])
|
| 149 |
+
rows.append([
|
| 150 |
+
len(indices), math.log1p(sum(len(strokes[index].get("points") or []) for index in indices)),
|
| 151 |
+
width / reference_height, height / reference_height,
|
| 152 |
+
math.log(max(width, 1e-6) / max(height, 1e-6)),
|
| 153 |
+
max(indices) - min(indices) + 1, float(max(indices) - min(indices) + 1 == len(indices)),
|
| 154 |
+
float(len(indices) == 1), sum(value.startswith("geometry:") for value in evidence),
|
| 155 |
+
float(any(value.startswith("temporal:") for value in evidence)),
|
| 156 |
+
float("spatial_pair" in evidence), float(np.mean(gaps)) if gaps else 0.0,
|
| 157 |
+
max(gaps, default=0.0), float(np.mean(widths)), float(np.std(widths)),
|
| 158 |
+
float(np.mean(heights)), float(np.std(heights)),
|
| 159 |
+
])
|
| 160 |
+
return np.asarray(rows, dtype=np.float32)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def lattice_ocr_features(candidates: Sequence[dict[str, Any]]) -> np.ndarray:
|
| 164 |
+
"""ํ์ ๋ณ์: OCR summary๊ฐ ๋ถ์ฐฉ๋ lattice. ์๋ ์๋ฆฌ: confidence์ singleton ๋๋น ๊ฒฐํฉ ์ด๋์ label-coarse feature๋ก ๋ฐ๊พผ๋ค."""
|
| 165 |
+
|
| 166 |
+
singleton = {
|
| 167 |
+
int(candidate["source_indices"][0]): candidate["ocr_summary"]
|
| 168 |
+
for candidate in candidates if len(candidate["source_indices"]) == 1
|
| 169 |
+
}
|
| 170 |
+
operators = {"+", "-", "=", "/", r"\times", r"\div", r"\pm", r"\neq", "<", ">"}
|
| 171 |
+
containers = {r"\sqrt{}", r"\sqrt", r"\sum", r"\int", r"\prod"}
|
| 172 |
+
delimiters = {"(", ")", "[", "]", r"\{", r"\}", r"\langle", r"\rangle", "|"}
|
| 173 |
+
dot_like = {".", ",", r"\dots", r"\dotsc", r"\cdot"}
|
| 174 |
+
rows = []
|
| 175 |
+
for candidate in candidates:
|
| 176 |
+
summary = candidate.get("ocr_summary")
|
| 177 |
+
if not isinstance(summary, dict):
|
| 178 |
+
raise ValueError("lattice OCR feature์๋ ocr_summary๊ฐ ํ์ํฉ๋๋ค.")
|
| 179 |
+
indices = [int(index) for index in candidate["source_indices"]]
|
| 180 |
+
components = [singleton[index] for index in indices if index in singleton]
|
| 181 |
+
component_top1 = float(np.mean([row["top1"] for row in components])) if components else 0.0
|
| 182 |
+
component_entropy = float(np.mean([row["entropy"] for row in components])) if components else 0.0
|
| 183 |
+
label = str(summary["top_label"])
|
| 184 |
+
rows.append([
|
| 185 |
+
float(summary["top1"]), float(summary["top1"] - summary["top2"]),
|
| 186 |
+
float(summary["top5_mass"]), float(summary["entropy"]), float(summary["family_top1"]),
|
| 187 |
+
float(summary["family_top1"] - summary["family_top2"]),
|
| 188 |
+
float(summary["top1"] - component_top1) if len(indices) > 1 else 0.0,
|
| 189 |
+
float(component_entropy - summary["entropy"]) if len(indices) > 1 else 0.0,
|
| 190 |
+
float(label.isalnum()), float(label in operators), float(label in containers),
|
| 191 |
+
float(label in delimiters), float(label in dot_like),
|
| 192 |
+
])
|
| 193 |
+
return np.asarray(rows, dtype=np.float32)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def select_lattice_partition(
|
| 197 |
+
candidates: Sequence[dict[str, Any]], scores: Sequence[float], stroke_count: int,
|
| 198 |
+
*, beam_width: int = 128, group_bias: float = 0.0, options_per_stroke: int = 32,
|
| 199 |
+
transition_scores: np.ndarray | None = None,
|
| 200 |
+
) -> list[frozenset[int]]:
|
| 201 |
+
"""ํ์ ๋ณ์: ํ๋ณดยทlogit scoreยทํ ์. ์๋ ์๋ฆฌ: ๋ชจ๋ ํ์ ์ ํํ ํ ๋ฒ ๋ฎ๋ ์ต๊ณ ์ ์ partition์ beam exact-cover๋ก ์ฐพ๋๋ค."""
|
| 202 |
+
|
| 203 |
+
if len(candidates) != len(scores):
|
| 204 |
+
raise ValueError("lattice ํ๋ณด์ score ์๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 205 |
+
if stroke_count < 1:
|
| 206 |
+
return []
|
| 207 |
+
if transition_scores is not None and transition_scores.shape != (len(candidates), len(candidates)):
|
| 208 |
+
raise ValueError("candidate transition score ํ๋ ฌ ํฌ๊ธฐ๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 209 |
+
full_mask = (1 << stroke_count) - 1
|
| 210 |
+
prepared = []
|
| 211 |
+
by_stroke: dict[int, list[int]] = defaultdict(list)
|
| 212 |
+
for candidate_index, (candidate, score) in enumerate(zip(candidates, scores, strict=True)):
|
| 213 |
+
indices = tuple(int(index) for index in candidate["source_indices"])
|
| 214 |
+
if not indices or any(index < 0 or index >= stroke_count for index in indices):
|
| 215 |
+
continue
|
| 216 |
+
mask = sum(1 << index for index in indices)
|
| 217 |
+
prepared.append((candidate_index, mask, frozenset(indices), float(score) + group_bias))
|
| 218 |
+
for index in indices:
|
| 219 |
+
by_stroke[index].append(len(prepared) - 1)
|
| 220 |
+
for index in by_stroke:
|
| 221 |
+
by_stroke[index].sort(key=lambda row: prepared[row][3], reverse=True)
|
| 222 |
+
by_stroke[index] = by_stroke[index][:options_per_stroke]
|
| 223 |
+
if transition_scores is None:
|
| 224 |
+
beams: dict[Any, tuple[float, tuple[frozenset[int], ...]]] = {0: (0.0, ())}
|
| 225 |
+
else:
|
| 226 |
+
beams = {(0, -1): (0.0, ())}
|
| 227 |
+
for _step in range(stroke_count):
|
| 228 |
+
expanded: dict[Any, tuple[float, tuple[frozenset[int], ...]]] = {}
|
| 229 |
+
for state_key, (total_score, groups) in beams.items():
|
| 230 |
+
used_mask, previous_candidate = (state_key, -1) if transition_scores is None else state_key
|
| 231 |
+
if used_mask == full_mask:
|
| 232 |
+
expanded[state_key] = max(expanded.get(state_key, (-math.inf, ())), (total_score, groups), key=lambda row: row[0])
|
| 233 |
+
continue
|
| 234 |
+
first_uncovered = next(index for index in range(stroke_count) if not used_mask & (1 << index))
|
| 235 |
+
for prepared_index in by_stroke.get(first_uncovered, []):
|
| 236 |
+
candidate_index, mask, group, score = prepared[prepared_index]
|
| 237 |
+
if used_mask & mask:
|
| 238 |
+
continue
|
| 239 |
+
new_mask = used_mask | mask
|
| 240 |
+
transition = 0.0 if transition_scores is None or previous_candidate < 0 else float(
|
| 241 |
+
transition_scores[previous_candidate, candidate_index]
|
| 242 |
+
)
|
| 243 |
+
proposal = (total_score + score + transition, groups + (group,))
|
| 244 |
+
new_key: Any = new_mask if transition_scores is None else (new_mask, candidate_index)
|
| 245 |
+
if new_key not in expanded or proposal[0] > expanded[new_key][0]:
|
| 246 |
+
expanded[new_key] = proposal
|
| 247 |
+
if not expanded:
|
| 248 |
+
raise ValueError("๋ชจ๋ stroke๋ฅผ ๋ฎ๋ lattice partition์ ์ฐพ์ง ๋ชปํ์ต๋๋ค.")
|
| 249 |
+
beams = dict(sorted(expanded.items(), key=lambda row: row[1][0], reverse=True)[:beam_width])
|
| 250 |
+
complete_keys = [key for key in beams if (key if transition_scores is None else key[0]) == full_mask]
|
| 251 |
+
if complete_keys and all((key if transition_scores is None else key[0]) == full_mask for key in beams):
|
| 252 |
+
break
|
| 253 |
+
complete = [value for key, value in beams.items() if (key if transition_scores is None else key[0]) == full_mask]
|
| 254 |
+
if not complete:
|
| 255 |
+
raise ValueError("beam width ์์์ ์์ partition์ ์ฐพ์ง ๋ชปํ์ต๋๋ค.")
|
| 256 |
+
return list(max(complete, key=lambda row: row[0])[1])
|
src/structure_relations.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""๊ธฐํธ ์ธ์๊ณผ ๋
๋ฆฝ์ ์ผ๋ก ์์์ 2D ๋ฐฐ์น ๊ด๊ณ๋ฅผ ์ถ๋ก ํ๋ ์ฐ๊ตฌ์ฉ ๋ ์ด์ด๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
from typing import Any, Sequence
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
from .math_tray import infer_math_trays
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
RELATION_TYPES = ("RIGHT", "SUPERSCRIPT", "SUBSCRIPT", "ABOVE", "BELOW", "CONTAINS")
|
| 14 |
+
STRUCTURAL_RELATION_TYPES = frozenset({"SUPERSCRIPT", "SUBSCRIPT", "ABOVE", "BELOW", "CONTAINS"})
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@dataclass(frozen=True, slots=True)
|
| 18 |
+
class RelationConfig:
|
| 19 |
+
"""ํ์ ๋ณ์: ์ฒจ์ ๊ธฐํ ์๊ณ๊ฐ. ์๋ ์๋ฆฌ: writer validation์์ ๊ณ ๋ฅผ ์ ์๋๋ก ๊ด๊ณ ํ์ ๊ฐ์ ์ธ๋ถํํ๋ค."""
|
| 20 |
+
|
| 21 |
+
script_min_height_ratio: float = 0.18
|
| 22 |
+
script_max_height_ratio: float = 0.82
|
| 23 |
+
script_vertical_shift_ratio: float = 0.18
|
| 24 |
+
script_max_horizontal_gap_ratio: float = 0.75
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _label(segment: dict[str, Any]) -> str:
|
| 28 |
+
"""ํ์ ๋ณ์: ์ธ์๋ segment. ์๋ ์๋ฆฌ: ๊ตฌ์กฐ ํํธ๋ฅผ ์ฐ์ ํ๊ณ ์ฒซ ์ ๋ฌธ๊ฐ์ top-1์ ๋ณด์กฐ ๋ผ๋ฒจ๋ก ์ฝ๋๋ค."""
|
| 29 |
+
|
| 30 |
+
hint = segment.get("structural_hint")
|
| 31 |
+
if hint:
|
| 32 |
+
return str(hint)
|
| 33 |
+
for result in (segment.get("results") or {}).values():
|
| 34 |
+
candidates = result.get("candidates") or []
|
| 35 |
+
if candidates:
|
| 36 |
+
return str(candidates[0]["label"])
|
| 37 |
+
return ""
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _geometry(segment: dict[str, Any]) -> dict[str, float]:
|
| 41 |
+
"""ํ์ ๋ณ์: segment bbox. ์๋ ์๋ฆฌ: ๊ด๊ณ ํ์ ์ ํ์ํ ํญยท๋์ดยท์ค์ฌ ์ขํ๋ฅผ ํ ๋ฒ ๊ณ์ฐํ๋ค."""
|
| 42 |
+
|
| 43 |
+
box = segment["box"]
|
| 44 |
+
left, top = float(box["left"]), float(box["top"])
|
| 45 |
+
right, bottom = float(box["right"]), float(box["bottom"])
|
| 46 |
+
return {
|
| 47 |
+
"left": left, "top": top, "right": right, "bottom": bottom,
|
| 48 |
+
"width": max(right - left, 1.0), "height": max(bottom - top, 1.0),
|
| 49 |
+
"cx": (left + right) * 0.5, "cy": (top + bottom) * 0.5,
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def infer_spatial_relations(
|
| 54 |
+
segments: Sequence[dict[str, Any]], config: RelationConfig | None = None,
|
| 55 |
+
*, trays: Sequence[dict[str, Any]] | None = None,
|
| 56 |
+
) -> list[dict[str, Any]]:
|
| 57 |
+
"""ํ์ ๋ณ์: geometry group๊ณผ ์ ํ์ OCR ๊ฒฐ๊ณผ. ์๋ ์๋ฆฌ: ๊ฐ ๊ธฐํธ๋ฅผ ํ ๋ฒ๋ง ์์์ผ๋ก ์๋นํด 2D ๊ด๊ณ ๊ทธ๋ํ๋ฅผ ๋ง๋ ๋ค."""
|
| 58 |
+
|
| 59 |
+
relation_config = config or RelationConfig()
|
| 60 |
+
if len(segments) < 2:
|
| 61 |
+
return []
|
| 62 |
+
boxes = [_geometry(segment) for segment in segments]
|
| 63 |
+
labels = [_label(segment) for segment in segments]
|
| 64 |
+
maximum_height = max(box["height"] for box in boxes)
|
| 65 |
+
body_heights = [box["height"] for box in boxes if box["height"] >= maximum_height * 0.25]
|
| 66 |
+
reference_height = float(np.median(body_heights))
|
| 67 |
+
relations: list[dict[str, Any]] = []
|
| 68 |
+
assigned: set[int] = set()
|
| 69 |
+
structural_parents: set[int] = set()
|
| 70 |
+
|
| 71 |
+
# ๊ทผํธ๋ ๋ด๋ถ ๊ธฐํธ๋ฅผ symbol group์ ํฉ์น์ง ์๊ณ CONTAINS edge๋ก๋ง ์์ ํ๋ค.
|
| 72 |
+
for parent, (box, label) in enumerate(zip(boxes, labels, strict=True)):
|
| 73 |
+
root_like = label in {r"\sqrt", r"\sqrt{}"}
|
| 74 |
+
if not root_like:
|
| 75 |
+
continue
|
| 76 |
+
structural_parents.add(parent)
|
| 77 |
+
for child, inner in enumerate(boxes):
|
| 78 |
+
if child == parent or child in assigned:
|
| 79 |
+
continue
|
| 80 |
+
inside_x = box["left"] + box["width"] * 0.20 <= inner["cx"] <= box["right"] + reference_height * 0.15
|
| 81 |
+
inside_y = inner["cy"] >= box["top"] and inner["top"] <= box["bottom"] + reference_height * 0.35
|
| 82 |
+
if inside_x and inside_y:
|
| 83 |
+
relations.append({"parent": parent, "child": child, "type": "CONTAINS", "confidence": 0.95})
|
| 84 |
+
assigned.add(child)
|
| 85 |
+
|
| 86 |
+
# ๋ถ์๋ bar ๋จ๋
์ด ์๋๋ผ ๋ถ์ยท๋ถ๋ชจ๊ฐ ๋ชจ๋ ์ ์ ๋ ์์์ Tray์ผ ๋๋ง ๊ด๊ณ๋ฅผ ๋ง๋ ๋ค.
|
| 87 |
+
tray_rows = list(trays) if trays is not None else infer_math_trays(segments)
|
| 88 |
+
fraction_trays = [tray for tray in tray_rows if tray["type"] == "FRACTION"]
|
| 89 |
+
for tray in fraction_trays:
|
| 90 |
+
parent = int(tray["anchor"])
|
| 91 |
+
structural_parents.add(parent)
|
| 92 |
+
for slot, relation in (("numerator", "ABOVE"), ("denominator", "BELOW")):
|
| 93 |
+
for child in tray["slots"].get(slot, []):
|
| 94 |
+
if child in assigned:
|
| 95 |
+
continue
|
| 96 |
+
relations.append({
|
| 97 |
+
"parent": parent, "child": child, "type": relation,
|
| 98 |
+
"confidence": float(tray["constraint_score"]),
|
| 99 |
+
})
|
| 100 |
+
assigned.add(child)
|
| 101 |
+
|
| 102 |
+
# ์ ๋ถยทํฉยท๊ณฑยท๊ทนํ์ ์ํํ์ ์ผ๋ฐ ์ต๊ทผ์ ์ฒจ์๋ณด๋ค ์ํ Tray์ anchor ์์ ๊ถ์ ์ฐ์ ํ๋ค.
|
| 103 |
+
operator_trays = [tray for tray in tray_rows if tray["type"] in {"INTEGRAL", "SUM", "PRODUCT", "LIM"}]
|
| 104 |
+
for tray in operator_trays:
|
| 105 |
+
parent = int(tray["anchor"])
|
| 106 |
+
structural_parents.add(parent)
|
| 107 |
+
for slot, relation in (("lower", "SUBSCRIPT"), ("upper", "SUPERSCRIPT")):
|
| 108 |
+
for child in tray["slots"].get(slot, []):
|
| 109 |
+
if child in assigned:
|
| 110 |
+
continue
|
| 111 |
+
relations.append({
|
| 112 |
+
"parent": parent, "child": int(child), "type": relation,
|
| 113 |
+
"confidence": float(tray["constraint_score"]),
|
| 114 |
+
})
|
| 115 |
+
assigned.add(int(child))
|
| 116 |
+
|
| 117 |
+
# ๋จ์ ๊ธฐํธ๋ ๊ฐ์ฅ ๊ฐ๊น์ด ์ผ์ชฝ ๊ธฐํธ๋ฅผ ๊ธฐ์ค์ผ๋ก ์ฒจ์ ๋๋ RIGHT ๊ด๊ณ๋ฅผ ๊ฐ๋๋ค.
|
| 118 |
+
for child, box in enumerate(boxes):
|
| 119 |
+
if child in assigned or child in structural_parents:
|
| 120 |
+
continue
|
| 121 |
+
candidates = [index for index, other in enumerate(boxes) if index != child and other["cx"] < box["cx"]]
|
| 122 |
+
if not candidates:
|
| 123 |
+
continue
|
| 124 |
+
parent = min(candidates, key=lambda index: max(0.0, box["left"] - boxes[index]["right"]))
|
| 125 |
+
base = boxes[parent]
|
| 126 |
+
horizontal_gap = max(0.0, box["left"] - base["right"])
|
| 127 |
+
script_sized = (
|
| 128 |
+
reference_height * relation_config.script_min_height_ratio
|
| 129 |
+
<= box["height"]
|
| 130 |
+
<= base["height"] * relation_config.script_max_height_ratio
|
| 131 |
+
)
|
| 132 |
+
script_near = horizontal_gap <= reference_height * relation_config.script_max_horizontal_gap_ratio
|
| 133 |
+
vertical_shift = base["height"] * relation_config.script_vertical_shift_ratio
|
| 134 |
+
if script_sized and script_near and box["cy"] < base["cy"] - vertical_shift:
|
| 135 |
+
relation = "SUPERSCRIPT"
|
| 136 |
+
confidence = 0.82
|
| 137 |
+
elif script_sized and script_near and box["cy"] > base["cy"] + vertical_shift:
|
| 138 |
+
relation = "SUBSCRIPT"
|
| 139 |
+
confidence = 0.82
|
| 140 |
+
else:
|
| 141 |
+
relation = "RIGHT"
|
| 142 |
+
confidence = 0.75
|
| 143 |
+
relations.append({"parent": parent, "child": child, "type": relation, "confidence": confidence})
|
| 144 |
+
assigned.add(child)
|
| 145 |
+
|
| 146 |
+
return sorted(relations, key=lambda item: (item["child"], item["parent"], item["type"]))
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def serialize_relation_graph(
|
| 150 |
+
labels: Sequence[str], segments: Sequence[dict[str, Any]], relations: Sequence[dict[str, Any]],
|
| 151 |
+
) -> str:
|
| 152 |
+
"""ํ์ ๋ณ์: group๋ณ ๋ผ๋ฒจยทbboxยท๊ด๊ณ edge. ์๋ ์๋ฆฌ: ๊ตฌ์กฐ ์์์ ํ ๋ฒ๋ง ์๋นํด 2D graph๋ฅผ LaTeX๋ก ์ง๋ ฌํํ๋ค."""
|
| 153 |
+
|
| 154 |
+
if len(labels) != len(segments):
|
| 155 |
+
raise ValueError("๊ด๊ณ serializer์ label๊ณผ segment ์๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 156 |
+
children: dict[int, dict[str, list[int]]] = {}
|
| 157 |
+
structural_children: set[int] = set()
|
| 158 |
+
for relation in relations:
|
| 159 |
+
relation_type = str(relation["type"])
|
| 160 |
+
if relation_type not in STRUCTURAL_RELATION_TYPES:
|
| 161 |
+
continue
|
| 162 |
+
parent, child = int(relation["parent"]), int(relation["child"])
|
| 163 |
+
if parent < 0 or child < 0 or parent >= len(labels) or child >= len(labels) or parent == child:
|
| 164 |
+
raise ValueError("๊ด๊ณ graph์ ์ ํจํ์ง ์์ node index๊ฐ ์์ต๋๋ค.")
|
| 165 |
+
children.setdefault(parent, {}).setdefault(relation_type, []).append(child)
|
| 166 |
+
structural_children.add(child)
|
| 167 |
+
|
| 168 |
+
def left(index: int) -> float:
|
| 169 |
+
"""ํ์ ๋ณ์: node index. ์๋ ์๋ฆฌ: ๊ฐ์ ๊ตฌ์กฐ ์ฌ๋กฏ ์์ ๊ธฐํธ๋ฅผ ์๋ x ์์๋ก ์ ๋ ฌํ๋ค."""
|
| 170 |
+
|
| 171 |
+
return float(segments[index]["box"]["left"])
|
| 172 |
+
|
| 173 |
+
def render_sequence(indices: Sequence[int], active: frozenset[int]) -> str:
|
| 174 |
+
"""ํ์ ๋ณ์: node index ์ดยท์ฌ๊ท ๊ฒฝ๋ก. ์๋ ์๋ฆฌ: ๊ฐ์ ์ฌ๋กฏ์ node๋ฅผ ์ขโ์ฐ๋ก ํ ๋ฒ์ฉ ์ถ๋ ฅํ๋ค."""
|
| 175 |
+
|
| 176 |
+
return "".join(render_node(index, active) for index in sorted(set(indices), key=left))
|
| 177 |
+
|
| 178 |
+
def render_node(index: int, active: frozenset[int]) -> str:
|
| 179 |
+
"""ํ์ ๋ณ์: ํ์ฌ nodeยท์ฌ๊ท ๊ฒฝ๋ก. ์๋ ์๋ฆฌ: root/fraction/script ๊ตฌ์กฐ๋ฅผ ์ค์ฒฉ ๊ฐ๋ฅํ LaTeX node๋ก ๋ง๋ ๋ค."""
|
| 180 |
+
|
| 181 |
+
if index in active:
|
| 182 |
+
raise ValueError("๊ด๊ณ graph์ cycle์ด ์์ต๋๋ค.")
|
| 183 |
+
next_active = active | {index}
|
| 184 |
+
slots = children.get(index, {})
|
| 185 |
+
above = slots.get("ABOVE", [])
|
| 186 |
+
below = slots.get("BELOW", [])
|
| 187 |
+
contained = slots.get("CONTAINS", [])
|
| 188 |
+
if above and below:
|
| 189 |
+
base = rf"\frac{{{render_sequence(above, next_active)}}}{{{render_sequence(below, next_active)}}}"
|
| 190 |
+
elif contained and labels[index] in {r"\sqrt", r"\sqrt{}"}:
|
| 191 |
+
base = rf"\sqrt{{{render_sequence(contained, next_active)}}}"
|
| 192 |
+
else:
|
| 193 |
+
base = labels[index]
|
| 194 |
+
superscript = slots.get("SUPERSCRIPT", [])
|
| 195 |
+
subscript = slots.get("SUBSCRIPT", [])
|
| 196 |
+
if subscript:
|
| 197 |
+
base += rf"_{{{render_sequence(subscript, next_active)}}}"
|
| 198 |
+
if superscript:
|
| 199 |
+
base += rf"^{{{render_sequence(superscript, next_active)}}}"
|
| 200 |
+
return base
|
| 201 |
+
|
| 202 |
+
roots = [index for index in range(len(labels)) if index not in structural_children]
|
| 203 |
+
return render_sequence(roots, frozenset())
|
src/tray_joint.py
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""OCR top-label๊ณผ Tray ์ฌ๋กฏ ๊ณ์ฝ์ lattice candidate score๋ก ๋ณํํ๋ค."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from dataclasses import dataclass
|
| 6 |
+
import math
|
| 7 |
+
from typing import Any, Sequence
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
from .cross_visual import CrossVisualModel, cross_pair_feature_rows
|
| 12 |
+
from .expression_contracts import horizontal_sides, segments_complete
|
| 13 |
+
from .equality_visual import EqualityVisualModel, equality_pair_feature_rows
|
| 14 |
+
from .math_tray import infer_math_trays
|
| 15 |
+
from .segmentation_lattice import LATTICE_FEATURE_NAMES
|
| 16 |
+
from .structure_relations import STRUCTURAL_RELATION_TYPES, infer_spatial_relations
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
ANCHOR_TYPES = {
|
| 20 |
+
r"\sqrt": "ROOT", r"\sqrt{}": "ROOT",
|
| 21 |
+
r"\int": "INTEGRAL", r"\oint": "INTEGRAL", r"\iint": "INTEGRAL", r"\iiint": "INTEGRAL",
|
| 22 |
+
r"\sum": "LARGE_OPERATOR", r"\prod": "LARGE_OPERATOR", r"\lim": "LARGE_OPERATOR",
|
| 23 |
+
}
|
| 24 |
+
RELIABLE_MULTISTROKE = {"i", r"\pi", r"\rightarrow", r"\pm", r"\leq", r"\geq"}
|
| 25 |
+
MULTISTROKE_FAMILY_LABELS = frozenset({
|
| 26 |
+
r"\sum", r"\Sigma", r"\pi", r"\Pi", r"\rightarrow", r"\shortrightarrow",
|
| 27 |
+
r"\longrightarrow", r"\Rightarrow", r"\neq",
|
| 28 |
+
r"\not\equiv", r"\pm",
|
| 29 |
+
})
|
| 30 |
+
MULTISTROKE_FAMILY_NAMES = frozenset({
|
| 31 |
+
r"\pi", r"\rightarrow", r"\shortrightarrow", r"\longrightarrow",
|
| 32 |
+
r"\Rightarrow", r"\neq", r"\not\equiv", r"\pm",
|
| 33 |
+
})
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _multistroke_family_geometry_guard(
|
| 37 |
+
label: str,
|
| 38 |
+
family: str,
|
| 39 |
+
feature_row: np.ndarray,
|
| 40 |
+
) -> bool:
|
| 41 |
+
"""ํ์ ๋ณ์: ํฉ์น OCR label/familyยทlattice geometry. ์๋ ์๋ฆฌ: ์์ฑ ๋จ์ผ๊ธฐํธ ์ด์ ๋จน๋ ๋์ alias ํ๋ณด๋ฅผ ์ฐจ๋จํ๋ค."""
|
| 42 |
+
|
| 43 |
+
width_ref = float(feature_row[2])
|
| 44 |
+
height_ref = float(feature_row[3])
|
| 45 |
+
pair_gap_max = float(feature_row[12])
|
| 46 |
+
if label == r"\Sigma" and width_ref < 2.0:
|
| 47 |
+
return False
|
| 48 |
+
if (label == r"\pm" or family == r"\pm") and height_ref > 2.0:
|
| 49 |
+
return False
|
| 50 |
+
arrow_family = {
|
| 51 |
+
r"\rightarrow", r"\shortrightarrow", r"\longrightarrow", r"\Rightarrow",
|
| 52 |
+
}
|
| 53 |
+
if (label in arrow_family or family in arrow_family) and pair_gap_max > 0.50:
|
| 54 |
+
return False
|
| 55 |
+
return True
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@dataclass(frozen=True, slots=True)
|
| 59 |
+
class TrayJointWeights:
|
| 60 |
+
"""ํ์ ๋ณ์: validation ์ ํ ๊ฐ์ค์น. ์๋ ์๋ฆฌ: ๊ตฌ์กฐ ๋ณด๋์ค์ ๋ must-not-link ๊ฐ์ ์ ํ ๊ณ์ฝ์ผ๋ก ๊ณ ์ ํ๋ค."""
|
| 61 |
+
|
| 62 |
+
tray: float = 4.0
|
| 63 |
+
symbol: float = 4.0
|
| 64 |
+
fraction: float = 8.0
|
| 65 |
+
infix: float = 8.0
|
| 66 |
+
competition: float = 0.0
|
| 67 |
+
local_baseline: float = 0.0
|
| 68 |
+
group_bias: float = -2.0
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def component_competition_penalties(
|
| 72 |
+
candidates: Sequence[dict[str, Any]],
|
| 73 |
+
features: np.ndarray,
|
| 74 |
+
*,
|
| 75 |
+
mode: str = "joint",
|
| 76 |
+
) -> np.ndarray:
|
| 77 |
+
"""ํ์ ๋ณ์: lattice ํ๋ณดยทOCR merge gain. ์๋ ์๋ฆฌ: ํฉ์น OCR์ด ๊ตฌ์ฑํ๋ณด๋ค ์ฝํ ๋คํ ํ๋ณด์ must-not-link ๊ฐ์ ์ค๋ค."""
|
| 78 |
+
|
| 79 |
+
if len(candidates) != len(features):
|
| 80 |
+
raise ValueError("component competition candidate/feature ์๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 81 |
+
if mode not in {"top1", "joint"}:
|
| 82 |
+
raise ValueError(f"์ง์ํ์ง ์๋ component competition mode์
๋๋ค: {mode}")
|
| 83 |
+
merge_index = len(LATTICE_FEATURE_NAMES) + 6
|
| 84 |
+
entropy_index = len(LATTICE_FEATURE_NAMES) + 7
|
| 85 |
+
penalties = np.zeros(len(candidates), dtype=np.float32)
|
| 86 |
+
for index, candidate in enumerate(candidates):
|
| 87 |
+
if len(candidate["source_indices"]) <= 1:
|
| 88 |
+
continue
|
| 89 |
+
merge_gain = float(features[index][merge_index])
|
| 90 |
+
deficit = -merge_gain
|
| 91 |
+
if mode == "joint":
|
| 92 |
+
deficit -= 0.5 * float(features[index][entropy_index])
|
| 93 |
+
penalties[index] = min(1.0, max(0.0, deficit))
|
| 94 |
+
return penalties
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _group_box(group: frozenset[int], strokes: Sequence[dict[str, Any]]) -> dict[str, float]:
|
| 98 |
+
"""ํ์ ๋ณ์: stroke groupยท์ขํ. ์๋ ์๋ฆฌ: candidate์ context symbol์ bbox๋ฅผ ๊ณ์ฐํ๋ค."""
|
| 99 |
+
|
| 100 |
+
points = [point for index in group for point in strokes[index]["points"]]
|
| 101 |
+
xs, ys = [float(point[0]) for point in points], [float(point[1]) for point in points]
|
| 102 |
+
return {"left": min(xs), "top": min(ys), "right": max(xs), "bottom": max(ys)}
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _recognized_segment(box: dict[str, float], label: str, score: float, family: str = "") -> dict[str, Any]:
|
| 106 |
+
"""ํ์ ๋ณ์: ํ๋ณด bboxยทtop label/familyยทํ๋ฅ . ์๋ ์๋ฆฌ: Tray์ ๋ถ๋ถ์ parser์ฉ ์ต์ OCR segment๋ฅผ ๋ง๋ ๋ค."""
|
| 107 |
+
|
| 108 |
+
return {"box": box, "results": {"expanded": {"candidates": [{
|
| 109 |
+
"label": label, "score": score, "visual_family": family,
|
| 110 |
+
}]}}}
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def local_baseline_boundary_penalties(
|
| 114 |
+
strokes: Sequence[dict[str, Any]],
|
| 115 |
+
candidates: Sequence[dict[str, Any]],
|
| 116 |
+
ocr_labels: Sequence[str],
|
| 117 |
+
features: np.ndarray,
|
| 118 |
+
base_partition: Sequence[frozenset[int]],
|
| 119 |
+
*,
|
| 120 |
+
ocr_families: Sequence[str] | None = None,
|
| 121 |
+
) -> np.ndarray:
|
| 122 |
+
"""ํ์ ๋ณ์: ์ด๊ธฐ partitionยทOCR segmentยท๊ณต๊ฐ ๊ด๊ณ. ์๋ ์๋ฆฌ: ์ฒจ์ยท๋ถ์ ๋ฑ ๊ตฌ์กฐ ๊ฒฝ๊ณ ์์ชฝ์ ๋์์ ๋จน๋ ๋ณํฉ๋ง ๊ฐ์ ํ๋ค."""
|
| 123 |
+
|
| 124 |
+
count = len(candidates)
|
| 125 |
+
if len(ocr_labels) != count or len(features) != count:
|
| 126 |
+
raise ValueError("local baseline candidate/label/feature ์๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 127 |
+
families = list(ocr_families or [""] * count)
|
| 128 |
+
if len(families) != count:
|
| 129 |
+
raise ValueError("local baseline candidate/family ์๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 130 |
+
index_by_group = {
|
| 131 |
+
frozenset(int(value) for value in candidate["source_indices"]): index
|
| 132 |
+
for index, candidate in enumerate(candidates)
|
| 133 |
+
}
|
| 134 |
+
partition_groups = [frozenset(group) for group in base_partition if frozenset(group) in index_by_group]
|
| 135 |
+
top1_index = len(LATTICE_FEATURE_NAMES)
|
| 136 |
+
segments = []
|
| 137 |
+
for group in partition_groups:
|
| 138 |
+
index = index_by_group[group]
|
| 139 |
+
segments.append(_recognized_segment(
|
| 140 |
+
_group_box(group, strokes), ocr_labels[index],
|
| 141 |
+
float(features[index][top1_index]), families[index],
|
| 142 |
+
))
|
| 143 |
+
trays = infer_math_trays(segments)
|
| 144 |
+
relations = infer_spatial_relations(segments, trays=trays)
|
| 145 |
+
structural_edges = [
|
| 146 |
+
(
|
| 147 |
+
partition_groups[int(relation["parent"])],
|
| 148 |
+
partition_groups[int(relation["child"])],
|
| 149 |
+
float(relation["confidence"]),
|
| 150 |
+
)
|
| 151 |
+
for relation in relations
|
| 152 |
+
if str(relation["type"]) in STRUCTURAL_RELATION_TYPES
|
| 153 |
+
]
|
| 154 |
+
penalties = np.zeros(count, dtype=np.float32)
|
| 155 |
+
for index, candidate in enumerate(candidates):
|
| 156 |
+
group = frozenset(int(value) for value in candidate["source_indices"])
|
| 157 |
+
if len(group) <= 1:
|
| 158 |
+
continue
|
| 159 |
+
penalties[index] = max(
|
| 160 |
+
(
|
| 161 |
+
confidence for parent, child, confidence in structural_edges
|
| 162 |
+
if group & parent and group & child
|
| 163 |
+
),
|
| 164 |
+
default=0.0,
|
| 165 |
+
)
|
| 166 |
+
return penalties
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _candidate_box(
|
| 170 |
+
candidate: dict[str, Any], group: frozenset[int], strokes: Sequence[dict[str, Any]],
|
| 171 |
+
) -> dict[str, float]:
|
| 172 |
+
"""ํ์ ๋ณ์: lattice ํ๋ณดยท์ stroke. ์๋ ์๋ฆฌ: cache bbox๋ฅผ ์ฐ์ ์ฌ์ฌ์ฉํ๊ณ ์ต์ ์
๋ ฅ์ ๊ณ์ฐ์ผ๋ก ํธํํ๋ค."""
|
| 173 |
+
|
| 174 |
+
box = candidate.get("box")
|
| 175 |
+
return dict(box) if isinstance(box, dict) else _group_box(group, strokes)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def _point_xy(point: Any) -> tuple[float, float]:
|
| 179 |
+
"""ํ์ ๋ณ์: ๋ฐฐ์ด ๋๋ ๊ฐ์ฒดํ point. ์๋ ์๋ฆฌ: ํ ํํ ๊ณ์ฐ์ฉ ์ขํ๋ฅผ ๊ณตํต ํ์์ผ๋ก ์ฝ๋๋ค."""
|
| 180 |
+
|
| 181 |
+
if isinstance(point, dict):
|
| 182 |
+
return float(point.get("x", 0.0)), float(point.get("y", 0.0))
|
| 183 |
+
return float(point[0]), float(point[1])
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def _stroke_direction(stroke: dict[str, Any]) -> tuple[float, float]:
|
| 187 |
+
"""ํ์ ๋ณ์: ๋ ์ ์ด์์ธ stroke. ์๋ ์๋ฆฌ: ์ฒซยท๋์ ๋ฒกํฐ๋ฅผ ๋ฐฉํฅ ์ ๊ทํํด ํํยท๊ต์ฐจ ํํ๋ฅผ ํ์ ํ๋ค."""
|
| 188 |
+
|
| 189 |
+
points = stroke.get("points") or []
|
| 190 |
+
if len(points) < 2:
|
| 191 |
+
return 0.0, 0.0
|
| 192 |
+
start, end = _point_xy(points[0]), _point_xy(points[-1])
|
| 193 |
+
dx, dy = end[0] - start[0], end[1] - start[1]
|
| 194 |
+
length = math.hypot(dx, dy)
|
| 195 |
+
return (dx / length, dy / length) if length > 1e-6 else (0.0, 0.0)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _completed_operand_context_score(
|
| 199 |
+
group: frozenset[int], group_box: dict[str, float], context: Sequence[tuple[frozenset[int], dict[str, Any]]],
|
| 200 |
+
) -> float:
|
| 201 |
+
"""ํ์ ๋ณ์: ์ค์ ํ๋ณด bboxยท์ด๊ธฐ partition. ์๋ ์๋ฆฌ: ์ข๋ณยท์ฐ๋ณ์ด ๋ชจ๋ ์๋ฃ๋ ํํ์ผ ๋๋ง ๊ตฌ์กฐ ์ ์๋ฅผ ์ฐ๋ค."""
|
| 202 |
+
|
| 203 |
+
usable = [(other_group, segment) for other_group, segment in context if not other_group & group]
|
| 204 |
+
left, right = horizontal_sides(group_box, usable)
|
| 205 |
+
return 1.0 if segments_complete(left) and segments_complete(right) else 0.0
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def _occupied_operand_context_score(
|
| 209 |
+
group: frozenset[int], group_box: dict[str, float], context: Sequence[tuple[frozenset[int], dict[str, Any]]],
|
| 210 |
+
) -> float:
|
| 211 |
+
"""ํ์ ๋ณ์: ๊ต์ฐจ์ ํ๋ณดยทํ์ฌ partition. ์๋ ์๋ฆฌ: ๊ณฑ์
์ OCR ๋ฌธ๋ฒ์ ๊ฐ์ ํ์ง ์๊ณ ๊ฐ์ ํ์ ์์ชฝ ์ ์ ๋ง ํ์ธํ๋ค."""
|
| 212 |
+
|
| 213 |
+
usable = [(other_group, segment) for other_group, segment in context if not other_group & group]
|
| 214 |
+
left, right = horizontal_sides(group_box, usable)
|
| 215 |
+
return 1.0 if left and right else 0.0
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def _infix_shape_score(
|
| 219 |
+
group: frozenset[int], strokes: Sequence[dict[str, Any]], stroke_boxes: Sequence[dict[str, float]],
|
| 220 |
+
) -> tuple[str, float]:
|
| 221 |
+
"""ํ์ ๋ณ์: ๋ ํ ํ๋ณดยท์ฌ์ ๊ณ์ฐ bbox. ์๋ ์๋ฆฌ: ํํ์ equality์ ๊ต์ฐจ์ multiply family๋ฅผ ๋ถ๋ฆฌํด ๋ฐํํ๋ค."""
|
| 222 |
+
|
| 223 |
+
if len(group) != 2:
|
| 224 |
+
return "", 0.0
|
| 225 |
+
first_index, second_index = sorted(group)
|
| 226 |
+
first, second = strokes[first_index], strokes[second_index]
|
| 227 |
+
first_direction, second_direction = _stroke_direction(first), _stroke_direction(second)
|
| 228 |
+
horizontal = min(abs(first_direction[0]), abs(second_direction[0]))
|
| 229 |
+
vertical_leak = max(abs(first_direction[1]), abs(second_direction[1]))
|
| 230 |
+
first_box, second_box = stroke_boxes[first_index], stroke_boxes[second_index]
|
| 231 |
+
first_width = max(first_box["right"] - first_box["left"], 1e-6)
|
| 232 |
+
second_width = max(second_box["right"] - second_box["left"], 1e-6)
|
| 233 |
+
x_overlap = max(0.0, min(first_box["right"], second_box["right"]) - max(first_box["left"], second_box["left"]))
|
| 234 |
+
overlap_ratio = x_overlap / min(first_width, second_width)
|
| 235 |
+
length_ratio = min(first_width, second_width) / max(first_width, second_width)
|
| 236 |
+
first_y = (first_box["top"] + first_box["bottom"]) * 0.5
|
| 237 |
+
second_y = (second_box["top"] + second_box["bottom"]) * 0.5
|
| 238 |
+
separation_ratio = abs(first_y - second_y) / max((first_width + second_width) * 0.5, 1e-6)
|
| 239 |
+
parallel = abs(first_direction[0] * second_direction[0] + first_direction[1] * second_direction[1])
|
| 240 |
+
if (
|
| 241 |
+
horizontal >= 0.92 and vertical_leak <= 0.38 and parallel >= 0.94
|
| 242 |
+
and overlap_ratio >= 0.60 and length_ratio >= 0.60 and 0.04 <= separation_ratio <= 0.65
|
| 243 |
+
):
|
| 244 |
+
return "equality", min(1.0, 0.35 + 0.25 * overlap_ratio + 0.20 * length_ratio + 0.20 * parallel)
|
| 245 |
+
|
| 246 |
+
diagonal = min(abs(first_direction[0] * first_direction[1]), abs(second_direction[0] * second_direction[1])) * 2.0
|
| 247 |
+
opposite_slopes = first_direction[0] * first_direction[1] * second_direction[0] * second_direction[1] < 0.0
|
| 248 |
+
x_intersects = max(first_box["left"], second_box["left"]) <= min(first_box["right"], second_box["right"])
|
| 249 |
+
y_intersects = max(first_box["top"], second_box["top"]) <= min(first_box["bottom"], second_box["bottom"])
|
| 250 |
+
if opposite_slopes and x_intersects and y_intersects and diagonal >= 0.55:
|
| 251 |
+
return "multiply", min(1.0, diagonal)
|
| 252 |
+
return "", 0.0
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def candidate_signals(
|
| 256 |
+
strokes: Sequence[dict[str, Any]], candidates: Sequence[dict[str, Any]],
|
| 257 |
+
ocr_labels: Sequence[str], features: np.ndarray, base_partition: Sequence[frozenset[int]],
|
| 258 |
+
*, ocr_families: Sequence[str] | None = None, strict_equality: bool = False,
|
| 259 |
+
equality_model: EqualityVisualModel | None = None, cross_model: CrossVisualModel | None = None,
|
| 260 |
+
cross_gap_ratio: float = 0.40,
|
| 261 |
+
multistroke_family_boost: float = 6.0,
|
| 262 |
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
| 263 |
+
"""ํ์ ๋ณ์: ์ strokeยทํ๋ณด/label/featureยทcross/family ์ค์ . ์๋ ์๋ฆฌ: Trayยท์ ๋ขฐ labelยท์ค์์ evidence๋ฅผ ๊ณ์ฐํ๋ค."""
|
| 264 |
+
|
| 265 |
+
count = len(candidates)
|
| 266 |
+
if len(ocr_labels) != count or len(features) != count:
|
| 267 |
+
raise ValueError("Tray joint candidate/label/feature ์๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 268 |
+
if cross_gap_ratio < 0.0:
|
| 269 |
+
raise ValueError("cross gap ratio๋ 0 ์ด์์ด์ด์ผ ํฉ๋๋ค.")
|
| 270 |
+
if multistroke_family_boost < 0.0:
|
| 271 |
+
raise ValueError("multistroke family boost๋ 0 ์ด์์ด์ด์ผ ํฉ๋๋ค.")
|
| 272 |
+
families = list(ocr_families or [""] * count)
|
| 273 |
+
if len(families) != count:
|
| 274 |
+
raise ValueError("Tray joint candidate/family ์๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 275 |
+
tray_signal = np.zeros(count, dtype=np.float32)
|
| 276 |
+
symbol_signal = np.zeros(count, dtype=np.float32)
|
| 277 |
+
infix_signal = np.zeros(count, dtype=np.float32)
|
| 278 |
+
top1 = features[:, len(LATTICE_FEATURE_NAMES)]
|
| 279 |
+
stroke_boxes = [_group_box(frozenset({index}), strokes) for index in range(len(strokes))]
|
| 280 |
+
index_by_group = {
|
| 281 |
+
frozenset(int(value) for value in candidate["source_indices"]): index
|
| 282 |
+
for index, candidate in enumerate(candidates)
|
| 283 |
+
}
|
| 284 |
+
pair_groups = [
|
| 285 |
+
frozenset(int(value) for value in candidate["source_indices"])
|
| 286 |
+
for candidate in candidates if len(candidate["source_indices"]) == 2
|
| 287 |
+
]
|
| 288 |
+
equality_probability: dict[frozenset[int], float] = {}
|
| 289 |
+
cross_probability: dict[frozenset[int], float] = {}
|
| 290 |
+
if equality_model is not None and pair_groups:
|
| 291 |
+
ordered_pairs = [tuple(sorted(group)) for group in pair_groups]
|
| 292 |
+
pair_rows = equality_pair_feature_rows(ordered_pairs, strokes)
|
| 293 |
+
equality_probability = {
|
| 294 |
+
group: equality_model.probability(row) for group, row in zip(pair_groups, pair_rows, strict=True)
|
| 295 |
+
}
|
| 296 |
+
if cross_model is not None and pair_groups:
|
| 297 |
+
nonzero_heights = [box["bottom"] - box["top"] for box in stroke_boxes if box["bottom"] - box["top"] > 1e-5]
|
| 298 |
+
reference = float(np.median(nonzero_heights)) if nonzero_heights else 1.0
|
| 299 |
+
plausible_groups = []
|
| 300 |
+
for group in pair_groups:
|
| 301 |
+
first_index, second_index = sorted(group)
|
| 302 |
+
first_box, second_box = stroke_boxes[first_index], stroke_boxes[second_index]
|
| 303 |
+
x_gap = max(first_box["left"] - second_box["right"], second_box["left"] - first_box["right"], 0.0)
|
| 304 |
+
y_gap = max(first_box["top"] - second_box["bottom"], second_box["top"] - first_box["bottom"], 0.0)
|
| 305 |
+
if x_gap <= reference * cross_gap_ratio and y_gap <= reference * cross_gap_ratio:
|
| 306 |
+
plausible_groups.append(group)
|
| 307 |
+
ordered_pairs = [tuple(sorted(group)) for group in plausible_groups]
|
| 308 |
+
pair_rows = cross_pair_feature_rows(ordered_pairs, strokes)
|
| 309 |
+
cross_probability = {
|
| 310 |
+
group: cross_model.probability(row) for group, row in zip(plausible_groups, pair_rows, strict=True)
|
| 311 |
+
}
|
| 312 |
+
context = [
|
| 313 |
+
(
|
| 314 |
+
group,
|
| 315 |
+
_recognized_segment(
|
| 316 |
+
_candidate_box(candidates[index_by_group[group]], group, strokes),
|
| 317 |
+
ocr_labels[index_by_group[group]], float(top1[index_by_group[group]]), families[index_by_group[group]],
|
| 318 |
+
),
|
| 319 |
+
)
|
| 320 |
+
for group in base_partition
|
| 321 |
+
]
|
| 322 |
+
for index, candidate in enumerate(candidates):
|
| 323 |
+
group = frozenset(int(value) for value in candidate["source_indices"])
|
| 324 |
+
label = ocr_labels[index]
|
| 325 |
+
group_box = _candidate_box(candidate, group, strokes)
|
| 326 |
+
shape_family, shape_score = _infix_shape_score(group, strokes, stroke_boxes)
|
| 327 |
+
learned_equality = equality_probability.get(group, 0.0)
|
| 328 |
+
learned_cross = cross_probability.get(group, 0.0)
|
| 329 |
+
if equality_model is not None and learned_equality >= equality_model.threshold:
|
| 330 |
+
shape_family, shape_score = "equality", learned_equality
|
| 331 |
+
if cross_model is not None and learned_cross >= cross_model.threshold and learned_cross > learned_equality:
|
| 332 |
+
# threshold ์ง์๋จ์ ๋ถํ์คํ pair๊ฐ ํฐ ๋ณํฉ ๋ณด๋์ค๋ฅผ ๋ฐ์ง ์๋๋ก ์ด๊ณผ margin๋ง ์ฌ์ฉํ๋ค.
|
| 333 |
+
margin = (learned_cross - cross_model.threshold) / max(1.0 - cross_model.threshold, 1e-6)
|
| 334 |
+
shape_family, shape_score = "cross_visual", min(1.0, max(0.0, margin))
|
| 335 |
+
if shape_family == "equality":
|
| 336 |
+
# ๋ฑํธ ์ธ์์ ์์ ์์ฑ ์ฌ๋ถ์ ๋
๋ฆฝ์ ์ด๋ค. parser๋ shadow ๋ถ์์์๋ง gatingํ๋ค.
|
| 337 |
+
context_score = _completed_operand_context_score(group, group_box, context) if strict_equality else 1.0
|
| 338 |
+
elif shape_family == "cross_visual":
|
| 339 |
+
context_score = 1.0
|
| 340 |
+
else:
|
| 341 |
+
context_score = _occupied_operand_context_score(group, group_box, context)
|
| 342 |
+
infix_signal[index] = shape_score * context_score
|
| 343 |
+
if len(group) > 1:
|
| 344 |
+
if label in RELIABLE_MULTISTROKE:
|
| 345 |
+
symbol_signal[index] = float(top1[index])
|
| 346 |
+
if (
|
| 347 |
+
label in MULTISTROKE_FAMILY_LABELS
|
| 348 |
+
or families[index] in MULTISTROKE_FAMILY_NAMES
|
| 349 |
+
) and _multistroke_family_geometry_guard(
|
| 350 |
+
label, families[index], features[index],
|
| 351 |
+
):
|
| 352 |
+
# ํฉ์น OCR family๊ฐ ์ด๋ฏธ ์ด์ ์์ ๋๋ง ์ถ๊ฐํ๋ค. ์ธ์ ํ์ด๋ผ๋ ์ด์ ๋ง์ผ๋ก ๋ณํฉํ์ง ์๋๋ค.
|
| 353 |
+
symbol_signal[index] += float(top1[index]) * multistroke_family_boost
|
| 354 |
+
tray_type = ANCHOR_TYPES.get(label)
|
| 355 |
+
if tray_type is None:
|
| 356 |
+
continue
|
| 357 |
+
anchor = _recognized_segment(_group_box(group, strokes), label, float(top1[index]), families[index])
|
| 358 |
+
segments = [anchor, *(segment for other_group, segment in context if not other_group & group)]
|
| 359 |
+
matches = (
|
| 360 |
+
tray for tray in infer_math_trays(segments)
|
| 361 |
+
if tray["anchor"] == 0 and tray["type"] == tray_type and tray["required_satisfied"]
|
| 362 |
+
)
|
| 363 |
+
tray_signal[index] = max((float(tray["constraint_score"]) for tray in matches), default=0.0)
|
| 364 |
+
return tray_signal, symbol_signal, infix_signal
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def adjusted_logits(
|
| 368 |
+
base_logits: np.ndarray, tray_signal: np.ndarray, symbol_signal: np.ndarray,
|
| 369 |
+
fraction_penalty: np.ndarray, weights: TrayJointWeights, infix_signal: np.ndarray | None = None,
|
| 370 |
+
competition_penalty: np.ndarray | None = None,
|
| 371 |
+
local_baseline_penalty: np.ndarray | None = None,
|
| 372 |
+
) -> np.ndarray:
|
| 373 |
+
"""ํ์ ๋ณ์: base logitยท๊ตฌ์กฐ signalยท์ธ penalty. ์๋ ์๋ฆฌ: joint selector ๋ชฉ์ ํจ์์ candidate logit์ ๋ง๋ ๋ค."""
|
| 374 |
+
|
| 375 |
+
effective_infix = np.zeros_like(base_logits) if infix_signal is None else infix_signal
|
| 376 |
+
effective_competition = (
|
| 377 |
+
np.zeros_like(base_logits) if competition_penalty is None else competition_penalty
|
| 378 |
+
)
|
| 379 |
+
effective_local_baseline = (
|
| 380 |
+
np.zeros_like(base_logits) if local_baseline_penalty is None else local_baseline_penalty
|
| 381 |
+
)
|
| 382 |
+
arrays = (
|
| 383 |
+
base_logits, tray_signal, symbol_signal, fraction_penalty,
|
| 384 |
+
effective_infix, effective_competition, effective_local_baseline,
|
| 385 |
+
)
|
| 386 |
+
if len({len(values) for values in arrays}) != 1:
|
| 387 |
+
raise ValueError("Tray joint logit๊ณผ signal ๊ธธ์ด๊ฐ ๋ค๋ฆ
๋๋ค.")
|
| 388 |
+
return (
|
| 389 |
+
base_logits + weights.tray * tray_signal + weights.symbol * symbol_signal
|
| 390 |
+
+ weights.infix * effective_infix - weights.fraction * fraction_penalty
|
| 391 |
+
- weights.competition * effective_competition
|
| 392 |
+
- weights.local_baseline * effective_local_baseline
|
| 393 |
+
)
|