| { |
| "purpose": "Stable Swift softmax validation; no model changes, training or new timing claims", |
| "dataset": { |
| "dataset_repository": "cua-ai/cua-s1-forms", |
| "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", |
| "path": "artifacts/test.jsonl", |
| "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/test.jsonl", |
| "sha256": "d63a7e0db195d4d20154a40b2f8dd09ce3bb65487a158c638da5c609d4475e7c", |
| "rows": 24370, |
| "bytes": 18710924, |
| "upstream_split_claim": "Synthetic test, form-signature-disjoint from train/validation. Train and validation signatures have not been independently re-audited here.", |
| "model_card_claim": { |
| "synthetic_test_top1": 0.9995, |
| "approximate_decisions": 15000, |
| "note": "The released file has 24370 rows. No matching full result ledger or exact 15000-row manifest is supplied by the linked model card." |
| } |
| }, |
| "conversions": { |
| "fp16": { |
| "model": "cua_s1_forms_fp16_options32.mlpackage", |
| "precision": "float16", |
| "minimum_target": "iOS17/macOS14", |
| "limits": { |
| "context_bytes": 224, |
| "option_bytes": 96, |
| "max_options": 32 |
| }, |
| "model_config": { |
| "context_tokens": 224, |
| "encoder": "tinyx", |
| "heads": 4, |
| "hf_model": "Qwen/Qwen2.5-0.5B", |
| "layers": 2, |
| "option_tokens": 96, |
| "rank": 128, |
| "width": 128 |
| }, |
| "parameters": 706048, |
| "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", |
| "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", |
| "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", |
| "trace_row": 0, |
| "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", |
| "export_seconds": 0.5374167499830946, |
| "python": "3.11.11", |
| "torch": "2.7.0", |
| "coremltools": "9.0", |
| "package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", |
| "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", |
| "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" |
| } |
| }, |
| "int8": { |
| "model": "cua_s1_forms_int8_options32.mlpackage", |
| "precision": "int8_weights_float16_compute", |
| "minimum_target": "iOS17/macOS14", |
| "limits": { |
| "context_bytes": 224, |
| "option_bytes": 96, |
| "max_options": 32 |
| }, |
| "model_config": { |
| "context_tokens": 224, |
| "encoder": "tinyx", |
| "heads": 4, |
| "hf_model": "Qwen/Qwen2.5-0.5B", |
| "layers": 2, |
| "option_tokens": 96, |
| "rank": 128, |
| "width": 128 |
| }, |
| "parameters": 706048, |
| "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", |
| "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", |
| "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", |
| "trace_row": 0, |
| "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", |
| "export_seconds": 0.5374167499830946, |
| "python": "3.11.11", |
| "torch": "2.7.0", |
| "coremltools": "9.0", |
| "package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a", |
| "Data/com.apple.CoreML/weights/weight.bin": "f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f", |
| "Manifest.json": "2206ec01787a85c12f1376fa5a832c5ac912f06172c6ff06b26ae8415470d246" |
| }, |
| "quantization": { |
| "name": "int8-weights", |
| "settings": { |
| "mode": "linear_symmetric", |
| "dtype": "int8", |
| "granularity": "per_channel", |
| "weight_threshold": 2048 |
| }, |
| "activations": "float16", |
| "calibration": "None; weight-only, data-free quantization", |
| "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", |
| "source_package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", |
| "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", |
| "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" |
| }, |
| "source_package_bytes": 1511163, |
| "package_bytes": 813422, |
| "compression_seconds": 0.3180606249952689, |
| "operation_counts": { |
| "add": 15, |
| "cast": 13, |
| "concat": 2, |
| "const": 226, |
| "constexpr_affine_dequantize": 19, |
| "equal": 2, |
| "expand_dims": 5, |
| "fill_like": 3, |
| "gather": 2, |
| "greater_equal": 4, |
| "layer_norm": 8, |
| "linear": 15, |
| "logical_not": 2, |
| "matmul": 8, |
| "maximum": 1, |
| "mul": 7, |
| "not_equal": 3, |
| "real_div": 1, |
| "reduce_sum": 3, |
| "relu": 3, |
| "reshape": 31, |
| "select": 9, |
| "slice_by_index": 13, |
| "softmax": 5, |
| "squeeze": 3, |
| "tile": 2, |
| "transpose": 21 |
| }, |
| "script_sha256": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97" |
| } |
| }, |
| "int4": { |
| "model": "cua_s1_forms_int4_options32.mlpackage", |
| "precision": "int4_weights_float16_compute", |
| "minimum_target": "iOS18/macOS15", |
| "limits": { |
| "context_bytes": 224, |
| "option_bytes": 96, |
| "max_options": 32 |
| }, |
| "model_config": { |
| "context_tokens": 224, |
| "encoder": "tinyx", |
| "heads": 4, |
| "hf_model": "Qwen/Qwen2.5-0.5B", |
| "layers": 2, |
| "option_tokens": 96, |
| "rank": 128, |
| "width": 128 |
| }, |
| "parameters": 706048, |
| "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", |
| "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", |
| "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", |
| "trace_row": 0, |
| "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", |
| "export_seconds": 0.5374167499830946, |
| "python": "3.11.11", |
| "torch": "2.7.0", |
| "coremltools": "9.0", |
| "package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "cadad6bfa6875383a7ee3de68d0d22871148aa21cf117e7fe44c2c971287c4e5", |
| "Data/com.apple.CoreML/weights/weight.bin": "0f766fcb9dbd612ed4f6c7a40117a5993a4789b3ae94afba0df37193c831f63c", |
| "Manifest.json": "fb91c8b5ec3168111567de663bb3140d2790c6e7c48f32e0df3f5135af0673af" |
| }, |
| "target_upgrade": { |
| "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", |
| "source_package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", |
| "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", |
| "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" |
| }, |
| "method": "Load original MIL at specification 9; empty pass pipeline; retain decomposed attention", |
| "operation_counts": { |
| "add": 13, |
| "cast": 11, |
| "concat": 2, |
| "const": 239, |
| "equal": 2, |
| "expand_dims": 5, |
| "fill_like": 3, |
| "gather": 2, |
| "greater_equal": 2, |
| "layer_norm": 8, |
| "linear": 15, |
| "logical_not": 2, |
| "matmul": 8, |
| "maximum": 1, |
| "mul": 7, |
| "not_equal": 3, |
| "real_div": 1, |
| "reduce_sum": 3, |
| "relu": 3, |
| "reshape": 31, |
| "select": 7, |
| "slice_by_index": 13, |
| "softmax": 5, |
| "squeeze": 3, |
| "tile": 2, |
| "transpose": 21 |
| }, |
| "script_sha256": "33db3295450428d90c1de5a804ab47bf3507c686fbbdef67cd276ce116758b5f" |
| }, |
| "quantization": { |
| "name": "int4-weights", |
| "settings": { |
| "mode": "linear_symmetric", |
| "dtype": "int4", |
| "granularity": "per_channel", |
| "weight_threshold": 2048 |
| }, |
| "activations": "float16", |
| "calibration": "None; weight-only, data-free quantization", |
| "source_conversion_sha256": "01105dc8bef7a30fc3430e150caa2a143dc0ba23a295b6280de3b8ff3b40af55", |
| "source_package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "6ab2df320df7081480b76876df0f4bfd2c841592d93f354c568ba504f7269627", |
| "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", |
| "Manifest.json": "e5b1c7dc66b2014b3985c180a75548a709788c6e314d9d2ae14e4e6d441c3c68" |
| }, |
| "source_package_bytes": 1511080, |
| "package_bytes": 451595, |
| "compression_seconds": 0.2996572910051327, |
| "packed_int4_tensors": 19, |
| "operation_counts": { |
| "add": 15, |
| "cast": 13, |
| "concat": 2, |
| "const": 226, |
| "constexpr_blockwise_shift_scale": 19, |
| "equal": 2, |
| "expand_dims": 5, |
| "fill_like": 3, |
| "gather": 2, |
| "greater_equal": 4, |
| "layer_norm": 8, |
| "linear": 15, |
| "logical_not": 2, |
| "matmul": 8, |
| "maximum": 1, |
| "mul": 7, |
| "not_equal": 3, |
| "real_div": 1, |
| "reduce_sum": 3, |
| "relu": 3, |
| "reshape": 31, |
| "select": 9, |
| "slice_by_index": 13, |
| "softmax": 5, |
| "squeeze": 3, |
| "tile": 2, |
| "transpose": 21 |
| }, |
| "script_sha256": "a91636e539d8934253b5e7628ca706346fbee538fa1aa9e3eaf794f440d49dcc" |
| } |
| } |
| }, |
| "references": { |
| "fp16": { |
| "report_sha256": "2ae85cfeb270b4f31be4f1d9082f4c73b3b9b084d6f1ddb661712a87d46b0f59", |
| "trace_sha256": "b6f6edc9de34fb37b10847a1f9568b530ce2b8e1bd241f51bf2e637f489d6554" |
| }, |
| "int8": { |
| "report_sha256": "952ebe281d0aef0c05667d3c0dc5f9f175be709639a542028940059393078b5d", |
| "trace_sha256": "69720b9e11430ecff7ea0718f14bf16792de6f39f114faba250cfe855e4e4156" |
| }, |
| "int4": { |
| "report_sha256": "9d6a691aeb2ddc352ab7c12327472c92814991c201b67c7da8db39a76c972b32", |
| "trace_sha256": "6922913041f95ac8872958b92312cb5b869b9ce88116c4684b36ea7753284674" |
| } |
| }, |
| "results": { |
| "fp16": { |
| "rows": 24370, |
| "correct": 24359, |
| "changed_choices": [], |
| "raw_choice_changes": [], |
| "raw_sum_failures": [ |
| 19270 |
| ], |
| "runtime_sum_failures": [], |
| "max_probability_change": 0.0010759234428405762, |
| "max_softmax_reference_error": 2.9801512702043453e-08, |
| "accuracy": 0.999548625359048, |
| "passed": true |
| }, |
| "int8": { |
| "rows": 24370, |
| "correct": 24359, |
| "changed_choices": [], |
| "raw_choice_changes": [], |
| "raw_sum_failures": [], |
| "runtime_sum_failures": [], |
| "max_probability_change": 0.0008352398872375488, |
| "max_softmax_reference_error": 2.980182234324502e-08, |
| "accuracy": 0.999548625359048, |
| "passed": true |
| }, |
| "int4": { |
| "rows": 24370, |
| "correct": 24353, |
| "changed_choices": [], |
| "raw_choice_changes": [], |
| "raw_sum_failures": [], |
| "runtime_sum_failures": [], |
| "max_probability_change": 0.0008838176727294922, |
| "max_softmax_reference_error": 2.9797999623326632e-08, |
| "accuracy": 0.9993024210094378, |
| "passed": true |
| } |
| }, |
| "passed": true, |
| "swift_source_sha256": { |
| "CuaS1FormsInput.swift": "b9ad5d74bac35f24155e3250bd08b18b812a308417f5a356bbb77bd8d78cb93c", |
| "CuaS1FormsManager.swift": "bb852f1f8f4d95eafbe04319e95a9be225052a02898db49913d8e1dd83ad036e", |
| "CuaS1FormsOutput.swift": "7f6c3225321a8eedb70af2cb8620e1e84b53e46b6af5f3908498e65ff50f8bd8", |
| "CuaS1FormsPackage.swift": "2a61b156523a42293cc1b077fd18dd064bdbd470b9f0404a093b138127b8276a", |
| "CuaS1FormsTypes.swift": "3aad906f508a699fb11684aba8a1f40325af33b3846bcd81b5ec32d16d0e1aa6" |
| }, |
| "harness_sha256": { |
| "validate-swift-runtime.py": "c06ca16c8b381a5bb08c9404cf6b6b500cc6102d43470beaa594a2542c9c87ce", |
| "swift-runtime-check.swift": "0ade2d3450e79adeb3380aad00ea854fd845b69f822aab3a7de16e580542ff59" |
| }, |
| "trace": { |
| "file": "decisions.jsonl.gz", |
| "sha256": "5d7ae5df86489b38e8f97167ab58c1d6bf5fb74c171a57b00d818b05d6c772b5", |
| "records": 73110 |
| }, |
| "limitations": [ |
| "Original raw conversion-parity failures remain in the unchanged benchmarks.", |
| "INT4 still has its previously measured accuracy loss.", |
| "CPU+ANE on the local Mac; no iPhone or older-OS validation." |
| ] |
| } |
|
|
