| { |
| "purpose": "Full published synthetic test; no fitting, filtering, resampling or test-based tuning", |
| "created_utc": "2026-09-19T22:07:40.987091+00:00", |
| "dataset": { |
| "dataset_repository": "cua-ai/cua-s1-forms", |
| "dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", |
| "path": "artifacts/test.jsonl", |
| "url": "https://huggingface.co/datasets/cua-ai/cua-s1-forms/resolve/8273f34778b99ac2e12d9f6e7d57dad99ae20845/test.jsonl", |
| "sha256": "d63a7e0db195d4d20154a40b2f8dd09ce3bb65487a158c638da5c609d4475e7c", |
| "rows": 24370, |
| "bytes": 18710924, |
| "upstream_split_claim": "Synthetic test, form-signature-disjoint from train/validation. Train and validation signatures have not been independently re-audited here.", |
| "model_card_claim": { |
| "synthetic_test_top1": 0.9995, |
| "approximate_decisions": 15000, |
| "note": "The released file has 24370 rows. No matching full result ledger or exact 15000-row manifest is supplied by the linked model card." |
| } |
| }, |
| "manifest_sha256": "e582b951ecc07809a0b06a2bc0790827b90a86ce57c70d5b6177ef04f0b4f1ca", |
| "input_census": { |
| "rows": 24370, |
| "unique_episode_seeds": 1040, |
| "maxima": { |
| "context_bytes": 217, |
| "option_bytes": 92, |
| "max_options": 32 |
| }, |
| "per_action": { |
| "check": 816, |
| "click": 1040, |
| "fill": 9802, |
| "skip": 12712 |
| }, |
| "option_count_histogram": { |
| "7": 17, |
| "8": 46, |
| "9": 80, |
| "10": 227, |
| "11": 361, |
| "12": 589, |
| "13": 888, |
| "14": 1254, |
| "15": 1513, |
| "16": 1761, |
| "17": 2373, |
| "18": 2014, |
| "19": 2277, |
| "20": 2506, |
| "21": 2396, |
| "22": 1818, |
| "23": 1422, |
| "24": 907, |
| "25": 688, |
| "26": 524, |
| "27": 483, |
| "28": 89, |
| "29": 33, |
| "30": 29, |
| "31": 35, |
| "32": 40 |
| }, |
| "excluded_rows": 0, |
| "truncated_contexts": 0, |
| "truncated_options": 0 |
| }, |
| "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", |
| "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", |
| "conversions": { |
| "baseline": { |
| "model": "cua_s1_forms_fp16_options32.mlpackage", |
| "precision": "float16", |
| "minimum_target": "iOS17/macOS14", |
| "limits": { |
| "context_bytes": 224, |
| "option_bytes": 96, |
| "max_options": 32 |
| }, |
| "model_config": { |
| "context_tokens": 224, |
| "encoder": "tinyx", |
| "heads": 4, |
| "hf_model": "Qwen/Qwen2.5-0.5B", |
| "layers": 2, |
| "option_tokens": 96, |
| "rank": 128, |
| "width": 128 |
| }, |
| "parameters": 706048, |
| "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", |
| "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", |
| "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", |
| "trace_row": 0, |
| "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", |
| "export_seconds": 0.5374167499830946, |
| "python": "3.11.11", |
| "torch": "2.7.0", |
| "coremltools": "9.0", |
| "package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", |
| "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", |
| "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" |
| } |
| }, |
| "int8-weights": { |
| "model": "cua_s1_forms_int8_options32.mlpackage", |
| "precision": "int8_weights_float16_compute", |
| "minimum_target": "iOS17/macOS14", |
| "limits": { |
| "context_bytes": 224, |
| "option_bytes": 96, |
| "max_options": 32 |
| }, |
| "model_config": { |
| "context_tokens": 224, |
| "encoder": "tinyx", |
| "heads": 4, |
| "hf_model": "Qwen/Qwen2.5-0.5B", |
| "layers": 2, |
| "option_tokens": 96, |
| "rank": 128, |
| "width": 128 |
| }, |
| "parameters": 706048, |
| "assets_lock_sha256": "8e65ad70af6bb814b571cdcfe828ba4bc339147da2d2211cbeac416163ef18ba", |
| "model_revision": "f54adbf447f4ca6ec259f529ee3f2e3e09f8cc71", |
| "source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145", |
| "trace_row": 0, |
| "trace_dataset_revision": "8273f34778b99ac2e12d9f6e7d57dad99ae20845", |
| "export_seconds": 0.5374167499830946, |
| "python": "3.11.11", |
| "torch": "2.7.0", |
| "coremltools": "9.0", |
| "package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "a7d9747308b8b754a46f3e70c8465b821f90e12376b3a71dc909f7364ae73c3a", |
| "Data/com.apple.CoreML/weights/weight.bin": "f600c4434b7f7e3e789de8f1a4aa54a55f84df51fa9b788e18f5c97b68e4840f", |
| "Manifest.json": "2206ec01787a85c12f1376fa5a832c5ac912f06172c6ff06b26ae8415470d246" |
| }, |
| "quantization": { |
| "name": "int8-weights", |
| "settings": { |
| "mode": "linear_symmetric", |
| "dtype": "int8", |
| "granularity": "per_channel", |
| "weight_threshold": 2048 |
| }, |
| "activations": "float16", |
| "calibration": "None; weight-only, data-free quantization", |
| "source_conversion_sha256": "ac94239751ceb7e19602eb7276940ff53446b1776d4900405af9be0382fe4a94", |
| "source_package_files": { |
| "Data/com.apple.CoreML/model.mlmodel": "70485fc18cbb21785df833cbddddc0b5b59acb00d22394b76e55307e2c135dd0", |
| "Data/com.apple.CoreML/weights/weight.bin": "4da9259f798e44f5a1b50769ee1916fd3747c4d723dd9997b516c7fe238c7895", |
| "Manifest.json": "2bc0f5f62337b27fb6b0ecde248f1e3dc269e1ba4b65516aaeede2a60e293dcc" |
| }, |
| "source_package_bytes": 1511163, |
| "package_bytes": 813422, |
| "compression_seconds": 0.3180606249952689, |
| "operation_counts": { |
| "add": 15, |
| "cast": 13, |
| "concat": 2, |
| "const": 226, |
| "constexpr_affine_dequantize": 19, |
| "equal": 2, |
| "expand_dims": 5, |
| "fill_like": 3, |
| "gather": 2, |
| "greater_equal": 4, |
| "layer_norm": 8, |
| "linear": 15, |
| "logical_not": 2, |
| "matmul": 8, |
| "maximum": 1, |
| "mul": 7, |
| "not_equal": 3, |
| "real_div": 1, |
| "reduce_sum": 3, |
| "relu": 3, |
| "reshape": 31, |
| "select": 9, |
| "slice_by_index": 13, |
| "softmax": 5, |
| "squeeze": 3, |
| "tile": 2, |
| "transpose": 21 |
| }, |
| "script_sha256": "22071babfcb02ef77225203a479d9bb14b9fa35462ba38bcc4bf07d29c125a97" |
| } |
| } |
| }, |
| "environment": { |
| "machine": "Apple M5 Pro", |
| "memory_bytes": 25769803776, |
| "os": "27.0", |
| "os_build": "26A428", |
| "python": "3.11.11", |
| "torch": "2.7.0", |
| "coremltools": "9.0", |
| "torch_cpu_threads": 2, |
| "torch_interop_threads": 1, |
| "torch_mha_fastpath": false |
| }, |
| "protocol": { |
| "batch_size": 1, |
| "warmups_per_model": 3, |
| "timed_passes": 1, |
| "coreml_compute_units": "CPU_AND_NE", |
| "coreml_order": "AB on even rows, BA on odd rows", |
| "timing": "PyTorch CPU forward+softmax / synchronous Core ML predict; excludes encoding, validation, GUI and network", |
| "preprocessing": "Upstream variable-length collator; independent fixed-shape Core ML encoder shared by both exports", |
| "load_note": "Process/model creation can use existing system caches; not a cold-start experiment", |
| "output_audit": "Imperfect finite probability sums are retained unchanged and fail the original normalization gate", |
| "calibration_note": "NLL/ECE use raw emitted scores; no renormalization or calibration was applied" |
| }, |
| "thresholds": { |
| "argmax_agreement": 1.0, |
| "max_abs_probability_error": 0.005, |
| "allow_accuracy_loss": false, |
| "probability_sum_atol": 0.001, |
| "probability_sum_rtol": 1e-05 |
| }, |
| "host_preprocessing_ms": { |
| "upstream_collator_ms": { |
| "count": 24370, |
| "median_ms": 0.09233300806954503, |
| "p95_ms": 0.12889755889773366, |
| "min_ms": 0.04445898230187595, |
| "max_ms": 1.0104170069098473 |
| }, |
| "shared_coreml_encoding_ms": { |
| "count": 24370, |
| "median_ms": 0.027417001547291875, |
| "p95_ms": 0.03995798761025071, |
| "min_ms": 0.012666976545006037, |
| "max_ms": 0.49662499804981053 |
| } |
| }, |
| "results": { |
| "upstream_pytorch": { |
| "correct": 24359, |
| "rows": 24370, |
| "accuracy": 0.999548625359048, |
| "model_call_latency_ms": { |
| "count": 24370, |
| "median_ms": 1.7357920005451888, |
| "p95_ms": 2.984285591810476, |
| "min_ms": 1.0271249921061099, |
| "max_ms": 30.681957985507324 |
| }, |
| "load_ms": 10.63050000811927, |
| "first_call_ms": 5.652791995089501, |
| "calibration": { |
| "nll": 0.0021302032038990836, |
| "ece": 0.00035450826133425167, |
| "equal_width_bins": 15, |
| "bins": [ |
| { |
| "bin": 7, |
| "count": 2, |
| "accuracy": 0.5, |
| "mean_confidence": 0.5216051042079926 |
| }, |
| { |
| "bin": 8, |
| "count": 5, |
| "accuracy": 0.8, |
| "mean_confidence": 0.575071656703949 |
| }, |
| { |
| "bin": 9, |
| "count": 7, |
| "accuracy": 0.7142857142857143, |
| "mean_confidence": 0.6404258779117039 |
| }, |
| { |
| "bin": 10, |
| "count": 4, |
| "accuracy": 1.0, |
| "mean_confidence": 0.6963506639003754 |
| }, |
| { |
| "bin": 11, |
| "count": 11, |
| "accuracy": 0.9090909090909091, |
| "mean_confidence": 0.7664791941642761 |
| }, |
| { |
| "bin": 12, |
| "count": 21, |
| "accuracy": 0.9523809523809523, |
| "mean_confidence": 0.8310647464933849 |
| }, |
| { |
| "bin": 13, |
| "count": 22, |
| "accuracy": 0.9545454545454546, |
| "mean_confidence": 0.9173552746122534 |
| }, |
| { |
| "bin": 14, |
| "count": 24298, |
| "accuracy": 0.9998353773973166, |
| "mean_confidence": 0.999802232897422 |
| } |
| ], |
| "nll_probability_floor": 1.1754943508222875e-38 |
| }, |
| "upstream_action_metrics": { |
| "examples": 24370, |
| "accuracy": 0.999548625359048, |
| "coverage": 0.4788264259335248, |
| "abstention_rate": 0.5211735740664751, |
| "selective_accuracy": 0.9990573313908647, |
| "wrong_action_rate": 0.00045137464095199015, |
| "wrong_target_rate": 0.0, |
| "unsafe_action_rate": 0.00045137464095199015, |
| "counts": { |
| "total": 24370, |
| "correct": 24359, |
| "abstained": 12701, |
| "acted": 11669, |
| "acted_correct": 11658, |
| "wrong_action": 11, |
| "unsafe_action": 11 |
| }, |
| "per_action": { |
| "abstain": { |
| "accuracy": 0.9991346758967904, |
| "abstention_rate": 0.9991346758967904, |
| "examples": 12712 |
| }, |
| "check": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 816 |
| }, |
| "click": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 1040 |
| }, |
| "fill": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 9802 |
| } |
| } |
| }, |
| "macro_action_accuracy": 0.9997836689741976, |
| "wrong_rows": [ |
| 137, |
| 3216, |
| 6631, |
| 6748, |
| 7280, |
| 10703, |
| 12255, |
| 13480, |
| 15214, |
| 21022, |
| 23671 |
| ] |
| }, |
| "baseline": { |
| "correct": 24359, |
| "rows": 24370, |
| "accuracy": 0.999548625359048, |
| "model_call_latency_ms": { |
| "count": 24370, |
| "median_ms": 0.9897080017253757, |
| "p95_ms": 1.1015420022886246, |
| "min_ms": 0.884292006958276, |
| "max_ms": 3.03837499814108 |
| }, |
| "load_ms": 640.2017919754144, |
| "first_call_ms": 2.522958006011322, |
| "calibration": { |
| "nll": 0.002108235388127147, |
| "ece": 0.000331678613561794, |
| "equal_width_bins": 15, |
| "bins": [ |
| { |
| "bin": 7, |
| "count": 2, |
| "accuracy": 0.5, |
| "mean_confidence": 0.5224609375 |
| }, |
| { |
| "bin": 8, |
| "count": 5, |
| "accuracy": 0.8, |
| "mean_confidence": 0.57607421875 |
| }, |
| { |
| "bin": 9, |
| "count": 6, |
| "accuracy": 0.8333333333333334, |
| "mean_confidence": 0.637451171875 |
| }, |
| { |
| "bin": 10, |
| "count": 5, |
| "accuracy": 0.8, |
| "mean_confidence": 0.69248046875 |
| }, |
| { |
| "bin": 11, |
| "count": 12, |
| "accuracy": 0.9166666666666666, |
| "mean_confidence": 0.7687581380208334 |
| }, |
| { |
| "bin": 12, |
| "count": 20, |
| "accuracy": 0.95, |
| "mean_confidence": 0.8322265625 |
| }, |
| { |
| "bin": 13, |
| "count": 22, |
| "accuracy": 0.9545454545454546, |
| "mean_confidence": 0.9176136363636364 |
| }, |
| { |
| "bin": 14, |
| "count": 24298, |
| "accuracy": 0.9998353773973166, |
| "mean_confidence": 0.9998245660008025 |
| } |
| ], |
| "nll_probability_floor": 1.1754943508222875e-38 |
| }, |
| "upstream_action_metrics": { |
| "examples": 24370, |
| "accuracy": 0.999548625359048, |
| "coverage": 0.4788264259335248, |
| "abstention_rate": 0.5211735740664751, |
| "selective_accuracy": 0.9990573313908647, |
| "wrong_action_rate": 0.00045137464095199015, |
| "wrong_target_rate": 0.0, |
| "unsafe_action_rate": 0.00045137464095199015, |
| "counts": { |
| "total": 24370, |
| "correct": 24359, |
| "abstained": 12701, |
| "acted": 11669, |
| "acted_correct": 11658, |
| "wrong_action": 11, |
| "unsafe_action": 11 |
| }, |
| "per_action": { |
| "abstain": { |
| "accuracy": 0.9991346758967904, |
| "abstention_rate": 0.9991346758967904, |
| "examples": 12712 |
| }, |
| "check": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 816 |
| }, |
| "click": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 1040 |
| }, |
| "fill": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 9802 |
| } |
| } |
| }, |
| "macro_action_accuracy": 0.9997836689741976, |
| "wrong_rows": [ |
| 137, |
| 3216, |
| 6631, |
| 6748, |
| 7280, |
| 10703, |
| 12255, |
| 13480, |
| 15214, |
| 21022, |
| 23671 |
| ], |
| "paired_with_upstream": { |
| "rows": 24370, |
| "argmax_agreement": 24370, |
| "agreement_rate": 1.0, |
| "reference_correct_candidate_wrong": 0, |
| "reference_wrong_candidate_correct": 0, |
| "both_wrong": 11, |
| "disagreement_rows": [] |
| }, |
| "max_abs_probability_error": 0.020487427711486816, |
| "mean_row_max_abs_probability_error": 4.673297645567538e-05, |
| "probability_tolerance_violation_rows": [ |
| 137, |
| 884, |
| 2945, |
| 3216, |
| 8706, |
| 9193, |
| 12725, |
| 13480, |
| 13768, |
| 15698, |
| 23992 |
| ], |
| "conversion_parity_passed": false, |
| "output_validation_failures": { |
| "19270": [ |
| "Live option probabilities do not sum to one" |
| ] |
| } |
| }, |
| "int8-weights": { |
| "correct": 24359, |
| "rows": 24370, |
| "accuracy": 0.999548625359048, |
| "model_call_latency_ms": { |
| "count": 24370, |
| "median_ms": 0.9902919991873205, |
| "p95_ms": 1.103792006324511, |
| "min_ms": 0.8826660050544888, |
| "max_ms": 49.30691700428724 |
| }, |
| "load_ms": 615.115666994825, |
| "first_call_ms": 1.328666985500604, |
| "calibration": { |
| "nll": 0.002145651629876076, |
| "ece": 0.00048729947809817673, |
| "equal_width_bins": 15, |
| "bins": [ |
| { |
| "bin": 7, |
| "count": 4, |
| "accuracy": 0.75, |
| "mean_confidence": 0.520751953125 |
| }, |
| { |
| "bin": 8, |
| "count": 4, |
| "accuracy": 1.0, |
| "mean_confidence": 0.576904296875 |
| }, |
| { |
| "bin": 9, |
| "count": 6, |
| "accuracy": 0.8333333333333334, |
| "mean_confidence": 0.6383463541666666 |
| }, |
| { |
| "bin": 10, |
| "count": 3, |
| "accuracy": 0.3333333333333333, |
| "mean_confidence": 0.7058919270833334 |
| }, |
| { |
| "bin": 11, |
| "count": 15, |
| "accuracy": 0.9333333333333333, |
| "mean_confidence": 0.7711263020833333 |
| }, |
| { |
| "bin": 12, |
| "count": 15, |
| "accuracy": 0.9333333333333333, |
| "mean_confidence": 0.8310546875 |
| }, |
| { |
| "bin": 13, |
| "count": 26, |
| "accuracy": 1.0, |
| "mean_confidence": 0.9116023137019231 |
| }, |
| { |
| "bin": 14, |
| "count": 24297, |
| "accuracy": 0.9997942132773593, |
| "mean_confidence": 0.9998235539624234 |
| } |
| ], |
| "nll_probability_floor": 1.1754943508222875e-38 |
| }, |
| "upstream_action_metrics": { |
| "examples": 24370, |
| "accuracy": 0.999548625359048, |
| "coverage": 0.4788264259335248, |
| "abstention_rate": 0.5211735740664751, |
| "selective_accuracy": 0.9990573313908647, |
| "wrong_action_rate": 0.00045137464095199015, |
| "wrong_target_rate": 0.0, |
| "unsafe_action_rate": 0.00045137464095199015, |
| "counts": { |
| "total": 24370, |
| "correct": 24359, |
| "abstained": 12701, |
| "acted": 11669, |
| "acted_correct": 11658, |
| "wrong_action": 11, |
| "unsafe_action": 11 |
| }, |
| "per_action": { |
| "abstain": { |
| "accuracy": 0.9991346758967904, |
| "abstention_rate": 0.9991346758967904, |
| "examples": 12712 |
| }, |
| "check": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 816 |
| }, |
| "click": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 1040 |
| }, |
| "fill": { |
| "accuracy": 1.0, |
| "abstention_rate": 0.0, |
| "examples": 9802 |
| } |
| } |
| }, |
| "macro_action_accuracy": 0.9997836689741976, |
| "wrong_rows": [ |
| 137, |
| 3216, |
| 6631, |
| 6748, |
| 7280, |
| 10703, |
| 12255, |
| 13480, |
| 15214, |
| 21022, |
| 23671 |
| ], |
| "paired_with_upstream": { |
| "rows": 24370, |
| "argmax_agreement": 24370, |
| "agreement_rate": 1.0, |
| "reference_correct_candidate_wrong": 0, |
| "reference_wrong_candidate_correct": 0, |
| "both_wrong": 11, |
| "disagreement_rows": [] |
| }, |
| "max_abs_probability_error": 0.0677376389503479, |
| "mean_row_max_abs_probability_error": 8.935863356371478e-05, |
| "probability_tolerance_violation_rows": [ |
| 137, |
| 410, |
| 1225, |
| 2076, |
| 2093, |
| 2818, |
| 2945, |
| 3064, |
| 3151, |
| 3216, |
| 3298, |
| 3479, |
| 3756, |
| 3978, |
| 4469, |
| 5294, |
| 5759, |
| 6426, |
| 6466, |
| 7177, |
| 7515, |
| 8243, |
| 8706, |
| 8714, |
| 8891, |
| 9193, |
| 9253, |
| 9977, |
| 9981, |
| 10703, |
| 11122, |
| 11257, |
| 11880, |
| 12725, |
| 12754, |
| 13370, |
| 13423, |
| 13693, |
| 13768, |
| 14656, |
| 15132, |
| 15214, |
| 15252, |
| 15698, |
| 16253, |
| 16920, |
| 16951, |
| 17304, |
| 18339, |
| 18549, |
| 18802, |
| 19308, |
| 19669, |
| 19957, |
| 20462, |
| 20476, |
| 21844, |
| 21871, |
| 22336, |
| 22714, |
| 22791, |
| 23276, |
| 23670, |
| 23992 |
| ], |
| "conversion_parity_passed": false, |
| "output_validation_failures": {} |
| } |
| }, |
| "failures": [ |
| { |
| "row": 137, |
| "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"Full name\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill Last: Novak", |
| "baseline": "fill Last: Novak", |
| "int8-weights": "fill Last: Novak" |
| } |
| }, |
| { |
| "row": 410, |
| "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Date of loss\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 884, |
| "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Health plan\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 1225, |
| "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Vehicle identification number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 2076, |
| "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Graduation year\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 2093, |
| "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Title\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 2818, |
| "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Your email\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 2945, |
| "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Organization\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 3064, |
| "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Make\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 3151, |
| "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Google Chrome\nELEMENT Edit \"Date of loss\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 3216, |
| "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Google Chrome\nELEMENT Edit \"Email\" value=\"oivanova66@yahoo.com\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill Email address: oivanova66@yahoo.com", |
| "baseline": "fill Email address: oivanova66@yahoo.com", |
| "int8-weights": "fill Email address: oivanova66@yahoo.com" |
| } |
| }, |
| { |
| "row": 3298, |
| "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Mozilla Firefox\nELEMENT Edit \"DL number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 3479, |
| "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Insurance company\" value=\"Centene\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 3756, |
| "context": "TASK fill the form from the document, then submit\nFORM Page 2 of 3\nELEMENT Edit \"Claim #\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 3978, |
| "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - Microsoft Edge\nELEMENT Edit \"Family doctor\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 4469, |
| "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"City\" value=\"Portland\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 5294, |
| "context": "TASK fill the form from the document, then submit\nFORM Untitled\nELEMENT Edit \"Email\" value=\"nadia.fischer@fastmail.com\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 5759, |
| "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form\nELEMENT Edit \"Relationship to patient\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 6426, |
| "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Microsoft Edge\nELEMENT CheckBox \"Consent to electronic communication (required)\" unchecked", |
| "gold_option": "check", |
| "selected_options": { |
| "upstream_pytorch": "check", |
| "baseline": "check", |
| "int8-weights": "check" |
| } |
| }, |
| { |
| "row": 6466, |
| "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - JevBrowser\nELEMENT Edit \"Details of loss\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 6631, |
| "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Given name\" value=\"Priya\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill Physician: Dr. Fischer", |
| "baseline": "fill Physician: Dr. Fischer", |
| "int8-weights": "fill Physician: Dr. Fischer" |
| } |
| }, |
| { |
| "row": 6748, |
| "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"State / Province\" value=\"NE\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill ST: NE", |
| "baseline": "fill ST: NE", |
| "int8-weights": "fill ST: NE" |
| } |
| }, |
| { |
| "row": 7177, |
| "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Claim #\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 7280, |
| "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Given name\" value=\"Tomas\" hint=\"First\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill First name: Tomas", |
| "baseline": "fill First name: Tomas", |
| "int8-weights": "fill First name: Tomas" |
| } |
| }, |
| { |
| "row": 7515, |
| "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"Province\" value=\"GA\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 8243, |
| "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Passport number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 8706, |
| "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Mozilla Firefox\nELEMENT CheckBox \"I authorize verification of the information provided\" unchecked", |
| "gold_option": "check", |
| "selected_options": { |
| "upstream_pytorch": "check", |
| "baseline": "check", |
| "int8-weights": "check" |
| } |
| }, |
| { |
| "row": 8714, |
| "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Registration number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 8891, |
| "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Google Chrome\nELEMENT Edit \"Last\" value=\"Ivanova\" hint=\"Family name\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 9193, |
| "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding\nELEMENT Edit \"DL number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 9253, |
| "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"Homepage\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 9977, |
| "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Country\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 9981, |
| "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"City\" value=\"Burlington\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 10703, |
| "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - JevBrowser\nELEMENT Edit \"Last\" value=\"Silva\" hint=\"Family name\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill Family name: Silva", |
| "baseline": "fill Family name: Silva", |
| "int8-weights": "fill Family name: Silva" |
| } |
| }, |
| { |
| "row": 11122, |
| "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"EIN\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 11257, |
| "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Microsoft Edge\nELEMENT CheckBox \"I am legally authorized to work in this country\" unchecked", |
| "gold_option": "check", |
| "selected_options": { |
| "upstream_pytorch": "check", |
| "baseline": "check", |
| "int8-weights": "check" |
| } |
| }, |
| { |
| "row": 11880, |
| "context": "TASK fill the form from the document, then submit\nFORM Blue Sun Freight - Vendor Onboarding - Google Chrome\nELEMENT Edit \"Date of loss\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 12255, |
| "context": "TASK fill the form from the document, then submit\nFORM Globex Insurance - Auto Claim Form - JevBrowser\nELEMENT Edit \"Taxpayer identification number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill National ID: 813-20-3361", |
| "baseline": "fill National ID: 813-20-3361", |
| "int8-weights": "fill National ID: 813-20-3361" |
| } |
| }, |
| { |
| "row": 12725, |
| "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Company\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 12754, |
| "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration - Microsoft Edge\nELEMENT CheckBox \"I consent to treatment and agree to the privacy policy\" checked", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 13370, |
| "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application - Microsoft Edge\nELEMENT Edit \"Mailing address\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 13423, |
| "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Current job title\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 13480, |
| "context": "TASK fill the form from the document, then submit\nFORM Evergreen Rentals - Tenant Application\nELEMENT Edit \"Current employer\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill Company reg: REG3878384", |
| "baseline": "fill Company reg: REG3878384", |
| "int8-weights": "fill Company reg: REG3878384" |
| } |
| }, |
| { |
| "row": 13693, |
| "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Mozilla Firefox\nELEMENT Edit \"Organization\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 13768, |
| "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"Legal name\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 14656, |
| "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Google Chrome\nELEMENT Edit \"Details of loss\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 15132, |
| "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Mozilla Firefox\nELEMENT Edit \"Town\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 15214, |
| "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Relation to applicant\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill Reg plate: WN9-8147", |
| "baseline": "fill Reg plate: WN9-8147", |
| "int8-weights": "fill Reg plate: WN9-8147" |
| } |
| }, |
| { |
| "row": 15252, |
| "context": "TASK fill the form from the document, then submit\nFORM Northwind Clinic - New Patient Registration\nELEMENT Edit \"VIN number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 15698, |
| "context": "TASK fill the form from the document, then submit\nFORM Orbit Telecom - Account Transfer - Mozilla Firefox\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 16253, |
| "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application\nELEMENT Edit \"Relation to applicant\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 16920, |
| "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application - Google Chrome\nELEMENT Edit \"Total years experience\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 16951, |
| "context": "TASK fill the form from the document, then submit\nFORM Form\nELEMENT Edit \"State\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 17304, |
| "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request - Google Chrome\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 18339, |
| "context": "TASK fill the form from the document, then submit\nFORM Summit Bank - Direct Deposit Enrollment - Mozilla Firefox\nELEMENT Edit \"Last name\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 18549, |
| "context": "TASK fill the form from the document, then submit\nFORM Lakeside Vet - Pet Owner Registration - Microsoft Edge\nELEMENT Edit \"ABA routing number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 18802, |
| "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment\nELEMENT Edit \"Unit number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 19270, |
| "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Plate number\" value=\"old\"", |
| "gold_option": "fill License plate: SJ3-1115", |
| "selected_options": { |
| "upstream_pytorch": "fill License plate: SJ3-1115", |
| "baseline": "fill License plate: SJ3-1115", |
| "int8-weights": "fill License plate: SJ3-1115" |
| } |
| }, |
| { |
| "row": 19308, |
| "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up - Google Chrome\nELEMENT Edit \"Last\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 19669, |
| "context": "TASK fill the form from the document, then submit\nFORM Acme Robotics - Job Application\nELEMENT Edit \"School\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 19957, |
| "context": "TASK fill the form from the document, then submit\nFORM Skyline Travel - Visa Assistance Form\nELEMENT Edit \"Plate number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 20462, |
| "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - Mozilla Firefox\nELEMENT CheckBox \"I am legally authorized to work in this country\" checked", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 20476, |
| "context": "TASK fill the form from the document, then submit\nFORM JevBrowser\nELEMENT Edit \"Sort code\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 21022, |
| "context": "TASK fill the form from the document, then submit\nFORM Harbor University - Graduate Application - JevBrowser\nELEMENT Edit \"State\" value=\"NE\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill National ID: 794-03-1212", |
| "baseline": "fill National ID: 794-03-1212", |
| "int8-weights": "fill National ID: 794-03-1212" |
| } |
| }, |
| { |
| "row": 21844, |
| "context": "TASK fill the form from the document, then submit\nFORM Cedar Pharmacy - Prescription Transfer - Mozilla Firefox\nELEMENT Edit \"Gender\" value=\"Female\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 21871, |
| "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake\nELEMENT Edit \"Sort code\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 22336, |
| "context": "TASK fill the form from the document, then submit\nFORM City Library - Membership Sign-up\nELEMENT Edit \"Car model\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 22714, |
| "context": "TASK fill the form from the document, then submit\nFORM Metro Utilities - New Service Request\nELEMENT Edit \"Describe your concern\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 22791, |
| "context": "TASK fill the form from the document, then submit\nFORM Riverside Dental - Patient Intake - Mozilla Firefox\nELEMENT CheckBox \"Consent to electronic communication (required)\" checked", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 23276, |
| "context": "TASK fill the form from the document, then submit\nFORM Pinewood Camp - Parent Consent Form - JevBrowser\nELEMENT Edit \"Relationship status\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 23670, |
| "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Member number\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| }, |
| { |
| "row": 23671, |
| "context": "TASK fill the form from the document, then submit\nFORM \nELEMENT Edit \"Email\" value=\"lchen6@yahoo.com\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "fill Email: lchen6@yahoo.com", |
| "baseline": "fill Email: lchen6@yahoo.com", |
| "int8-weights": "fill Email: lchen6@yahoo.com" |
| } |
| }, |
| { |
| "row": 23992, |
| "context": "TASK fill the form from the document, then submit\nFORM Apex Gym - Member Enrollment - Microsoft Edge\nELEMENT Edit \"Total years experience\" value=\"\"", |
| "gold_option": "skip", |
| "selected_options": { |
| "upstream_pytorch": "skip", |
| "baseline": "skip", |
| "int8-weights": "skip" |
| } |
| } |
| ], |
| "trace": { |
| "file": "int8-synthetic-test-decisions.jsonl.gz", |
| "sha256": "69720b9e11430ecff7ea0718f14bf16792de6f39f114faba250cfe855e4e4156", |
| "rows": 24370 |
| }, |
| "upstream_evaluator_sha256": "49dccdeb4e2b456187cdc4491a6d78fabac6a45384272f8e3ec802a58872aa6e", |
| "harness_sha256": { |
| "benchmark-synthetic.py": "891da8408156e2b80849b4d799c77c9a31c091dfd999fcd044639deefac3e888", |
| "synthetic_test.py": "e2b68725236e6827eb422480c8cc8aa189b2490c1594650d8f2928771d1bc626", |
| "score-report.py": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e" |
| }, |
| "conversion_parity_passed": false, |
| "limitations": [ |
| "The release file has 24370 rows; the model card claims approximately 15000. This is not a reconstruction of that unspecified manifest.", |
| "Upstream describes form-disjoint synthetic data; this is not an unseen real-world GUI or task-completion benchmark.", |
| "No fresh hosted Jev calls are included. Published hosted-Jev percentages are not a matched baseline for this run.", |
| "Local timings use CPU for PyTorch and CPU+ANE for Core ML; they are backend/deployment measurements, not equal-hardware algorithm comparisons." |
| ] |
| } |
|
|
