{ "package": "laya_multilingual_w8head_L128_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "4cd189517c286d7bb7b302194f9ce47856c4d37ccd3c4130d9481d79d4b38a22", "Data/com.apple.CoreML/weights/weight.bin": "6af60903cde6e7c2f3d69b5a2c3256d4e1f8e4b01200b59f2b054f72c9ea831c", "Manifest.json": "2bcb3450a489648654b757599cf5165372e9cff7b19c46ed68e67bb7c709c17c" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 128, "max_options": 32, "precision": "w8head", "package_bytes": 629370771, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 1.3937852500239387, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.0021350979804992676, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.881749988067895, "p95": 4.301029801717959, "min": 3.7419579457491636, "max": 5.2461669547483325 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0002027377486228943, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.9013955101836473 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00014770030975341797, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998931884765625, "median_ms": 3.899583505699411 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005466341972351074, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.8867290131747723 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00026895105838775635, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.8466039986815304 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0021350979804992676, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.8555419887416065 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 5.608797073364258e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000054955482483, "median_ms": 3.81277097039856 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0008405148983001709, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.865333506837487 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 4.1962601244449615e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000038146972656, "median_ms": 3.893625020282343 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997756481170654, "median_ms": 4.3462289904709905 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00011944770812988281, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998931884765625, "median_ms": 3.9221664774231613 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001788139343262, "median_ms": 3.8190629857126623 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998931884765625, "median_ms": 3.8404999941121787 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.0006826519966125488, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000295639038086, "median_ms": 3.883999976096675 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0011126399040222168, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.891729487804696 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 6.240047514438629e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999908447265625, "median_ms": 3.8686875195708126 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0010861754417419434, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999313354492188, "median_ms": 3.864958503982052 } ], "passed": true }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 3.136163458984811, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.6686249950435013, "p95": 3.834012206061743, "min": 3.6094170063734055, "max": 3.9680000045336783 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0004076361656188965, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 3.7582289951387793 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00034058094024658203, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0004119873046875, "median_ms": 3.659229027107358 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0033596158027648926, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.649458463769406 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0018558651208877563, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.6580205196514726 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.003983199596405029, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.7758539838250726 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004321932792663574, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9995625019073486, "median_ms": 3.6599170125555247 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.001478821039199829, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.648062003776431 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999771118164062, "median_ms": 3.6501249705906957 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999778151512146, "median_ms": 3.6388119915500283 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004856809973716736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0003662109375, "median_ms": 3.67008350440301 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001773834228516, "median_ms": 3.835582989268005 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.757666505407542 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.004688441753387451, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0002021789550781, "median_ms": 3.6604584893211722 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00048828125, "median_ms": 3.680854511912912 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0003953631967306137, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0003662109375, "median_ms": 3.670312522444874 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0022311657667160034, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0003395080566406, "median_ms": 3.6394580092746764 } ], "passed": true } }, "passed": true }