{ "package": "laya_multilingual_w8e_L128_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "6b64793cb28ca42ff8fb485ac9f4d3fffff6b83f7154ad74c833651697e70704", "Data/com.apple.CoreML/weights/weight.bin": "7e8f717e9ae24f9e7849ae6411df0d20b4d1e4d2090511597210f3e59652b8f1", "Manifest.json": "cc4e600199e5f1220eaf701ccfd436c0ff191219155fdb84182f9442dc498289" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 128, "max_options": 32, "precision": "w8e", "package_bytes": 519369445, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 0.3818642499973066, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.029359042644500732, "max_action_probability_error": 0.0, "latency_ms": { "p50": 4.109186993446201, "p95": 7.231489886180504, "min": 3.7559160264208913, "max": 29.032292019110173 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 8.066743612289429e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 4.101333499420434 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0006817486137151718, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000457763671875, "median_ms": 4.100457997992635 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.005801022052764893, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.924895980162546 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.005982756614685059, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.843708516797051 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.029359042644500732, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 8.767875522607937 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00013147492427378893, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99992436170578, "median_ms": 4.394833522383124 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.009995788335800171, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99981689453125, "median_ms": 3.9516875112894922 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00015640351921319962, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001182556152344, "median_ms": 4.1966664721257985 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00025540590286254883, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000189185142517, "median_ms": 4.051833035191521 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0020015016198158264, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998321533203125, "median_ms": 4.216625005938113 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000185251235962, "median_ms": 4.073020973009989 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0009060502052307129, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998893737792969, "median_ms": 4.18681200244464 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.02070218324661255, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997787475585938, "median_ms": 4.019979474833235 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.02378973364830017, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 4.726208513602614 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0010694805532693863, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 4.220062546664849 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.015346959233283997, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001373291015625, "median_ms": 4.182229022262618 } ], "passed": false }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 0.05620520899537951, "questions": 16, "argmax_agreements": 11, "max_probability_error": 0.6655978560447693, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.6921665305271745, "p95": 3.90408328094054, "min": 3.6110419896431267, "max": 6.501417024992406 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": false, "coreml_argmax": 1, "reference_argmax": 0, "max_probability_error": 0.4751783013343811, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99951171875, "median_ms": 3.7319585098885 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.17641723155975342, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9993896484375, "median_ms": 3.688833530759439 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": false, "coreml_argmax": 2, "reference_argmax": 1, "max_probability_error": 0.0827004462480545, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0006103515625, "median_ms": 3.7430415104608983 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.22194863855838776, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.673228988191113 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.14068716764450073, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.666521020932123 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.6146900057792664, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00048828125, "median_ms": 3.6692084977403283 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": false, "coreml_argmax": 2, "reference_argmax": 1, "max_probability_error": 0.28802502155303955, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99951171875, "median_ms": 3.764437511563301 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": false, "coreml_argmax": 1, "reference_argmax": 0, "max_probability_error": 0.5356826782226562, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.9228539972100407 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997237324714661, "median_ms": 3.771416493691504 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.23254135251045227, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.784479486057535 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.5684245228767395, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.7379164714366198 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": false, "coreml_argmax": 2, "reference_argmax": 0, "max_probability_error": 0.6655978560447693, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.66466652485542 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.34282320737838745, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99945068359375, "median_ms": 3.648666985100135 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.04918032884597778, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.6654994764830917 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.43704086542129517, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999755859375, "median_ms": 3.6537709820549935 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.10411351919174194, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.6801249952986836 } ], "passed": false } }, "passed": false }