{ "package": "laya_multilingual_fp16_L512_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "a9efa197dc468765e5074399dd9cbad131052e8f241654aefeea390d1faa57ac", "Data/com.apple.CoreML/weights/weight.bin": "b232c8f3e17396c1077b42633c3098daa5bf94b27668accd22042ee7a49e0b49", "Manifest.json": "b75c4c456a85d54c38eda45ebe88560e208079d0eec16d568a39abebfbc6be04" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 512, "max_options": 32, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 2.4458846659981646, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.0021350979804992676, "max_action_probability_error": 0.0, "latency_ms": { "p50": 9.00429199100472, "p95": 9.368602131144144, "min": 8.766749990172684, "max": 13.464874995406717 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0001417025923728943, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 9.15208351216279 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00014770030975341797, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998626708984375, "median_ms": 9.204000001773238 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005466341972351074, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 9.063625504495576 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00014688074588775635, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 9.09689551917836 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0021350979804992676, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 9.05550000607036 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 5.608797073364258e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000602006912231, "median_ms": 8.988271001726389 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0008405148983001709, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 9.089228988159448 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999732971191406, "median_ms": 8.93558349343948 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997743368148804, "median_ms": 8.88558299629949 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00011944770812988281, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999786376953125, "median_ms": 8.887521020369604 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000178337097168, "median_ms": 9.004458494018763 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998912811279297, "median_ms": 8.942167012719437 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 157, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.0014334022998809814, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998855590820312, "median_ms": 9.010562498588115 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0011126399040222168, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 8.911187498597428 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 174, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0001442432403564453, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000152587890625, "median_ms": 8.846083976095542 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 158, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0009380578994750977, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999969482421875, "median_ms": 9.015603980515152 } ], "passed": true }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 4.784726334037259, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "latency_ms": { "p50": 27.451770496554673, "p95": 27.900547938770615, "min": 27.101749961730093, "max": 30.147792014759034 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0004076361656188965, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 27.682187472237274 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00034058094024658203, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0003662109375, "median_ms": 27.74883297388442 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0033596158027648926, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 27.54443750018254 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0014896541833877563, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 27.727770997444168 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0034949183464050293, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 27.766666520619765 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004321932792663574, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9995682239532471, "median_ms": 27.386936999391764 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.001600891351699829, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99981689453125, "median_ms": 27.639291482046247 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999771118164062, "median_ms": 27.325041504809633 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999773383140564, "median_ms": 27.280833513941616 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004093870520591736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0002899169921875, "median_ms": 27.29837500373833 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001769065856934, "median_ms": 27.201541524846107 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 27.596458472544327 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 157, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.001290060579776764, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998397827148438, "median_ms": 27.25768749951385 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 27.227271028095856 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 174, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0003440380096435547, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999664306640625, "median_ms": 27.25012498558499 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 158, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0005267858505249023, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999908447265625, "median_ms": 27.342583489371464 } ], "passed": true } }, "passed": true }