{ "package": "laya_multilingual_fp16_L1024_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "88a745e53bdb53abdf600fa235fd7930ee8f9d59e7f1485f7402f036aaa17d77", "Data/com.apple.CoreML/weights/weight.bin": "a99aadbb5f3e3a6ad0ebf28df2027fdc9fae2717e2c7a2fb4c1e9ec8be1e9529", "Manifest.json": "a4cb1a599ac7ee392f09181ff0f340df4a05540b9ff1566f15ee35646fb11e43" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 1024, "max_options": 32, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 1.9747608749894425, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.002333313226699829, "max_action_probability_error": 0.0, "latency_ms": { "p50": 17.886937508592382, "p95": 18.32043749454897, "min": 17.64466700842604, "max": 23.31449999473989 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 8.064508438110352e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 17.863854009192437 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00014770030975341797, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999237060546875, "median_ms": 18.013292021350935 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0007520020008087158, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 17.79310347046703 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00046347081661224365, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 17.958333482965827 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00038102269172668457, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 17.863124987343326 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 5.608797073364258e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000487565994263, "median_ms": 17.887624504510313 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005022585391998291, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 17.897104495204985 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999771118164062, "median_ms": 17.81887552351691 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999762773513794, "median_ms": 17.968854488572106 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00012467056512832642, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999755859375, "median_ms": 17.823145462898538 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000178337097168, "median_ms": 17.794958519516513 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999675750732422, "median_ms": 17.900292004924268 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 157, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.000586189329624176, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999237060546875, "median_ms": 17.867436981759965 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.002333313226699829, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 17.970292014069855 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 174, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0001442432403564453, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 17.907041998114437 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 158, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.00044977664947509766, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 17.80535452417098 } ], "passed": true }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 6.936569582961965, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "latency_ms": { "p50": 80.06945901433937, "p95": 80.43368989310693, "min": 79.77154199033976, "max": 80.92245802981779 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0004076361656188965, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 80.10460447985679 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00034058094024658203, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0003662109375, "median_ms": 80.07691701641306 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0033596158027648926, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 80.06479148752987 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0014896541833877563, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 80.11591649847105 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0034949183464050293, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 80.14208299573511 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004321932792663574, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9995682239532471, "median_ms": 80.23072901414707 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.001600891351699829, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99981689453125, "median_ms": 80.22379150497727 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999771118164062, "median_ms": 80.06289551849477 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999773383140564, "median_ms": 80.02085401676595 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004093870520591736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0002899169921875, "median_ms": 79.99004202429205 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001769065856934, "median_ms": 80.35356248728931 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 80.08831250481308 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 157, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.001290060579776764, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998397827148438, "median_ms": 79.97343697934411 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 79.98441651579924 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 174, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0003440380096435547, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999664306640625, "median_ms": 80.0024795171339 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 158, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0005267858505249023, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999908447265625, "median_ms": 79.98839602805674 } ], "passed": true } }, "passed": true }