{ "package": "laya_multilingual_fp16_L256_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "e6afbab8c933fc317626764a872bbbede233faeb4591222b0742330d77b829f8", "Data/com.apple.CoreML/weights/weight.bin": "41e287a96c4260faec5c1d2aa37209666279a69541652ea450879e5853881d73", "Manifest.json": "4a04e53c9966399b9b1b38f93ac2688db2ef79a3b24d4866bdb93015c162c13a" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 256, "max_options": 32, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 1.829418750014156, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.0021350979804992676, "max_action_probability_error": 0.0, "latency_ms": { "p50": 5.186791997402906, "p95": 5.620151464245282, "min": 5.075542023405433, "max": 6.205584038980305 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0001417025923728943, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 5.240916507318616 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00014770030975341797, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998626708984375, "median_ms": 5.267270491458476 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005466341972351074, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 5.622666503768414 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00014688074588775635, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 5.3875620069447905 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0021350979804992676, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 5.193667020648718 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 5.608797073364258e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000602006912231, "median_ms": 5.3670419729314744 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0008405148983001709, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 5.145104514667764 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999732971191406, "median_ms": 5.1331460126675665 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997743368148804, "median_ms": 5.142499983776361 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00011944770812988281, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999786376953125, "median_ms": 5.46637549996376 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000178337097168, "median_ms": 5.178166495170444 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998912811279297, "median_ms": 5.163750000065193 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 157, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.0014334022998809814, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998855590820312, "median_ms": 5.150437500560656 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0011126399040222168, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 5.1465209980960935 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 174, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0001442432403564453, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000152587890625, "median_ms": 5.122436996316537 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 158, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0009380578994750977, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999969482421875, "median_ms": 5.1526665047276765 } ], "passed": true }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 4.439885583007708, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "latency_ms": { "p50": 9.878875018330291, "p95": 10.141766036394984, "min": 9.49920795392245, "max": 11.666958045680076 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0004076361656188965, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 9.773458004929125 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00034058094024658203, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0003662109375, "median_ms": 9.998270979849622 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0033596158027648926, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 10.107375041116029 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0014896541833877563, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 9.907895990181714 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0034949183464050293, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 9.904021018883213 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004321932792663574, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9995682239532471, "median_ms": 9.766937495442107 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.001600891351699829, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99981689453125, "median_ms": 9.736333013279364 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999771118164062, "median_ms": 9.809916489757597 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999773383140564, "median_ms": 9.79193698731251 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004093870520591736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0002899169921875, "median_ms": 9.886458021355793 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001769065856934, "median_ms": 9.888791479170322 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 9.867791522992775 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 157, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.001290060579776764, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998397827148438, "median_ms": 9.76256252033636 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 9.754520520800725 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 174, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0003440380096435547, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999664306640625, "median_ms": 9.926104015903547 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 158, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0005267858505249023, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999908447265625, "median_ms": 9.970562008675188 } ], "passed": true } }, "passed": true }