{ "package": "laya_multilingual_e8_L256_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "d16209d9b4654f87f20d9a5cbe7d8ee508c045ab3d9261d1bfc9d09392f0ade9", "Data/com.apple.CoreML/weights/weight.bin": "1cf24b8e0a545a1ee18a881ae1d9ff175eefc144d9da2e4674d2bd219be15a50", "Manifest.json": "9af9f12550091738ae8b873e4de9de5dd80e948f477bdbb3ca408875a0d4562c" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 256, "max_options": 32, "precision": "e8", "package_bytes": 448664434, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 6.394701374985743, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.006270706653594971, "max_action_probability_error": 0.0, "latency_ms": { "p50": 5.778561986517161, "p95": 6.196189246838912, "min": 5.249582987744361, "max": 7.637374976184219 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0025831088423728943, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 5.771229014499113 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004168841987848282, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999237060546875, "median_ms": 5.793874501250684 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0018501728773117065, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 5.76785349403508 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0011222213506698608, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 5.752563010901213 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.006270706653594971, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 5.8532295224722475 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 5.608797073364258e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000249147415161, "median_ms": 5.717207997804508 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.001695007085800171, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 5.461541499244049 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00012970063835382462, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000091552734375, "median_ms": 5.813874973682687 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00025540590286254883, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0002155303955078, "median_ms": 5.779874976724386 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0003025755286216736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 5.733354017138481 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001797676086426, "median_ms": 5.7553540100343525 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0004576537758111954, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 5.898542003706098 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 157, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.0022539570927619934, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 5.863646016223356 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.004014343023300171, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999755859375, "median_ms": 5.812771007185802 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 174, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0003440380096435547, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998931884765625, "median_ms": 5.861729005118832 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 158, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0014263391494750977, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 5.755438032792881 } ], "passed": true }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 8.776065583981108, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.013535827398300171, "max_action_probability_error": 0.0, "latency_ms": { "p50": 9.623270976589993, "p95": 9.869244421133772, "min": 9.491292003076524, "max": 12.103457993362099 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0022779330611228943, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 9.617979492759332 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005542133003473282, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999786376953125, "median_ms": 9.594645991455764 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0025046318769454956, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 9.62677100324072 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0018546432256698608, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 9.69827052904293 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.006285488605499268, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99951171875, "median_ms": 9.767146519152448 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004321932792663574, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9995424151420593, "median_ms": 9.632125002099201 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0009625852108001709, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 9.647416474763304 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 9.155366569757462e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000534057617188, "median_ms": 9.634103975258768 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997386932373047, "median_ms": 9.819416998652741 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005619749426841736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0004425048828125, "median_ms": 9.638833027565852 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001797676086426, "median_ms": 9.553520503686741 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0009060502052307129, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0005512237548828, "median_ms": 9.560791979311034 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 157, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.0027727559208869934, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 9.57918749190867 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.013535827398300171, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99951171875, "median_ms": 9.776624996447936 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 174, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0003440380096435547, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999847412109375, "median_ms": 9.59639551001601 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 158, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0009380578994750977, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 9.579125500749797 } ], "passed": true } }, "passed": true }