{ "package": "laya_multilingual_w8enc_L128_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "5879590bd22681a203478520fa46233185dc14bead3c7fb3c542ca50cb7f46f3", "Data/com.apple.CoreML/weights/weight.bin": "badc17e87d0f012690822536e23a99420ea255e3315f11bebed0a13a3f4bff3b", "Manifest.json": "1254ad9d54bbd2c0bbb6a7a407e20b138a159b3ccc6e8e622e9490bf0463c90c" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 128, "max_options": 32, "precision": "w8enc", "package_bytes": 534276639, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 1.2120079589658417, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.03180044889450073, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.898020484484732, "p95": 5.602850587456487, "min": 3.751541953533888, "max": 11.005959007889032 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0001634731888771057, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999755859375, "median_ms": 4.372228984721005 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0006970074027776718, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 3.9413750055246055 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.007510006427764893, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.8934164913371205 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.006471037864685059, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.8704585167579353 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.03180044889450073, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.8540005043614656 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00015507836360484362, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999006986618042, "median_ms": 3.916291519999504 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.009751647710800171, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.8382704951800406 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0002784738317131996, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0002403259277344, "median_ms": 6.378958001732826 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00025540590286254883, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001884698867798, "median_ms": 4.003583511803299 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0023829713463783264, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99993896484375, "median_ms": 3.9288749685510993 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000187873840332, "median_ms": 3.880603995639831 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0009060502052307129, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998207092285156, "median_ms": 3.8895210018381476 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.02412015199661255, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000040054321289, "median_ms": 3.889312531100586 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.01963934302330017, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999755859375, "median_ms": 3.8546455034520477 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0010694805532693863, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.8585829897783697 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.014614537358283997, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001907348632812, "median_ms": 3.9253539871424437 } ], "passed": false }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 3.062132292019669, "questions": 16, "argmax_agreements": 11, "max_probability_error": 0.6651095747947693, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.6724585224874318, "p95": 3.8027315720682964, "min": 3.6111659719608724, "max": 3.8808330427855253 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": false, "coreml_argmax": 1, "reference_argmax": 0, "max_probability_error": 0.4756666123867035, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.727874980540946 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.17544066905975342, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999755859375, "median_ms": 3.653666499303654 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": false, "coreml_argmax": 2, "reference_argmax": 1, "max_probability_error": 0.0822121649980545, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.651562496088445 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.22170449793338776, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 3.6659374891314656 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.14068716764450073, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 3.668145975098014 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.6149341464042664, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.6606459761969745 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": false, "coreml_argmax": 2, "reference_argmax": 1, "max_probability_error": 0.29022228717803955, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9993896484375, "median_ms": 3.6990624794270843 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": false, "coreml_argmax": 1, "reference_argmax": 0, "max_probability_error": 0.537635862827301, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.6667500098701566 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997236728668213, "median_ms": 3.6832710029557347 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.23302960395812988, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99951171875, "median_ms": 3.757104481337592 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.5689128041267395, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.7534795119427145 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": false, "coreml_argmax": 2, "reference_argmax": 0, "max_probability_error": 0.6651095747947693, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.655479522421956 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.34367769956588745, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9994659423828125, "median_ms": 3.664686984848231 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.04918035864830017, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.648479003459215 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.43704086542129517, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99951171875, "median_ms": 3.730521013494581 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.10460180044174194, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 3.674479521578178 } ], "passed": false } }, "passed": false }