{ "package": "laya_multilingual_e8_L128_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "a702d6e956cfecd233b8e9573c2bb3a018c71c3fcac29d60868e5b01e7a6a6d3", "Data/com.apple.CoreML/weights/weight.bin": "ca1e5da71f1498eac7b68722c9f5c3d43d1ce299aaf0c29743585749f3cc9677", "Manifest.json": "fff2cdff70403a2b33aff04c4b93ba1d443873e796c3cd72cc8f59a82b8d96bc" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 128, "max_options": 32, "precision": "e8", "package_bytes": 448435058, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 6.075320333999116, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.015331089496612549, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.8387505046557635, "p95": 4.020178844803013, "min": 3.719458996783942, "max": 4.489124985411763 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0025831088423728943, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 3.8522914983332157 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004168841987848282, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999237060546875, "median_ms": 3.8293334946502 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0018501728773117065, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.821459016762674 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0011222213506698608, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.8383124920073897 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.006026566028594971, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.847000014502555 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 5.608797073364258e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000249147415161, "median_ms": 3.7663340044673532 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.001695007085800171, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.8036454934626818 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00012970063835382462, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000091552734375, "median_ms": 3.8716669951099902 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00025540590286254883, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0002155303955078, "median_ms": 3.9035830122884363 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0003025755286216736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 3.8823539798613638 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001797676086426, "median_ms": 3.782583517022431 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0004576537758111954, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.77122953068465 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.015331089496612549, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000648498535156, "median_ms": 3.791604482103139 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.004014343023300171, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999755859375, "median_ms": 3.802395978709683 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00012070499360561371, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000091552734375, "median_ms": 3.8319380255416036 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.004284918308258057, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997673034667969, "median_ms": 3.7830204819329083 } ], "passed": true }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 8.86458433396183, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.014354526996612549, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.638145513832569, "p95": 3.827884842758067, "min": 3.580958000384271, "max": 626.2272089952603 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0022779330611228943, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.7582709919661283 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005542133003473282, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999786376953125, "median_ms": 3.6189999955240637 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0025046318769454956, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 3.6314789904281497 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0018546432256698608, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 3.6308334674686193 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.006285488605499268, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99951171875, "median_ms": 3.6329374997876585 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004321932792663574, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9995424151420593, "median_ms": 3.7671044701710343 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0009625852108001709, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.6246664822101593 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 9.155366569757462e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000534057617188, "median_ms": 3.6396670329850167 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997386932373047, "median_ms": 3.6316874902695417 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005619749426841736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0004425048828125, "median_ms": 3.7328749895095825 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001797676086426, "median_ms": 3.6238130123820156 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0009060502052307129, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0005512237548828, "median_ms": 3.632687497884035 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.014354526996612549, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998035430908203, "median_ms": 3.6104585451539606 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.013535827398300171, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99951171875, "median_ms": 3.628916514571756 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 5.966983735561371e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000030517578125, "median_ms": 3.823729493888095 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.004299595952033997, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996833801269531, "median_ms": 3.7571460125036538 } ], "passed": true } }, "passed": true }