{ "package": "laya_multilingual_fp16_L128_options32.mlpackage", "package_files_sha256": { "Data/com.apple.CoreML/model.mlmodel": "a536e177f174cc14a0dbe539357db29016a10ad579cdbf040b98525f32e7a4a7", "Data/com.apple.CoreML/weights/weight.bin": "506eb53e74aaebcdfb3f55304fbdd0b9c1678026efaef60454a1c66557f35d90", "Manifest.json": "9957a84277c0a722bcdbbc69ae3ce065122d9dd14de85ee9c76a2e44b2cb7146" }, "source_repo": "convaiinnovations/laya", "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "variant": "multilingual", "length": 128, "max_options": 32, "environment": { "chip": "Apple M5 Pro", "os": "27.0", "python": "3.12.13", "torch": "2.7.0", "coremltools": "9.0" }, "gates": { "argmax_agreement": "100%", "max_probability_error": 0.02, "max_action_probability_error": 0.02, "finite": true, "padding_masked": true }, "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", "runs": { "ALL": { "compute_units": "ALL", "load_seconds": 1.9205929159652442, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.0021350979804992676, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.874978981912136, "p95": 4.150477421353571, "min": 3.734458005055785, "max": 6.0334589798003435 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0001417025923728943, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 3.948333003791049 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00014770030975341797, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998626708984375, "median_ms": 3.872645494993776 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0005466341972351074, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001220703125, "median_ms": 4.006020986707881 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00014688074588775635, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.9132919919211417 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0021350979804992676, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.8335000281222165 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 5.608797073364258e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0000602006912231, "median_ms": 3.799437516136095 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0008405148983001709, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00006103515625, "median_ms": 3.865458507789299 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999732971191406, "median_ms": 3.829437482636422 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9997743368148804, "median_ms": 3.8122705300338566 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00011944770812988281, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999786376953125, "median_ms": 3.9022920245770365 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000178337097168, "median_ms": 3.817520511802286 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998912811279297, "median_ms": 3.8392704736907035 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.0005449950695037842, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999656677246094, "median_ms": 4.023146029794589 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0011126399040222168, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0, "median_ms": 3.868666506605223 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0001234356313943863, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999847412109375, "median_ms": 3.8872919976711273 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0010861754417419434, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999961853027344, "median_ms": 3.830937494058162 } ], "passed": true }, "CPU_AND_NE": { "compute_units": "CPU_AND_NE", "load_seconds": 3.0328945000073873, "questions": 16, "argmax_agreements": 16, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "latency_ms": { "p50": 3.6408329906407744, "p95": 3.7919832742772996, "min": 3.5940000088885427, "max": 6.553583953063935 }, "skipped": [], "rows": [ { "case": "tetris_clean", "question": "clean", "type": "noul", "tokens": 80, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0004076361656188965, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 3.6329374997876585 }, { "case": "tetris_flat", "question": "clean", "type": "noul", "tokens": 84, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.00034058094024658203, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0003662109375, "median_ms": 3.6503754963632673 }, { "case": "tetris_lane", "question": "lane", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0033596158027648926, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.6310830037109554 }, { "case": "tetris_lane", "question": "risk", "type": "score", "tokens": 64, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0014896541833877563, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 3.644520998932421 }, { "case": "form_field", "question": "action", "type": "choice", "tokens": 104, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.0034949183464050293, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9996337890625, "median_ms": 3.6403754784259945 }, { "case": "support_triage", "question": "intent", "type": "choice", "tokens": 62, "options": 4, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004321932792663574, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9995682239532471, "median_ms": 3.631958010373637 }, { "case": "support_triage", "question": "frustration", "type": "score", "tokens": 69, "options": 3, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.001600891351699829, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.99981689453125, "median_ms": 3.6398124939296395 }, { "case": "support_triage", "question": "churn", "type": "noul", "tokens": 71, "options": 2, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 3.820657730102539e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9999771118164062, "median_ms": 3.6401874967850745 }, { "case": "zh", "question": "intent", "type": "choice", "tokens": 37, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00023287534713745117, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.999773383140564, "median_ms": 3.7934584834147245 }, { "case": "zh", "question": "urgent", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0004093870520591736, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0002899169921875, "median_ms": 3.753063007025048 }, { "case": "ja", "question": "intent", "type": "choice", "tokens": 45, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 0.00017899274826049805, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0001769065856934, "median_ms": 3.62402101745829 }, { "case": "de", "question": "intent", "type": "choice", "tokens": 44, "options": 3, "argmax_agrees": true, "coreml_argmax": 0, "reference_argmax": 0, "max_probability_error": 7.051229476928711e-05, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 0.9998779296875, "median_ms": 3.6391039902810007 }, { "case": "many_options", "question": "lane", "type": "choice", "tokens": 128, "options": 20, "argmax_agrees": true, "coreml_argmax": 3, "reference_argmax": 3, "max_probability_error": 0.004200160503387451, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0005359649658203, "median_ms": 3.6420005199033767 }, { "case": "guard", "question": "injection", "type": "noul", "tokens": 48, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.012559235095977783, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.000244140625, "median_ms": 3.6424169957172126 }, { "case": "long_state", "question": "ready", "type": "noul", "tokens": 128, "options": 2, "argmax_agrees": true, "coreml_argmax": 1, "reference_argmax": 1, "max_probability_error": 0.0002122577279806137, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.00018310546875, "median_ms": 3.634604043327272 }, { "case": "long_state", "question": "owner", "type": "choice", "tokens": 128, "options": 3, "argmax_agrees": true, "coreml_argmax": 2, "reference_argmax": 2, "max_probability_error": 0.0022311657667160034, "max_action_probability_error": 0.0, "finite": true, "padding_masked": true, "probability_sum": 1.0003395080566406, "median_ms": 3.628145990660414 } ], "passed": true } }, "passed": true }