| { |
| "package": "laya_multilingual_w4_L128_options32.mlpackage", |
| "package_files_sha256": { |
| "Data/com.apple.CoreML/model.mlmodel": "c63a2e91dca84518e5dfac91b506daa42d35e0577b0fc7db93538b813f99d3fa", |
| "Data/com.apple.CoreML/weights/weight.bin": "573c65383a9b8ded8f6c5bb0e7f10b137cdb454cd6e0ccc3a5a1e2f5611c52af", |
| "Manifest.json": "e5d5e20a35f0d1d94974834915b607c1eee46cdb91ad7a350f96926ca0255263" |
| }, |
| "source_repo": "convaiinnovations/laya", |
| "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", |
| "variant": "multilingual", |
| "length": 128, |
| "max_options": 32, |
| "precision": "w4", |
| "package_bytes": 456441713, |
| "environment": { |
| "chip": "Apple M5 Pro", |
| "os": "27.0", |
| "python": "3.12.13", |
| "torch": "2.7.0", |
| "coremltools": "9.0" |
| }, |
| "gates": { |
| "argmax_agreement": "100%", |
| "max_probability_error": 0.02, |
| "max_action_probability_error": 0.02, |
| "finite": true, |
| "padding_masked": true |
| }, |
| "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", |
| "runs": { |
| "ALL": { |
| "compute_units": "ALL", |
| "load_seconds": 1.0729787919553928, |
| "questions": 16, |
| "argmax_agreements": 12, |
| "max_probability_error": 0.7274829745292664, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 3.9001669792924076, |
| "p95": 4.311243767733686, |
| "min": 3.7357090041041374, |
| "max": 5.792083975393325 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0635572075843811, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 4.182292002951726 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.15346801280975342, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.9166875067166984 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.09560063481330872, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 3.966332966228947 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.08459603786468506, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 3.85877100052312 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.15770003199577332, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000244140625, |
| "median_ms": 3.7997290201019496 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.7274829745292664, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99993896484375, |
| "median_ms": 3.8310835079755634 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.27412307262420654, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.865125501761213 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.055702269077301025, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999908447265625, |
| "median_ms": 3.889250016072765 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": false, |
| "coreml_argmax": 1, |
| "reference_argmax": 0, |
| "max_probability_error": 0.6696664690971375, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.258249478880316 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.043149255216121674, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00006103515625, |
| "median_ms": 3.951812512241304 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.1131022572517395, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999908447265625, |
| "median_ms": 3.922499978216365 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.3140353560447693, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 3.9491454954259098 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 3, |
| "max_probability_error": 0.42387789487838745, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9999847412109375, |
| "median_ms": 3.9071874925866723 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.18882879614830017, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.856478986563161 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.15432602167129517, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.878874995280057 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.030788853764533997, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000091552734375, |
| "median_ms": 3.8517920474987477 |
| } |
| ], |
| "passed": false |
| }, |
| "CPU_AND_NE": { |
| "compute_units": "CPU_AND_NE", |
| "load_seconds": 0.05808512499788776, |
| "questions": 16, |
| "argmax_agreements": 12, |
| "max_probability_error": 0.7284595370292664, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 3.720854001585394, |
| "p95": 3.8386354368412867, |
| "min": 3.6097500124014914, |
| "max": 4.019500047434121 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0650220513343811, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.7071249971631914 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.15420041978359222, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000244140625, |
| "median_ms": 3.79056204110384 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.09523442387580872, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.764145978493616 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.08459603786468506, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9996337890625, |
| "median_ms": 3.7676040083169937 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.15599104762077332, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.7193335010670125 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.7284595370292664, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99951171875, |
| "median_ms": 3.682729002321139 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.27216994762420654, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00048828125, |
| "median_ms": 3.704917005961761 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.055702269077301025, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99981689453125, |
| "median_ms": 3.703791997395456 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": false, |
| "coreml_argmax": 1, |
| "reference_argmax": 0, |
| "max_probability_error": 0.6625863909721375, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00018310546875, |
| "median_ms": 3.665396012365818 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.04259991645812988, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.712687495863065 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.1179850697517395, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99969482421875, |
| "median_ms": 3.6586874921340495 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.3145236372947693, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.660646005300805 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 3, |
| "max_probability_error": 0.42399996519088745, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.998809814453125, |
| "median_ms": 3.7595834874082357 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.18785223364830017, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99951171875, |
| "median_ms": 3.777874488150701 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.15481430292129517, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.7151249998714775 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.033504918217659, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0006256103515625, |
| "median_ms": 3.673125000204891 |
| } |
| ], |
| "passed": false |
| } |
| }, |
| "passed": false |
| } |
|
|
