| { |
| "package": "laya_multilingual_w4e8_L128_options32.mlpackage", |
| "package_files_sha256": { |
| "Data/com.apple.CoreML/model.mlmodel": "5c0ee3881561d52854c9ff3c094713cac4797f056ef7162021ea3f1b5614979b", |
| "Data/com.apple.CoreML/weights/weight.bin": "98c39cc41bbf495cc933648522fb345709b4062822c3322fefba9d3e8c5c40aa", |
| "Manifest.json": "fb608932bd9ea1b5c41621f5aa67768ad8e98f812de50f010e946d063fd871ee" |
| }, |
| "source_repo": "convaiinnovations/laya", |
| "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", |
| "variant": "multilingual", |
| "length": 128, |
| "max_options": 32, |
| "precision": "w4e8", |
| "package_bytes": 260602440, |
| "environment": { |
| "chip": "Apple M5 Pro", |
| "os": "27.0", |
| "python": "3.12.13", |
| "torch": "2.7.0", |
| "coremltools": "9.0" |
| }, |
| "gates": { |
| "argmax_agreement": "100%", |
| "max_probability_error": 0.02, |
| "max_action_probability_error": 0.02, |
| "finite": true, |
| "padding_masked": true |
| }, |
| "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", |
| "runs": { |
| "ALL": { |
| "compute_units": "ALL", |
| "load_seconds": 5.722255375003442, |
| "questions": 16, |
| "argmax_agreements": 11, |
| "max_probability_error": 0.7338754534721375, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 4.154979513259605, |
| "p95": 5.602722565527074, |
| "min": 3.771624993532896, |
| "max": 7.908458006568253 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.049885354936122894, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.993958031060174 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.16909301280975342, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 4.158916504820809 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.10023930668830872, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.195916495518759 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.08752572536468506, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.104916530195624 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": false, |
| "coreml_argmax": 1, |
| "reference_argmax": 0, |
| "max_probability_error": 0.18113753199577332, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 4.256458021700382 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.7035571932792664, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 4.523917014012113 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.24116408824920654, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.675916512496769 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.07236480712890625, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00006103515625, |
| "median_ms": 5.404833500506356 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": false, |
| "coreml_argmax": 1, |
| "reference_argmax": 0, |
| "max_probability_error": 0.7338754534721375, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 4.865312483161688 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.038815759122371674, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.0588130068499595 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0989421010017395, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.04439601697959 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.3438205122947693, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 4.084146028617397 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 3, |
| "max_probability_error": 0.42412203550338745, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9999847412109375, |
| "median_ms": 3.9592709799762815 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.17125067114830017, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.90997898648493 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.16909652948379517, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.13027050672099 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.05040258169174194, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000213623046875, |
| "median_ms": 3.8169795006979257 |
| } |
| ], |
| "passed": false |
| }, |
| "CPU_AND_NE": { |
| "compute_units": "CPU_AND_NE", |
| "load_seconds": 8.592580166994594, |
| "questions": 16, |
| "argmax_agreements": 11, |
| "max_probability_error": 0.7287485003471375, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 3.7338540132623166, |
| "p95": 4.254380951169878, |
| "min": 3.609207982663065, |
| "max": 5.975832988042384 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.050861917436122894, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00048828125, |
| "median_ms": 3.767687507206574 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.16958129405975342, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.648437501396984 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.09975102543830872, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.6624585336539894 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.08752572536468506, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.6646875087171793 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": false, |
| "coreml_argmax": 1, |
| "reference_argmax": 0, |
| "max_probability_error": 0.17942854762077332, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.6670624976977706 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.7045337557792664, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00048828125, |
| "median_ms": 3.6677290045190603 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.24067580699920654, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000244140625, |
| "median_ms": 3.6528539785649627 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.07169342041015625, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0003662109375, |
| "median_ms": 3.706833493197337 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": false, |
| "coreml_argmax": 1, |
| "reference_argmax": 0, |
| "max_probability_error": 0.7287485003471375, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.679521003505215 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.039304040372371674, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 3.8889585121069103 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.1013835072517395, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000091552734375, |
| "median_ms": 3.7854375259485096 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.3467501997947693, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 3.774520999286324 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 3, |
| "max_probability_error": 0.42216891050338745, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99951171875, |
| "median_ms": 4.269541997928172 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.16880923509597778, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000244140625, |
| "median_ms": 4.042041546199471 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.16848617792129517, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00048828125, |
| "median_ms": 3.7245624989736825 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.05040258169174194, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999664306640625, |
| "median_ms": 3.689062490593642 |
| } |
| ], |
| "passed": false |
| } |
| }, |
| "passed": false |
| } |
|
|
