| { |
| "package": "laya_multilingual_w6e8_L128_options32.mlpackage", |
| "package_files_sha256": { |
| "Data/com.apple.CoreML/model.mlmodel": "329ca89bebd8fe69df5445421c4c7efd64247391f5d6c7aa145b67d30385cb15", |
| "Data/com.apple.CoreML/weights/weight.bin": "f576015e91ea145b1268168b6dc0c6ac97a5bbfd6923ee1620cb3a38a6525be9", |
| "Manifest.json": "ef4c5d2e529ff4bba0d39f9509fc4895c5a6f18dce9a11c30688890021eca773" |
| }, |
| "source_repo": "convaiinnovations/laya", |
| "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", |
| "variant": "multilingual", |
| "length": 128, |
| "max_options": 32, |
| "precision": "w6e8", |
| "package_bytes": 291918867, |
| "environment": { |
| "chip": "Apple M5 Pro", |
| "os": "27.0", |
| "python": "3.12.13", |
| "torch": "2.7.0", |
| "coremltools": "9.0" |
| }, |
| "gates": { |
| "argmax_agreement": "100%", |
| "max_probability_error": 0.02, |
| "max_action_probability_error": 0.02, |
| "finite": true, |
| "padding_masked": true |
| }, |
| "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", |
| "runs": { |
| "ALL": { |
| "compute_units": "ALL", |
| "load_seconds": 6.012676417012699, |
| "questions": 16, |
| "argmax_agreements": 15, |
| "max_probability_error": 0.11705142259597778, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 4.070562485139817, |
| "p95": 4.799069106229581, |
| "min": 3.748167015146464, |
| "max": 6.246125034522265 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0006299838423728943, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00006103515625, |
| "median_ms": 3.9375209889840335 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0012615825980901718, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001373291015625, |
| "median_ms": 3.934312961064279 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.06732445955276489, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.815457981545478 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.04846322536468506, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 4.293645499274135 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.007645308971405029, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.9634164713788778 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0002473463537171483, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998081922531128, |
| "median_ms": 4.605561989592388 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.01239025592803955, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00006103515625, |
| "median_ms": 4.601292021106929 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 9.536836296319962e-05, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0000572204589844, |
| "median_ms": 4.162083467235789 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00025540590286254883, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0002191066741943, |
| "median_ms": 4.251854494214058 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.014951013028621674, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00018310546875, |
| "median_ms": 4.1342704789713025 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00017899274826049805, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0002022981643677, |
| "median_ms": 4.035437508719042 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0007172729820013046, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001888275146484, |
| "median_ms": 3.949687525164336 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": true, |
| "coreml_argmax": 3, |
| "reference_argmax": 3, |
| "max_probability_error": 0.026790112257003784, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998741149902344, |
| "median_ms": 4.269541503163055 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.11705142259597778, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000244140625, |
| "median_ms": 4.126771003939211 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.004853665828704834, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000030517578125, |
| "median_ms": 3.9383749826811254 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.025011956691741943, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998321533203125, |
| "median_ms": 3.8940419908612967 |
| } |
| ], |
| "passed": false |
| }, |
| "CPU_AND_NE": { |
| "compute_units": "CPU_AND_NE", |
| "load_seconds": 8.698386332951486, |
| "questions": 16, |
| "argmax_agreements": 15, |
| "max_probability_error": 0.11998111009597778, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 3.672833467135206, |
| "p95": 3.8521848036907613, |
| "min": 3.614499990362674, |
| "max": 4.668750043492764 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0005689263343811035, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99981689453125, |
| "median_ms": 3.8271665398497134 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.001612544059753418, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999847412109375, |
| "median_ms": 3.7480210012290627 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 0, |
| "reference_argmax": 1, |
| "max_probability_error": 0.06854516267776489, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.646500001195818 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.04846322536468506, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9993896484375, |
| "median_ms": 3.6573539837263525 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.01252812147140503, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.671353973913938 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0005443692207336426, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0002832412719727, |
| "median_ms": 3.661396011011675 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.00994884967803955, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.6549170035868883 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00011062715202569962, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0000724792480469, |
| "median_ms": 3.662938019260764 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00023287534713745117, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999731183052063, |
| "median_ms": 3.7158330087549984 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.015744447708129883, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999481201171875, |
| "median_ms": 3.6696250026579946 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00017899274826049805, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0002055168151855, |
| "median_ms": 3.810978989349678 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0010470747947692871, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998512268066406, |
| "median_ms": 3.7581874639727175 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": true, |
| "coreml_argmax": 3, |
| "reference_argmax": 3, |
| "max_probability_error": 0.023860424757003784, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000925064086914, |
| "median_ms": 3.732874523848295 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.11998111009597778, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 3.666708478704095 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.004929954186081886, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9999237060546875, |
| "median_ms": 3.648000507382676 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.021105706691741943, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0004081726074219, |
| "median_ms": 3.6529169883579016 |
| } |
| ], |
| "passed": false |
| } |
| }, |
| "passed": false |
| } |
|
|
