| { |
| "package": "laya_multilingual_w8_L128_options32.mlpackage", |
| "package_files_sha256": { |
| "Data/com.apple.CoreML/model.mlmodel": "4d1358ac20b62ec9aa90d76b3ce22e04f76505d3fbc2242f9ea83a47527f709d", |
| "Data/com.apple.CoreML/weights/weight.bin": "7c0bf0c97c3dfa524571c349f24241d1499514c90f5bef5bfccceb7c1c3cfee4", |
| "Manifest.json": "4bdc36727345b7bf6c23248407c45a0d5d466b20170506b5ef9c34273b55a68e" |
| }, |
| "source_repo": "convaiinnovations/laya", |
| "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", |
| "variant": "multilingual", |
| "length": 128, |
| "max_options": 32, |
| "precision": "w8", |
| "package_bytes": 323530193, |
| "environment": { |
| "chip": "Apple M5 Pro", |
| "os": "27.0", |
| "python": "3.12.13", |
| "torch": "2.7.0", |
| "coremltools": "9.0" |
| }, |
| "gates": { |
| "argmax_agreement": "100%", |
| "max_probability_error": 0.02, |
| "max_action_probability_error": 0.02, |
| "finite": true, |
| "padding_masked": true |
| }, |
| "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", |
| "runs": { |
| "ALL": { |
| "compute_units": "ALL", |
| "load_seconds": 0.30971424997551367, |
| "questions": 16, |
| "argmax_agreements": 16, |
| "max_probability_error": 0.04169827699661255, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 4.134541988605633, |
| "p95": 4.69923158234451, |
| "min": 3.8557079969905317, |
| "max": 5.924416007474065 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0025831088423728943, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00006103515625, |
| "median_ms": 4.497124988120049 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.00016294978559017181, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0000152587890625, |
| "median_ms": 4.4412079732865095 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.004824459552764893, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 4.317542014177889 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.005494475364685059, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 4.138145508477464 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.025696933269500732, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.157624993240461 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0001596083166077733, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998961687088013, |
| "median_ms": 4.174895962933078 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.010239928960800171, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00006103515625, |
| "median_ms": 4.044833505759016 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0005264878273010254, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998092651367188, |
| "median_ms": 4.042041982756928 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00025540590286254883, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001870393753052, |
| "median_ms": 4.069791495567188 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0013606324791908264, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9999847412109375, |
| "median_ms": 4.184874997008592 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00017899274826049805, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001893043518066, |
| "median_ms": 4.190374515019357 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0013943314552307129, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001106262207031, |
| "median_ms": 4.013374971691519 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": true, |
| "coreml_argmax": 3, |
| "reference_argmax": 3, |
| "max_probability_error": 0.04169827699661255, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001888275146484, |
| "median_ms": 4.021686996566132 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.02745184302330017, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.948749479604885 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.001435697078704834, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00018310546875, |
| "median_ms": 4.0304374997504056 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.021206334233283997, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000213623046875, |
| "median_ms": 4.040937987156212 |
| } |
| ], |
| "passed": false |
| }, |
| "CPU_AND_NE": { |
| "compute_units": "CPU_AND_NE", |
| "load_seconds": 0.058956374996341765, |
| "questions": 16, |
| "argmax_agreements": 11, |
| "max_probability_error": 0.6675509810447693, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 3.8277290004771203, |
| "p95": 3.96077292971313, |
| "min": 3.718333027791232, |
| "max": 5.271999980323017 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": false, |
| "coreml_argmax": 1, |
| "reference_argmax": 0, |
| "max_probability_error": 0.4746900498867035, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.7765210145153105 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.17495238780975342, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.7898749869782478 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": false, |
| "coreml_argmax": 2, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0824563056230545, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.790062037296593 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.22170449793338776, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0003662109375, |
| "median_ms": 3.7895625282544643 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.14325064420700073, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9996337890625, |
| "median_ms": 3.744395973626524 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.6139575839042664, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9991455078125, |
| "median_ms": 3.7628124991897494 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": false, |
| "coreml_argmax": 2, |
| "reference_argmax": 1, |
| "max_probability_error": 0.29144299030303955, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000244140625, |
| "median_ms": 3.837042022496462 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": false, |
| "coreml_argmax": 1, |
| "reference_argmax": 0, |
| "max_probability_error": 0.533485472202301, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999755859375, |
| "median_ms": 3.8239584828261286 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00023287534713745117, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9997035264968872, |
| "median_ms": 3.8753960106987506 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.23302960395812988, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99951171875, |
| "median_ms": 3.8337705191224813 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.5684245228767395, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000244140625, |
| "median_ms": 3.8206460303626955 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": false, |
| "coreml_argmax": 2, |
| "reference_argmax": 0, |
| "max_probability_error": 0.6675509810447693, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99951171875, |
| "median_ms": 3.8565624854527414 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": true, |
| "coreml_argmax": 3, |
| "reference_argmax": 3, |
| "max_probability_error": 0.33147066831588745, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.840166493318975 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.04869207739830017, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.8751665269955993 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.43752914667129517, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99951171875, |
| "median_ms": 3.910416504368186 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.10606664419174194, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99969482421875, |
| "median_ms": 3.908667014911771 |
| } |
| ], |
| "passed": false |
| } |
| }, |
| "passed": false |
| } |
|
|
