| { |
| "package": "laya_multilingual_fp16_L128_options32.mlpackage", |
| "package_files_sha256": { |
| "Data/com.apple.CoreML/model.mlmodel": "a536e177f174cc14a0dbe539357db29016a10ad579cdbf040b98525f32e7a4a7", |
| "Data/com.apple.CoreML/weights/weight.bin": "506eb53e74aaebcdfb3f55304fbdd0b9c1678026efaef60454a1c66557f35d90", |
| "Manifest.json": "9957a84277c0a722bcdbbc69ae3ce065122d9dd14de85ee9c76a2e44b2cb7146" |
| }, |
| "source_repo": "convaiinnovations/laya", |
| "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", |
| "variant": "multilingual", |
| "length": 128, |
| "max_options": 32, |
| "environment": { |
| "chip": "Apple M5 Pro", |
| "os": "27.0", |
| "python": "3.12.13", |
| "torch": "2.7.0", |
| "coremltools": "9.0" |
| }, |
| "gates": { |
| "argmax_agreement": "100%", |
| "max_probability_error": 0.02, |
| "max_action_probability_error": 0.02, |
| "finite": true, |
| "padding_masked": true |
| }, |
| "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed", |
| "runs": { |
| "ALL": { |
| "compute_units": "ALL", |
| "load_seconds": 1.9205929159652442, |
| "questions": 16, |
| "argmax_agreements": 16, |
| "max_probability_error": 0.0021350979804992676, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 3.874978981912136, |
| "p95": 4.150477421353571, |
| "min": 3.734458005055785, |
| "max": 6.0334589798003435 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0001417025923728943, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00006103515625, |
| "median_ms": 3.948333003791049 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.00014770030975341797, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998626708984375, |
| "median_ms": 3.872645494993776 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0005466341972351074, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001220703125, |
| "median_ms": 4.006020986707881 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.00014688074588775635, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.9132919919211417 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0021350979804992676, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.8335000281222165 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 5.608797073364258e-05, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0000602006912231, |
| "median_ms": 3.799437516136095 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0008405148983001709, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00006103515625, |
| "median_ms": 3.865458507789299 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 3.820657730102539e-05, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9999732971191406, |
| "median_ms": 3.829437482636422 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00023287534713745117, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9997743368148804, |
| "median_ms": 3.8122705300338566 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.00011944770812988281, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999786376953125, |
| "median_ms": 3.9022920245770365 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00017899274826049805, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000178337097168, |
| "median_ms": 3.817520511802286 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 7.051229476928711e-05, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998912811279297, |
| "median_ms": 3.8392704736907035 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": true, |
| "coreml_argmax": 3, |
| "reference_argmax": 3, |
| "max_probability_error": 0.0005449950695037842, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9999656677246094, |
| "median_ms": 4.023146029794589 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0011126399040222168, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0, |
| "median_ms": 3.868666506605223 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0001234356313943863, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999847412109375, |
| "median_ms": 3.8872919976711273 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.0010861754417419434, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9999961853027344, |
| "median_ms": 3.830937494058162 |
| } |
| ], |
| "passed": true |
| }, |
| "CPU_AND_NE": { |
| "compute_units": "CPU_AND_NE", |
| "load_seconds": 3.0328945000073873, |
| "questions": 16, |
| "argmax_agreements": 16, |
| "max_probability_error": 0.012559235095977783, |
| "max_action_probability_error": 0.0, |
| "latency_ms": { |
| "p50": 3.6408329906407744, |
| "p95": 3.7919832742772996, |
| "min": 3.5940000088885427, |
| "max": 6.553583953063935 |
| }, |
| "skipped": [], |
| "rows": [ |
| { |
| "case": "tetris_clean", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 80, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0004076361656188965, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00018310546875, |
| "median_ms": 3.6329374997876585 |
| }, |
| { |
| "case": "tetris_flat", |
| "question": "clean", |
| "type": "noul", |
| "tokens": 84, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.00034058094024658203, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0003662109375, |
| "median_ms": 3.6503754963632673 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0033596158027648926, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.6310830037109554 |
| }, |
| { |
| "case": "tetris_lane", |
| "question": "risk", |
| "type": "score", |
| "tokens": 64, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0014896541833877563, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9996337890625, |
| "median_ms": 3.644520998932421 |
| }, |
| { |
| "case": "form_field", |
| "question": "action", |
| "type": "choice", |
| "tokens": 104, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.0034949183464050293, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9996337890625, |
| "median_ms": 3.6403754784259945 |
| }, |
| { |
| "case": "support_triage", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 62, |
| "options": 4, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0004321932792663574, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9995682239532471, |
| "median_ms": 3.631958010373637 |
| }, |
| { |
| "case": "support_triage", |
| "question": "frustration", |
| "type": "score", |
| "tokens": 69, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.001600891351699829, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.99981689453125, |
| "median_ms": 3.6398124939296395 |
| }, |
| { |
| "case": "support_triage", |
| "question": "churn", |
| "type": "noul", |
| "tokens": 71, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 3.820657730102539e-05, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9999771118164062, |
| "median_ms": 3.6401874967850745 |
| }, |
| { |
| "case": "zh", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 37, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00023287534713745117, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.999773383140564, |
| "median_ms": 3.7934584834147245 |
| }, |
| { |
| "case": "zh", |
| "question": "urgent", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0004093870520591736, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0002899169921875, |
| "median_ms": 3.753063007025048 |
| }, |
| { |
| "case": "ja", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 45, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 0.00017899274826049805, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0001769065856934, |
| "median_ms": 3.62402101745829 |
| }, |
| { |
| "case": "de", |
| "question": "intent", |
| "type": "choice", |
| "tokens": 44, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 0, |
| "reference_argmax": 0, |
| "max_probability_error": 7.051229476928711e-05, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 0.9998779296875, |
| "median_ms": 3.6391039902810007 |
| }, |
| { |
| "case": "many_options", |
| "question": "lane", |
| "type": "choice", |
| "tokens": 128, |
| "options": 20, |
| "argmax_agrees": true, |
| "coreml_argmax": 3, |
| "reference_argmax": 3, |
| "max_probability_error": 0.004200160503387451, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0005359649658203, |
| "median_ms": 3.6420005199033767 |
| }, |
| { |
| "case": "guard", |
| "question": "injection", |
| "type": "noul", |
| "tokens": 48, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.012559235095977783, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.000244140625, |
| "median_ms": 3.6424169957172126 |
| }, |
| { |
| "case": "long_state", |
| "question": "ready", |
| "type": "noul", |
| "tokens": 128, |
| "options": 2, |
| "argmax_agrees": true, |
| "coreml_argmax": 1, |
| "reference_argmax": 1, |
| "max_probability_error": 0.0002122577279806137, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.00018310546875, |
| "median_ms": 3.634604043327272 |
| }, |
| { |
| "case": "long_state", |
| "question": "owner", |
| "type": "choice", |
| "tokens": 128, |
| "options": 3, |
| "argmax_agrees": true, |
| "coreml_argmax": 2, |
| "reference_argmax": 2, |
| "max_probability_error": 0.0022311657667160034, |
| "max_action_probability_error": 0.0, |
| "finite": true, |
| "padding_masked": true, |
| "probability_sum": 1.0003395080566406, |
| "median_ms": 3.628145990660414 |
| } |
| ], |
| "passed": true |
| } |
| }, |
| "passed": true |
| } |
|
|
