{
  "package": "laya_multilingual_w6_L128_options32.mlpackage",
  "package_files_sha256": {
    "Data/com.apple.CoreML/model.mlmodel": "437fcb102aa75b0b4298d6884594f3a633843b40634e72d8cefeabdf092ffb5f",
    "Data/com.apple.CoreML/weights/weight.bin": "2b9ba8dd6c102d1982b66fe9c18b15f155a1231fa299ebc3659d7f799e087e14",
    "Manifest.json": "3ca5e9231730455da0b2ab6c913ad8f722018cd6e18a10e961fd3eebe8a26178"
  },
  "source_repo": "convaiinnovations/laya",
  "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
  "variant": "multilingual",
  "length": 128,
  "max_options": 32,
  "precision": "w6",
  "package_bytes": 487758065,
  "environment": {
    "chip": "Apple M5 Pro",
    "os": "27.0",
    "python": "3.12.13",
    "torch": "2.7.0",
    "coremltools": "9.0"
  },
  "gates": {
    "argmax_agreement": "100%",
    "max_probability_error": 0.02,
    "max_action_probability_error": 0.02,
    "finite": true,
    "padding_masked": true
  },
  "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
  "runs": {
    "ALL": {
      "compute_units": "ALL",
      "load_seconds": 0.40539787500165403,
      "questions": 16,
      "argmax_agreements": 15,
      "max_probability_error": 0.11412176489830017,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 4.023020999738947,
        "p95": 4.896181236836128,
        "min": 3.7430000375024974,
        "max": 6.122291961219162
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0023607611656188965,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000244140625,
          "median_ms": 4.197187488898635
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.002100825309753418,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9999847412109375,
          "median_ms": 4.224250471452251
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": false,
          "coreml_argmax": 0,
          "reference_argmax": 1,
          "max_probability_error": 0.06976586580276489,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 4.11027148948051
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.05334603786468506,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 4.552416998194531
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.003983199596405029,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 4.148854000959545
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0002187361242249608,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998368620872498,
          "median_ms": 4.186771489912644
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.00848400592803955,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 4.011520504718646
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017547700554132462,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001373291015625,
          "median_ms": 3.8826040108688176
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00023287534713745117,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999935507774353,
          "median_ms": 3.886792022967711
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.015897057950496674,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000152587890625,
          "median_ms": 3.8653119991067797
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017899274826049805,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001966953277588,
          "median_ms": 3.8469999853987247
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0015353560447692871,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9997730255126953,
          "median_ms": 3.8703329919371754
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 128,
          "options": 20,
          "argmax_agrees": true,
          "coreml_argmax": 3,
          "reference_argmax": 3,
          "max_probability_error": 0.06206148862838745,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001564025878906,
          "median_ms": 3.8793960120528936
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.11412176489830017,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 4.9398334813304245
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 128,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.004853665828704834,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00006103515625,
          "median_ms": 3.991229517851025
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 128,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.03331273794174194,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0002517700195312,
          "median_ms": 3.9200414903461933
        }
      ],
      "passed": false
    },
    "CPU_AND_NE": {
      "compute_units": "CPU_AND_NE",
      "load_seconds": 0.06528987496858463,
      "questions": 16,
      "argmax_agreements": 15,
      "max_probability_error": 0.11753970384597778,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 3.664499963633716,
        "p95": 3.813516642549075,
        "min": 3.6027920432388783,
        "max": 4.030583018902689
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0013841986656188965,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000244140625,
          "median_ms": 3.6517085100058466
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.002589106559753418,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9996337890625,
          "median_ms": 3.6495624808594584
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": false,
          "coreml_argmax": 0,
          "reference_argmax": 1,
          "max_probability_error": 0.07001000642776489,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9996337890625,
          "median_ms": 3.6430624895729125
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.05334603786468506,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999755859375,
          "median_ms": 3.673332976177335
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.007645308971405029,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 3.665978991193697
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0005443692207336426,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0003440380096436,
          "median_ms": 3.6652290145866573
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.006042599678039551,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00006103515625,
          "median_ms": 3.835354495095089
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 9.155366569757462e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0000534057617188,
          "median_ms": 3.758832986932248
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00023287534713745117,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9999407529830933,
          "median_ms": 3.7352295184973627
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.016965173184871674,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000732421875,
          "median_ms": 3.6477295216172934
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017899274826049805,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001966953277588,
          "median_ms": 3.6439785035327077
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0012475159019231796,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0002479553222656,
          "median_ms": 3.663499985123053
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 128,
          "options": 20,
          "argmax_agrees": true,
          "coreml_argmax": 3,
          "reference_argmax": 3,
          "max_probability_error": 0.06425875425338745,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998111724853516,
          "median_ms": 3.671749960631132
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.11753970384597778,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0003662109375,
          "median_ms": 3.6952500231564045
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 128,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.004853665828704834,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000274658203125,
          "median_ms": 3.651062026619911
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 128,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.03282445669174194,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9995269775390625,
          "median_ms": 3.650186990853399
        }
      ],
      "passed": false
    }
  },
  "passed": false
}