{
  "package": "laya_multilingual_w4e8_L128_options32.mlpackage",
  "package_files_sha256": {
    "Data/com.apple.CoreML/model.mlmodel": "5c0ee3881561d52854c9ff3c094713cac4797f056ef7162021ea3f1b5614979b",
    "Data/com.apple.CoreML/weights/weight.bin": "98c39cc41bbf495cc933648522fb345709b4062822c3322fefba9d3e8c5c40aa",
    "Manifest.json": "fb608932bd9ea1b5c41621f5aa67768ad8e98f812de50f010e946d063fd871ee"
  },
  "source_repo": "convaiinnovations/laya",
  "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
  "variant": "multilingual",
  "length": 128,
  "max_options": 32,
  "precision": "w4e8",
  "package_bytes": 260602440,
  "environment": {
    "chip": "Apple M5 Pro",
    "os": "27.0",
    "python": "3.12.13",
    "torch": "2.7.0",
    "coremltools": "9.0"
  },
  "gates": {
    "argmax_agreement": "100%",
    "max_probability_error": 0.02,
    "max_action_probability_error": 0.02,
    "finite": true,
    "padding_masked": true
  },
  "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
  "runs": {
    "ALL": {
      "compute_units": "ALL",
      "load_seconds": 5.722255375003442,
      "questions": 16,
      "argmax_agreements": 11,
      "max_probability_error": 0.7338754534721375,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 4.154979513259605,
        "p95": 5.602722565527074,
        "min": 3.771624993532896,
        "max": 7.908458006568253
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.049885354936122894,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 3.993958031060174
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.16909301280975342,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 4.158916504820809
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": false,
          "coreml_argmax": 0,
          "reference_argmax": 1,
          "max_probability_error": 0.10023930668830872,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 4.195916495518759
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.08752572536468506,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 4.104916530195624
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": false,
          "coreml_argmax": 1,
          "reference_argmax": 0,
          "max_probability_error": 0.18113753199577332,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 4.256458021700382
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": false,
          "coreml_argmax": 0,
          "reference_argmax": 1,
          "max_probability_error": 0.7035571932792664,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 4.523917014012113
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.24116408824920654,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 4.675916512496769
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.07236480712890625,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00006103515625,
          "median_ms": 5.404833500506356
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": false,
          "coreml_argmax": 1,
          "reference_argmax": 0,
          "max_probability_error": 0.7338754534721375,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 4.865312483161688
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.038815759122371674,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 4.0588130068499595
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0989421010017395,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 4.04439601697959
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.3438205122947693,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 4.084146028617397
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 128,
          "options": 20,
          "argmax_agrees": false,
          "coreml_argmax": 0,
          "reference_argmax": 3,
          "max_probability_error": 0.42412203550338745,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9999847412109375,
          "median_ms": 3.9592709799762815
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.17125067114830017,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 3.90997898648493
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 128,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.16909652948379517,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 4.13027050672099
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 128,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.05040258169174194,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000213623046875,
          "median_ms": 3.8169795006979257
        }
      ],
      "passed": false
    },
    "CPU_AND_NE": {
      "compute_units": "CPU_AND_NE",
      "load_seconds": 8.592580166994594,
      "questions": 16,
      "argmax_agreements": 11,
      "max_probability_error": 0.7287485003471375,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 3.7338540132623166,
        "p95": 4.254380951169878,
        "min": 3.609207982663065,
        "max": 5.975832988042384
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.050861917436122894,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00048828125,
          "median_ms": 3.767687507206574
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.16958129405975342,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 3.648437501396984
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": false,
          "coreml_argmax": 0,
          "reference_argmax": 1,
          "max_probability_error": 0.09975102543830872,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 3.6624585336539894
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.08752572536468506,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 3.6646875087171793
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": false,
          "coreml_argmax": 1,
          "reference_argmax": 0,
          "max_probability_error": 0.17942854762077332,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999755859375,
          "median_ms": 3.6670624976977706
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": false,
          "coreml_argmax": 0,
          "reference_argmax": 1,
          "max_probability_error": 0.7045337557792664,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00048828125,
          "median_ms": 3.6677290045190603
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.24067580699920654,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000244140625,
          "median_ms": 3.6528539785649627
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.07169342041015625,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0003662109375,
          "median_ms": 3.706833493197337
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": false,
          "coreml_argmax": 1,
          "reference_argmax": 0,
          "max_probability_error": 0.7287485003471375,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 3.679521003505215
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.039304040372371674,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 3.8889585121069103
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.1013835072517395,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000091552734375,
          "median_ms": 3.7854375259485096
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.3467501997947693,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 3.774520999286324
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 128,
          "options": 20,
          "argmax_agrees": false,
          "coreml_argmax": 0,
          "reference_argmax": 3,
          "max_probability_error": 0.42216891050338745,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.99951171875,
          "median_ms": 4.269541997928172
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.16880923509597778,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000244140625,
          "median_ms": 4.042041546199471
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 128,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.16848617792129517,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00048828125,
          "median_ms": 3.7245624989736825
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 128,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.05040258169174194,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999664306640625,
          "median_ms": 3.689062490593642
        }
      ],
      "passed": false
    }
  },
  "passed": false
}