{
  "package": "laya_multilingual_e8_L1024_options32.mlpackage",
  "package_files_sha256": {
    "Data/com.apple.CoreML/model.mlmodel": "1be66f2b54a08ea6a07f93c195777c4b105f7b2d9c09e47983ffecbb9f986f49",
    "Data/com.apple.CoreML/weights/weight.bin": "441cefaa5768572327ba89214566c6cb9a27aaacd479523b453f442c62f08eb2",
    "Manifest.json": "3dba274c81ab7dc9e42a34cbf43ad6af5329692dc2d3c3e2ce21a78ee73cbea9"
  },
  "source_repo": "convaiinnovations/laya",
  "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
  "variant": "multilingual",
  "length": 1024,
  "max_options": 32,
  "precision": "e8",
  "package_bytes": 452793209,
  "environment": {
    "chip": "Apple M5 Pro",
    "os": "27.0",
    "python": "3.12.13",
    "torch": "2.7.0",
    "coremltools": "9.0"
  },
  "gates": {
    "argmax_agreement": "100%",
    "max_probability_error": 0.02,
    "max_action_probability_error": 0.02,
    "finite": true,
    "padding_masked": true
  },
  "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
  "runs": {
    "ALL": {
      "compute_units": "ALL",
      "load_seconds": 6.339008749986533,
      "questions": 16,
      "argmax_agreements": 16,
      "max_probability_error": 0.005797207355499268,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 17.89439597632736,
        "p95": 18.36130567244254,
        "min": 17.64220802579075,
        "max": 23.747125000227243
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0025831088423728943,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00006103515625,
          "median_ms": 17.95933351968415
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0004474017769098282,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998931884765625,
          "median_ms": 17.946458974620327
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0013618916273117065,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 17.91014600894414
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0008780807256698608,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 17.874291486805305
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.005797207355499268,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999755859375,
          "median_ms": 17.784750496502966
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 5.608797073364258e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0000473260879517,
          "median_ms": 18.000207986915484
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.002427428960800171,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.99993896484375,
          "median_ms": 17.78575050411746
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00012970063835382462,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000091552734375,
          "median_ms": 17.90124998660758
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00025540590286254883,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00022292137146,
          "median_ms": 17.889228998683393
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0006077289581298828,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999908447265625,
          "median_ms": 17.80060399323702
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017899274826049805,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001795291900635,
          "median_ms": 17.893854004796594
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0004958007484674454,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0000476837158203,
          "median_ms": 17.79702099156566
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 157,
          "options": 20,
          "argmax_agrees": true,
          "coreml_argmax": 3,
          "reference_argmax": 3,
          "max_probability_error": 0.0021624043583869934,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.99993896484375,
          "median_ms": 17.815499508287758
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.004502624273300171,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999755859375,
          "median_ms": 18.24512452003546
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 174,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0003440380096435547,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998931884765625,
          "median_ms": 17.888771020807326
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 158,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.0009380578994750977,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999786376953125,
          "median_ms": 17.976166476728395
        }
      ],
      "passed": true
    },
    "CPU_AND_NE": {
      "compute_units": "CPU_AND_NE",
      "load_seconds": 11.53603254200425,
      "questions": 16,
      "argmax_agreements": 16,
      "max_probability_error": 0.013535827398300171,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 80.1227499905508,
        "p95": 81.49640440824443,
        "min": 79.91241605486721,
        "max": 92.36383298411965
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0022779330611228943,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000244140625,
          "median_ms": 80.32110400381498
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0005542133003473282,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999786376953125,
          "median_ms": 80.10449950234033
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0025046318769454956,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9996337890625,
          "median_ms": 80.17289600684308
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0018546432256698608,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 80.06041651242413
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.006285488605499268,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.99951171875,
          "median_ms": 80.15079150209203
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0004321932792663574,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9995424151420593,
          "median_ms": 80.05002100253478
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0009625852108001709,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 80.20041699637659
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 9.155366569757462e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0000534057617188,
          "median_ms": 80.10068751173094
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00023287534713745117,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9997386932373047,
          "median_ms": 80.0608120043762
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0005619749426841736,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0004425048828125,
          "median_ms": 81.02227098424919
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017899274826049805,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001797676086426,
          "median_ms": 80.29522901051678
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0009060502052307129,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0005512237548828,
          "median_ms": 80.08233297732659
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 157,
          "options": 20,
          "argmax_agrees": true,
          "coreml_argmax": 3,
          "reference_argmax": 3,
          "max_probability_error": 0.0027727559208869934,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00018310546875,
          "median_ms": 80.27691699680872
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.013535827398300171,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.99951171875,
          "median_ms": 80.1208334742114
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 174,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0003440380096435547,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999847412109375,
          "median_ms": 80.04837500629947
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 158,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.0009380578994750977,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9996337890625,
          "median_ms": 80.1351044792682
        }
      ],
      "passed": true
    }
  },
  "passed": true
}