{
  "package": "laya_multilingual_e8_L512_options32.mlpackage",
  "package_files_sha256": {
    "Data/com.apple.CoreML/model.mlmodel": "908a392aa7d3382b5c1c178e812a9fcfe0e741c240a9a10ce96458ef047c2c60",
    "Data/com.apple.CoreML/weights/weight.bin": "70c299c32d118ab1e2549e9c0088d712bfe2e936ad37ab8f9b7366f277afed4b",
    "Manifest.json": "a2cf370109daa1108d2880de4c8686191fa76926643f8f56cacafae7d0a246b9"
  },
  "source_repo": "convaiinnovations/laya",
  "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
  "variant": "multilingual",
  "length": 512,
  "max_options": 32,
  "precision": "e8",
  "package_bytes": 449516404,
  "environment": {
    "chip": "Apple M5 Pro",
    "os": "27.0",
    "python": "3.12.13",
    "torch": "2.7.0",
    "coremltools": "9.0"
  },
  "gates": {
    "argmax_agreement": "100%",
    "max_probability_error": 0.02,
    "max_action_probability_error": 0.02,
    "finite": true,
    "padding_masked": true
  },
  "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
  "runs": {
    "ALL": {
      "compute_units": "ALL",
      "load_seconds": 5.690649084048346,
      "questions": 16,
      "argmax_agreements": 16,
      "max_probability_error": 0.006270706653594971,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 9.148353972705081,
        "p95": 9.470572863938287,
        "min": 8.71966703562066,
        "max": 13.645083992742002
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0025831088423728943,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00006103515625,
          "median_ms": 9.161458496237174
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0004168841987848282,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9999237060546875,
          "median_ms": 9.129562502494082
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0018501728773117065,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 9.263979503884912
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0011222213506698608,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 9.095000015804544
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.006270706653594971,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 9.031604480696842
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 5.608797073364258e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0000249147415161,
          "median_ms": 9.217895974870771
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.001695007085800171,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 9.165416500763968
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00012970063835382462,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000091552734375,
          "median_ms": 9.23962498200126
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00025540590286254883,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0002155303955078,
          "median_ms": 8.99200001731515
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0003025755286216736,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00018310546875,
          "median_ms": 9.181979519780725
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017899274826049805,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001797676086426,
          "median_ms": 8.985167020000517
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0004576537758111954,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 9.146521013462916
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 157,
          "options": 20,
          "argmax_agrees": true,
          "coreml_argmax": 3,
          "reference_argmax": 3,
          "max_probability_error": 0.0022539570927619934,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00006103515625,
          "median_ms": 9.232708514900878
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.004014343023300171,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999755859375,
          "median_ms": 8.971625502454117
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 174,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0003440380096435547,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998931884765625,
          "median_ms": 9.061104501597583
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 158,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.0014263391494750977,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 9.162145521258935
        }
      ],
      "passed": true
    },
    "CPU_AND_NE": {
      "compute_units": "CPU_AND_NE",
      "load_seconds": 10.31118995900033,
      "questions": 16,
      "argmax_agreements": 16,
      "max_probability_error": 0.013535827398300171,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 27.725584019208327,
        "p95": 28.007108659949154,
        "min": 27.269584010355175,
        "max": 32.14670799206942
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0022779330611228943,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000244140625,
          "median_ms": 27.705749991582707
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0005542133003473282,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999786376953125,
          "median_ms": 27.664041990647092
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0025046318769454956,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9996337890625,
          "median_ms": 27.79045849456452
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0018546432256698608,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 27.8251460113097
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.006285488605499268,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.99951171875,
          "median_ms": 27.81237449380569
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0004321932792663574,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9995424151420593,
          "median_ms": 27.647562499623746
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0009625852108001709,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 27.70072899875231
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 9.155366569757462e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0000534057617188,
          "median_ms": 27.73627103306353
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00023287534713745117,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9997386932373047,
          "median_ms": 27.639124979032204
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0005619749426841736,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0004425048828125,
          "median_ms": 27.770020999014378
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017899274826049805,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001797676086426,
          "median_ms": 27.599229477345943
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0009060502052307129,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0005512237548828,
          "median_ms": 27.759166987380013
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 157,
          "options": 20,
          "argmax_agrees": true,
          "coreml_argmax": 3,
          "reference_argmax": 3,
          "max_probability_error": 0.0027727559208869934,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00018310546875,
          "median_ms": 27.80645849998109
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.013535827398300171,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.99951171875,
          "median_ms": 27.632729499600828
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 174,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0003440380096435547,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999847412109375,
          "median_ms": 27.62754147988744
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 158,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.0009380578994750977,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9996337890625,
          "median_ms": 27.861104463227093
        }
      ],
      "passed": true
    }
  },
  "passed": true
}