{
  "chip" : "Apple M5 Pro",
  "elapsed_s" : 22.913529992103577,
  "latency_ms" : {
    "p50" : 5.2010420000000002,
    "p95" : 17.967292
  },
  "lengths" : [
    128,
    256,
    512,
    1024
  ],
  "load_s" : 2.2602130174636841,
  "questions" : 3899,
  "suites" : {
    "app.email_spam" : {
      "accuracy" : 0.99250000000000005,
      "buckets" : {
        "L128" : 98,
        "L256" : 117,
        "L512" : 93,
        "L1024" : 92
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.8082500000000001,
        "p95" : 18.628333000000001
      },
      "max_probability_delta_vs_reference" : 0.0070431232452392578,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 9
    },
    "app.guardrails_jailbreak" : {
      "accuracy" : 0.80500000000000005,
      "buckets" : {
        "L128" : 285,
        "L256" : 59,
        "L512" : 53,
        "L1024" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 3.8389169999999999,
        "p95" : 9.5504999999999995
      },
      "max_probability_delta_vs_reference" : 0.80200904607772827,
      "n" : 400,
      "reference_argmax_agreement" : 0.995,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.model_routing_domain" : {
      "accuracy" : 0.44110275689223055,
      "buckets" : {
        "L128" : 137,
        "L256" : 261,
        "L512" : 1
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.2591669999999997,
        "p95" : 5.8784999999999998
      },
      "max_probability_delta_vs_reference" : 0.027976274490356445,
      "n" : 399,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 399,
      "state_truncated" : 0
    },
    "app.moderation_toxicity" : {
      "accuracy" : 0.53500000000000003,
      "buckets" : {
        "L128" : 306,
        "L256" : 54,
        "L512" : 37,
        "L1024" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 3.8188749999999998,
        "p95" : 9.387041
      },
      "max_probability_delta_vs_reference" : 0.0043558478355407715,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.phishing" : {
      "accuracy" : 0.99250000000000005,
      "buckets" : {
        "L128" : 76,
        "L256" : 111,
        "L512" : 104,
        "L1024" : 109
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 9.0027089999999994,
        "p95" : 19.462250000000001
      },
      "max_probability_delta_vs_reference" : 0.003923460841178894,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 7
    },
    "app.rag_relevance" : {
      "accuracy" : 0.67249999999999999,
      "buckets" : {
        "L128" : 135,
        "L256" : 262,
        "L512" : 3
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.3082500000000001,
        "p95" : 6.2931670000000004
      },
      "max_probability_delta_vs_reference" : 0.0088013410568237305,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "app.support_triage" : {
      "accuracy" : 0.54249999999999998,
      "buckets" : {
        "L256" : 377,
        "L512" : 23
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.297625,
        "p95" : 8.8684999999999992
      },
      "max_probability_delta_vs_reference" : 0.0048961937427520752,
      "n" : 400,
      "reference_argmax_agreement" : 0.99750000000000005,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "jev.ag_news" : {
      "accuracy" : 0.93500000000000005,
      "buckets" : {
        "L128" : 368,
        "L256" : 32
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 3.7891249999999999,
        "p95" : 6.5880419999999997
      },
      "max_probability_delta_vs_reference" : 0.010490596294403076,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "jev.emotion" : {
      "accuracy" : 0.53749999999999998,
      "buckets" : {
        "L128" : 400
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 3.7059579999999999,
        "p95" : 3.8140000000000001
      },
      "max_probability_delta_vs_reference" : 0.020582765340805054,
      "n" : 400,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 400,
      "state_truncated" : 0
    },
    "massive_intent.en" : {
      "accuracy" : 0.65666666666666662,
      "buckets" : {
        "L256" : 300
      },
      "dropped" : 0,
      "latency_ms" : {
        "p50" : 5.2149999999999999,
        "p95" : 5.4784579999999998
      },
      "max_probability_delta_vs_reference" : 0.019814342260360718,
      "n" : 300,
      "reference_argmax_agreement" : 1,
      "reference_compared" : 300,
      "state_truncated" : 0
    }
  }
}