{
  "variant": "9B-MLX-8bit",
  "backend": "mlx 0.32.2 on Linux x86_64 (Device(gpu, 0))",
  "rows": 80,
  "correct": 69,
  "reference_correct": 68,
  "decision_agreement": 79,
  "changed_panel_indexes": [
    28
  ],
  "max_abs_probability_delta": 0.062176525592803955,
  "mean_abs_probability_delta": 0.003565970142608421,
  "peak_memory_gb": 11.28,
  "seconds": 206.09,
  "by_family": {
    "browser": {
      "rows": 16,
      "correct": 13
    },
    "hs3": {
      "rows": 16,
      "correct": 14
    },
    "boolq": {
      "rows": 16,
      "correct": 16
    },
    "mnli": {
      "rows": 16,
      "correct": 13
    },
    "score": {
      "rows": 16,
      "correct": 13
    }
  }
}