{
  "variant": "35B-A3B-MLX-8bit",
  "backend": "mlx 0.32.2 on Linux x86_64 (Device(gpu, 0))",
  "rows": 80,
  "correct": 71,
  "reference_correct": 71,
  "decision_agreement": 80,
  "changed_panel_indexes": [],
  "max_abs_probability_delta": 0.1930832862854004,
  "mean_abs_probability_delta": 0.00731312684911245,
  "peak_memory_gb": 38.66,
  "seconds": 279.82,
  "by_family": {
    "browser": {
      "rows": 16,
      "correct": 13
    },
    "hs3": {
      "rows": 16,
      "correct": 14
    },
    "boolq": {
      "rows": 16,
      "correct": 16
    },
    "mnli": {
      "rows": 16,
      "correct": 13
    },
    "score": {
      "rows": 16,
      "correct": 15
    }
  }
}