{
  "protocol_sha256": "b6dde5e425239455e9e31a44bc22b151b0ee66ec827c538aaca3308e29441c15",
  "requests_sha256": "201812a5401f469975046eb9c96e2a471469f03e03ca464367425bf42c96daf5",
  "methods": {
    "spark-a/trained": {
      "model_load_seconds": 2.0727300819999073,
      "peak_cuda_allocated_bytes": 16475382784,
      "peak_cuda_reserved_bytes": 16519266304,
      "accuracy": {
        "diagnostics": {
          "count": 383,
          "correct": 303,
          "accuracy": 0.7911227154046997,
          "per_family": {
            "commonsenseqa": {
              "count": 128,
              "correct": 98,
              "accuracy": 0.765625
            },
            "hellaswag": {
              "count": 128,
              "correct": 97,
              "accuracy": 0.7578125
            },
            "piqa": {
              "count": 127,
              "correct": 108,
              "accuracy": 0.8503937007874016
            }
          }
        },
        "heldout": {
          "count": 320,
          "correct": 285,
          "accuracy": 0.890625,
          "per_family": {
            "arc": {
              "count": 64,
              "correct": 62,
              "accuracy": 0.96875
            },
            "banking": {
              "count": 64,
              "correct": 62,
              "accuracy": 0.96875
            },
            "boolq": {
              "count": 64,
              "correct": 61,
              "accuracy": 0.953125
            },
            "snli": {
              "count": 64,
              "correct": 53,
              "accuracy": 0.828125
            },
            "social": {
              "count": 64,
              "correct": 47,
              "accuracy": 0.734375
            }
          }
        }
      },
      "timing_cases": 12
    },
    "spark-a/base_verifier": {
      "model_load_seconds": 1.9338143200002378,
      "peak_cuda_allocated_bytes": 16409322496,
      "peak_cuda_reserved_bytes": 16473128960,
      "accuracy": {
        "diagnostics": {
          "count": 383,
          "correct": 298,
          "accuracy": 0.7780678851174935,
          "per_family": {
            "commonsenseqa": {
              "count": 128,
              "correct": 90,
              "accuracy": 0.703125
            },
            "hellaswag": {
              "count": 128,
              "correct": 101,
              "accuracy": 0.7890625
            },
            "piqa": {
              "count": 127,
              "correct": 107,
              "accuracy": 0.84251968503937
            }
          }
        },
        "heldout": {
          "count": 320,
          "correct": 259,
          "accuracy": 0.809375,
          "per_family": {
            "arc": {
              "count": 64,
              "correct": 58,
              "accuracy": 0.90625
            },
            "banking": {
              "count": 64,
              "correct": 54,
              "accuracy": 0.84375
            },
            "boolq": {
              "count": 64,
              "correct": 58,
              "accuracy": 0.90625
            },
            "snli": {
              "count": 64,
              "correct": 45,
              "accuracy": 0.703125
            },
            "social": {
              "count": 64,
              "correct": 44,
              "accuracy": 0.6875
            }
          }
        }
      },
      "timing_cases": 12
    },
    "spark-a/base_label": {
      "model_load_seconds": 1.957351696997648,
      "peak_cuda_allocated_bytes": 16495933440,
      "peak_cuda_reserved_bytes": 16554917888,
      "accuracy": {
        "diagnostics": {
          "count": 383,
          "correct": 303,
          "accuracy": 0.7911227154046997,
          "per_family": {
            "commonsenseqa": {
              "count": 128,
              "correct": 90,
              "accuracy": 0.703125
            },
            "hellaswag": {
              "count": 128,
              "correct": 109,
              "accuracy": 0.8515625
            },
            "piqa": {
              "count": 127,
              "correct": 104,
              "accuracy": 0.8188976377952756
            }
          }
        },
        "heldout": {
          "count": 320,
          "correct": 276,
          "accuracy": 0.8625,
          "per_family": {
            "arc": {
              "count": 64,
              "correct": 60,
              "accuracy": 0.9375
            },
            "banking": {
              "count": 64,
              "correct": 62,
              "accuracy": 0.96875
            },
            "boolq": {
              "count": 64,
              "correct": 57,
              "accuracy": 0.890625
            },
            "snli": {
              "count": 64,
              "correct": 48,
              "accuracy": 0.75
            },
            "social": {
              "count": 64,
              "correct": 49,
              "accuracy": 0.765625
            }
          }
        }
      },
      "timing_cases": 12
    },
    "spark-b/trained": {
      "model_load_seconds": 2.059389772999566,
      "peak_cuda_allocated_bytes": 16269351936,
      "peak_cuda_reserved_bytes": 16309551104,
      "accuracy": {
        "diagnostics": {
          "count": 383,
          "correct": 312,
          "accuracy": 0.814621409921671,
          "per_family": {
            "commonsenseqa": {
              "count": 128,
              "correct": 96,
              "accuracy": 0.75
            },
            "hellaswag": {
              "count": 128,
              "correct": 106,
              "accuracy": 0.828125
            },
            "piqa": {
              "count": 127,
              "correct": 110,
              "accuracy": 0.8661417322834646
            }
          }
        },
        "heldout": {
          "count": 320,
          "correct": 284,
          "accuracy": 0.8875,
          "per_family": {
            "arc": {
              "count": 64,
              "correct": 62,
              "accuracy": 0.96875
            },
            "banking": {
              "count": 64,
              "correct": 62,
              "accuracy": 0.96875
            },
            "boolq": {
              "count": 64,
              "correct": 61,
              "accuracy": 0.953125
            },
            "snli": {
              "count": 64,
              "correct": 52,
              "accuracy": 0.8125
            },
            "social": {
              "count": 64,
              "correct": 47,
              "accuracy": 0.734375
            }
          }
        }
      }
    }
  },
  "comparisons": {
    "selected_vs_base_verifier": {
      "diagnostics": {
        "count": 383,
        "left_only_correct": 36,
        "right_only_correct": 31,
        "accuracy_difference_left_minus_right": 0.013054830287206266
      },
      "heldout": {
        "count": 320,
        "left_only_correct": 38,
        "right_only_correct": 12,
        "accuracy_difference_left_minus_right": 0.08125
      }
    },
    "selected_vs_base_label": {
      "diagnostics": {
        "count": 383,
        "left_only_correct": 32,
        "right_only_correct": 32,
        "accuracy_difference_left_minus_right": 0.0
      },
      "heldout": {
        "count": 320,
        "left_only_correct": 19,
        "right_only_correct": 10,
        "accuracy_difference_left_minus_right": 0.028125
      }
    },
    "expanded159_vs_selected": {
      "diagnostics": {
        "count": 383,
        "left_only_correct": 14,
        "right_only_correct": 5,
        "accuracy_difference_left_minus_right": 0.02349869451697128
      },
      "heldout": {
        "count": 320,
        "left_only_correct": 2,
        "right_only_correct": 3,
        "accuracy_difference_left_minus_right": -0.003125
      }
    }
  },
  "all_prediction_and_timing_checks_passed": true
}