v0007-math/manifest.json

{
  "version": "v0007-math",
  "parent": "v0007",
  "parents": [
    "v0007"
  ],
  "kind": "lmhead",
  "backbone": "Qwen/Qwen2.5-1.5B-Instruct",
  "max_len": 1024,
  "created": "2026-09-22T13:55:28+00:00",
  "tasks": [
    "math",
    "gsm8k",
    "svamp",
    "aqua"
  ],
  "trained_on": [
    "math",
    "gsm8k",
    "svamp",
    "aqua"
  ],
  "holdout": [],
  "steps": 3684,
  "train_examples": 57771,
  "args": {
    "cmd": "lmtrain",
    "lora_r": 32,
    "loss": "mix",
    "max_per_task": 30000,
    "epochs": 1,
    "lr": 0.0001,
    "anchor": 0.1
  },
  "metrics": {
    "math": {
      "n": 300,
      "acc": 0.8733333333333333,
      "nll": 0.2491237256136509,
      "brier": 0.15517219901605317,
      "ece": 0.04117123444875081,
      "mean_conf": 0.8545713822046915,
      "cov@0.5": 1.0,
      "acc@0.5": 0.8733333333333333,
      "cov@0.7": 0.7533333333333333,
      "acc@0.7": 0.9690265486725663,
      "cov@0.9": 0.7066666666666667,
      "acc@0.9": 1.0
    },
    "gsm8k": {
      "n": 300,
      "acc": 0.8166666666666667,
      "nll": 0.4634518215859158,
      "brier": 0.24914535577158609,
      "ece": 0.08174108674128854,
      "mean_conf": 0.7950625959038734,
      "cov@0.5": 0.9,
      "acc@0.5": 0.8666666666666667,
      "cov@0.7": 0.7,
      "acc@0.7": 0.9238095238095239,
      "cov@0.9": 0.4266666666666667,
      "acc@0.9": 0.9921875
    },
    "svamp": {
      "n": 100,
      "acc": 0.68,
      "nll": 0.7193946922798429,
      "brier": 0.41086004277348953,
      "ece": 0.07071352988481525,
      "mean_conf": 0.7261623141169548,
      "cov@0.5": 0.83,
      "acc@0.5": 0.7349397590361446,
      "cov@0.7": 0.58,
      "acc@0.7": 0.8275862068965517,
      "cov@0.9": 0.2,
      "acc@0.9": 1.0
    },
    "aqua": {
      "n": 253,
      "acc": 0.391304347826087,
      "nll": 1.414271918680051,
      "brier": 0.7115495715758425,
      "ece": 0.0654898358899143,
      "mean_conf": 0.3998582398467384,
      "cov@0.5": 0.20553359683794467,
      "acc@0.5": 0.6538461538461539,
      "cov@0.7": 0.06324110671936758,
      "acc@0.7": 0.8125,
      "cov@0.9": 0.007905138339920948,
      "acc@0.9": 1.0
    }
  },
  "unseen_test_uncalibrated": {},
  "summary": {
    "mean_acc": 0.6903260869565218,
    "mean_ece": 0.06477892174119222
  },
  "history": []
}