{
 "ref_ppl": {
  "chat": 8.5587,
  "code": 5.6556,
  "long-book": 12.1535,
  "short-all": 7.1382
 },
 "gsm8k_bf16_cpu_acc": 0.98,
 "rows": [
  {
   "config": "bf16",
   "plan": "plans/bf16.json",
   "weights": "IT",
   "read_GB": 23.8446,
   "floor_ms@128": 46.66,
   "chat": {
    "ppl_delta_pct": -0.77,
    "dnll": -0.0077,
    "se": 0.0014,
    "top1": 0.975,
    "kl": 0.0095,
    "kl_p99": 0.103
   },
   "code": {
    "ppl_delta_pct": -0.05,
    "dnll": -0.0005,
    "se": 0.0014,
    "top1": 0.9843,
    "kl": 0.0139,
    "kl_p99": 0.061
   },
   "long-book": {
    "ppl_delta_pct": -0.14,
    "dnll": -0.0014,
    "se": 0.0013,
    "top1": 0.9764,
    "kl": 0.0048,
    "kl_p99": 0.053
   },
   "short-all": {
    "ppl_delta_pct": -0.45,
    "dnll": -0.0045,
    "se": 0.001,
    "top1": 0.9791,
    "kl": 0.0114,
    "kl_p99": 0.081
   }
  },
  {
   "config": "a-bfp8",
   "plan": "plans/a-bfp8.json",
   "weights": "IT",
   "read_GB": 12.6675,
   "floor_ms@128": 24.83,
   "chat": {
    "ppl_delta_pct": -0.29,
    "dnll": -0.0029,
    "se": 0.0015,
    "top1": 0.9713,
    "kl": 0.0122,
    "kl_p99": 0.139
   },
   "code": {
    "ppl_delta_pct": 1.01,
    "dnll": 0.01,
    "se": 0.0014,
    "top1": 0.9827,
    "kl": 0.0057,
    "kl_p99": 0.08
   },
   "long-book": {
    "ppl_delta_pct": 1.61,
    "dnll": 0.016,
    "se": 0.0015,
    "top1": 0.9723,
    "kl": 0.007,
    "kl_p99": 0.068
   },
   "short-all": {
    "ppl_delta_pct": 0.28,
    "dnll": 0.0028,
    "se": 0.001,
    "top1": 0.9763,
    "kl": 0.0093,
    "kl_p99": 0.105
   }
  },
  {
   "config": "a-lm4",
   "plan": "plans/a-lm4.json",
   "weights": "IT",
   "read_GB": 12.1641,
   "floor_ms@128": 23.84,
   "chat": {
    "ppl_delta_pct": 0.61,
    "dnll": 0.0061,
    "se": 0.0018,
    "top1": 0.955,
    "kl": 0.0208,
    "kl_p99": 0.244
   },
   "code": {
    "ppl_delta_pct": 1.98,
    "dnll": 0.0196,
    "se": 0.0022,
    "top1": 0.9703,
    "kl": 0.0169,
    "kl_p99": 0.289
   },
   "long-book": {
    "ppl_delta_pct": 2.55,
    "dnll": 0.0252,
    "se": 0.0023,
    "top1": 0.9605,
    "kl": 0.0149,
    "kl_p99": 0.128
   },
   "short-all": {
    "ppl_delta_pct": 1.21,
    "dnll": 0.012,
    "se": 0.0014,
    "top1": 0.9617,
    "kl": 0.0191,
    "kl_p99": 0.264
   }
  },
  {
   "config": "f-dn4",
   "plan": "plans/f-dn4.json",
   "weights": "IT",
   "read_GB": 11.2519,
   "floor_ms@128": 22.06,
   "chat": {
    "ppl_delta_pct": 2.61,
    "dnll": 0.0257,
    "se": 0.004,
    "top1": 0.9102,
    "kl": 0.0942,
    "kl_p99": 1.456
   },
   "code": {
    "ppl_delta_pct": 2.43,
    "dnll": 0.024,
    "se": 0.005,
    "top1": 0.9333,
    "kl": 0.101,
    "kl_p99": 1.949
   },
   "long-book": {
    "ppl_delta_pct": -0.64,
    "dnll": -0.0065,
    "se": 0.0049,
    "top1": 0.912,
    "kl": 0.0732,
    "kl_p99": 0.819
   },
   "short-all": {
    "ppl_delta_pct": 2.53,
    "dnll": 0.025,
    "se": 0.0031,
    "top1": 0.9203,
    "kl": 0.0972,
    "kl_p99": 1.65
   }
  },
  {
   "config": "f-gu4",
   "plan": "plans/f-gu4.json",
   "weights": "IT",
   "read_GB": 9.8363,
   "floor_ms@128": 19.3,
   "chat": {
    "ppl_delta_pct": 1.56,
    "dnll": 0.0155,
    "se": 0.004,
    "top1": 0.9021,
    "kl": 0.0962,
    "kl_p99": 1.461
   },
   "code": {
    "ppl_delta_pct": 2.43,
    "dnll": 0.024,
    "se": 0.0051,
    "top1": 0.9286,
    "kl": 0.1111,
    "kl_p99": 2.099
   },
   "long-book": {
    "ppl_delta_pct": 9.73,
    "dnll": 0.0929,
    "se": 0.0053,
    "top1": 0.9053,
    "kl": 0.0848,
    "kl_p99": 0.863
   },
   "short-all": {
    "ppl_delta_pct": 1.94,
    "dnll": 0.0192,
    "se": 0.0032,
    "top1": 0.9137,
    "kl": 0.1028,
    "kl_p99": 1.702
   }
  },
  {
   "config": "g-rank3072",
   "plan": "plans/g-rank3072.json",
   "weights": "IT",
   "read_GB": 11.6058,
   "floor_ms@128": 22.75,
   "chat": {
    "ppl_delta_pct": -0.04,
    "dnll": -0.0004,
    "se": 0.0031,
    "top1": 0.9308,
    "kl": 0.0548,
    "kl_p99": 0.837
   },
   "code": {
    "ppl_delta_pct": 2.05,
    "dnll": 0.0203,
    "se": 0.0041,
    "top1": 0.9478,
    "kl": 0.0688,
    "kl_p99": 1.243
   },
   "long-book": {
    "ppl_delta_pct": 7.3,
    "dnll": 0.0704,
    "se": 0.0037,
    "top1": 0.9422,
    "kl": 0.036,
    "kl_p99": 0.353
   },
   "short-all": {
    "ppl_delta_pct": 0.87,
    "dnll": 0.0087,
    "se": 0.0025,
    "top1": 0.9382,
    "kl": 0.061,
    "kl_p99": 1.024
   }
  },
  {
   "config": "g-rank2048",
   "plan": "plans/g-rank2048.json",
   "weights": "IT",
   "read_GB": 10.5441,
   "floor_ms@128": 20.68,
   "chat": {
    "ppl_delta_pct": 1.25,
    "dnll": 0.0125,
    "se": 0.0037,
    "top1": 0.9126,
    "kl": 0.082,
    "kl_p99": 1.28
   },
   "code": {
    "ppl_delta_pct": 1.82,
    "dnll": 0.018,
    "se": 0.0048,
    "top1": 0.9349,
    "kl": 0.0971,
    "kl_p99": 1.813
   },
   "long-book": {
    "ppl_delta_pct": 8.86,
    "dnll": 0.0849,
    "se": 0.0048,
    "top1": 0.9169,
    "kl": 0.0671,
    "kl_p99": 0.713
   },
   "short-all": {
    "ppl_delta_pct": 1.5,
    "dnll": 0.0149,
    "se": 0.003,
    "top1": 0.9224,
    "kl": 0.0886,
    "kl_p99": 1.542
   }
  },
  {
   "config": "b-mlp4",
   "plan": "plans/b-mlp4.json",
   "weights": "IT",
   "read_GB": 8.4207,
   "floor_ms@128": 16.53,
   "chat": {
    "ppl_delta_pct": 5.17,
    "dnll": 0.0504,
    "se": 0.0052,
    "top1": 0.8706,
    "kl": 0.1747,
    "kl_p99": 2.743
   },
   "code": {
    "ppl_delta_pct": 3.91,
    "dnll": 0.0383,
    "se": 0.0066,
    "top1": 0.9083,
    "kl": 0.1758,
    "kl_p99": 3.069
   },
   "long-book": {
    "ppl_delta_pct": 9.91,
    "dnll": 0.0945,
    "se": 0.007,
    "top1": 0.8769,
    "kl": 0.1502,
    "kl_p99": 1.589
   },
   "short-all": {
    "ppl_delta_pct": 4.61,
    "dnll": 0.0451,
    "se": 0.0041,
    "top1": 0.8871,
    "kl": 0.1752,
    "kl_p99": 2.848
   }
  },
  {
   "config": "c-mlp4-lm4",
   "plan": "plans/c-mlp4-lm4.json",
   "weights": "IT",
   "read_GB": 7.9174,
   "floor_ms@128": 15.55,
   "chat": {
    "ppl_delta_pct": 6.21,
    "dnll": 0.0603,
    "se": 0.0054,
    "top1": 0.8667,
    "kl": 0.186,
    "kl_p99": 2.85
   },
   "code": {
    "ppl_delta_pct": 4.99,
    "dnll": 0.0487,
    "se": 0.0069,
    "top1": 0.9057,
    "kl": 0.1882,
    "kl_p99": 3.352
   },
   "long-book": {
    "ppl_delta_pct": 10.8,
    "dnll": 0.1026,
    "se": 0.0072,
    "top1": 0.874,
    "kl": 0.1582,
    "kl_p99": 1.699
   },
   "short-all": {
    "ppl_delta_pct": 5.67,
    "dnll": 0.0552,
    "se": 0.0043,
    "top1": 0.8838,
    "kl": 0.1869,
    "kl_p99": 3.089
   }
  },
  {
   "config": "e-all4",
   "plan": "plans/e-all4.json",
   "weights": "IT",
   "read_GB": 7.2096,
   "floor_ms@128": 14.17,
   "chat": {
    "ppl_delta_pct": 7.24,
    "dnll": 0.0699,
    "se": 0.0068,
    "top1": 0.8266,
    "kl": 0.2946,
    "kl_p99": 4.066
   },
   "code": {
    "ppl_delta_pct": 8.32,
    "dnll": 0.0799,
    "se": 0.0085,
    "top1": 0.8736,
    "kl": 0.2975,
    "kl_p99": 4.776
   },
   "long-book": {
    "ppl_delta_pct": 19.16,
    "dnll": 0.1753,
    "se": 0.0107,
    "top1": 0.8009,
    "kl": 0.3806,
    "kl_p99": 3.648
   },
   "short-all": {
    "ppl_delta_pct": 7.71,
    "dnll": 0.0743,
    "se": 0.0053,
    "top1": 0.8472,
    "kl": 0.2959,
    "kl_p99": 4.394
   }
  },
  {
   "config": "e-all4-lm4",
   "plan": "plans/e-all4-lm4.json",
   "weights": "IT",
   "read_GB": 6.7063,
   "floor_ms@128": 13.18,
   "chat": {
    "ppl_delta_pct": 8.48,
    "dnll": 0.0814,
    "se": 0.0069,
    "top1": 0.824,
    "kl": 0.3089,
    "kl_p99": 4.273
   },
   "code": {
    "ppl_delta_pct": 9.76,
    "dnll": 0.0931,
    "se": 0.0087,
    "top1": 0.8709,
    "kl": 0.3114,
    "kl_p99": 5.03
   },
   "long-book": {
    "ppl_delta_pct": 19.92,
    "dnll": 0.1816,
    "se": 0.0108,
    "top1": 0.7996,
    "kl": 0.3899,
    "kl_p99": 3.703
   },
   "short-all": {
    "ppl_delta_pct": 9.04,
    "dnll": 0.0865,
    "se": 0.0055,
    "top1": 0.8446,
    "kl": 0.31,
    "kl_p99": 4.618
   }
  },
  {
   "config": "d-qat-a",
   "plan": "plans/a-bfp8.json",
   "weights": "QAT",
   "read_GB": 12.6675,
   "floor_ms@128": 24.83,
   "chat": {
    "ppl_delta_pct": -2.86,
    "dnll": -0.029,
    "se": 0.0053,
    "top1": 0.9074,
    "kl": 0.1289,
    "kl_p99": 2.267
   },
   "code": {
    "ppl_delta_pct": -8.9,
    "dnll": -0.0932,
    "se": 0.0059,
    "top1": 0.928,
    "kl": 0.1018,
    "kl_p99": 1.497
   },
   "long-book": {
    "ppl_delta_pct": -4.87,
    "dnll": -0.0499,
    "se": 0.0054,
    "top1": 0.9055,
    "kl": 0.0771,
    "kl_p99": 0.782
   },
   "short-all": {
    "ppl_delta_pct": -5.55,
    "dnll": -0.0571,
    "se": 0.0039,
    "top1": 0.9164,
    "kl": 0.117,
    "kl_p99": 2.004
   }
  },
  {
   "config": "d-qat-b",
   "plan": "plans/b-mlp4.json",
   "weights": "QAT",
   "read_GB": 8.4207,
   "floor_ms@128": 16.53,
   "chat": {
    "ppl_delta_pct": 0.98,
    "dnll": 0.0098,
    "se": 0.0062,
    "top1": 0.8658,
    "kl": 0.2104,
    "kl_p99": 3.608
   },
   "code": {
    "ppl_delta_pct": -0.85,
    "dnll": -0.0086,
    "se": 0.0075,
    "top1": 0.8987,
    "kl": 0.1855,
    "kl_p99": 3.064
   },
   "long-book": {
    "ppl_delta_pct": 6.68,
    "dnll": 0.0647,
    "se": 0.0074,
    "top1": 0.8719,
    "kl": 0.1512,
    "kl_p99": 1.517
   },
   "short-all": {
    "ppl_delta_pct": 0.18,
    "dnll": 0.0018,
    "se": 0.0048,
    "top1": 0.8802,
    "kl": 0.1995,
    "kl_p99": 3.406
   }
  }
 ],
 "gsm8k_50": {
  "a-lm4": {
   "acc": 0.98,
   "agree": 0.96
  },
  "a-bfp8": {
   "acc": 1.0,
   "agree": 0.98
  },
  "b-mlp4": {
   "acc": 0.98,
   "agree": 0.96
  }
 },
 "pick": "a-lm4"
}