{
  "bench": "../benchmarks/csn-3.12-licensed/bench.jsonl",
  "n": 600,
  "size_axis": "rep_lines = lines of the disassembly handed to the model (bench row `input`)",
  "best_of_n_included": true,
  "self_test": {
    "preflight_pct": 100.0,
    "mutation_kill_rate_pct": 100.0,
    "SOUND": true
  },
  "rep_lines_distribution": {
    "min": 19,
    "p25": 39,
    "median": 59,
    "p75": 101,
    "p90": 167,
    "p99": 478,
    "max": 1622
  },
  "by_rep_lines": [
    {
      "bucket": "0-49",
      "n": 224,
      "v3_greedy": {
        "certified": 215,
        "n": 224,
        "pct": 95.98,
        "ci95": [
          93.01,
          98.51
        ],
        "ci_method": "repo-clustered bootstrap"
      },
      "base_greedy": {
        "certified": 4,
        "n": 224,
        "pct": 1.79,
        "ci95": [
          0.43,
          3.62
        ],
        "ci_method": "repo-clustered bootstrap"
      },
      "v3_boN32": {
        "certified": 220,
        "n": 224,
        "pct": 98.21,
        "ci95": [
          96.31,
          99.58
        ],
        "ci_method": "repo-clustered bootstrap"
      }
    },
    {
      "bucket": "50-99",
      "n": 224,
      "v3_greedy": {
        "certified": 201,
        "n": 224,
        "pct": 89.73,
        "ci95": [
          85.17,
          93.93
        ],
        "ci_method": "repo-clustered bootstrap"
      },
      "base_greedy": {
        "certified": 0,
        "n": 224,
        "pct": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "ci_method": "repo-clustered bootstrap"
      },
      "v3_boN32": {
        "certified": 220,
        "n": 224,
        "pct": 98.21,
        "ci95": [
          96.26,
          99.57
        ],
        "ci_method": "repo-clustered bootstrap"
      }
    },
    {
      "bucket": "100-199",
      "n": 112,
      "v3_greedy": {
        "certified": 73,
        "n": 112,
        "pct": 65.18,
        "ci95": [
          55.36,
          74.14
        ],
        "ci_method": "repo-clustered bootstrap"
      },
      "base_greedy": {
        "certified": 0,
        "n": 112,
        "pct": 0.0,
        "ci95": [
          0.0,
          0.0
        ],
        "ci_method": "repo-clustered bootstrap"
      },
      "v3_boN32": {
        "certified": 96,
        "n": 112,
        "pct": 85.71,
        "ci95": [
          77.57,
          92.98
        ],
        "ci_method": "repo-clustered bootstrap"
      }
    },
    {
      "bucket": "200-299",
      "n": 27,
      "v3_greedy": {
        "certified": 14,
        "n": 27,
        "pct": 51.85,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      },
      "base_greedy": {
        "certified": 0,
        "n": 27,
        "pct": 0.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      },
      "v3_boN32": {
        "certified": 21,
        "n": 27,
        "pct": 77.78,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      }
    },
    {
      "bucket": "300-399",
      "n": 5,
      "v3_greedy": {
        "certified": 3,
        "n": 5,
        "pct": 60.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      },
      "base_greedy": {
        "certified": 0,
        "n": 5,
        "pct": 0.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      },
      "v3_boN32": {
        "certified": 5,
        "n": 5,
        "pct": 100.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      }
    },
    {
      "bucket": "400-599",
      "n": 5,
      "v3_greedy": {
        "certified": 0,
        "n": 5,
        "pct": 0.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      },
      "base_greedy": {
        "certified": 0,
        "n": 5,
        "pct": 0.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      },
      "v3_boN32": {
        "certified": 0,
        "n": 5,
        "pct": 0.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      }
    },
    {
      "bucket": "600+",
      "n": 3,
      "v3_greedy": {
        "certified": 0,
        "n": 3,
        "pct": 0.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      },
      "base_greedy": {
        "certified": 0,
        "n": 3,
        "pct": 0.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      },
      "v3_boN32": {
        "certified": 0,
        "n": 3,
        "pct": 0.0,
        "ci95": null,
        "ci_method": "omitted: too few rows/repos to estimate"
      }
    }
  ],
  "share_of_certifications_below_knee": {
    "v3_greedy": {
      "knee_rep_lines": 200,
      "certified_total": 506,
      "certified_below": 489,
      "pct": 96.64
    },
    "base_greedy": {
      "knee_rep_lines": 200,
      "certified_total": 4,
      "certified_below": 4,
      "pct": 100.0
    },
    "v3_boN32": {
      "knee_rep_lines": 200,
      "certified_total": 562,
      "certified_below": 536,
      "pct": 95.37
    }
  }
}