results/problem_solving_n500.json

{
  "schema": "supermix-v65-problem-solving-accuracy-v1",
  "checkpoint": "output\\v74_broad\\v74_broad.pt",
  "dev_loss": 0.065107,
  "problems": 500,
  "unparsed_replies": 0,
  "by_source": {
    "novel": {
      "n": 500,
      "correct": 447,
      "tasks": {
        "algebra_one_step": {
          "n": 56,
          "correct": 50,
          "accuracy": 0.8929,
          "median_relative_error": 0.0,
          "within_10_percent": 0.8929
        },
        "arithmetic": {
          "n": 56,
          "correct": 50,
          "accuracy": 0.8929,
          "median_relative_error": 0.0,
          "within_10_percent": 1.0
        },
        "average": {
          "n": 56,
          "correct": 33,
          "accuracy": 0.5893,
          "median_relative_error": 0.0,
          "within_10_percent": 0.9821
        },
        "division": {
          "n": 55,
          "correct": 55,
          "accuracy": 1.0,
          "median_relative_error": 0.0,
          "within_10_percent": 1.0
        },
        "multiplication": {
          "n": 55,
          "correct": 55,
          "accuracy": 1.0,
          "median_relative_error": 0.0,
          "within_10_percent": 1.0
        },
        "percent": {
          "n": 56,
          "correct": 42,
          "accuracy": 0.75,
          "median_relative_error": 0.0,
          "within_10_percent": 0.7679
        },
        "sequence": {
          "n": 55,
          "correct": 54,
          "accuracy": 0.9818,
          "median_relative_error": 0.0,
          "within_10_percent": 0.9818
        },
        "two_step": {
          "n": 55,
          "correct": 54,
          "accuracy": 0.9818,
          "median_relative_error": 0.0,
          "within_10_percent": 0.9818
        },
        "word_problem": {
          "n": 56,
          "correct": 54,
          "accuracy": 0.9643,
          "median_relative_error": 0.0,
          "within_10_percent": 0.9643
        }
      },
      "accuracy": 0.894,
      "median_relative_error": 0.0,
      "within_10_percent": 0.952
    }
  },
  "memorisation_gap": null,
  "examples": [
    {
      "source": "novel",
      "task": "arithmetic",
      "prompt": "Solve this basic math problem: 524 - 305",
      "reply": "500 - 300 = 200, 24 - 5 = 19, total 219",
      "expected": 219.0,
      "predicted": 219.0,
      "correct": true
    },
    {
      "source": "novel",
      "task": "percent",
      "prompt": "What is 20% of 1749?",
      "reply": "1 percent of 1749 = 17.49, times 20 = 349.8, total 349.8",
      "expected": 349.8,
      "predicted": 349.8,
      "correct": true
    },
    {
      "source": "novel",
      "task": "average",
      "prompt": "Find the average (mean) of these numbers: 61, 63, 72, 61",
      "reply": "sum: 61 then 124 then 196 then 257, total 257, divide by 4, total 64.25",
      "expected": 64.25,
      "predicted": 64.25,
      "correct": true
    },
    {
      "source": "novel",
      "task": "algebra_one_step",
      "prompt": "Solve for x: x + 29 = 34",
      "reply": "subtract 29 from both sides, 34 - 29 = 5, total 5",
      "expected": 5.0,
      "predicted": 5.0,
      "correct": true
    },
    {
      "source": "novel",
      "task": "word_problem",
      "prompt": "A student has 63 stickers. They get 57 more and then give away 45. How many stickers do th",
      "reply": "63 + 57 = 120, 120 - 45 = 75, total 75",
      "expected": 75.0,
      "predicted": 75.0,
      "correct": true
    },
    {
      "source": "novel",
      "task": "multiplication",
      "prompt": "Solve this basic math problem: 39 x 3",
      "reply": "30 x 3 = 90, 9 x 3 = 27, total 117",
      "expected": 117.0,
      "predicted": 117.0,
      "correct": true
    },
    {
      "source": "novel",
      "task": "division",
      "prompt": "Solve this basic math problem: 243 / 9",
      "reply": "180 / 9 = 20, 63 / 9 = 7, total 27",
      "expected": 27.0,
      "predicted": 27.0,
      "correct": true
    },
    {
      "source": "novel",
      "task": "sequence",
      "prompt": "What comes next in the sequence: 15, 27, 39, 51?",
      "reply": "27 - 15 = 12, difference 12, 51 + 12 = 63, total 63",
      "expected": 63.0,
      "predicted": 63.0,
      "correct": true
    },
    {
      "source": "novel",
      "task": "two_step",
      "prompt": "What is 50% of 678, then add 49?",
      "reply": "1 percent of 678 = 6.78, times 50 = 339, then 339 + 49 = 388, total 388",
      "expected": 388.0,
      "predicted": 388.0,
      "correct": true
    },
    {
      "source": "novel",
      "task": "arithmetic",
      "prompt": "Solve this basic math problem: 215 + 173",
      "reply": "200 + 100 = 300, 15 + 73 = 88, total 388",
      "expected": 388.0,
      "predicted": 388.0,
      "correct": true
    },
    {
      "source": "novel",
      "task": "percent",
      "prompt": "What is 25% of 1707?",
      "reply": "1 percent of 1707 = 17.07, times 20 = 341.4, times 5 = 85.35, total 426.75",
      "expected": 426.75,
      "predicted": 426.75,
      "correct": true
    },
    {
      "source": "novel",
      "task": "average",
      "prompt": "Find the average (mean) of these numbers: 22, 93, 49, 52",
      "reply": "sum: 22 then 115 then 164 then 216, total 216, divide by 4, total 54.0",
      "expected": 54.0,
      "predicted": 54.0,
      "correct": true
    }
  ],
  "non_claims": [
    "Accuracy on generated arithmetic is not general problem solving. It covers five task types with small operands and nothing else.",
    "A high 'seen' score with a low 'novel' score is recall, not skill; the gap is the finding, not either number alone.",
    "Answer extraction takes the last number in the reply. A reply that reasons correctly and then trails off into another number scores wrong, so this is a lower bound on the model."
  ]
}