eval/results-pytorch.json

{
  "datasets": {
    "handwritten_demo": "/Users/hermes-tool/builder-work/one-pass-specialists-67k19_tm/data/sv-10000/demo-handwritten.jsonl",
    "synthetic_test": "/Users/hermes-tool/builder-work/one-pass-specialists-67k19_tm/data/sv-10000/test.jsonl"
  },
  "device": "mps",
  "specialist": {
    "checkpoint": "/Users/hermes-tool/builder-work/one-pass-specialists-67k19_tm/runs/sv-tinyx-10000/model/model.safetensors",
    "checkpoint_bytes": 2828784,
    "config": {
      "context_tokens": 224,
      "encoder": "tinyx",
      "heads": 4,
      "layers": 2,
      "option_tokens": 96,
      "rank": 128,
      "width": 128
    },
    "handwritten_demo": {
      "action_mix": {
        "check": 3,
        "click": 3,
        "fill": 32,
        "skip": 12
      },
      "ece": 0.000869,
      "examples": 50,
      "majority_baseline": 0.64,
      "per_action": {
        "check": {
          "acc": 1.0,
          "n": 3
        },
        "click": {
          "acc": 1.0,
          "n": 3
        },
        "fill": {
          "acc": 1.0,
          "n": 32
        },
        "skip": {
          "acc": 1.0,
          "n": 12
        }
      },
      "skipped_when_fill_expected": 0,
      "top1": 1.0
    },
    "handwritten_demo__shuffled_context": {
      "action_mix": {
        "check": 3,
        "click": 3,
        "fill": 32,
        "skip": 12
      },
      "ece": 0.819131,
      "examples": 50,
      "majority_baseline": 0.64,
      "per_action": {
        "check": {
          "acc": 0.0,
          "n": 3
        },
        "click": {
          "acc": 0.0,
          "n": 3
        },
        "fill": {
          "acc": 0.0,
          "n": 32
        },
        "skip": {
          "acc": 0.25,
          "n": 12
        }
      },
      "skipped_when_fill_expected": 3,
      "top1": 0.06
    },
    "label": "specialist",
    "synthetic_test": {
      "action_mix": {
        "check": 855,
        "click": 1009,
        "fill": 13844,
        "skip": 14131
      },
      "ece": 0.001373,
      "examples": 29839,
      "majority_baseline": 0.47357,
      "per_action": {
        "check": {
          "acc": 1.0,
          "n": 855
        },
        "click": {
          "acc": 1.0,
          "n": 1009
        },
        "fill": {
          "acc": 0.99003,
          "n": 13844
        },
        "skip": {
          "acc": 0.99476,
          "n": 14131
        }
      },
      "skipped_when_fill_expected": 1,
      "top1": 0.9929
    },
    "synthetic_test__shuffled_context": {
      "action_mix": {
        "check": 855,
        "click": 1009,
        "fill": 13844,
        "skip": 14131
      },
      "ece": 0.607719,
      "examples": 29839,
      "majority_baseline": 0.47357,
      "per_action": {
        "check": {
          "acc": 0.20936,
          "n": 855
        },
        "click": {
          "acc": 0.0,
          "n": 1009
        },
        "fill": {
          "acc": 0.00079,
          "n": 13844
        },
        "skip": {
          "acc": 0.66209,
          "n": 14131
        }
      },
      "skipped_when_fill_expected": 3774,
      "top1": 0.31992
    },
    "trainable_params": 706048
  }
}