{
  "device": "mps",
  "reference": {
    "checkpoint": "/tmp/cua1ckpt/cua-s1-forms.safetensors",
    "checkpoint_bytes": 2828784,
    "config": {
      "context_tokens": 224,
      "encoder": "tinyx",
      "heads": 4,
      "hf_model": "Qwen/Qwen2.5-0.5B",
      "layers": 2,
      "option_tokens": 96,
      "rank": 128,
      "width": 128
    },
    "label": "cua-s1-forms-english",
    "sv_handwritten_demo": {
      "action_mix": {
        "check": 3,
        "click": 3,
        "fill": 32,
        "skip": 12
      },
      "ece": 0.660977,
      "examples": 50,
      "majority_baseline": 0.64,
      "per_action": {
        "check": {
          "acc": 0.66667,
          "n": 3
        },
        "click": {
          "acc": 0.33333,
          "n": 3
        },
        "fill": {
          "acc": 0.15625,
          "n": 32
        },
        "skip": {
          "acc": 0.16667,
          "n": 12
        }
      },
      "skipped_when_fill_expected": 16,
      "top1": 0.2
    },
    "sv_handwritten_demo__shuffled_context": {
      "action_mix": {
        "check": 3,
        "click": 3,
        "fill": 32,
        "skip": 12
      },
      "ece": 0.760258,
      "examples": 50,
      "majority_baseline": 0.64,
      "per_action": {
        "check": {
          "acc": 0.66667,
          "n": 3
        },
        "click": {
          "acc": 0.0,
          "n": 3
        },
        "fill": {
          "acc": 0.0,
          "n": 32
        },
        "skip": {
          "acc": 0.25,
          "n": 12
        }
      },
      "skipped_when_fill_expected": 13,
      "top1": 0.1
    },
    "sv_synthetic_test": {
      "action_mix": {
        "check": 58,
        "click": 79,
        "fill": 1010,
        "skip": 1168
      },
      "ece": 0.640725,
      "examples": 2315,
      "majority_baseline": 0.50454,
      "per_action": {
        "check": {
          "acc": 0.93103,
          "n": 58
        },
        "click": {
          "acc": 0.26582,
          "n": 79
        },
        "fill": {
          "acc": 0.05545,
          "n": 1010
        },
        "skip": {
          "acc": 0.31164,
          "n": 1168
        }
      },
      "skipped_when_fill_expected": 371,
      "top1": 0.21382
    },
    "sv_synthetic_test__shuffled_context": {
      "action_mix": {
        "check": 58,
        "click": 79,
        "fill": 1010,
        "skip": 1168
      },
      "ece": 0.679541,
      "examples": 2315,
      "majority_baseline": 0.50454,
      "per_action": {
        "check": {
          "acc": 0.60345,
          "n": 58
        },
        "click": {
          "acc": 0.06329,
          "n": 79
        },
        "fill": {
          "acc": 0.01782,
          "n": 1010
        },
        "skip": {
          "acc": 0.29709,
          "n": 1168
        }
      },
      "skipped_when_fill_expected": 363,
      "top1": 0.17495
    },
    "trainable_params": 706048
  },
  "sv": {
    "checkpoint": "runs/sv-tinyx/model/model.safetensors",
    "checkpoint_bytes": 2828784,
    "config": {
      "context_tokens": 224,
      "encoder": "tinyx",
      "heads": 4,
      "layers": 2,
      "option_tokens": 96,
      "rank": 128,
      "width": 128
    },
    "label": "swedish-specialist",
    "sv_handwritten_demo": {
      "action_mix": {
        "check": 3,
        "click": 3,
        "fill": 32,
        "skip": 12
      },
      "ece": 0.09176,
      "examples": 50,
      "majority_baseline": 0.64,
      "per_action": {
        "check": {
          "acc": 1.0,
          "n": 3
        },
        "click": {
          "acc": 1.0,
          "n": 3
        },
        "fill": {
          "acc": 0.8125,
          "n": 32
        },
        "skip": {
          "acc": 0.91667,
          "n": 12
        }
      },
      "skipped_when_fill_expected": 3,
      "top1": 0.86
    },
    "sv_handwritten_demo__shuffled_context": {
      "action_mix": {
        "check": 3,
        "click": 3,
        "fill": 32,
        "skip": 12
      },
      "ece": 0.785789,
      "examples": 50,
      "majority_baseline": 0.64,
      "per_action": {
        "check": {
          "acc": 0.0,
          "n": 3
        },
        "click": {
          "acc": 0.0,
          "n": 3
        },
        "fill": {
          "acc": 0.0,
          "n": 32
        },
        "skip": {
          "acc": 0.25,
          "n": 12
        }
      },
      "skipped_when_fill_expected": 6,
      "top1": 0.06
    },
    "sv_synthetic_test": {
      "action_mix": {
        "check": 58,
        "click": 79,
        "fill": 1010,
        "skip": 1168
      },
      "ece": 0.016565,
      "examples": 2315,
      "majority_baseline": 0.50454,
      "per_action": {
        "check": {
          "acc": 0.98276,
          "n": 58
        },
        "click": {
          "acc": 0.93671,
          "n": 79
        },
        "fill": {
          "acc": 0.75545,
          "n": 1010
        },
        "skip": {
          "acc": 0.88014,
          "n": 1168
        }
      },
      "skipped_when_fill_expected": 76,
      "top1": 0.83024
    },
    "sv_synthetic_test__shuffled_context": {
      "action_mix": {
        "check": 58,
        "click": 79,
        "fill": 1010,
        "skip": 1168
      },
      "ece": 0.497346,
      "examples": 2315,
      "majority_baseline": 0.50454,
      "per_action": {
        "check": {
          "acc": 0.15517,
          "n": 58
        },
        "click": {
          "acc": 0.0,
          "n": 79
        },
        "fill": {
          "acc": 0.01287,
          "n": 1010
        },
        "skip": {
          "acc": 0.65668,
          "n": 1168
        }
      },
      "skipped_when_fill_expected": 264,
      "top1": 0.34082
    },
    "trainable_params": 706048
  }
}