{
  "checkpoint": "runs/sv-tinyx/model",
  "limits": {
    "context_bytes": 224,
    "max_options": 40,
    "option_bytes": 96
  },
  "torch_baseline": {
    "per_action": {
      "check": {
        "acc": 0.9827586206896551,
        "n": 58
      },
      "click": {
        "acc": 0.9367088607594937,
        "n": 79
      },
      "fill": {
        "acc": 0.7554455445544555,
        "n": 1010
      },
      "skip": {
        "acc": 0.8801369863013698,
        "n": 1168
      }
    },
    "rows": 2315,
    "silent_skip_rate_of_fills": 0.07524752475247524,
    "silent_skips": 76,
    "top1": 0.8302375809935205
  },
  "variants": {
    "fp16": {
      "handwritten_demo__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "latency_ms_median": 1.2992918491363525,
        "latency_ms_p95": 1.4392081648111343,
        "mismatches_vs_torch": 0,
        "parity_with_torch": 1.0,
        "per_action": {
          "check": {
            "acc": 1.0,
            "n": 3
          },
          "click": {
            "acc": 1.0,
            "n": 3
          },
          "fill": {
            "acc": 0.7931034482758621,
            "n": 29
          },
          "skip": {
            "acc": 0.9166666666666666,
            "n": 12
          }
        },
        "rows": 47,
        "silent_skip_rate_of_fills": 0.10344827586206896,
        "silent_skips": 3,
        "top1": 0.851063829787234
      },
      "package_bytes": 1507481,
      "synthetic_test__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "latency_ms_median": 1.3071454595774412,
        "latency_ms_p95": 1.4601247385144234,
        "mismatches_vs_torch": 3,
        "parity_with_torch": 0.9987024221453287,
        "per_action": {
          "check": {
            "acc": 0.9827586206896551,
            "n": 58
          },
          "click": {
            "acc": 0.9367088607594937,
            "n": 79
          },
          "fill": {
            "acc": 0.753968253968254,
            "n": 1008
          },
          "skip": {
            "acc": 0.8808911739502999,
            "n": 1167
          }
        },
        "rows": 2312,
        "silent_skip_rate_of_fills": 0.07539682539682539,
        "silent_skips": 76,
        "top1": 0.8300173010380623
      },
      "synthetic_test__cpu": {
        "compute_units": "ComputeUnit.CPU_ONLY",
        "latency_ms_median": 1.733562909066677,
        "latency_ms_p95": 1.9167079590260983,
        "mismatches_vs_torch": 2,
        "parity_with_torch": 0.9991349480968859,
        "per_action": {
          "check": {
            "acc": 0.9827586206896551,
            "n": 58
          },
          "click": {
            "acc": 0.9367088607594937,
            "n": 79
          },
          "fill": {
            "acc": 0.7549603174603174,
            "n": 1008
          },
          "skip": {
            "acc": 0.8808911739502999,
            "n": 1167
          }
        },
        "rows": 2312,
        "silent_skip_rate_of_fills": 0.07539682539682539,
        "silent_skips": 76,
        "top1": 0.8304498269896193
      }
    },
    "int4": {
      "handwritten_demo__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "latency_ms_median": 1.8591247498989105,
        "latency_ms_p95": 1.993041019886732,
        "mismatches_vs_torch": 32,
        "parity_with_torch": 0.3191489361702128,
        "per_action": {
          "check": {
            "acc": 0.0,
            "n": 3
          },
          "click": {
            "acc": 0.0,
            "n": 3
          },
          "fill": {
            "acc": 0.034482758620689655,
            "n": 29
          },
          "skip": {
            "acc": 1.0,
            "n": 12
          }
        },
        "rows": 47,
        "silent_skip_rate_of_fills": 0.9655172413793104,
        "silent_skips": 28,
        "top1": 0.2765957446808511
      },
      "package_bytes": 492584,
      "synthetic_test__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "latency_ms_median": 1.9683334976434708,
        "latency_ms_p95": 2.099000383168459,
        "mismatches_vs_torch": 1186,
        "parity_with_torch": 0.4870242214532872,
        "per_action": {
          "check": {
            "acc": 0.0,
            "n": 58
          },
          "click": {
            "acc": 0.0,
            "n": 79
          },
          "fill": {
            "acc": 0.018849206349206348,
            "n": 1008
          },
          "skip": {
            "acc": 0.9914310197086547,
            "n": 1167
          }
        },
        "rows": 2312,
        "silent_skip_rate_of_fills": 0.9464285714285714,
        "silent_skips": 954,
        "top1": 0.5086505190311419
      }
    },
    "int8": {
      "handwritten_demo__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "latency_ms_median": 1.31650036200881,
        "latency_ms_p95": 1.4507090672850609,
        "mismatches_vs_torch": 0,
        "parity_with_torch": 1.0,
        "per_action": {
          "check": {
            "acc": 1.0,
            "n": 3
          },
          "click": {
            "acc": 1.0,
            "n": 3
          },
          "fill": {
            "acc": 0.7931034482758621,
            "n": 29
          },
          "skip": {
            "acc": 0.9166666666666666,
            "n": 12
          }
        },
        "rows": 47,
        "silent_skip_rate_of_fills": 0.10344827586206896,
        "silent_skips": 3,
        "top1": 0.851063829787234
      },
      "package_bytes": 806132,
      "synthetic_test__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "latency_ms_median": 1.315583474934101,
        "latency_ms_p95": 1.401625107973814,
        "mismatches_vs_torch": 9,
        "parity_with_torch": 0.9961072664359861,
        "per_action": {
          "check": {
            "acc": 0.9827586206896551,
            "n": 58
          },
          "click": {
            "acc": 0.9367088607594937,
            "n": 79
          },
          "fill": {
            "acc": 0.753968253968254,
            "n": 1008
          },
          "skip": {
            "acc": 0.882604970008569,
            "n": 1167
          }
        },
        "rows": 2312,
        "silent_skip_rate_of_fills": 0.07539682539682539,
        "silent_skips": 76,
        "top1": 0.8308823529411765
      }
    }
  }
}