{
  "checkpoint": "runs/sv-tinyx-10000/model",
  "limits": {
    "context_bytes": 224,
    "max_options": 40,
    "option_bytes": 96
  },
  "torch_baseline": {
    "examples": 29839,
    "per_action": {
      "check": {
        "acc": 1.0,
        "n": 855
      },
      "click": {
        "acc": 1.0,
        "n": 1009
      },
      "fill": {
        "acc": 0.9900317827217567,
        "n": 13844
      },
      "skip": {
        "acc": 0.9947632863916213,
        "n": 14131
      }
    },
    "silent_skip_rate_of_fills": 7.22334585379948e-05,
    "skipped_when_fill_expected": 1,
    "top1": 0.9928952042628775
  },
  "variants": {
    "fp16": {
      "handwritten_demo__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "examples": 47,
        "latency_ms_median": 1.3022092171013355,
        "latency_ms_p95": 1.6720420680940151,
        "mismatches_vs_torch": 0,
        "parity_with_torch": 1.0,
        "per_action": {
          "check": {
            "acc": 1.0,
            "n": 3
          },
          "click": {
            "acc": 1.0,
            "n": 3
          },
          "fill": {
            "acc": 1.0,
            "n": 29
          },
          "skip": {
            "acc": 1.0,
            "n": 12
          }
        },
        "silent_skip_rate_of_fills": 0.0,
        "skipped_when_fill_expected": 0,
        "top1": 1.0
      },
      "package_bytes": 1507474,
      "synthetic_test__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "examples": 29836,
        "latency_ms_median": 1.313916640356183,
        "latency_ms_p95": 1.412208192050457,
        "mismatches_vs_torch": 15,
        "parity_with_torch": 0.9994972516423113,
        "per_action": {
          "check": {
            "acc": 1.0,
            "n": 855
          },
          "click": {
            "acc": 1.0,
            "n": 1009
          },
          "fill": {
            "acc": 0.9899580985406733,
            "n": 13842
          },
          "skip": {
            "acc": 0.994762915782024,
            "n": 14130
          }
        },
        "silent_skip_rate_of_fills": 7.224389539083947e-05,
        "skipped_when_fill_expected": 1,
        "top1": 0.9928609733208205
      },
      "synthetic_test__cpu": {
        "compute_units": "ComputeUnit.CPU_ONLY",
        "examples": 29836,
        "latency_ms_median": 1.7267910297960043,
        "latency_ms_p95": 1.9287499599158764,
        "mismatches_vs_torch": 14,
        "parity_with_torch": 0.9995307681994905,
        "per_action": {
          "check": {
            "acc": 1.0,
            "n": 855
          },
          "click": {
            "acc": 1.0,
            "n": 1009
          },
          "fill": {
            "acc": 0.9897413668545008,
            "n": 13842
          },
          "skip": {
            "acc": 0.994762915782024,
            "n": 14130
          }
        },
        "silent_skip_rate_of_fills": 7.224389539083947e-05,
        "skipped_when_fill_expected": 1,
        "top1": 0.9927604236492827
      }
    },
    "int4": {
      "handwritten_demo__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "examples": 47,
        "latency_ms_median": 1.8585827201604843,
        "latency_ms_p95": 2.018083818256855,
        "mismatches_vs_torch": 32,
        "parity_with_torch": 0.3191489361702128,
        "per_action": {
          "check": {
            "acc": 0.0,
            "n": 3
          },
          "click": {
            "acc": 0.0,
            "n": 3
          },
          "fill": {
            "acc": 0.10344827586206896,
            "n": 29
          },
          "skip": {
            "acc": 1.0,
            "n": 12
          }
        },
        "silent_skip_rate_of_fills": 0.896551724137931,
        "skipped_when_fill_expected": 26,
        "top1": 0.3191489361702128
      },
      "package_bytes": 492577,
      "synthetic_test__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "examples": 29836,
        "latency_ms_median": 1.9839999731630087,
        "latency_ms_p95": 2.11525009945035,
        "mismatches_vs_torch": 15015,
        "parity_with_torch": 0.4967488939536131,
        "per_action": {
          "check": {
            "acc": 0.012865497076023392,
            "n": 855
          },
          "click": {
            "acc": 0.0,
            "n": 1009
          },
          "fill": {
            "acc": 0.06408033521167461,
            "n": 13842
          },
          "skip": {
            "acc": 0.9905166312809625,
            "n": 14130
          }
        },
        "silent_skip_rate_of_fills": 0.9143909839618553,
        "skipped_when_fill_expected": 12657,
        "top1": 0.4991956026276981
      }
    },
    "int8": {
      "handwritten_demo__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "examples": 47,
        "latency_ms_median": 1.3158749788999557,
        "latency_ms_p95": 1.4334581792354584,
        "mismatches_vs_torch": 0,
        "parity_with_torch": 1.0,
        "per_action": {
          "check": {
            "acc": 1.0,
            "n": 3
          },
          "click": {
            "acc": 1.0,
            "n": 3
          },
          "fill": {
            "acc": 1.0,
            "n": 29
          },
          "skip": {
            "acc": 1.0,
            "n": 12
          }
        },
        "silent_skip_rate_of_fills": 0.0,
        "skipped_when_fill_expected": 0,
        "top1": 1.0
      },
      "package_bytes": 806125,
      "synthetic_test__ane": {
        "compute_units": "ComputeUnit.CPU_AND_NE",
        "examples": 29836,
        "latency_ms_median": 1.3136658817529678,
        "latency_ms_p95": 1.4113341458141804,
        "mismatches_vs_torch": 13,
        "parity_with_torch": 0.9995642847566698,
        "per_action": {
          "check": {
            "acc": 1.0,
            "n": 855
          },
          "click": {
            "acc": 1.0,
            "n": 1009
          },
          "fill": {
            "acc": 0.990102586331455,
            "n": 13842
          },
          "skip": {
            "acc": 0.994762915782024,
            "n": 14130
          }
        },
        "silent_skip_rate_of_fills": 7.224389539083947e-05,
        "skipped_when_fill_expected": 1,
        "top1": 0.992928006435179
      }
    }
  }
}