| { |
| "checkpoint": "runs/sv-tinyx-10000/model", |
| "limits": { |
| "context_bytes": 224, |
| "max_options": 40, |
| "option_bytes": 96 |
| }, |
| "torch_baseline": { |
| "examples": 29839, |
| "per_action": { |
| "check": { |
| "acc": 1.0, |
| "n": 855 |
| }, |
| "click": { |
| "acc": 1.0, |
| "n": 1009 |
| }, |
| "fill": { |
| "acc": 0.9900317827217567, |
| "n": 13844 |
| }, |
| "skip": { |
| "acc": 0.9947632863916213, |
| "n": 14131 |
| } |
| }, |
| "silent_skip_rate_of_fills": 7.22334585379948e-05, |
| "skipped_when_fill_expected": 1, |
| "top1": 0.9928952042628775 |
| }, |
| "variants": { |
| "fp16": { |
| "handwritten_demo__ane": { |
| "compute_units": "ComputeUnit.CPU_AND_NE", |
| "examples": 47, |
| "latency_ms_median": 1.3022092171013355, |
| "latency_ms_p95": 1.6720420680940151, |
| "mismatches_vs_torch": 0, |
| "parity_with_torch": 1.0, |
| "per_action": { |
| "check": { |
| "acc": 1.0, |
| "n": 3 |
| }, |
| "click": { |
| "acc": 1.0, |
| "n": 3 |
| }, |
| "fill": { |
| "acc": 1.0, |
| "n": 29 |
| }, |
| "skip": { |
| "acc": 1.0, |
| "n": 12 |
| } |
| }, |
| "silent_skip_rate_of_fills": 0.0, |
| "skipped_when_fill_expected": 0, |
| "top1": 1.0 |
| }, |
| "package_bytes": 1507474, |
| "synthetic_test__ane": { |
| "compute_units": "ComputeUnit.CPU_AND_NE", |
| "examples": 29836, |
| "latency_ms_median": 1.313916640356183, |
| "latency_ms_p95": 1.412208192050457, |
| "mismatches_vs_torch": 15, |
| "parity_with_torch": 0.9994972516423113, |
| "per_action": { |
| "check": { |
| "acc": 1.0, |
| "n": 855 |
| }, |
| "click": { |
| "acc": 1.0, |
| "n": 1009 |
| }, |
| "fill": { |
| "acc": 0.9899580985406733, |
| "n": 13842 |
| }, |
| "skip": { |
| "acc": 0.994762915782024, |
| "n": 14130 |
| } |
| }, |
| "silent_skip_rate_of_fills": 7.224389539083947e-05, |
| "skipped_when_fill_expected": 1, |
| "top1": 0.9928609733208205 |
| }, |
| "synthetic_test__cpu": { |
| "compute_units": "ComputeUnit.CPU_ONLY", |
| "examples": 29836, |
| "latency_ms_median": 1.7267910297960043, |
| "latency_ms_p95": 1.9287499599158764, |
| "mismatches_vs_torch": 14, |
| "parity_with_torch": 0.9995307681994905, |
| "per_action": { |
| "check": { |
| "acc": 1.0, |
| "n": 855 |
| }, |
| "click": { |
| "acc": 1.0, |
| "n": 1009 |
| }, |
| "fill": { |
| "acc": 0.9897413668545008, |
| "n": 13842 |
| }, |
| "skip": { |
| "acc": 0.994762915782024, |
| "n": 14130 |
| } |
| }, |
| "silent_skip_rate_of_fills": 7.224389539083947e-05, |
| "skipped_when_fill_expected": 1, |
| "top1": 0.9927604236492827 |
| } |
| }, |
| "int4": { |
| "handwritten_demo__ane": { |
| "compute_units": "ComputeUnit.CPU_AND_NE", |
| "examples": 47, |
| "latency_ms_median": 1.8585827201604843, |
| "latency_ms_p95": 2.018083818256855, |
| "mismatches_vs_torch": 32, |
| "parity_with_torch": 0.3191489361702128, |
| "per_action": { |
| "check": { |
| "acc": 0.0, |
| "n": 3 |
| }, |
| "click": { |
| "acc": 0.0, |
| "n": 3 |
| }, |
| "fill": { |
| "acc": 0.10344827586206896, |
| "n": 29 |
| }, |
| "skip": { |
| "acc": 1.0, |
| "n": 12 |
| } |
| }, |
| "silent_skip_rate_of_fills": 0.896551724137931, |
| "skipped_when_fill_expected": 26, |
| "top1": 0.3191489361702128 |
| }, |
| "package_bytes": 492577, |
| "synthetic_test__ane": { |
| "compute_units": "ComputeUnit.CPU_AND_NE", |
| "examples": 29836, |
| "latency_ms_median": 1.9839999731630087, |
| "latency_ms_p95": 2.11525009945035, |
| "mismatches_vs_torch": 15015, |
| "parity_with_torch": 0.4967488939536131, |
| "per_action": { |
| "check": { |
| "acc": 0.012865497076023392, |
| "n": 855 |
| }, |
| "click": { |
| "acc": 0.0, |
| "n": 1009 |
| }, |
| "fill": { |
| "acc": 0.06408033521167461, |
| "n": 13842 |
| }, |
| "skip": { |
| "acc": 0.9905166312809625, |
| "n": 14130 |
| } |
| }, |
| "silent_skip_rate_of_fills": 0.9143909839618553, |
| "skipped_when_fill_expected": 12657, |
| "top1": 0.4991956026276981 |
| } |
| }, |
| "int8": { |
| "handwritten_demo__ane": { |
| "compute_units": "ComputeUnit.CPU_AND_NE", |
| "examples": 47, |
| "latency_ms_median": 1.3158749788999557, |
| "latency_ms_p95": 1.4334581792354584, |
| "mismatches_vs_torch": 0, |
| "parity_with_torch": 1.0, |
| "per_action": { |
| "check": { |
| "acc": 1.0, |
| "n": 3 |
| }, |
| "click": { |
| "acc": 1.0, |
| "n": 3 |
| }, |
| "fill": { |
| "acc": 1.0, |
| "n": 29 |
| }, |
| "skip": { |
| "acc": 1.0, |
| "n": 12 |
| } |
| }, |
| "silent_skip_rate_of_fills": 0.0, |
| "skipped_when_fill_expected": 0, |
| "top1": 1.0 |
| }, |
| "package_bytes": 806125, |
| "synthetic_test__ane": { |
| "compute_units": "ComputeUnit.CPU_AND_NE", |
| "examples": 29836, |
| "latency_ms_median": 1.3136658817529678, |
| "latency_ms_p95": 1.4113341458141804, |
| "mismatches_vs_torch": 13, |
| "parity_with_torch": 0.9995642847566698, |
| "per_action": { |
| "check": { |
| "acc": 1.0, |
| "n": 855 |
| }, |
| "click": { |
| "acc": 1.0, |
| "n": 1009 |
| }, |
| "fill": { |
| "acc": 0.990102586331455, |
| "n": 13842 |
| }, |
| "skip": { |
| "acc": 0.994762915782024, |
| "n": 14130 |
| } |
| }, |
| "silent_skip_rate_of_fills": 7.224389539083947e-05, |
| "skipped_when_fill_expected": 1, |
| "top1": 0.992928006435179 |
| } |
| } |
| } |
| } |
|
|
