| { |
| "experiment": "STRATA-PROMPT-SERIALIZATION-COMPARISON-1", |
| "unique_records": 4096, |
| "repeats": 2, |
| "registration_sha256": "f94beb6d736a3178f651ddc196b6a90d2cefffe2b9979b19129777afa2c1c782", |
| "arms": { |
| "PROMPT_SERIALIZATION": { |
| "correct": 8102, |
| "total": 8192, |
| "reference_exact": 0, |
| "input_tokens": 1116502, |
| "truncated": 16, |
| "accuracy": 0.989013671875, |
| "mean_input_tokens": 136.291748046875, |
| "seconds": 962.6804165355861, |
| "amortized_ms_per_record": 117.51469928412916, |
| "median_batch_amortized_ms": 62.35561228822917, |
| "p95_batch_amortized_ms": 417.074111988768, |
| "repeated_response_identity": 1.0 |
| }, |
| "STRATA_NATIVE_V1": { |
| "correct": 8192, |
| "total": 8192, |
| "reference_exact": 8110, |
| "input_tokens": 528536, |
| "truncated": 0, |
| "accuracy": 1.0, |
| "mean_input_tokens": 64.5185546875, |
| "seconds": 309.70214548148215, |
| "amortized_ms_per_record": 37.80543768084499, |
| "median_batch_amortized_ms": 37.54055639728904, |
| "p95_batch_amortized_ms": 42.57394268643111, |
| "repeated_response_identity": 1.0 |
| } |
| }, |
| "slices": [ |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "graph_depth", |
| "value": "1", |
| "correct": 6566, |
| "total": 6656, |
| "accuracy": 0.9864783653846154 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "graph_depth", |
| "value": "2", |
| "correct": 512, |
| "total": 512, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "graph_depth", |
| "value": "3", |
| "correct": 512, |
| "total": 512, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "graph_depth", |
| "value": "4", |
| "correct": 512, |
| "total": 512, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "kind", |
| "value": "binding", |
| "correct": 5120, |
| "total": 5120, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "kind", |
| "value": "heldout", |
| "correct": 1446, |
| "total": 1536, |
| "accuracy": 0.94140625 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "kind", |
| "value": "join", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema", |
| "value": "approval", |
| "correct": 492, |
| "total": 494, |
| "accuracy": 0.9959514170040485 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema", |
| "value": "calendar", |
| "correct": 1024, |
| "total": 1024, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema", |
| "value": "crm", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema", |
| "value": "deployment", |
| "correct": 516, |
| "total": 524, |
| "accuracy": 0.9847328244274809 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema", |
| "value": "incident", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema", |
| "value": "inventory", |
| "correct": 1024, |
| "total": 1024, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema", |
| "value": "project", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema", |
| "value": "shipment", |
| "correct": 438, |
| "total": 518, |
| "accuracy": 0.8455598455598455 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema_partition", |
| "value": "heldout", |
| "correct": 1446, |
| "total": 1536, |
| "accuracy": 0.94140625 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "schema_partition", |
| "value": "registered", |
| "correct": 6656, |
| "total": 6656, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "value_type", |
| "value": "datetime", |
| "correct": 654, |
| "total": 654, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "value_type", |
| "value": "entity", |
| "correct": 3434, |
| "total": 3436, |
| "accuracy": 0.9994179278230501 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "value_type", |
| "value": "enum", |
| "correct": 2008, |
| "total": 2008, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "value_type", |
| "value": "integer", |
| "correct": 612, |
| "total": 692, |
| "accuracy": 0.884393063583815 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "value_type", |
| "value": "location", |
| "correct": 956, |
| "total": 964, |
| "accuracy": 0.991701244813278 |
| }, |
| { |
| "arm": "PROMPT_SERIALIZATION", |
| "axis": "value_type", |
| "value": "string", |
| "correct": 438, |
| "total": 438, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "graph_depth", |
| "value": "1", |
| "correct": 6656, |
| "total": 6656, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "graph_depth", |
| "value": "2", |
| "correct": 512, |
| "total": 512, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "graph_depth", |
| "value": "3", |
| "correct": 512, |
| "total": 512, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "graph_depth", |
| "value": "4", |
| "correct": 512, |
| "total": 512, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "kind", |
| "value": "binding", |
| "correct": 5120, |
| "total": 5120, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "kind", |
| "value": "heldout", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "kind", |
| "value": "join", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema", |
| "value": "approval", |
| "correct": 494, |
| "total": 494, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema", |
| "value": "calendar", |
| "correct": 1024, |
| "total": 1024, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema", |
| "value": "crm", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema", |
| "value": "deployment", |
| "correct": 524, |
| "total": 524, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema", |
| "value": "incident", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema", |
| "value": "inventory", |
| "correct": 1024, |
| "total": 1024, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema", |
| "value": "project", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema", |
| "value": "shipment", |
| "correct": 518, |
| "total": 518, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema_partition", |
| "value": "heldout", |
| "correct": 1536, |
| "total": 1536, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "schema_partition", |
| "value": "registered", |
| "correct": 6656, |
| "total": 6656, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "value_type", |
| "value": "datetime", |
| "correct": 654, |
| "total": 654, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "value_type", |
| "value": "entity", |
| "correct": 3436, |
| "total": 3436, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "value_type", |
| "value": "enum", |
| "correct": 2008, |
| "total": 2008, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "value_type", |
| "value": "integer", |
| "correct": 692, |
| "total": 692, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "value_type", |
| "value": "location", |
| "correct": 964, |
| "total": 964, |
| "accuracy": 1.0 |
| }, |
| { |
| "arm": "STRATA_NATIVE_V1", |
| "axis": "value_type", |
| "value": "string", |
| "correct": 438, |
| "total": 438, |
| "accuracy": 1.0 |
| } |
| ] |
| } |
