Accelerate mixed SystemOne decisions with verified typed scheduling

.gitattributes

@@ -10,3 +10,4 @@ assets/candidate-readout.png filter=lfs diff=lfs merge=lfs -text
 assets/decision-lex-header.png filter=lfs diff=lfs merge=lfs -text
 assets/residual-layers.pdf filter=lfs diff=lfs merge=lfs -text
 assets/residual-layers.png filter=lfs diff=lfs merge=lfs -text
+assets/mixed-question-scaling.png filter=lfs diff=lfs merge=lfs -text

MIXED_QUESTION_SCALING.json

@@ -0,0 +1,1056 @@
+{
+  "model": "Lex",
+  "native_manifest_sha256": "f288d873999832a3f37c6a7c4268c2ab309691e621794dbf7acab891acbbb7e6",
+  "weights_unchanged": true,
+  "runtime": "SystemOne default stable grouping by decision type, physicalB8, restored caller order; native predictor/optionalauto unchanged",
+  "final_package_measurement": {
+    "status": "COMPLETE_ACTUAL_RELEASE_PACKAGE_LATENCY",
+    "model": "Lex",
+    "timing": {
+      "1": {
+        "previous": {
+          "p50_ms": 13.217147497925907,
+          "p95_ms": 13.322706916369498,
+          "samples_ms": [
+            13.282976928167045,
+            13.237406965345144,
+            13.211747980676591,
+            13.152898056432605,
+            13.230256969109178,
+            13.178368099033833,
+            13.25502700638026,
+            13.151048100553453,
+            13.099667965434492,
+            13.143047923222184,
+            13.119807932525873,
+            13.172856997698545,
+            13.215948012657464,
+            13.322706916369498,
+            13.251267024315894,
+            13.226117007434368,
+            13.22932809125632,
+            13.1907369941473,
+            13.43417598400265,
+            13.178187073208392,
+            13.225198024883866,
+            13.218346983194351,
+            13.236007071100175,
+            13.224758091382682,
+            13.257008045911789,
+            13.164428062736988,
+            13.149028061889112,
+            13.277907040901482,
+            13.13608803320676,
+            13.126858044415712
+          ],
+          "forwards": 1
+        },
+        "current": {
+          "p50_ms": 13.203607522882521,
+          "p95_ms": 13.400446041487157,
+          "samples_ms": [
+            13.205877039581537,
+            13.137917965650558,
+            13.263097032904625,
+            13.192676939070225,
+            13.159338035620749,
+            13.066528015770018,
+            13.224957045167685,
+            13.128897990100086,
+            13.172497972846031,
+            13.362155994400382,
+            13.400446041487157,
+            13.369087013415992,
+            13.508685980923474,
+            13.245807029306889,
+            13.190687051974237,
+            13.201338006183505,
+            13.176057022064924,
+            13.362827012315392,
+            13.350806897506118,
+            13.200717978179455,
+            13.09696794487536,
+            13.282197061926126,
+            13.206628034822643,
+            13.207547017373145,
+            13.197846943512559,
+            13.18042806815356,
+            13.190128025598824,
+            13.355995994061232,
+            13.315456919372082,
+            13.173048035241663
+          ],
+          "forwards": 1
+        },
+        "p50_ratio": 0.9989755750970085
+      },
+      "8": {
+        "previous": {
+          "p50_ms": 28.06903002783656,
+          "p95_ms": 28.322704951278865,
+          "samples_ms": [
+            28.004087042063475,
+            28.014626004733145,
+            28.206884977407753,
+            28.091166052035987,
+            28.09197606984526,
+            28.103135991841555,
+            27.947345981374383,
+            27.99994603265077,
+            28.176164953038096,
+            28.125606011599302,
+            27.949147042818367,
+            28.065624996088445,
+            28.087166021578014,
+            28.040915960446,
+            28.449213947169483,
+            28.018205892294645,
+            27.994325035251677,
+            28.044176986441016,
+            28.243384906090796,
+            28.170255944132805,
+            28.322704951278865,
+            27.86880696658045,
+            28.18244497757405,
+            28.012576047331095,
+            28.072435059584677,
+            27.985455002635717,
+            28.23424502275884,
+            27.97982608899474,
+            28.03374594077468,
+            28.174914070405066
+          ],
+          "forwards": 1
+        },
+        "current": {
+          "p50_ms": 28.02164654713124,
+          "p95_ms": 28.305554995313287,
+          "samples_ms": [
+            27.882487047463655,
+            27.982397004961967,
+            28.524363064207137,
+            28.012415976263583,
+            28.066674945876002,
+            28.09819602407515,
+            28.139916015788913,
+            28.162325033918023,
+            28.012767084874213,
+            27.914006961509585,
+            27.955345925875008,
+            28.200226020999253,
+            28.069816064089537,
+            27.95354591216892,
+            28.061965946108103,
+            27.974725933745503,
+            28.034597053192556,
+            28.27983594033867,
+            27.997806086204946,
+            28.006776934489608,
+            28.305554995313287,
+            28.011246002279222,
+            27.974336058832705,
+            28.092985041439533,
+            28.070215019397438,
+            27.96054701320827,
+            28.062126017175615,
+            27.959375991486013,
+            27.941217995248735,
+            28.030526009388268
+          ],
+          "forwards": 1
+        },
+        "p50_ratio": 0.9983118946162967
+      },
+      "32": {
+        "previous": {
+          "p50_ms": 93.75805454328656,
+          "p95_ms": 94.06405396293849,
+          "samples_ms": [
+            93.82304409518838,
+            93.8560439972207,
+            93.62955600954592,
+            93.10267795808613,
+            93.90996396541595,
+            93.64896605256945,
+            93.62079505808651,
+            93.67714496329427,
+            93.83229492232203,
+            93.16505899187177,
+            93.54266698937863,
+            93.6963950516656,
+            93.74855505302548,
+            93.54403603356332,
+            93.50503596942872,
+            93.71798590291291,
+            93.91635493375361,
+            93.67461502552032,
+            93.6862260568887,
+            93.78432505764067,
+            94.01922405231744,
+            94.0071539953351,
+            94.18116300366819,
+            94.06405396293849,
+            93.63992500584573,
+            93.92943407874554,
+            93.9940030220896,
+            93.76755403354764,
+            93.8123851083219,
+            93.78303401172161
+          ],
+          "forwards": 4
+        },
+        "current": {
+          "p50_ms": 53.42581600416452,
+          "p95_ms": 53.77058498561382,
+          "samples_ms": [
+            53.39325708337128,
+            53.337397053837776,
+            53.106967941857874,
+            53.07436909060925,
+            53.77058498561382,
+            53.8342340150848,
+            53.71403496246785,
+            53.636976052075624,
+            53.13433799892664,
+            52.87010897882283,
+            53.569856099784374,
+            53.04888903629035,
+            53.360027028247714,
+            53.35383699275553,
+            53.595576086081564,
+            53.40233608148992,
+            53.43425599858165,
+            53.34917700383812,
+            53.44594607595354,
+            53.55702596716583,
+            52.97180905472487,
+            53.135977941565216,
+            53.637044969946146,
+            53.51222597528249,
+            53.74323495198041,
+            53.64776495844126,
+            53.65436489228159,
+            53.38411801494658,
+            53.417376009747386,
+            53.46064700279385
+          ],
+          "forwards": 4
+        },
+        "p50_ratio": 0.5698264140015692
+      },
+      "64": {
+        "previous": {
+          "p50_ms": 181.77667557029054,
+          "p95_ms": 184.529556077905,
+          "samples_ms": [
+            181.30511394701898,
+            182.10042896680534,
+            183.3407029043883,
+            181.7122820066288,
+            181.48475303314626,
+            182.18554893974215,
+            181.45461298990995,
+            182.56243702489883,
+            181.425362941809,
+            181.56275304500014,
+            181.81476008612663,
+            182.31657799333334,
+            181.52347404975444,
+            181.47010309621692,
+            181.0953450622037,
+            181.81708198972046,
+            182.10425099823624,
+            182.6307469746098,
+            182.9800249543041,
+            184.529556077905,
+            184.77372406050563,
+            182.89187492337078,
+            182.06122005358338,
+            181.52308301068842,
+            181.62922305054963,
+            181.69329199008644,
+            181.3240039627999,
+            181.9472999777645,
+            181.73859105445445,
+            181.2681140145287
+          ],
+          "forwards": 8
+        },
+        "current": {
+          "p50_ms": 87.38938998430967,
+          "p95_ms": 88.08800508268178,
+          "samples_ms": [
+            87.14881108608097,
+            87.20357099082321,
+            87.67556794919074,
+            87.43762003723532,
+            87.8160479478538,
+            87.28525089100003,
+            87.12115103844553,
+            87.49875891953707,
+            87.27059105876833,
+            87.15924096759409,
+            87.37221104092896,
+            87.13559003081173,
+            87.83565799240023,
+            87.46328996494412,
+            87.40672003477812,
+            86.9746939279139,
+            88.08800508268178,
+            87.65111805405468,
+            87.68402808345854,
+            88.8292919844389,
+            87.40467997267842,
+            87.23169099539518,
+            87.6758280210197,
+            87.31710002757609,
+            87.82118710223585,
+            87.21276093274355,
+            87.51507895067334,
+            87.28954102844,
+            87.37409999594092,
+            86.76742296665907
+          ],
+          "forwards": 8
+        },
+        "p50_ratio": 0.480751392939395
+      },
+      "128": {
+        "previous": {
+          "p50_ms": 355.9657375444658,
+          "p95_ms": 360.2475899970159,
+          "samples_ms": [
+            356.70211003161967,
+            357.4898249935359,
+            356.45368101540953,
+            355.5921650258824,
+            356.4777700230479,
+            355.7541760383174,
+            331.34153904393315,
+            355.33637704793364,
+            355.07863899692893,
+            354.9016900360584,
+            360.2475899970159,
+            355.7152450084686,
+            356.9431280484423,
+            361.3199539249763,
+            356.9705080008134,
+            355.9749521082267,
+            358.1079120049253,
+            356.542830937542,
+            356.62822995800525,
+            355.24915694259107,
+            355.9565229807049,
+            358.03848202340305,
+            357.22736606840044,
+            354.29402289446443,
+            354.5909799868241,
+            355.8202739804983,
+            355.5644250009209,
+            354.77684903889894,
+            354.4976110570133,
+            356.43676901236176
+          ],
+          "forwards": 16
+        },
+        "current": {
+          "p50_ms": 154.4099859893322,
+          "p95_ms": 155.9316230705008,
+          "samples_ms": [
+            154.1334129869938,
+            154.66729004401714,
+            154.6833390602842,
+            154.5290610520169,
+            154.0722930803895,
+            153.57524703722447,
+            148.0596859473735,
+            150.81938996445388,
+            154.5265899039805,
+            153.9489240385592,
+            156.18523210287094,
+            154.25303194206208,
+            154.7749990131706,
+            154.7707199351862,
+            155.18246707506478,
+            154.5011909911409,
+            155.9316230705008,
+            154.60049000103027,
+            154.8181800171733,
+            154.3837309582159,
+            154.88766902126372,
+            154.30260205175728,
+            154.4362410204485,
+            153.7249149987474,
+            154.51833105180413,
+            154.30224197916687,
+            154.14311294443905,
+            153.9313340326771,
+            153.8708939915523,
+            154.12014396861196
+          ],
+          "forwards": 16
+        },
+        "p50_ratio": 0.4337776637001305
+      }
+    },
+    "gate_pass": true,
+    "actual_forwards": 2400,
+    "weights_unchanged": true,
+    "native_manifest_sha256": "f288d873999832a3f37c6a7c4268c2ab309691e621794dbf7acab891acbbb7e6",
+    "plan_sha256": "c10e0c4fbe385577ffece0e1d18b274f6870079e80c29fdfec92927a515009a7",
+    "environment": {
+      "gpu": "",
+      "torch": "2.12.0+git6bbd260",
+      "hip": "7.2.53211",
+      "precision": "FP32",
+      "batch_size": 8
+    },
+    "workload": {
+      "state": "The cafe has excellent coffee and friendly staff. Weekend queues are long. The customer wants more staff during busy hours, and says they will return.",
+      "questions": [
+        {
+          "type": "choice",
+          "instructions": "Which type of business is reviewed?",
+          "criteria": {
+            "cafe": "Coffee and refreshments",
+            "hotel": "Overnight accommodation",
+            "bank": "Financial services"
+          }
+        },
+        {
+          "type": "noul",
+          "instructions": "Does the customer intend to return?"
+        },
+        {
+          "type": "score",
+          "instructions": "Rate the quality of the coffee.",
+          "criteria": [
+            "Poor",
+            "Average",
+            "Excellent"
+          ]
+        }
+      ],
+      "ordering": "Cycle three fixed question types, one shared context, fresh IDs only",
+      "warmup_pairs": 10,
+      "measured_pairs": 30,
+      "clock": "Synchronized resident local SystemOne including conversion/tokenization/forward/answers; excludes HTTP/Studio/network"
+    },
+    "source_hashes": {
+      "Lex/prepared/decision_inference/_grouped.py": "80551e1bebd10634abcbbfce1aecf6cafab93b7e178399c428dd97022c6f9d95",
+      "Lex/prepared/decision_inference/_system_one.py": "0dc61f7558eed31c9ac186827b2050c0d391c69ea3684c02e8cd9c71400b1690"
+    },
+    "public_release": false
+  },
+  "full_source_qualification": {
+    "sha256": "59a6914eef31575f31e1fbb68e26d7dc2e9b44fc4ae755babd91668c0329a920",
+    "content": {
+      "status": "COMPLETE_ACTUAL_AMD_SYSTEMONE_SCHEDULE_QUALIFICATION",
+      "model": "Lex",
+      "panels": {
+        "old_core": {
+          "requested_requests": 880,
+          "accepted_requests": 880,
+          "refused_requests": 0,
+          "compared_decisions": 880,
+          "hard_flips": 0,
+          "max_probability_delta": 3.6656856536865234e-06,
+          "mismatched_order_or_usage": 0
+        },
+        "v3_core": {
+          "requested_requests": 880,
+          "accepted_requests": 880,
+          "refused_requests": 0,
+          "compared_decisions": 880,
+          "hard_flips": 0,
+          "max_probability_delta": 0.0,
+          "mismatched_order_or_usage": 0
+        },
+        "v4": {
+          "requested_requests": 480,
+          "accepted_requests": 480,
+          "refused_requests": 0,
+          "compared_decisions": 480,
+          "hard_flips": 0,
+          "max_probability_delta": 0.0,
+          "mismatched_order_or_usage": 0
+        },
+        "v5": {
+          "requested_requests": 480,
+          "accepted_requests": 480,
+          "refused_requests": 0,
+          "compared_decisions": 480,
+          "hard_flips": 0,
+          "max_probability_delta": 4.1425228118896484e-06,
+          "mismatched_order_or_usage": 0
+        },
+        "old_native": {
+          "requested_requests": 68,
+          "accepted_requests": 29,
+          "refused_requests": 39,
+          "compared_decisions": 29,
+          "hard_flips": 0,
+          "max_probability_delta": 1.1920928955078125e-06,
+          "mismatched_order_or_usage": 0
+        },
+        "v3_native": {
+          "requested_requests": 68,
+          "accepted_requests": 50,
+          "refused_requests": 18,
+          "compared_decisions": 202,
+          "hard_flips": 0,
+          "max_probability_delta": 3.2782554626464844e-06,
+          "mismatched_order_or_usage": 0
+        },
+        "transfer_v9_test": {
+          "requested_requests": 1209,
+          "accepted_requests": 1209,
+          "refused_requests": 0,
+          "compared_decisions": 1209,
+          "hard_flips": 0,
+          "max_probability_delta": 2.3245811462402344e-06,
+          "mismatched_order_or_usage": 0
+        }
+      },
+      "gate_pass": true,
+      "actual_forwards": 1044,
+      "weights_unchanged": true,
+      "request_order_and_full_admission_preserved": true,
+      "gold_access": false,
+      "public_release": false
+    }
+  },
+  "package_qualification": {
+    "status": "PASS_ACTUAL_STAGED_SYSTEMONE_PACKAGE",
+    "model": "Lex",
+    "checks": [
+      {
+        "case": "observed192_first128",
+        "decisions": 128,
+        "hard_flips": 0,
+        "max_probability_delta": 2.4139881134033203e-06
+      },
+      {
+        "case": "observed192_last64",
+        "decisions": 64,
+        "hard_flips": 0,
+        "max_probability_delta": 8.940696716308594e-06
+      },
+      {
+        "case": "studio:lex-inbox-contexts-v1",
+        "decisions": 8,
+        "hard_flips": 0,
+        "max_probability_delta": 0.0
+      },
+      {
+        "case": "studio:lex-ticket-batch",
+        "decisions": 8,
+        "hard_flips": 0,
+        "max_probability_delta": 0.0
+      },
+      {
+        "case": "studio:lex-security-six-v3",
+        "decisions": 6,
+        "hard_flips": 0,
+        "max_probability_delta": 1.6808509826660156e-05
+      },
+      {
+        "case": "studio:lex-urgency-six-v3",
+        "decisions": 6,
+        "hard_flips": 0,
+        "max_probability_delta": 0.0
+      },
+      {
+        "case": "studio:lex-access-six-v4",
+        "decisions": 6,
+        "hard_flips": 0,
+        "max_probability_delta": 5.960464477539063e-08
+      },
+      {
+        "case": "studio:lex-request-contexts-v4",
+        "decisions": 6,
+        "hard_flips": 0,
+        "max_probability_delta": 0.0
+      },
+      {
+        "case": "four_contexts_512_decisions",
+        "decisions": 512,
+        "hard_flips": 0,
+        "max_probability_delta": 5.066394805908203e-07
+      },
+      {
+        "case": "optional_auto_preserved",
+        "decisions": 32,
+        "hard_flips": 0,
+        "max_probability_delta": 0.0
+      },
+      {
+        "case": "external_id_alias",
+        "decisions": 128,
+        "probabilities_exact": true
+      },
+      {
+        "case": "late_overflow",
+        "same_error_before_forward": true
+      },
+      {
+        "case": "over_512",
+        "same_error_before_forward": true
+      },
+      {
+        "case": "over_128_requests",
+        "same_error_before_forward": true
+      }
+    ],
+    "actual_forwards": 228,
+    "weights_unchanged": true,
+    "native_manifest_sha256": "f288d873999832a3f37c6a7c4268c2ab309691e621794dbf7acab891acbbb7e6",
+    "plan_sha256": "855ea4bf964db543ae45b3c0201d288248f1745576b6393a3bdbb24920275873",
+    "source_hashes": {
+      "Lex/prepared/decision_inference/_grouped.py": "80551e1bebd10634abcbbfce1aecf6cafab93b7e178399c428dd97022c6f9d95",
+      "Lex/prepared/decision_inference/_system_one.py": "0dc61f7558eed31c9ac186827b2050c0d391c69ea3684c02e8cd9c71400b1690"
+    },
+    "studio_examples": 6,
+    "public_release": false
+  },
+  "earlier_concurrent_measurement": {
+    "status": "COMPLETE_ACTUAL_RELEASE_PACKAGE_LATENCY",
+    "model": "Lex",
+    "timing": {
+      "1": {
+        "previous": {
+          "p50_ms": 14.571013452950865,
+          "p95_ms": 106.91890399903059,
+          "samples_ms": [
+            13.361250050365925,
+            140.63826599158347,
+            13.696447014808655,
+            27.827713056467474,
+            13.735816930420697,
+            62.6358789158985,
+            19.878904917277396,
+            18.178944010287523,
+            106.91890399903059,
+            13.263568980619311,
+            13.29776004422456,
+            42.206557001918554,
+            13.289999915286899,
+            15.967766055837274,
+            13.450457947328687,
+            19.57047695759684,
+            94.55266897566617,
+            13.303969986736774,
+            13.46319995354861,
+            13.324889005161822,
+            13.705997029319406,
+            13.202580972574651,
+            70.73775609023869,
+            66.92217499949038,
+            13.30288895405829,
+            15.406209975481033,
+            13.376468908973038,
+            13.277549995109439,
+            23.668864043429494,
+            16.42183295916766
+          ],
+          "forwards": 1
+        },
+        "current": {
+          "p50_ms": 13.378849485889077,
+          "p95_ms": 97.0877748914063,
+          "samples_ms": [
+            17.63901603408158,
+            13.316189986653626,
+            74.40948602743447,
+            165.44692497700453,
+            13.580687926150858,
+            16.24104392249137,
+            13.239049934782088,
+            13.382968958467245,
+            13.367859064601362,
+            13.338689925149083,
+            15.964004909619689,
+            13.253489974886179,
+            13.302339008077979,
+            13.258340070024133,
+            56.361171999014914,
+            13.369528925977647,
+            13.29931989312172,
+            17.52129604574293,
+            19.00173898320645,
+            13.478238019160926,
+            97.0877748914063,
+            13.302690000273287,
+            13.37473001331091,
+            13.357058051042259,
+            13.258830062113702,
+            16.982758999802172,
+            23.39470700826496,
+            13.19589908234775,
+            13.20873899385333,
+            13.39158893097192
+          ],
+          "forwards": 1
+        },
+        "p50_ratio": 0.9181824949300039
+      },
+      "8": {
+        "previous": {
+          "p50_ms": 34.73892150213942,
+          "p95_ms": 172.8683450492099,
+          "samples_ms": [
+            31.131225056014955,
+            28.410759987309575,
+            85.56938706897199,
+            41.561309015378356,
+            30.786116956733167,
+            38.726025028154254,
+            145.64779901411384,
+            82.93773094192147,
+            28.462579008191824,
+            68.68544709868729,
+            34.76842597592622,
+            30.330618959851563,
+            34.70941702835262,
+            33.41492300387472,
+            190.743581042625,
+            31.30575397517532,
+            105.50514201167971,
+            87.7190459286794,
+            28.843987034633756,
+            31.655543018132448,
+            36.35207703337073,
+            170.99016509018838,
+            31.442093080841005,
+            73.0566029669717,
+            28.541999054141343,
+            33.372384030371904,
+            85.321647929959,
+            172.8683450492099,
+            31.578613095916808,
+            30.789837008342147
+          ],
+          "forwards": 1
+        },
+        "current": {
+          "p50_ms": 43.07004599831998,
+          "p95_ms": 282.32143609784544,
+          "samples_ms": [
+            41.653947904706,
+            28.454498969949782,
+            28.578040073625743,
+            286.50921303778887,
+            28.733848012052476,
+            87.18010690063238,
+            32.454418018460274,
+            34.42092798650265,
+            187.3356889700517,
+            28.99436606094241,
+            122.0393639523536,
+            28.320749988779426,
+            71.88508997205645,
+            31.886521028354764,
+            33.56740204617381,
+            282.32143609784544,
+            32.90519502479583,
+            255.91422605793923,
+            28.52455899119377,
+            75.28732100035995,
+            223.87088497634977,
+            63.35321499500424,
+            72.95793399680406,
+            28.87122705578804,
+            31.580012990161777,
+            44.486144091933966,
+            94.85666803084314,
+            153.9103949908167,
+            110.09115702472627,
+            29.06379592604935
+          ],
+          "forwards": 1
+        },
+        "p50_ratio": 1.2398210461331531
+      },
+      "32": {
+        "previous": {
+          "p50_ms": 198.33825452951714,
+          "p95_ms": 286.02026600856334,
+          "samples_ms": [
+            208.20390700828284,
+            286.02026600856334,
+            116.49416398722678,
+            167.50254202634096,
+            246.3420460699126,
+            345.44017002917826,
+            145.68236900959164,
+            167.11201495490968,
+            206.77826495375484,
+            107.46569093316793,
+            158.0538930138573,
+            193.4651560150087,
+            243.90959809534252,
+            182.90021095890552,
+            221.07835905626416,
+            187.00264894869179,
+            244.45490492507815,
+            172.653064946644,
+            125.88770198635757,
+            227.9231830034405,
+            259.72379301674664,
+            263.34351405967027,
+            163.20275398902595,
+            203.2113530440256,
+            275.67597897723317,
+            208.6803240235895,
+            119.42932708188891,
+            211.07498207129538,
+            151.2328089447692,
+            170.27989693451673
+          ],
+          "forwards": 4
+        },
+        "current": {
+          "p50_ms": 133.98809952195734,
+          "p95_ms": 376.0747379856184,
+          "samples_ms": [
+            100.60080699622631,
+            355.66331702284515,
+            111.73206905368716,
+            68.3719179360196,
+            335.7284109806642,
+            64.3072499660775,
+            208.7552340235561,
+            117.52685799729079,
+            160.36495100706816,
+            121.48911599069834,
+            376.0747379856184,
+            58.965447009541094,
+            136.99047500267625,
+            122.37974093295634,
+            59.73403400275856,
+            212.6397140091285,
+            63.47907392773777,
+            171.72804998699576,
+            163.32766390405595,
+            168.65042597055435,
+            136.3753970945254,
+            66.5766280144453,
+            74.80555400252342,
+            59.44409500807524,
+            71.25750207342207,
+            223.68835506495088,
+            289.2386970343068,
+            409.67884799465537,
+            131.60080194938928,
+            287.98995399847627
+          ],
+          "forwards": 4
+        },
+        "p50_ratio": 0.675553487348135
+      },
+      "64": {
+        "previous": {
+          "p50_ms": 488.5446400148794,
+          "p95_ms": 860.3939639870077,
+          "samples_ms": [
+            588.6253280332312,
+            356.67536908295006,
+            594.9330650037155,
+            692.8134460467845,
+            396.07378002256155,
+            527.112663956359,
+            336.06783696450293,
+            456.5627579577267,
+            326.6586670652032,
+            752.5941269705072,
+            860.3939639870077,
+            681.654252926819,
+            430.87316304445267,
+            428.6890249932185,
+            404.57855199929327,
+            538.390701985918,
+            561.1566319130361,
+            469.90221505984664,
+            674.7355579864234,
+            500.82891003694385,
+            914.364974014461,
+            534.5511310733855,
+            477.23246505483985,
+            550.633255043067,
+            499.85681497491896,
+            198.40770598966628,
+            211.90554497297853,
+            182.23994190339,
+            182.4631510535255,
+            182.32450098730624
+          ],
+          "forwards": 8
+        },
+        "current": {
+          "p50_ms": 240.10508198989555,
+          "p95_ms": 546.4967269217595,
+          "samples_ms": [
+            237.67261998727918,
+            197.24152295384556,
+            170.06384895648807,
+            191.1246069939807,
+            316.73357996623963,
+            374.13932604249567,
+            242.53754399251193,
+            244.4212429691106,
+            200.5026569822803,
+            435.37590198684484,
+            453.023066977039,
+            593.5483209323138,
+            399.27821105811745,
+            243.32293902989477,
+            246.88830005470663,
+            145.2575579751283,
+            233.21076203137636,
+            403.36408896837384,
+            166.22657689731568,
+            229.93817902170122,
+            276.7654510680586,
+            126.66724692098796,
+            546.4967269217595,
+            448.94893595483154,
+            465.4096569865942,
+            106.47687502205372,
+            121.47569505032152,
+            108.83200191892684,
+            88.08667189441621,
+            87.95632293913513
+          ],
+          "forwards": 8
+        },
+        "p50_ratio": 0.49147009776339534
+      },
+      "128": {
+        "previous": {
+          "p50_ms": 359.8495615296997,
+          "p95_ms": 371.3875049725175,
+          "samples_ms": [
+            359.17470196727663,
+            359.7314669750631,
+            359.6855190116912,
+            359.19495997950435,
+            358.3321350160986,
+            358.4663450019434,
+            359.8091179737821,
+            359.1001909226179,
+            358.5895940195769,
+            359.10174099262804,
+            360.01692595891654,
+            360.8712210552767,
+            361.5627179387957,
+            360.6750110629946,
+            360.5347230331972,
+            359.8734770203009,
+            360.36473501008004,
+            361.02003895211965,
+            360.0647649727762,
+            371.3875049725175,
+            379.1667229961604,
+            360.60995201114565,
+            359.8042860394344,
+            359.8256460390985,
+            360.15939398203045,
+            361.5629649721086,
+            359.4064770732075,
+            359.6939370036125,
+            359.90800498984754,
+            359.70397596247494
+          ],
+          "forwards": 16
+        },
+        "current": {
+          "p50_ms": 156.12808446167037,
+          "p95_ms": 158.11885800212622,
+          "samples_ms": [
+            155.28889501001686,
+            155.16494493931532,
+            155.34398599993438,
+            155.51191405393183,
+            155.16564599238336,
+            155.8415510226041,
+            155.35369399003685,
+            155.15018592122942,
+            155.40425409562886,
+            156.3086489913985,
+            156.2130000675097,
+            156.37198893819004,
+            156.19600901845843,
+            156.0601599048823,
+            155.79097100999206,
+            156.46956791169941,
+            156.37390792835504,
+            156.02733101695776,
+            156.69844602234662,
+            156.47772699594498,
+            155.98976996261626,
+            156.3491279957816,
+            156.2649189727381,
+            156.37214807793498,
+            156.49046702310443,
+            155.80621093977243,
+            156.68375696986914,
+            155.72028001770377,
+            160.92158399987966,
+            158.11885800212622
+          ],
+          "forwards": 16
+        },
+        "p50_ratio": 0.4338704312935075
+      }
+    },
+    "gate_pass": false,
+    "actual_forwards": 1920,
+    "weights_unchanged": true,
+    "native_manifest_sha256": "f288d873999832a3f37c6a7c4268c2ab309691e621794dbf7acab891acbbb7e6",
+    "plan_sha256": "74d011196525925c371bf73aaf40c69a7b2c35249f84769543a68f429f8200bd",
+    "environment": {
+      "gpu": "",
+      "torch": "2.12.0+git6bbd260",
+      "hip": "7.2.53211",
+      "precision": "FP32",
+      "batch_size": 8
+    },
+    "workload": {
+      "state": "The cafe has excellent coffee and friendly staff. Weekend queues are long. The customer wants more staff during busy hours, and says they will return.",
+      "questions": [
+        {
+          "type": "choice",
+          "instructions": "Which type of business is reviewed?",
+          "criteria": {
+            "cafe": "Coffee and refreshments",
+            "hotel": "Overnight accommodation",
+            "bank": "Financial services"
+          }
+        },
+        {
+          "type": "noul",
+          "instructions": "Does the customer intend to return?"
+        },
+        {
+          "type": "score",
+          "instructions": "Rate the quality of the coffee.",
+          "criteria": [
+            "Poor",
+            "Average",
+            "Excellent"
+          ]
+        }
+      ],
+      "ordering": "Cycle three fixed question types, one shared context, fresh IDs only",
+      "warmup_pairs": 2,
+      "measured_pairs": 30,
+      "clock": "Synchronized resident local SystemOne including conversion/tokenization/forward/answers; excludes HTTP/Studio/network"
+    },
+    "source_hashes": {
+      "Lex/prepared/decision_inference/_grouped.py": "80551e1bebd10634abcbbfce1aecf6cafab93b7e178399c428dd97022c6f9d95",
+      "Lex/prepared/decision_inference/_system_one.py": "0dc61f7558eed31c9ac186827b2050c0d391c69ea3684c02e8cd9c71400b1690"
+    },
+    "public_release": false
+  },
+  "measurement_note": "Initial concurrent final-package run had substantial nonstationary stalls and LexQ8 failed its gate. Its samples are retained. Final fixed serial measurement follows termination of own training/data producers,10warmup and30measured alternating pairs. No sample filtering; unchanged acceptance gates."
+}

PACKAGE_MANIFEST.json

@@ -1,7 +1,7 @@
 {
-  "bytes_excluding_this_manifest": 2327366131,
+  "bytes_excluding_this_manifest": 2327567235,
   "display_name": "Decision-1.0-Lex-0.6B",
-  "file_count_excluding_this_manifest": 102,
+  "file_count_excluding_this_manifest": 107,
   "files": {
     ".gitattributes": {
       "bytes": 753,
@@ -63,6 +63,10 @@
       "bytes": 2543,
       "sha256": "a7dd6dec55dfe463cc0211c3dffa114b88bb95db90816b70c18b62b5d7be7ad1"
     },
+    "MIXED_QUESTION_SCALING.json": {
+      "bytes": 32763,
+      "sha256": "ed4a9e7a9aac0385f8a4516694a6d9ea843aacefde6c7189b9980bac86c9d773"
+    },
     "NOTICE": {
       "bytes": 4371,
       "sha256": "e3f71eb2a262084fa572ccbf9b65110a4feb9d8c625dc26e792beba282cdf032"
@@ -72,20 +76,20 @@
       "sha256": "2e5936d3ea196e00968cc37b7662a6afbe8cfd6fd11d04b9fecba554b1e1e37b"
     },
     "PERFORMANCE.md": {
-      "bytes": 4368,
-      "sha256": "7e275e73c18b756a20b886272750b58416ead2c345b12393e162050231aa8f64"
+      "bytes": 6351,
+      "sha256": "0f5c84a90b2c82be6146b1b7032a9f836e5858d59e29bd6c94f016be36d8b3b6"
     },
     "README.md": {
-      "bytes": 5993,
-      "sha256": "5b89d9013b9890381f169c496081d23818836119417bcf1a5f9af1893297d668"
+      "bytes": 6230,
+      "sha256": "6ff9eb7a6ac0c6b90b017f497d75bd3366c9c598ce3c5f813dfa0f192d49b1ec"
     },
     "RUNTIME_UPDATE.json": {
       "bytes": 1087,
       "sha256": "0a77c012fe1bd23c5f1a06c106c1a29ae5ee156248db218fbbc6d631d1c5aff4"
     },
     "SYSTEM_ONE.md": {
-      "bytes": 5735,
-      "sha256": "414e12680a6ec2dbbbc660d1f6ba0d601fa941388476a62696b7f5e8a3e11ef9"
+      "bytes": 6177,
+      "sha256": "c59c5d1fbab6c18033b1339701aadddc34653f9fb93f79aee93534bb34d66648"
     },
     "SYSTEM_ONE_VALIDATION.json": {
       "bytes": 2583,
@@ -104,8 +108,8 @@
       "sha256": "5c95cf851a9480e6037bc190d0deee87da35576489362262ad4151bafbb96990"
     },
     "USAGE.md": {
-      "bytes": 7866,
-      "sha256": "652a6f94b27e352a3bd899c8fdb1a0ff4ed4b7598824a598a2947e124b73cd33"
+      "bytes": 8308,
+      "sha256": "f7b6fe124f34667480f324f125d92d72e35045ea043c8b45da7184b9c7a28dcb"
     },
     "VALIDATION.md": {
       "bytes": 2120,
@@ -155,6 +159,18 @@
       "bytes": 2188963,
       "sha256": "9980c6b901aff89b69e9cc6de6de40408b95c7378f18e931401198f42d32e40b"
     },
+    "assets/mixed-question-scaling.pdf": {
+      "bytes": 27984,
+      "sha256": "e1261c998cdcdf6b72f57b4df4cb56cbf47fa521210c4778a0577e2f3a47c1cf"
+    },
+    "assets/mixed-question-scaling.png": {
+      "bytes": 121576,
+      "sha256": "4efadb4888c43122d461ca432081f2b5bd67c350f9605940b96f156829d65a54"
+    },
+    "assets/mixed-question-scaling.svg": {
+      "bytes": 14623,
+      "sha256": "0fcea2c3e7fd1c649a1dd7acf206c0d41e4f6fcab5e51abd838d6bb18a2560ff"
+    },
     "assets/residual-layers.pdf": {
       "bytes": 131144,
       "sha256": "2c5b50bb840ae115a1264b66d535a1048172e891a33d02d507c193525cc45630"
@@ -203,13 +219,17 @@
       "bytes": 1325,
       "sha256": "3602e3a06386daab5494305413a08adb581f23007ffb5ca6e5df62fa23d11bd6"
     },
+    "decision_inference/_grouped.py": {
+      "bytes": 1016,
+      "sha256": "80551e1bebd10634abcbbfce1aecf6cafab93b7e178399c428dd97022c6f9d95"
+    },
     "decision_inference/_request.py": {
       "bytes": 3026,
       "sha256": "85b8349cdf08a3550027606b3108778955f84d66c52fa11dd908ea50311e69e5"
     },
     "decision_inference/_system_one.py": {
-      "bytes": 10690,
-      "sha256": "d2ef9aa1d0badcf168065a782f8a96316045455a676c08bdb5dd384a651e2194"
+      "bytes": 10728,
+      "sha256": "0dc61f7558eed31c9ac186827b2050c0d391c69ea3684c02e8cd9c71400b1690"
     },
     "decision_inference/profile.py": {
       "bytes": 2071,
@@ -428,7 +448,7 @@
   },
   "parameter_tensors": 489,
   "parameters": 571909635,
-  "public_non_native_bytes_excluding_this_manifest": 5106920,
+  "public_non_native_bytes_excluding_this_manifest": 5308024,
   "publication": {
     "download_access": "public_ungated",
     "new_contribution_license": "Apache-2.0",
@@ -443,7 +463,9 @@
   "status": "READY_FOR_ROOT_PUBLICATION",
   "subject_manifest_sha256": "f288d873999832a3f37c6a7c4268c2ab309691e621794dbf7acab891acbbb7e6",
   "system_one_api": {
+    "current_scheduling_validation": "MIXED_QUESTION_SCALING.json",
     "default_physical_batch_limit": 8,
+    "default_scheduling": "stable typed groups",
     "entry": "decision_inference.SystemOne",
     "finetune_conversion": "decision_finetune.system_one.system_one_training_rows",
     "max_batch_decisions": 512,
@@ -457,5 +479,15 @@
     "weights_unchanged": true
   },
   "technical_validation": "TECHNICAL_VALIDATION.json",
-  "training_provenance": "TRAINING_PROVENANCE.json"
+  "training_provenance": "TRAINING_PROVENANCE.json",
+  "typed_scheduling_update": {
+    "complete_input_limit_unchanged": 1024,
+    "default_schedule": "Stable group by decision type; physicalB8; restore original caller order",
+    "entry": "decision_inference.SystemOne",
+    "native_weights_unchanged": true,
+    "new_chat_api": false,
+    "public_api_unchanged": true,
+    "subject_native_manifest_sha256": "f288d873999832a3f37c6a7c4268c2ab309691e621794dbf7acab891acbbb7e6",
+    "validation": "MIXED_QUESTION_SCALING.json"
+  }
 }

PERFORMANCE.md

@@ -1,18 +1,44 @@
-# Lex: AMD batch capacity
+# Lex: AMD SystemOne latency
 
-## Optional larger batches
+## Mixed questions, faster decisions
+
+**128 mixed questions: 154.41 ms median, 56.6% lower than the previous default runtime.**
+
+![Lex: latency as mixed question count increases](assets/mixed-question-scaling.png)
+
+| Questions | Previous p50 / p95 (ms) | Typed scheduling p50 / p95 (ms) |
+|---:|---:|---:|
+| 1 | 13.22 / 13.32 | 13.20 / 13.40 |
+| 8 | 28.07 / 28.32 | 28.02 / 28.31 |
+| 32 | 93.76 / 94.06 | 53.43 / 53.77 |
+| 64 | 181.78 / 184.53 | 87.39 / 88.09 |
+| 128 | 355.97 / 360.25 | 154.41 / 155.93 |
+
+The request repeats three fixed questions—Choice, Noul and Score—over one unchanged context. Only question count and bookkeeping IDs change. The default SystemOne path groups admitted rows by type so each encoder path processes fuller batches, then restores the original answer order. Every question still has its own contextual computation. Weights, precision, complete-input limit and request schema are unchanged.
+
+Measured using the exact release package on AMD ROCm, FP32, physical batch size eight: 10 warmup pairs and 30 alternating AB/BA measurement pairs per point, with GPU synchronization. Measurements include local request conversion, tokenization, model execution and answer assembly; they exclude transport and Studio. Models ran serially after our training and data jobs completed. Results describe this fixed workload, not a service latency guarantee.
+
+Both models passed seven source panels: 4,160 admitted decisions without an argmax change and 57 identical whole-request refusals before any forward. Exact-package checks additionally cover 512 decisions, six current Studio examples, caller IDs/order, optional auto batching and input limits. Small floating-point probability differences remain possible.
+
+[All samples, workload, checks and earlier concurrent measurements](MIXED_QUESTION_SCALING.json) · [SVG](assets/mixed-question-scaling.svg) · [PDF](assets/mixed-question-scaling.pdf)
+
+## Earlier measurements
+
+## Lex: AMD batch capacity
+
+### Optional larger batches
 
 ```python
 from decision_inference import predict_1k, predict_auto_1k
 
-# native and records use the same objects as the Python usage examples.
+## native and records use the same objects as the Python usage examples.
 outputs = predict_1k(native, records)       # unchanged default: up to 8
 outputs = predict_auto_1k(native, records)  # opt in: up to 32 with the padding guard
 ```
 
 The optional entry admits every complete input before the first forward. It uses consecutive batches up to 32 only when the request has one decision type and total padded tokens do not increase relative to B8. Mixed types or increased padding fall back to B8; requests of eight or fewer take the unchanged default path. Input order, candidates, full 1,024-token limit and FP32 weights remain unchanged. It does not share contextual activations across questions. Larger physical batches can change floating-point rounding and peak memory.
 
-### Paired B8/B32 study
+#### Paired B8/B32 study
 
 These are synchronized resident **Python API measurements**, not Studio or network latency. This earlier study compared default B8 with a homogeneous cap32 policy before the final padding guard was added. All eight measured points satisfy that guard, but **the final `predict_auto_1k` entry was validated separately and was not timed**. Existing default-B8 measurements above, if present, remain their original separate run.
 
@@ -33,7 +59,7 @@ Choice count curves repeat the same synthetic four-candidate question with opaqu
 
 At 1024×32, peak allocated memory increased from approximately 2.652 to 3.824 GiB. Shared allocator reserved-memory observations are not independent per-mode estimates. Equal total padding does not imply equal peak memory.
 
-### Public-entry validation
+#### Public-entry validation
 
 The actual imported package passed 46 forwards per model (92 total across Kai and Lex) over 225 technical row-occurrences/model, including all three types, dynamic candidate counts, enabled32, tail33, unequal-length fallback and mixed64. Late invalid and empty requests added zero forwards. The original strict bounds were 1e-4 logits, 2e-5 probability and normalized Score error, with zero hard-decision flips. Both models passed; fallback outputs were exact. All 489 parameters and 66 buffers retained identical contents and versions. No training, quality evaluation or timing was added in this integration check. Fixed repetition is not independent quality support.
 

README.md

@@ -48,6 +48,8 @@ Lex is evaluated as an English specialist on these four workflows. For broader m
 
 **Optional larger batches:** `predict_auto_1k` uses up to 32 same-type questions when padding does not increase. Earlier B8/B32 comparisons showed approximately 23% lower median latency for 32 short questions and 35% for the multi-context fixture; the final guard was validated separately, not timed. [Usage and measurements](PERFORMANCE.md#optional-larger-batches).
 
+**128 mixed questions in 154 ms — 57% lower latency.** Automatic typed scheduling accelerates the default SystemOne path with the same weights. Paired local AMD measurements on a fixed workload. [Latency and scaling](PERFORMANCE.md).
+
 ## Use
 
 Replace the placeholder with a SystemOne-compatible endpoint configured to serve `Decision-1.0-Lex-0.6B`, and set `DECISION_API_KEY` to that endpoint's key.

SYSTEM_ONE.md

@@ -127,3 +127,8 @@ source/component ID and stay in one split; the CLI checks component and exact
 input overlap. Optional `hard_target_ids` supplies separate evaluation labels
 under the same question IDs. Noul supports soft yes probabilities; Choice and
 Score support complete soft distributions in the original candidate order.
+
+
+### Default typed scheduling
+
+The default `SystemOne` path groups complete, admitted questions by decision type in physical batches of eight, then restores the original request and question order. This works for many questions over one state and questions across multiple contexts. No API changes or application-side sorting are needed. The optional `batching="auto"` policy is unchanged. [Measured mixed-question scaling](PERFORMANCE.md).

USAGE.md

@@ -114,3 +114,8 @@ The Python API returns original record/candidate identities, logits and probabil
 All state/question/candidate text and markers count toward the complete 1024 limit. `predict_1k` checks every input before the first forward and rejects overflow without truncation. Batch sizes 1–8 are supported by this wrapper. Use it rather than directly invoking the native research-capacity entrypoint.
 
 A compatible fine-tuned export can be used with `--native <directory> --manifest-sha256 <trusted hash>`. Only the pinned runtime revision and three-path architecture are accepted. Downloaded HF symlinks must be materialized into real files before loading. The manifest hash is an integrity check, not a reason to execute arbitrary untrusted Python.
+
+
+### Default typed scheduling
+
+The default `SystemOne` path groups complete, admitted questions by decision type in physical batches of eight, then restores the original request and question order. This works for many questions over one state and questions across multiple contexts. No API changes or application-side sorting are needed. The optional `batching="auto"` policy is unchanged. [Measured mixed-question scaling](PERFORMANCE.md).

decision_inference/_grouped.py

@@ -0,0 +1,16 @@
+"""Stable typed scheduling behind SystemOne; preserve request and answer order."""
+from dataclasses import replace
+
+def predict_grouped_1k(native,records,*,batch_size=8):
+ from decision_inference._request import request_collator
+ from decision_runtime import predict
+ if type(batch_size) is not int or batch_size!=8:raise ValueError('SystemOne typed scheduling uses physical batch size 8')
+ records=list(records);guard=request_collator(native.collator,records)
+ # Admit every original row before scheduling or any forward.
+ order=sorted(range(len(records)),key=lambda i:records[i]['question']['type'])
+ sorted_rows=[records[i]for i in order]
+ guard._records=tuple(sorted_rows);guard._encoded=[guard._encoded[i]for i in order]
+ predictions=predict(replace(native,collator=guard),sorted_rows,batch_size=8);guard.finish();restored=[None]*len(records)
+ for i,result in zip(order,predictions):restored[i]=result
+ if any(x is None for x in restored):raise RuntimeError('Incomplete scheduled predictions')
+ return restored

decision_inference/_system_one.py

@@ -150,7 +150,7 @@ class SystemOne:
 
     evaluate(request) accepts the HTTP body shape; system_one(**request) is its
     Python equivalent. batch(requests) flattens independent states into GPU
-    batches and restores the original request/question order. B8 is the default;
+    batches and restores the original request/question order. Default B8 groups rows by decision type;
     batching='auto' opts into the published homogeneous padding-aware B32 path.
     """
     def __init__(self, native, *, model=None, batching="default"):
@@ -202,8 +202,8 @@ class SystemOne:
             from ._auto import predict_auto_1k
             predictions = predict_auto_1k(self.native, records)
         else:
-            from ._request import predict_1k
-            predictions = predict_1k(self.native, records, batch_size=8)
+            from ._grouped import predict_grouped_1k
+            predictions = predict_grouped_1k(self.native, records, batch_size=8)
         if len(predictions) != len(records):
             raise RuntimeError("Incomplete model result; no partial answers returned")
         values = [[None] * len(group) for group in groups]