fix: bind all inputs for codec decoder

- Similarly as previous commit, all inputs'
shapes to codec decoder are static
and can be bound.
- Using in-place to copy numpy array to
the buffers.
- The `chunk_tokens` need additional `.copy()`
before passed on to the function to run
the graph. Without this additional `.copy()`,
not using CUDA graph and/or not using IOBinding
is fine, but if we bind it and/or enable CUDA
graph, it has to use the additional `.copy()`,
otherwise the output is messed up.
- Right now, all main modules: talker, local,
local lm_head, and codec decoder are able
to be run with the accelerated CUDA graphs.
- Still need to further improve by removing
unnecessary `Ops`.

src/inference/qwen3_tts_inferencer_onnx.py

@@ -440,8 +440,7 @@ class Qwen3TTSInferencerONNX:
 
         # Non-graphed utility sessions
         log.info(f"  codec decoder   <- {codec_decoder_model_path}")
-        # self._codec_decoder_sess = _sess(codec_decoder_model_path, graph=self._enable_cuda_graph)
-        self._codec_decoder_sess = _sess(codec_decoder_model_path, graph=False)
+        self._codec_decoder_sess = _sess(codec_decoder_model_path, graph=self._enable_cuda_graph)
         log.info(f"  speaker encoder <- {speaker_encoder_model_path}")
         self._speaker_encoder = _sess(speaker_encoder_model_path, graph=False)
         log.info(f"  codec embed     <- {talker_codec_embed_model_path}")
@@ -765,7 +764,7 @@ class Qwen3TTSInferencerONNX:
         b.bind_output_device("current_hidden_state_cache", self._codec_hidden_cache_ov)
         b.bind_output_device("current_pre_conv_hidden_state_cache", self._codec_pre_conv_cache_ov)
 
-        # b.bind_input_device("codes", self._codec_codes_ov)
+        b.bind_input_device("codes", self._codec_codes_ov)
         b.bind_input_device("cache_position", self._codec_cache_position_ov)
         b.bind_input_device("hidden_state_cache", self._codec_hidden_cache_ov)
         b.bind_input_device("pre_conv_hidden_state_cache", self._codec_pre_conv_cache_ov)
@@ -1319,17 +1318,12 @@ class Qwen3TTSInferencerONNX:
     ) -> np.ndarray:
         """Execute one Codec Decoder step via IOBinding and return wav as NumPy."""
         b = self._codec_decoder_bound
-        # Re-bind the two small dynamic inputs (H→D copy, ~4 KB total)
-        b.bind_input_cpu("codes", chunk_tokens)
-        # log.info(f"run_codec_decoder chunk_tokens {chunk_tokens.shape} {chunk_tokens.dtype}")
-        # b.bind_input_cpu("cache_position", pos)
-        # log.info(f"run_codec_decoder pos {pos.shape} {pos.dtype}")
-        # _copy_numpy_to_ortvalue(chunk_tokens, self._codec_codes_ov)
+        # Copy numpy inputs to buffer in-place
+        _copy_numpy_to_ortvalue(chunk_tokens, self._codec_codes_ov)
         _copy_numpy_to_ortvalue(pos, self._codec_cache_position_ov)
         # All KV and output bindings are already wired statically
         b.run()
         # Copy wav to CPU
-        # return b.get_output_numpy("wav")
         return self._codec_wav_ov.numpy()
 
     # ── Main autoregressive step ──────────────────────────────────────────────
@@ -1481,8 +1475,8 @@ class Qwen3TTSInferencerONNX:
             None, :, :
         ]  # [1, 1, 16]
 
-        self._last_audio_tokens = audio_tokens
-        self._generated_tokens = np.concatenate([self._generated_tokens, audio_tokens], axis=1)
+        self._last_audio_tokens = audio_tokens.copy()
+        self._generated_tokens = np.concatenate([self._generated_tokens, audio_tokens.copy()], axis=1)
 
     # ── Warmup (triggers CUDA graph capture) ─────────────────────────────────
 
@@ -1753,7 +1747,8 @@ class Qwen3TTSInferencerONNX:
                 axis=-1,
             )
             log.info(
-                f"codec-idx-{self._codec_step_idx} len_pad_right {len_pad_right} chunk_tokens {chunk_tokens} {chunk_tokens.shape}"
+                f"codec-idx-{self._codec_step_idx} len_pad_right {len_pad_right} "
+                f"chunk_tokens {chunk_tokens} {chunk_tokens.shape}"
             )
 
         # cache_position for the decoder transformer
@@ -1762,7 +1757,8 @@ class Qwen3TTSInferencerONNX:
         self._codec_step_idx += chunk_tokens.shape[-1]
 
         # Run the wired graph
-        wav = self._run_codec_decoder(chunk_tokens, pos).copy()
+        # chunk_tokens need to be .copy(), otherwise it messed up if we bind input and/or use CUDA graph
+        wav = self._run_codec_decoder(chunk_tokens.copy(), pos).copy()
         # new memory address, so that output wav not only the last repeated
 
         # The decoder output already discards the upsampled left-context frames (25 frames)