feat: further improve the handling of cont. stream

- Previously, as `step_idx` was reset after exceeding
limit of next turn length, that next turn will actually
use the very first audio reference instead of continuing
from the generated wav list. This causes discontinuity
and confusion on the generation. Now, it has been fixed.
- Further, the very first audio reference is now included
in the list of reference wav array. This is to maintain
coherence from the very first reference, to the first
generated audio chunks, and so on. Therefore the list
buffer ring of the reference wav for subsequent rounds is
initiated by the very first audio ref., then populated
more by the generated audio. It is now with max. length
of 8 s., the list ring.
- Fix the sampling parameters to make the generation more
robust and consistent, as we are feeding out-of-distribution
sequence structure to the model: temperature to 0.75,
top_p to 0.85, repetition_penalty to 9.5, repetition_window to 75.
- Now, the system works robustly with respect to the pronunciation
and hallucination.
- There is still a bug on the audio initial pulse between rounds
of text. This most likely caused by the reset of codec cache
every rounds due to the pad right of the last chunk for each round.
Need to further investigate, if that is the case, the last chunk
of each round needs to be generated by another onnx session
with dynamic axes on the frame length. Otherwise, it is probably
needing crossfade.

src/inference/qwen3_tts_inferencer_onnx.py

@@ -119,7 +119,7 @@ _NUM_CODE_GROUPS = 16
 
 # For continuous streaming with multi turn texts
 _LENGTH_RESET_LIMIT_FOR_START_OF_MULTI_TURN = 50
-_REF_WAV_LENGTH_CUT_LIMIT_FOR_START_OF_MULTI_TURN = 144000
+_REF_WAV_LENGTH_CUT_LIMIT_FOR_START_OF_MULTI_TURN = 192000
 
 
 # ── CPU sampling helpers ──────────────────────────────────────────────────────
@@ -650,24 +650,6 @@ class Qwen3TTSInferencerONNX:
         self._local_prefill_sin_rope_ov = _make_device_ortvalue(
             (1, 2, self._head_dim), np.float32, self._device, cuda_device_id
         )
-        # self._local_prefill_cache_write_mask_ov = _make_device_ortvalue(
-        #     (1, self._local_num_key_value_heads, self._num_code_groups, self._local_head_dim),
-        #     np.bool,
-        #     self._device,
-        #     cuda_device_id,
-        # )
-        # self._local_prefill_key_gqa_expanded_ov = _make_device_ortvalue(
-        #     (1, self._local_num_attention_heads, self._num_code_groups, self._local_head_dim),
-        #     np.float32,
-        #     self._device,
-        #     cuda_device_id,
-        # )
-        # self._local_prefill_value_gqa_expanded_ov = _make_device_ortvalue(
-        #     (1, self._local_num_attention_heads, self._num_code_groups, self._local_head_dim),
-        #     np.float32,
-        #     self._device,
-        #     cuda_device_id,
-        # )
 
         # Pre-allocate output OrtValues for Local Talker prefill
         # Backbone outputs last_hidden [1, cp_hidden_size]; logits come from lm_head
@@ -689,24 +671,6 @@ class Qwen3TTSInferencerONNX:
         self._local_step_sin_rope_ov = _make_device_ortvalue(
             (1, 1, self._head_dim), np.float32, self._device, cuda_device_id
         )
-        # self._local_step_cache_write_mask_ov = _make_device_ortvalue(
-        #     (1, self._local_num_key_value_heads, self._num_code_groups, self._local_head_dim),
-        #     np.bool,
-        #     self._device,
-        #     cuda_device_id,
-        # )
-        # self._local_step_key_gqa_expanded_ov = _make_device_ortvalue(
-        #     (1, self._local_num_attention_heads, self._num_code_groups, self._local_head_dim),
-        #     np.float32,
-        #     self._device,
-        #     cuda_device_id,
-        # )
-        # self._local_step_value_gqa_expanded_ov = _make_device_ortvalue(
-        #     (1, self._local_num_attention_heads, self._num_code_groups, self._local_head_dim),
-        #     np.float32,
-        #     self._device,
-        #     cuda_device_id,
-        # )
 
         # Pre-allocate output OrtValue for the unified Local Talker backbone step
         self._local_step_hidden_ov = _make_device_ortvalue(
@@ -775,9 +739,6 @@ class Qwen3TTSInferencerONNX:
         self._codec_codes_ov = _make_device_ortvalue(
             (1, self._num_code_groups, self.chunk_frames), np.int64, self._device, cuda_device_id
         )
-        # self._codec_cache_position_ov = _make_device_ortvalue(
-        #     (self.chunk_frames), np.int64, self._device, cuda_device_id
-        # )
         self._codec_attention_mask_ov = _make_device_ortvalue(
             (1, 1, self.chunk_frames, self._speech_tokenizer_sliding_window), np.float32, self._device, cuda_device_id
         )
@@ -819,7 +780,7 @@ class Qwen3TTSInferencerONNX:
         self._last_hidden_states_np: Optional[np.ndarray] = None
         self._step_idx = 0
         self._talker_seq_len = 0
-        self._generated_wav_list = None
+        self._ref_wav_history_list = None
 
         # ── Streaming text state ──────────────────────────────────────────────
         self._text_cache = ""
@@ -878,7 +839,6 @@ class Qwen3TTSInferencerONNX:
         b.bind_input_device("codes", self._codec_codes_ov)
         b.bind_input_device("hidden_state_cache", self._codec_hidden_cache_ov)
         b.bind_input_device("pre_conv_hidden_state_cache", self._codec_pre_conv_cache_ov)
-        # b.bind_input_device("cache_position", self._codec_cache_position_ov)
         b.bind_input_device("attention_mask", self._codec_attention_mask_ov)
         b.bind_input_device("cos_rope", self._codec_cos_rope_ov)
         b.bind_input_device("sin_rope", self._codec_sin_rope_ov)
@@ -929,9 +889,6 @@ class Qwen3TTSInferencerONNX:
         b.bind_input_device("attention_mask", self._local_prefill_attention_mask_ov)
         b.bind_input_device("cos_rope", self._local_prefill_cos_rope_ov)
         b.bind_input_device("sin_rope", self._local_prefill_sin_rope_ov)
-        # b.bind_input_device("cache_write_mask", self._local_prefill_cache_write_mask_ov)
-        # b.bind_input_device("key_gqa_expanded", self._local_prefill_key_gqa_expanded_ov)
-        # b.bind_input_device("value_gqa_expanded", self._local_prefill_value_gqa_expanded_ov)
         for i in range(self._local_num_hidden_layers):
             b.bind_input_device(f"past_key_{i}", self._kv_local_ov[2 * i])
             b.bind_input_device(f"past_value_{i}", self._kv_local_ov[2 * i + 1])
@@ -977,9 +934,6 @@ class Qwen3TTSInferencerONNX:
         b.bind_input_device("attention_mask", self._local_step_attention_mask_ov)
         b.bind_input_device("cos_rope", self._local_step_cos_rope_ov)
         b.bind_input_device("sin_rope", self._local_step_sin_rope_ov)
-        # b.bind_input_device("cache_write_mask", self._local_step_cache_write_mask_ov)
-        # b.bind_input_device("key_gqa_expanded", self._local_step_key_gqa_expanded_ov)
-        # b.bind_input_device("value_gqa_expanded", self._local_step_value_gqa_expanded_ov)
         for i in range(self._local_num_hidden_layers):
             b.bind_input_device(f"past_key_{i}", self._kv_local_ov[2 * i])
             b.bind_input_device(f"past_value_{i}", self._kv_local_ov[2 * i + 1])
@@ -1227,34 +1181,6 @@ class Qwen3TTSInferencerONNX:
 
         return cos.astype(np.float32), sin.astype(np.float32)
 
-    # ── Other helper functions ────────────────────────────────────────────────
-
-    # def _make_local_cache_write_mask(self, batch_size=1):
-    #     t = np.arange(self._num_code_groups, dtype=np.int64).reshape(batch_size, 1, 1, self._num_code_groups)
-
-    #     cache_position = np.asarray([0, 1], dtype=np.int64).reshape(batch_size, 1, -1, 1)
-    #     mask = (t == cache_position).any(axis=2)  # shape [1, 1, max_seq_len]
-    #     mask = mask[..., None]  # shape [1, 1, max_seq_len, 1]
-    #     mask_list = [
-    #         np.broadcast_to(
-    #             mask, (batch_size, self._local_num_key_value_heads, self._num_code_groups, self._local_head_dim)
-    #         )
-    #     ]
-
-    #     # cache write mask for each codebook from 1 to 15 (0 is generated by talker)
-    #     for pos in range(2, self._num_code_groups):
-    #         cache_position = np.asarray([pos], dtype=np.int64).reshape(1, 1, -1, 1)
-    #         mask = (t == cache_position).any(axis=2)  # shape [1, 1, max_seq_len]
-    #         mask = mask[..., None]  # shape [1, 1, max_seq_len, 1]
-    #         mask_list.append(
-    #             np.broadcast_to(
-    #                 mask, (batch_size, self._local_num_key_value_heads, self._num_code_groups, self._local_head_dim)
-    #             )
-    #         )
-    #     # this is used for torch.where inside attention function to update KV-cache buffer in-place
-
-    #     return mask_list
-
     # ── CPU sampling ──────────────────────────────────────────────────────────
 
     def _sample(
@@ -1296,7 +1222,7 @@ class Qwen3TTSInferencerONNX:
                 wav = wav.mean(axis=1)
             return wav.astype(np.float32), int(sr)
 
-    def create_voice_clone_spkemb(self, ref_audio: AudioLike) -> NDArrayFloat:
+    def create_voice_clone_spkemb(self, ref_audio: AudioLike, init_cont_stream: bool = False) -> NDArrayFloat:
         wav, sr = self._normalize_audio_inputs(ref_audio)
         if sr != self._speaker_encoder_sample_rate:
             wav = librosa.resample(
@@ -1304,6 +1230,10 @@ class Qwen3TTSInferencerONNX:
                 orig_sr=int(sr),
                 target_sr=self._speaker_encoder_sample_rate,
             )
+        if self._ref_wav_history_list is None or len(self._ref_wav_history_list) == 0:
+            # split ref wavs into chunks of 0.32s audio for ref wav history ring list buffer
+            # for the continuous streaming
+            self._ref_wav_history_list = [wav[None, None, i : i + 7680] for i in range(0, len(wav), 7680)]
         mels = mel_spectrogram_numpy(
             wav,
             n_fft=1024,
@@ -1321,9 +1251,6 @@ class Qwen3TTSInferencerONNX:
     def _build_assistant_text(self) -> str:
         return "<|im_start|>assistant\n"
 
-    def _build_ending_assistant_text(self) -> str:
-        return "<|im_end|>\n"
-
     def _prefill_embeds(self, audio_info: AudioLike, language: str) -> np.ndarray:
         """Build the 9-token prefill embedding for the Talker.
 
@@ -1409,28 +1336,35 @@ class Qwen3TTSInferencerONNX:
 
     # ── Talker prefill (IOBinding + CUDA graph) ─────────────────────────────
 
-    def prefill(self) -> None:
+    def prefill(self, cont_stream: bool = False) -> None:
         """Run the 9-token prefill pass (IOBinding + CUDA graph) and snapshot KV."""
-        if self._step_idx == 0:
+        if not cont_stream:
             audio_info = self._audio_ref_path
         else:
-            log.info(f"prefill cont generated_wav_list {self._generated_wav_list} {len(self._generated_wav_list)}")
-            wav_history = np.concatenate(self._generated_wav_list, axis=-1)[0, 0]
+            # use previously generated audio sequence
+            # from previous list of texts with the window limit 50 as reference
+            log.info(
+                f"prefill cont ref_wav_history_list {self._ref_wav_history_list} {len(self._ref_wav_history_list)}"
+            )
+            wav_history = np.concatenate(self._ref_wav_history_list, axis=-1)[0, 0]
+            log.info(f"prefill cont wav_history {wav_history} {wav_history.shape} {wav_history.dtype}")
             cut_idx = 0
             while len(wav_history) > _REF_WAV_LENGTH_CUT_LIMIT_FOR_START_OF_MULTI_TURN:
-                wav_history = wav_history[self._generated_wav_list[cut_idx].shape[-1] :]
+                wav_history = wav_history[self._ref_wav_history_list[cut_idx].shape[-1] :]
                 cut_idx += 1
-            log.info(f"prefill cont wav_history {wav_history} {wav_history.shape} {wav_history.dtype}")
-            self._generated_wav_list = self._generated_wav_list[cut_idx:]
             log.info(
-                f"prefill cont generated_wav_list_after cut_idx {cut_idx} {self._generated_wav_list} {len(self._generated_wav_list)}"
+                f"prefill cont wav_history_after cut_idx {cut_idx} "
+                f"{wav_history} {wav_history.shape} {wav_history.dtype}"
+            )
+            self._ref_wav_history_list = self._ref_wav_history_list[cut_idx:]
+            log.info(
+                f"prefill cont ref_wav_history_list_after cut_idx {cut_idx} "
+                f"{self._ref_wav_history_list} {len(self._ref_wav_history_list)}"
             )
             audio_info = (
                 wav_history,
                 24000,
             )
-        # use previously generated audio sequence
-        # from previous list of texts with the window limit 50 as reference
         inputs_embeds = self._prefill_embeds(audio_info, self._language)
         if self._step_idx == 0:
             cache_position = np.arange(_TALKER_PREFILL_LEN, dtype=np.int64)
@@ -1460,69 +1394,7 @@ class Qwen3TTSInferencerONNX:
         [codec_eos],[None],[codec_pad] -> continue with codec_bos and generation
         """
 
-        # text_ids = np.array([[self._tts_pad_token_id, self._tts_bos_token_id]], dtype=np.int64)
-        # text_ids = np.array([[self._tts_pad_token_id]], dtype=np.int64)
-        # log.info(f"prefill_cont text_ids {text_ids} {text_ids.shape}")
-        # outputs = self._text_embed_proj.run(["text_emb_out"], {"text_ids": text_ids})  # 3
-        # # embeds = outputs[0]
-        # # tts_pad_embed, tts_bos_embed = embeds[:, :1], embeds[:, 1:]  # 2 * [1 1 d]
-        # tts_pad_embed = outputs[0]
-        # # log.info(f"prefill_cont tts_bos_embed {tts_bos_embed} {tts_bos_embed.shape} {tts_bos_embed.dtype}")
-        # log.info(f"prefill_cont tts_pad_embed {tts_pad_embed} {tts_pad_embed.shape} {tts_pad_embed.dtype}")
-
-        # # codec_ids = np.array([[self._codec_eos_token_id, self._codec_pad_id]], dtype=np.int64)
-        # codec_ids = np.array([[self._codec_eos_token_id]], dtype=np.int64)
-        # log.info(f"prefill_cont codec_ids  {codec_ids}  {codec_ids.shape}")
-        # outputs = self._talker_codec_embed.run(["codec_emb"], {"codec_ids": codec_ids})
-        # # embeds = outputs[0]
-        # # codec_eos_embed, codec_pad_embed = embeds[:, :1], embeds[:, 1:]  # 2 * [1 1 d]
-        # codec_eos_embed = outputs[0]
-        # log.info(f"prefill_cont codec_eos_embed {codec_eos_embed} {codec_eos_embed.shape} {codec_eos_embed.dtype}")
-        # # log.info(f"prefill_cont codec_pad_embed {codec_pad_embed} {codec_pad_embed.shape} {codec_pad_embed.dtype}")
-
-        # tts_pad_codec_eos_embed = tts_pad_embed + codec_eos_embed
-        # # tts_bos_codec_pad_embed = tts_bos_embed + codec_pad_embed
-
-        # self._tokenize_texts([self._build_ending_assistant_text(), self._build_assistant_text()]),
-        # self._tokenize_texts([self._build_assistant_text()]),
-        # prefix_tokens = np.expand_dims(
-        #     np.array(
-        #         self._tokenize_texts([self._build_ending_assistant_text()]),
-        #         dtype=np.int64,
-        #     ),
-        #     axis=0,
-        # )
-        # log.info(f"prefill_cont prefix_tokens {prefix_tokens} {prefix_tokens.shape}")
-        # outputs = self._text_embed_proj.run(["text_emb_out"], {"text_ids": prefix_tokens})  # 3
-        # _talker_input_embed_role = outputs[0]
-        # log.info(
-        #     f"prefill_cont _talker_input_embed_role {_talker_input_embed_role} {_talker_input_embed_role.shape} {_talker_input_embed_role.dtype}"
-        # )
-
-        # talker_input_embed = _talker_input_embed_role
-        # talker_input_embed = np.concatenate(
-        #     (_talker_input_embed_role, tts_bos_codec_pad_embed), axis=1
-        # )  # 6
-        # (tts_pad_codec_eos_embed, _talker_input_embed_role, tts_bos_codec_pad_embed), axis=1
-        # talker_input_embed = np.concatenate((tts_pad_codec_eos_embed, tts_bos_codec_pad_embed), axis=1)  # 2
-        # talker_input_embed = np.concatenate((tts_pad_codec_eos_embed, _talker_input_embed_role), axis=1)  # 3
-        # log.info(
-        #     f"prefill_cont talker_input_embed {talker_input_embed} {talker_input_embed.shape} {talker_input_embed.dtype}"
-        # )
-
-        # log.info(f"prefill_cont talker_seq_len {self._talker_seq_len} step_idx {self._step_idx}")
-        # for i in range(talker_input_embed.shape[1]):
-        #     inputs_embeds = talker_input_embed[:, i : i + 1]
-        #     cache_position = np.array([self._talker_seq_len], dtype=np.int64)
-        #     causal_rows = self._causal_tril[:, :, cache_position, :]  # [1,1,q_len,MAX]
-        #     attn_mask = np.where(causal_rows, 0.0, -np.inf).astype(np.float32)  # [1, 1, q_len, MAX_SEQ_LEN]
-        #     cos_rope = self._cos_rope[:, :, cache_position].copy()
-        #     sin_rope = self._sin_rope[:, :, cache_position].copy()
-        #     self._run_talker_step(inputs_embeds, cache_position, attn_mask, cos_rope, sin_rope)
-        #     self._talker_seq_len += 1
-        #     self._step_idx += 1
-
-        self.prefill()
+        self.prefill(cont_stream=True)
         # Snapshot the post-prefill KV state for turn resets
         self._kv_talker_prefill_np = self._snapshot_talker_kv()
         self._prefilled = True
@@ -1599,7 +1471,6 @@ class Qwen3TTSInferencerONNX:
         _copy_numpy_to_ortvalue(attention_mask, self._local_prefill_attention_mask_ov)
         _copy_numpy_to_ortvalue(cos_rope, self._local_prefill_cos_rope_ov)
         _copy_numpy_to_ortvalue(sin_rope, self._local_prefill_sin_rope_ov)
-        # _copy_numpy_to_ortvalue(cache_write_mask, self._local_prefill_cache_write_mask_ov)
         b.run()
         return self._local_prefill_hidden_ov.numpy()  # [1, cp_hidden_size]
 
@@ -1625,7 +1496,6 @@ class Qwen3TTSInferencerONNX:
         _copy_numpy_to_ortvalue(attention_mask, self._local_step_attention_mask_ov)
         _copy_numpy_to_ortvalue(cos_rope, self._local_step_cos_rope_ov)
         _copy_numpy_to_ortvalue(sin_rope, self._local_step_sin_rope_ov)
-        # _copy_numpy_to_ortvalue(cache_write_mask, self._local_step_cache_write_mask_ov)
         b.run()
         return self._local_step_hidden_ov.numpy()  # [1, cp_hidden_size]
 
@@ -1654,12 +1524,10 @@ class Qwen3TTSInferencerONNX:
         cos_rope: np.ndarray,  # [1, chunk_length, 64]
         sin_rope: np.ndarray,  # [1, chunk_length, 64]
     ) -> np.ndarray:
-        # pos: np.ndarray,  # [chunk_length]
         """Execute one Codec Decoder step via IOBinding and return wav as NumPy."""
         b = self._codec_decoder_bound
         # Copy numpy inputs to buffer in-place
         _copy_numpy_to_ortvalue(chunk_tokens, self._codec_codes_ov)
-        # _copy_numpy_to_ortvalue(pos, self._codec_cache_position_ov)
         _copy_numpy_to_ortvalue(attention_mask, self._codec_attention_mask_ov)
         _copy_numpy_to_ortvalue(cos_rope, self._codec_cos_rope_ov)
         _copy_numpy_to_ortvalue(sin_rope, self._codec_sin_rope_ov)
@@ -1703,14 +1571,9 @@ class Qwen3TTSInferencerONNX:
         cos_rope = self._cos_rope[:, :, cache_position].copy()
         sin_rope = self._sin_rope[:, :, cache_position].copy()
         logits = self._run_talker_step(inputs_embeds, cache_position, attn_mask, cos_rope, sin_rope)  # [1, vocab]
-        # log.info(f"step-idx-{self._step_idx} logits {logits} {logits.shape} {logits.dtype}")
 
         # Retrieve hidden states for Local Talker prefill
         self._last_hidden_states_np = self._talker_hidden_ov.numpy()  # [1, 1, H]
-        # log.info(
-        #     f"step-idx-{self._step_idx} self._last_hidden_states_np {self._last_hidden_states_np} "
-        #     f"{self._last_hidden_states_np.shape} {self._last_hidden_states_np.dtype}"
-        # )
 
         # ── Sample first token (CPU) ──────────────────────────────────────────
         history = self._generated_tokens[:, :, 0]
@@ -1768,7 +1631,6 @@ class Qwen3TTSInferencerONNX:
         attn_mask_prefill = np.where(causal_rows, 0.0, -np.inf).astype(np.float32)  # [1, 1, q_len, MAX_SEQ_LEN]
         cos_rope_prefill = self._cos_rope_local[:, cache_position_prefill].copy()
         sin_rope_prefill = self._sin_rope_local[:, cache_position_prefill].copy()
-        # cache_write_mask_prefill = self._local_cache_write_mask_list[0]
 
         # Backbone prefill → last_hidden [1, cp_hidden_size]
         hidden = self._run_local_prefill(
@@ -1778,7 +1640,6 @@ class Qwen3TTSInferencerONNX:
             cos_rope_prefill,
             sin_rope_prefill,
         )
-        # cache_write_mask_prefill,
         # Batched lm_head → [15, local_vocab_size]; select group 0 (codebook group 1)
         all_logits = self._run_local_lm_head(hidden)  # [15, local_vocab_size]
         logits_g1 = all_logits[0:1, :]  # [1, local_vocab_size]
@@ -1806,7 +1667,6 @@ class Qwen3TTSInferencerONNX:
             attn_mask_step = np.where(causal_rows, 0.0, -np.inf).astype(np.float32)  # [1, 1, q_len, MAX_SEQ_LEN]
             cos_rope_step = self._cos_rope_local[:, cache_position_step].copy()
             sin_rope_step = self._sin_rope_local[:, cache_position_step].copy()
-            # cache_write_mask_step = self._local_cache_write_mask_list[step_i - 1]
 
             # Unified backbone step → last_hidden [1, cp_hidden_size]
             hidden = self._run_local_step(
@@ -1816,7 +1676,6 @@ class Qwen3TTSInferencerONNX:
                 cos_rope_step,
                 sin_rope_step,
             )
-            # cache_write_mask_step,
             # Batched lm_head → [15, local_vocab_size]; select the current group
             all_logits = self._run_local_lm_head(hidden)
             logits_gi = all_logits[head_idx : head_idx + 1, :]  # [1, local_vocab_size]
@@ -1948,11 +1807,9 @@ class Qwen3TTSInferencerONNX:
         attn_mask_ls = np.where(causal_rows, 0.0, -np.inf).astype(np.float32)  # [1, 1, q_len, MAX_SEQ_LEN]
         cos_rope_ls = self._cos_rope_local[:, cache_pos_ls].copy()
         sin_rope_ls = self._sin_rope_local[:, cache_pos_ls].copy()
-        # cache_write_mask_ls = self._local_cache_write_mask_list[1]
         for _ in range(n_iter):
             t0 = time.perf_counter()
             self._run_local_step(dummy_ls_embeds, cache_pos_ls, attn_mask_ls, cos_rope_ls, sin_rope_ls)
-            # cache_write_mask_ls
             times["local_step"].append((time.perf_counter() - t0) * 1000)
         exec_time_tot = sum(times["local_step"]) / 1000
         avg_exec_time = exec_time_tot / len(times["local_step"]) * 1000
@@ -2081,7 +1938,8 @@ class Qwen3TTSInferencerONNX:
         while self._pending_tokens and not self.is_finished:
             token = self._pending_tokens.pop(0)
             log.info(
-                f"drain_pending_tokens token {token} -> pending_tokens {self._pending_tokens} {len(self._pending_tokens)}"
+                f"drain_pending_tokens token {token} -> "
+                f"pending_tokens {self._pending_tokens} {len(self._pending_tokens)}"
             )
             output = self.step(token)
             if output is not None:
@@ -2094,7 +1952,8 @@ class Qwen3TTSInferencerONNX:
         for segment in self._extract_text_segments(force=False):
             tokenized_segment = self._tokenize_texts([segment])
             log.info(
-                f"push_text segment {segment} {len(segment)} -> tokenized_segment {tokenized_segment} {len(tokenized_segment)}"
+                f"push_text segment {segment} {len(segment)} -> "
+                f"tokenized_segment {tokenized_segment} {len(tokenized_segment)}"
             )
             self._pending_tokens.extend(tokenized_segment)
             log.info(f"push_text pending_tokens {self._pending_tokens}")
@@ -2206,10 +2065,10 @@ class Qwen3TTSInferencerONNX:
         )
 
         self._codec_step_idx += self.chunk_frames
-        if self._generated_wav_list is not None:
-            self._generated_wav_list.append(wav.copy())
+        if self._ref_wav_history_list is not None:
+            self._ref_wav_history_list.append(wav.copy())
         else:
-            self._generated_wav_list = [wav.copy()]
+            self._ref_wav_history_list = [wav.copy()]
         return wav
 
     def _overlap_samples(self, wav: NDArrayFloat) -> int:

test_qwen3-tts-streaming_onnx.py

@@ -131,11 +131,11 @@ _TEXT = [
 ]
 _OUTPUT_SAMPLE_RATE = 24000
 _CHUNK_FRAMES = 4
-_TEMPERATURE = 0.85
-_TOP_P = 0.8
+_TEMPERATURE = 0.75
+_TOP_P = 0.85
 _TOP_K = 50
-_REPETITION_PENALTY = 1.9
-_REPETITION_WINDOW = 50
+_REPETITION_PENALTY = 9.5
+_REPETITION_WINDOW = 75
 _WARMUP_ITERS = 50
 
 # ---------------------------------------------------------------------------