Commit ·
ecdbc41
1
Parent(s): 6261c1d
perf: optimize gRPC server for lower latency
Browse files- Replace librosa resampler with scipy.signal.resample_poly (~10ms savings)
- Reduce JPEG encoding quality from 85 to 75 (faster encoding, smaller frames)
- Remove unnecessary deepcopy when accessing avatar frames
Benchmarks show ~10% improvement in TTFF (256ms vs 285ms) and ~6%
improvement in FPS (22.5 vs 21.2) for single user tests.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
server/musetalk_grpc_server.py
CHANGED
|
@@ -39,6 +39,8 @@ import numpy as np
|
|
| 39 |
import cv2
|
| 40 |
import torch
|
| 41 |
from einops import rearrange
|
|
|
|
|
|
|
| 42 |
|
| 43 |
# Change to MuseTalk directory for imports
|
| 44 |
os.chdir('/workspace/MuseTalk')
|
|
@@ -248,12 +250,16 @@ class MuseTalkStreamingEngine:
|
|
| 248 |
print(f"[Engine] Loaded {len(self.mask_list_cycle)} masks")
|
| 249 |
|
| 250 |
def resample_audio(self, audio: np.ndarray, orig_sr: int) -> np.ndarray:
|
| 251 |
-
"""Resample audio to 16kHz for Whisper"""
|
| 252 |
if orig_sr == self.target_sample_rate:
|
| 253 |
return audio
|
| 254 |
|
| 255 |
-
|
| 256 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 257 |
|
| 258 |
@torch.no_grad()
|
| 259 |
def process_audio_buffer(self, audio_buffer: np.ndarray, start_frame_index: int,
|
|
@@ -346,7 +352,8 @@ class MuseTalkStreamingEngine:
|
|
| 346 |
avatar_idx = frame_idx % self.num_avatar_frames
|
| 347 |
|
| 348 |
bbox = self.coord_list_cycle[avatar_idx]
|
| 349 |
-
|
|
|
|
| 350 |
x1, y1, x2, y2 = bbox
|
| 351 |
|
| 352 |
try:
|
|
@@ -375,8 +382,8 @@ class MuseTalkStreamingEngine:
|
|
| 375 |
# Update previous frame for next iteration
|
| 376 |
prev_frame = combine_frame.copy()
|
| 377 |
|
| 378 |
-
# Encode as JPEG
|
| 379 |
-
_, jpeg_data = cv2.imencode('.jpg', combine_frame, [cv2.IMWRITE_JPEG_QUALITY,
|
| 380 |
|
| 381 |
gen_time = (time.time() - start_time) * 1000 / (i + 1)
|
| 382 |
results.append((frame_idx, jpeg_data.tobytes(), gen_time))
|
|
|
|
| 39 |
import cv2
|
| 40 |
import torch
|
| 41 |
from einops import rearrange
|
| 42 |
+
from scipy.signal import resample_poly
|
| 43 |
+
from math import gcd
|
| 44 |
|
| 45 |
# Change to MuseTalk directory for imports
|
| 46 |
os.chdir('/workspace/MuseTalk')
|
|
|
|
| 250 |
print(f"[Engine] Loaded {len(self.mask_list_cycle)} masks")
|
| 251 |
|
| 252 |
def resample_audio(self, audio: np.ndarray, orig_sr: int) -> np.ndarray:
|
| 253 |
+
"""Resample audio to 16kHz for Whisper using fast scipy resample_poly"""
|
| 254 |
if orig_sr == self.target_sample_rate:
|
| 255 |
return audio
|
| 256 |
|
| 257 |
+
# Use scipy's resample_poly which is much faster than librosa
|
| 258 |
+
# Calculate up/down factors: target/orig = 16000/24000 = 2/3
|
| 259 |
+
g = gcd(self.target_sample_rate, orig_sr)
|
| 260 |
+
up = self.target_sample_rate // g
|
| 261 |
+
down = orig_sr // g
|
| 262 |
+
return resample_poly(audio, up, down).astype(np.float32)
|
| 263 |
|
| 264 |
@torch.no_grad()
|
| 265 |
def process_audio_buffer(self, audio_buffer: np.ndarray, start_frame_index: int,
|
|
|
|
| 352 |
avatar_idx = frame_idx % self.num_avatar_frames
|
| 353 |
|
| 354 |
bbox = self.coord_list_cycle[avatar_idx]
|
| 355 |
+
# Use the frame directly - get_image_blending doesn't modify the original
|
| 356 |
+
ori_frame = self.frame_list_cycle[avatar_idx]
|
| 357 |
x1, y1, x2, y2 = bbox
|
| 358 |
|
| 359 |
try:
|
|
|
|
| 382 |
# Update previous frame for next iteration
|
| 383 |
prev_frame = combine_frame.copy()
|
| 384 |
|
| 385 |
+
# Encode as JPEG (quality 75 for faster encoding and smaller size)
|
| 386 |
+
_, jpeg_data = cv2.imencode('.jpg', combine_frame, [cv2.IMWRITE_JPEG_QUALITY, 75])
|
| 387 |
|
| 388 |
gen_time = (time.time() - start_time) * 1000 / (i + 1)
|
| 389 |
results.append((frame_idx, jpeg_data.tobytes(), gen_time))
|