Update app.py

app.py

@@ -9,24 +9,19 @@ import warnings
 
 warnings.filterwarnings('ignore')
 
-# ── CONFIGURATION ──
 REPO_ID = "anggars/neural-mathrock"
 SR = 16000
 DURATION = 10
-# Pakai urutan yang sama dengan training
 TARGET_COLS = ['mbti', 'emotion', 'vibe', 'intensity', 'tempo']
 
 print("Downloading model from HF Hub...")
 model_path = hf_hub_download(repo_id=REPO_ID, filename="model.pt")
-# Load checkpoint
 ckpt = torch.load(model_path, map_location=torch.device('cpu'), weights_only=False)
 
-# ── LABEL ENCODER PATCH (Sesuai output training lo) ──
 class DummyEncoder:
     def __init__(self, classes_list):
         self.classes_ = np.array(classes_list)
 
-# Ambil encoder dari checkpoint, kalo gak ada pake manual sesuai kategori di dataset
 label_encoders = {
     'mbti': ckpt.get('le_mbti', DummyEncoder(['ENFJ', 'ENFP', 'ENTJ', 'ENTP', 'ESFJ', 'ESFP', 'ESTJ', 'ESTP', 'INFJ', 'INFP', 'INTJ', 'INTP', 'ISFJ', 'ISFP', 'ISTJ', 'ISTP'])),
     'emotion': ckpt.get('le_emotion', DummyEncoder(['amusement', 'anger', 'annoyance', 'approval', 'caring', 'confusion', 'curiosity', 'desire', 'disappointment', 'disapproval', 'disgust', 'embarrassment', 'excitement', 'fear', 'gratitude', 'grief', 'love', 'nervousness', 'neutral', 'pride', 'realization', 'relief', 'remorse', 'sadness'])),
@@ -37,25 +32,16 @@ label_encoders = {
 
 num_classes = {col: len(le.classes_) for col, le in label_encoders.items()}
 
-# ============================================================
-# ── ARCHITECTURE (MATCHING EPOCH 10 WEIGHTS) ──
-# ============================================================
 class MultimodalMathRock(nn.Module):
     def __init__(self):
         super().__init__()
-        # Encoder Audio & Text
         self.audio_encoder = WavLMModel.from_pretrained("microsoft/wavlm-base")
         self.text_encoder  = AutoModel.from_pretrained('xlm-roberta-base')
-        
-        # Fusion Layer (Linear 1536 -> 512)
         self.fusion = nn.Sequential(
             nn.Dropout(0.3),
             nn.Linear(768 + 768, 512),
             nn.ReLU()
         )
-        
-        # Classification Heads
-        # Gunakan ModuleDict agar key-nya cocok dengan state_dict "heads.mbti.weight" dst.
         self.heads = nn.ModuleDict({
             'mbti': nn.Linear(512, num_classes['mbti']),
             'emotion': nn.Linear(512, num_classes['emotion']),
@@ -65,23 +51,13 @@ class MultimodalMathRock(nn.Module):
         })
 
     def forward(self, audio_values, input_ids, attention_mask):
-        # Audio Feature Extraction
         x_a = self.audio_encoder(audio_values).last_hidden_state.mean(dim=1)
-        # Text Feature Extraction (CLS Token)
         x_t = self.text_encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state[:, 0, :]
-        
-        # Late Fusion
-        combined = torch.cat([x_a, x_t], dim=1)
-        fused = self.fusion(combined)
-        
+        fused = self.fusion(torch.cat([x_a, x_t], dim=1))
         return {col: self.heads[col](fused) for col in TARGET_COLS}
 
-# ============================================================
-
 print("Initializing architecture and loading weights...")
 model = MultimodalMathRock()
-
-# Load state dict dengan filter prefix jika perlu
 state_dict = ckpt['model_state'] if 'model_state' in ckpt else ckpt
 model.load_state_dict(state_dict, strict=False)
 model.eval()
@@ -89,35 +65,43 @@ model.eval()
 tokenizer = AutoTokenizer.from_pretrained('xlm-roberta-base')
 audio_processor = AutoFeatureExtractor.from_pretrained("microsoft/wavlm-base")
 
-# ── INFERENCE PIPELINE ──
 def predict_multimodal(audio_path, lyrics):
     if audio_path is None:
         return [None]*5
         
     try:
-        # Load audio 10 detik awal
-        y, _ = librosa.load(audio_path, sr=SR, duration=DURATION, mono=True)
-        inputs = audio_processor(y, sampling_rate=SR, return_tensors="pt")
+        y_full, _ = librosa.load(audio_path, sr=SR, mono=True)
+        total_dur = librosa.get_duration(y=y_full, sr=SR)
+        
+        num_windows = min(8, int(total_dur // DURATION))
+        if num_windows == 0: num_windows = 1
+        offsets = np.linspace(0, total_dur - DURATION, num_windows)
         
-        # Handle lyrics kosong
         if not lyrics or lyrics.strip() == "":
-            lyrics = "instrumental math rock music"
+            lyrics = "instrumental math rock"
             
-        enc = tokenizer(
-            lyrics, max_length=128, padding='max_length',
-            truncation=True, return_tensors='pt'
-        )
+        enc = tokenizer(lyrics, max_length=128, padding='max_length', truncation=True, return_tensors='pt')
+        all_logits = {col: [] for col in TARGET_COLS}
         
         with torch.no_grad():
-            out = model(inputs.input_values, enc['input_ids'], enc['attention_mask'])
-            
+            for offset in offsets:
+                start = int(offset * SR)
+                end = start + int(DURATION * SR)
+                y_chunk = y_full[start:end]
+                if len(y_chunk) < int(DURATION * SR):
+                    y_chunk = np.pad(y_chunk, (0, int(DURATION * SR) - len(y_chunk)))
+                
+                audio_inputs = audio_processor(y_chunk, sampling_rate=SR, return_tensors="pt")
+                out = model(audio_inputs.input_values, enc['input_ids'], enc['attention_mask'])
+                for col in TARGET_COLS:
+                    all_logits[col].append(out[col][0])
+        
         final_results = []
         for col in TARGET_COLS:
-            logits = out[col][0]
-            probs = torch.nn.functional.softmax(logits, dim=0).numpy()
+            avg_logits = torch.stack(all_logits[col]).mean(dim=0)
+            probs = torch.nn.functional.softmax(avg_logits, dim=0).numpy()
             classes = label_encoders[col].classes_
             
-            # Buat dict untuk Gradio Label (Top 3)
             pred_dict = {str(classes[i]): float(probs[i]) for i in range(len(classes)) if i < len(probs)}
             sorted_preds = dict(sorted(pred_dict.items(), key=lambda item: item[1], reverse=True)[:3])
             final_results.append(sorted_preds)
@@ -127,23 +111,23 @@ def predict_multimodal(audio_path, lyrics):
     except Exception as e:
         return [{"Error": str(e)}]*5
 
-# ── GRADIO UI ──
-with gr.Blocks(theme=gr.themes.Soft()) as demo:
+# ── GRADIO UI (Glass Theme) ──
+with gr.Blocks(theme=gr.themes.Glass()) as demo:
     gr.Markdown("# Neural Math Rock & Midwest Emo Analysis")
-    gr.Markdown("Analyze audio and lyrics to predict MBTI, Emotion, Vibe, Intensity, and Tempo using Multimodal Transformers.")
+    gr.Markdown("Multimodal Affective Computing System for Math Rock & Midwest Emo Genres.")
     
     with gr.Row():
         with gr.Column():
-            audio_input = gr.Audio(type="filepath", label="Upload Song (10s will be analyzed)")
-            lyrics_input = gr.Textbox(lines=5, label="Lyrics", placeholder="Paste lyrics here...")
-            btn = gr.Button("Analyze Personality & Emotion", variant="primary")
+            audio_input = gr.Audio(type="filepath", label="Upload Song (Full Analysis)")
+            lyrics_input = gr.Textbox(lines=5, label="Lyrics Content", placeholder="Paste lyrics here...")
+            btn = gr.Button("START MULTIMODAL ANALYSIS", variant="primary")
         
         with gr.Column():
-            out_mbti = gr.Label(label="Predicted MBTI")
-            out_emotion = gr.Label(label="Predicted Emotion")
-            out_vibe = gr.Label(label="Music Vibe")
-            out_intensity = gr.Label(label="Energy Intensity")
-            out_tempo = gr.Label(label="Estimated Tempo")
+            out_mbti = gr.Label(label="Personality Type (MBTI)")
+            out_emotion = gr.Label(label="Emotional State")
+            out_vibe = gr.Label(label="Acoustic Vibe")
+            out_intensity = gr.Label(label="Energy Level")
+            out_tempo = gr.Label(label="Tempo Classification")
 
     btn.click(
         fn=predict_multimodal,
@@ -151,10 +135,6 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
         outputs=[out_mbti, out_emotion, out_vibe, out_intensity, out_tempo]
     )
 
-    gr.Examples(
-        examples=[["example.mp3", "I'm not sad, I'm just tired of being alone in this basement."]],
-        inputs=[audio_input, lyrics_input]
-    )
-
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()
+