Upload folder using huggingface_hub

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+# HuggingFace Spaces requires user ID 1000
+RUN useradd -m -u 1000 user
+
+WORKDIR /home/user/app
+
+# Install dependencies (CPU-only torch to keep image small)
+COPY --chown=user requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt \
+    --extra-index-url https://download.pytorch.org/whl/cpu
+
+# Copy application code
+COPY --chown=user symbio_model.py .
+COPY --chown=user vocab_fusion_model.py .
+COPY --chown=user server.py .
+
+# Switch to non-root user
+USER user
+ENV HOME=/home/user
+
+EXPOSE 7860
+
+CMD ["python", "server.py"]

README.md

@@ -1,10 +1,38 @@
 ---
 title: VocabFusion Pythia160m Fused
-emoji: 💻
-colorFrom: indigo
-colorTo: green
+emoji: 🧬
+colorFrom: purple
+colorTo: blue
 sdk: docker
+app_port: 7860
 pinned: false
+license: apache-2.0
+tags:
+  - symbiogenesis
+  - vocab-fusion
+  - pythia
+  - pca-fusion
+  - openai-compatible
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# VocabFusion-Pythia160m-Fused
+
+VocabFusion-SymbioGPT architecture with knowledge from Pythia-160M (12 layers, d=768) projected via PCA decomposition and blended at alpha=0.3 (92% variance retained). 17.5M params.
+
+The donor embedding (GPTNeoX, ~50k tokens, 128-dim) bridges to the 320-dim SymbioGPT body (4-organelle decoder) via a junction layer (Linear + LayerNorm). Output is weight-tied to the donor embedding.
+
+## API
+
+OpenAI-compatible endpoints:
+
+- `GET /` — model info
+- `GET /v1/models` — list models
+- `POST /v1/chat/completions` — generate (supports `stream: true`)
+
+## Links
+
+- **Fused model**: [LisaMegaWatts/VocabFusion-Pythia160m-Fused](https://huggingface.co/LisaMegaWatts/VocabFusion-Pythia160m-Fused)
+- **Base VocabFusion**: [LisaMegaWatts/VocabFusion-SymbioGPT-10M](https://huggingface.co/LisaMegaWatts/VocabFusion-SymbioGPT-10M)
+- **Source model**: [EleutherAI/pythia-160m](https://huggingface.co/EleutherAI/pythia-160m)
+- **Base architecture**: [LisaMegaWatts/SymbioGPT-10M](https://huggingface.co/LisaMegaWatts/SymbioGPT-10M)
+- **GitHub**: [DavinciDreams/SymbioGPT](https://github.com/DavinciDreams/SymbioGPT)

requirements.txt

@@ -0,0 +1,6 @@
+torch>=2.0.0
+numpy
+huggingface_hub
+tokenizers
+fastapi
+uvicorn

server.py

@@ -0,0 +1,322 @@
+"""VocabFusion-Pythia160m-Fused — OpenAI-compatible inference server.
+
+VocabFusion-SymbioGPT architecture (donor embedding + junction + SymbioGPT body)
+with knowledge from Pythia-160M (12 layers, d=768) projected in via PCA
+decomposition and blended at alpha=0.3. 17.5M params.
+
+Downloads checkpoint from model repo and tokenizer from VocabFusion-SymbioGPT-10M.
+"""
+import json as json_mod
+import math
+import os
+import time
+import uuid
+
+import torch
+import torch.nn.functional as F
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse, StreamingResponse
+from huggingface_hub import hf_hub_download
+from tokenizers import Tokenizer
+
+from symbio_model import SymbioConfig
+from vocab_fusion_model import VocabFusionSymbioGPT
+
+# ═══════════════════════════════════════════════════════════════════
+# Configuration
+# ═══════════════════════════════════════════════════════════════════
+
+MODEL_REPO = os.environ.get("MODEL_REPO", "LisaMegaWatts/VocabFusion-Pythia160m-Fused")
+TOKENIZER_REPO = os.environ.get("TOKENIZER_REPO", "LisaMegaWatts/VocabFusion-SymbioGPT-10M")
+PORT = int(os.environ.get("PORT", "7860"))
+CHECKPOINT_FILE = "vocabfusion_pythia160m_fused.pt"
+
+DONOR_VOCAB_SIZE = 50304
+DONOR_DIM = 128
+
+MODEL_CONFIG = SymbioConfig(
+    d_model=320,
+    n_layers=8,
+    n_heads=5,
+    head_dim=64,
+    ffn_mult=4,
+    context_length=256,
+    vocab_size=2000,
+    weight_tying=True,
+    organelles=("causal_conv", "monarch", "long_conv", "attention"),
+    conv_kernel_size=4,
+    n_monarch_heads=1,
+    gate_temperature_init=1.0,
+    free_energy_beta=0.001,
+)
+
+# ═══════════════════════════════════════════════════════════════════
+# Load model and tokenizer
+# ═══════════════════════════════════════════════════════════════════
+
+print(f"Downloading checkpoint from {MODEL_REPO} ...")
+ckpt_path = hf_hub_download(repo_id=MODEL_REPO, filename=CHECKPOINT_FILE)
+
+print(f"Downloading tokenizer from {TOKENIZER_REPO} ...")
+tok_path = hf_hub_download(repo_id=TOKENIZER_REPO, filename="tokenizer/tokenizer.json")
+
+print("Loading tokenizer ...")
+tokenizer = Tokenizer.from_file(tok_path)
+actual_vocab_size = tokenizer.get_vocab_size()
+print(f"  Tokenizer vocab_size = {actual_vocab_size}")
+print(f"  Donor embedding size = {DONOR_VOCAB_SIZE} (padded)")
+
+print("Loading VocabFusion-Pythia160m-Fused model ...")
+model = VocabFusionSymbioGPT(MODEL_CONFIG, donor_vocab_size=DONOR_VOCAB_SIZE, donor_dim=DONOR_DIM)
+
+state_dict = torch.load(ckpt_path, map_location="cpu", weights_only=True)
+if "model_state_dict" in state_dict:
+    state_dict = state_dict["model_state_dict"]
+elif "state_dict" in state_dict:
+    state_dict = state_dict["state_dict"]
+state_dict = {k.replace("_orig_mod.", ""): v for k, v in state_dict.items()}
+model.load_state_dict(state_dict)
+
+model.eval()
+n_params = sum(p.numel() for p in model.parameters())
+print(f"  Model ready: {n_params/1e6:.1f}M params (Pythia-160M knowledge fused)")
+
+# ═══════════════════════════════════════════════════════════════════
+# Generation
+# ═══════════════════════════════════════════════════════════════════
+
+
+@torch.no_grad()
+def generate_streaming(
+    prompt: str,
+    max_tokens: int = 200,
+    temperature: float = 0.8,
+    top_k: int = 40,
+    top_p: float = 1.0,
+):
+    """Generator yielding token strings one at a time for real SSE streaming."""
+    encoding = tokenizer.encode(prompt)
+    tokens = encoding.ids
+    if not tokens:
+        tokens = [0]
+    idx = torch.tensor([tokens], dtype=torch.long)
+
+    generated_ids = []
+    prev_text = ""
+
+    for _ in range(max_tokens):
+        idx_cond = idx[:, -MODEL_CONFIG.context_length:]
+        logits = model(idx_cond)
+        logits_last = logits[0, -1, :].float()
+
+        if temperature > 0.01:
+            logits_last = logits_last / temperature
+        else:
+            logits_last = logits_last / 0.01
+
+        if 0 < top_k < logits_last.size(0):
+            threshold = torch.topk(logits_last, top_k).values[-1]
+            logits_last[logits_last < threshold] = float("-inf")
+
+        if top_p < 1.0:
+            sorted_logits, sorted_indices = torch.sort(logits_last, descending=True)
+            probs_sorted = F.softmax(sorted_logits, dim=-1)
+            cumprobs = torch.cumsum(probs_sorted, dim=-1)
+            cutoff_mask = cumprobs - probs_sorted > top_p
+            sorted_logits[cutoff_mask] = float("-inf")
+            logits_last = sorted_logits.scatter(0, sorted_indices, sorted_logits)
+
+        probs = F.softmax(logits_last, dim=-1)
+        next_id = torch.multinomial(probs, 1).item()
+
+        if next_id == 0:
+            break
+
+        idx = torch.cat([idx, torch.tensor([[next_id]])], dim=1)
+
+        generated_ids.append(next_id)
+        full_text = tokenizer.decode(generated_ids, skip_special_tokens=True)
+        delta = full_text[len(prev_text):]
+        prev_text = full_text
+        if delta:
+            yield delta
+
+
+@torch.no_grad()
+def generate(
+    prompt: str,
+    max_tokens: int = 200,
+    temperature: float = 0.8,
+    top_k: int = 40,
+    top_p: float = 1.0,
+) -> str:
+    """Generate complete text (non-streaming wrapper)."""
+    return "".join(generate_streaming(prompt, max_tokens, temperature, top_k, top_p))
+
+
+# ═══════════════════════════════════════════════════════════════════
+# FastAPI server
+# ═══════════════════════════════════════════════════════════════════
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+MODEL_CREATED_AT = int(time.time())
+MODEL_ID = "vocabfusion-pythia160m-fused"
+
+
+def extract_prompt(messages):
+    if not messages:
+        return ""
+    for msg in reversed(messages):
+        if msg.get("role") == "user":
+            return msg.get("content", "")
+    return messages[-1].get("content", "")
+
+
+@app.get("/")
+def health():
+    return {
+        "name": "VocabFusion-Pythia160m-Fused",
+        "version": "1.0.0",
+        "description": (
+            "VocabFusion-SymbioGPT with knowledge from Pythia-160M (12 layers, d=768) "
+            "projected via PCA decomposition and blended at alpha=0.3. "
+            "The donor embedding (GPTNeoX, ~50k tokens, 128-dim) bridges to "
+            "the 320-dim SymbioGPT body via a junction layer."
+        ),
+        "architecture": "VocabFusion wrapper + 4-organelle decoder "
+                        "(CausalConv + Monarch + LongConv + Attention) "
+                        "+ OrganelleGate + SwiGLU + RoPE + RMSNorm",
+        "model": {
+            "d_model": MODEL_CONFIG.d_model,
+            "n_layers": MODEL_CONFIG.n_layers,
+            "n_heads": MODEL_CONFIG.n_heads,
+            "context_length": MODEL_CONFIG.context_length,
+            "donor_vocab_size": DONOR_VOCAB_SIZE,
+            "donor_dim": DONOR_DIM,
+            "params": f"{n_params/1e6:.1f}M",
+        },
+        "fusion": {
+            "source": "EleutherAI/pythia-160m (12 layers, d=768)",
+            "method": "PCA projection + blend",
+            "blend_alpha": 0.3,
+            "pca_avg_variance": 0.921,
+        },
+        "organelles": list(MODEL_CONFIG.organelles),
+        "endpoints": ["/v1/models", "/v1/chat/completions"],
+        "features": ["streaming", "OpenAI-compatible", "top-k", "top-p"],
+        "github": "https://github.com/DavinciDreams/SymbioGPT",
+    }
+
+
+@app.get("/v1/models")
+def list_models():
+    return {
+        "object": "list",
+        "data": [{
+            "id": MODEL_ID,
+            "object": "model",
+            "created": MODEL_CREATED_AT,
+            "owned_by": "symbiogpt",
+        }],
+    }
+
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    try:
+        body = await request.json()
+    except Exception:
+        return JSONResponse(status_code=400, content={
+            "error": {"message": "Invalid JSON", "type": "invalid_request_error"}
+        })
+
+    temperature = max(0.01, min(2.0, body.get("temperature", 0.8)))
+    max_tokens = max(1, min(MODEL_CONFIG.context_length, body.get("max_tokens", 200)))
+    top_k_val = max(0, min(DONOR_VOCAB_SIZE, body.get("top_k", 40)))
+    top_p_val = max(0.0, min(1.0, body.get("top_p", 1.0)))
+    stream = body.get("stream", False)
+    messages = body.get("messages", [])
+    prompt_text = extract_prompt(messages)
+    prompt_tokens = len(tokenizer.encode(prompt_text).ids) if prompt_text else 0
+
+    completion_id = f"chatcmpl-{uuid.uuid4()}"
+    created = int(time.time())
+
+    if stream:
+        def sse_stream():
+            initial = {
+                "id": completion_id,
+                "object": "chat.completion.chunk",
+                "created": created,
+                "model": MODEL_ID,
+                "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}],
+            }
+            yield f"data: {json_mod.dumps(initial)}\n\n"
+
+            token_count = 0
+            for token_str in generate_streaming(
+                prompt_text, max_tokens=max_tokens, temperature=temperature,
+                top_k=top_k_val, top_p=top_p_val,
+            ):
+                token_count += 1
+                chunk = {
+                    "id": completion_id,
+                    "object": "chat.completion.chunk",
+                    "created": created,
+                    "model": MODEL_ID,
+                    "choices": [{"index": 0, "delta": {"content": token_str}, "finish_reason": None}],
+                }
+                yield f"data: {json_mod.dumps(chunk)}\n\n"
+
+            finish = {
+                "id": completion_id,
+                "object": "chat.completion.chunk",
+                "created": created,
+                "model": MODEL_ID,
+                "choices": [{"index": 0, "delta": {}, "finish_reason": "length" if token_count >= max_tokens else "stop"}],
+                "usage": {
+                    "prompt_tokens": prompt_tokens,
+                    "completion_tokens": token_count,
+                    "total_tokens": prompt_tokens + token_count,
+                },
+            }
+            yield f"data: {json_mod.dumps(finish)}\n\n"
+            yield "data: [DONE]\n\n"
+
+        return StreamingResponse(sse_stream(), media_type="text/event-stream",
+                                 headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
+    else:
+        text = generate(prompt_text, max_tokens=max_tokens, temperature=temperature,
+                        top_k=top_k_val, top_p=top_p_val)
+        completion_tokens = len(tokenizer.encode(text).ids)
+
+        return {
+            "id": completion_id,
+            "object": "chat.completion",
+            "created": created,
+            "model": MODEL_ID,
+            "choices": [{
+                "index": 0,
+                "message": {"role": "assistant", "content": text},
+                "finish_reason": "length",
+            }],
+            "usage": {
+                "prompt_tokens": prompt_tokens,
+                "completion_tokens": completion_tokens,
+                "total_tokens": prompt_tokens + completion_tokens,
+            },
+            "system_fingerprint": "vocabfusion-pythia160m-fused-v1",
+        }
+
+
+if __name__ == "__main__":
+    print(f"\nVocabFusion-Pythia160m-Fused server starting on 0.0.0.0:{PORT} ...")
+    uvicorn.run(app, host="0.0.0.0", port=PORT)

symbio_model.py

@@ -0,0 +1,355 @@
+"""SymbioGPT — Multi-organelle GPT with learned per-channel gating.
+
+4 organelles: CausalConv + Monarch + LongConv + CausalSelfAttention
+fused via OrganelleGate with learnable temperature.
+
+Architecture: RoPE, RMSNorm, SwiGLU, SkipGate, weight-tied output.
+"""
+import math
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# ═══════════════════════════════════════════════════════════════════
+# Configuration
+# ═══════════════════════════════════════════════════════════════════
+
+
+@dataclass
+class SymbioConfig:
+    d_model: int = 320
+    n_layers: int = 8
+    n_heads: int = 5
+    head_dim: int = 64
+    ffn_mult: int = 4
+    dropout: float = 0.0
+    context_length: int = 256
+    vocab_size: int = 2000
+    weight_tying: bool = True
+    organelles: Tuple[str, ...] = ("causal_conv", "monarch", "long_conv", "attention")
+    conv_kernel_size: int = 4
+    n_monarch_heads: int = 1
+    gate_temperature_init: float = 1.0
+    free_energy_beta: float = 0.001
+    per_layer_organelles: Optional[List[Tuple[str, ...]]] = None
+
+    @property
+    def p(self) -> int:
+        return int(math.isqrt(self.context_length))
+
+    @property
+    def n_organelles(self) -> int:
+        return len(self.organelles)
+
+
+# ═══════════════════════════════════════════════════════════════════
+# Building blocks
+# ═══════════════════════════════════════════════════════════════════
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.eps = eps
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        rms = torch.sqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+        return x / rms * self.weight
+
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, dim: int, max_seq_len: int = 2048):
+        super().__init__()
+        freqs = 1.0 / (10000.0 ** (torch.arange(0, dim, 2).float() / dim))
+        positions = torch.arange(max_seq_len).float()
+        angles = torch.outer(positions, freqs)
+        self.register_buffer("cos_cache", angles.cos())
+        self.register_buffer("sin_cache", angles.sin())
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        seq_len = x.size(2)
+        half = x.size(-1) // 2
+        x1, x2 = x[..., :half], x[..., half:]
+        cos = self.cos_cache[:seq_len, :half].unsqueeze(0).unsqueeze(0)
+        sin = self.sin_cache[:seq_len, :half].unsqueeze(0).unsqueeze(0)
+        return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1)
+
+
+class SwiGLU(nn.Module):
+    def __init__(self, d_model: int, ffn_mult: int = 4):
+        super().__init__()
+        raw_hidden = 2 * d_model * ffn_mult // 3
+        hidden_dim = max(64, (raw_hidden // 64) * 64)
+        self.w1 = nn.Linear(d_model, hidden_dim, bias=False)
+        self.v = nn.Linear(d_model, hidden_dim, bias=False)
+        self.w2 = nn.Linear(hidden_dim, d_model, bias=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.w2(F.silu(self.w1(x)) * self.v(x))
+
+
+class CausalSelfAttention(nn.Module):
+    def __init__(self, d_model: int, n_heads: int, head_dim: int, dropout: float = 0.0):
+        super().__init__()
+        self.n_heads = n_heads
+        self.head_dim = head_dim
+        total_dim = n_heads * head_dim
+        self.wq = nn.Linear(d_model, total_dim, bias=False)
+        self.wk = nn.Linear(d_model, total_dim, bias=False)
+        self.wv = nn.Linear(d_model, total_dim, bias=False)
+        self.wo = nn.Linear(total_dim, d_model, bias=False)
+        self.attn_dropout = nn.Dropout(dropout) if dropout > 0.0 else nn.Identity()
+
+    def forward(self, x, rope, mask=None):
+        B, T, D = x.shape
+        H, HD = self.n_heads, self.head_dim
+        q = self.wq(x).view(B, T, H, HD).transpose(1, 2)
+        k = self.wk(x).view(B, T, H, HD).transpose(1, 2)
+        v = self.wv(x).view(B, T, H, HD).transpose(1, 2)
+        q = rope(q)
+        k = rope(k)
+        scale = 1.0 / math.sqrt(HD)
+        attn = torch.matmul(q, k.transpose(-2, -1)) * scale
+        if mask is not None:
+            attn = attn + mask
+        attn = F.softmax(attn, dim=-1)
+        attn = self.attn_dropout(attn)
+        out = torch.matmul(attn, v)
+        out = out.transpose(1, 2).contiguous().view(B, T, H * HD)
+        return self.wo(out)
+
+
+# ════════════════════���══════════════════════════════════════════════
+# Organelles
+# ═══════════════════════════════════════════════════════════════════
+
+
+class CausalDepthwiseConv1d(nn.Module):
+    def __init__(self, channels: int, kernel_size: int = 4):
+        super().__init__()
+        self.channels = channels
+        self.kernel_size = kernel_size
+        self.weight = nn.Parameter(torch.empty(channels, 1, kernel_size))
+        nn.init.normal_(self.weight, std=math.sqrt(1.0 / kernel_size))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        B, T, D = x.shape
+        x_t = x.transpose(1, 2)
+        x_padded = F.pad(x_t, (self.kernel_size - 1, 0))
+        out = F.conv1d(x_padded, self.weight, groups=D)
+        return out.transpose(1, 2)
+
+
+class MonarchMatrix(nn.Module):
+    def __init__(self, seq_len: int):
+        super().__init__()
+        p = int(math.isqrt(seq_len))
+        self.seq_len = seq_len
+        self.p = p
+        scale = math.sqrt(2.0 / (p + p))
+        self.L1 = nn.Parameter(torch.randn(p, p, p) * scale)
+        self.L2 = nn.Parameter(torch.randn(p, p, p) * scale)
+
+    @staticmethod
+    def _julia_batched_mul(A, B):
+        return torch.bmm(A.permute(2, 0, 1), B.permute(2, 0, 1)).permute(1, 2, 0)
+
+    def realize(self):
+        p = self.p
+        T = self.seq_len
+        I_T = torch.eye(T, device=self.L1.device, dtype=self.L1.dtype)
+        x = I_T.reshape(p, p, T)
+        x = x.permute(0, 2, 1)
+        x = self._julia_batched_mul(self.L2, x)
+        x = x.permute(0, 2, 1)
+        x = x.permute(1, 0, 2)
+        x = x.permute(0, 2, 1)
+        x = self._julia_batched_mul(self.L1, x)
+        x = x.permute(0, 2, 1)
+        x = x.permute(1, 0, 2)
+        return x.reshape(T, T)
+
+    def forward(self, x, causal_mask=None):
+        B, T, D_head = x.shape
+        M = self.realize()[:T, :T]
+        if causal_mask is not None:
+            M = M * causal_mask[:T, :T]
+        x_flat = x.permute(1, 0, 2).reshape(T, B * D_head)
+        y_flat = M @ x_flat
+        return y_flat.reshape(T, B, D_head).permute(1, 0, 2)
+
+
+class LongConv(nn.Module):
+    def __init__(self, channels: int, seq_len: int):
+        super().__init__()
+        self.channels = channels
+        self.seq_len = seq_len
+        self.kernel = nn.Parameter(torch.empty(channels, 1, seq_len))
+        self._init_weights()
+
+    def _init_weights(self):
+        scale = math.sqrt(1.0 / self.seq_len)
+        nn.init.normal_(self.kernel, mean=0.0, std=scale)
+        with torch.no_grad():
+            decay = torch.exp(-0.1 * torch.arange(self.seq_len, dtype=torch.float32))
+            self.kernel.mul_(decay.unsqueeze(0).unsqueeze(0))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        B, T, D = x.shape
+        K = self.seq_len
+        x_t = x.transpose(1, 2)
+        x_padded = F.pad(x_t, (K - 1, 0))
+        out = F.conv1d(x_padded, self.kernel, groups=D)
+        return out.transpose(1, 2)
+
+
+# ═══════════════════════════════════════════════════════════════════
+# Gate and mixer
+# ═══════════════════════════════════════════════════════════════════
+
+
+class OrganelleGate(nn.Module):
+    def __init__(self, dim: int, n_organelles: int, temperature_init: float = 1.0):
+        super().__init__()
+        self.n_organelles = n_organelles
+        self.logits = nn.Parameter(torch.zeros(n_organelles, dim))
+        self.temperature = nn.Parameter(torch.tensor([temperature_init]))
+
+    def forward(self, organelle_outputs, organelle_mask=None):
+        logits = self.logits
+        if organelle_mask is not None:
+            mask_additive = torch.zeros_like(logits)
+            for i in range(self.n_organelles):
+                if not organelle_mask[i]:
+                    mask_additive[i, :] = -1e10
+            logits = logits + mask_additive
+        tau = self.temperature.clamp(min=0.01)
+        weights = F.softmax(logits / tau, dim=0)
+        out = torch.zeros_like(organelle_outputs[0])
+        for i in range(self.n_organelles):
+            w = weights[i].unsqueeze(0).unsqueeze(0)
+            out = out + w * organelle_outputs[i]
+        return out
+
+
+class SkipGate(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.scale = nn.Parameter(torch.ones(1))
+
+    def forward(self, x):
+        return self.scale * x
+
+
+class SymbioSequenceMixer(nn.Module):
+    def __init__(self, config: SymbioConfig):
+        super().__init__()
+        self.config = config
+        d = config.d_model
+        T = config.context_length
+        self.organelle_names = list(config.organelles)
+        self.organelle_modules = nn.ModuleDict()
+        for name in self.organelle_names:
+            if name == "causal_conv":
+                self.organelle_modules[name] = CausalDepthwiseConv1d(d, config.conv_kernel_size)
+            elif name == "monarch":
+                self.organelle_modules[name] = nn.ModuleList(
+                    [MonarchMatrix(T) for _ in range(config.n_monarch_heads)]
+                )
+            elif name == "long_conv":
+                self.organelle_modules[name] = LongConv(d, T)
+            elif name == "attention":
+                self.organelle_modules[name] = CausalSelfAttention(
+                    d, config.n_heads, config.head_dim, config.dropout
+                )
+        self.gate = OrganelleGate(d, len(self.organelle_names), config.gate_temperature_init)
+        if "monarch" in self.organelle_names:
+            self.register_buffer("monarch_causal_mask", torch.tril(torch.ones(T, T)))
+
+    def forward(self, x, rope, attn_mask=None, organelle_mask=None):
+        B, T, D = x.shape
+        outputs = []
+        for name in self.organelle_names:
+            if name == "causal_conv":
+                out = self.organelle_modules[name](x)
+            elif name == "monarch":
+                heads = self.organelle_modules[name]
+                n_mh = len(heads)
+                hd = D // n_mh
+                slices = []
+                for i, monarch in enumerate(heads):
+                    x_slice = x[:, :, i * hd : (i + 1) * hd]
+                    y_slice = monarch(x_slice, self.monarch_causal_mask)
+                    slices.append(y_slice)
+                out = torch.cat(slices, dim=-1)
+            elif name == "long_conv":
+                out = self.organelle_modules[name](x)
+            elif name == "attention":
+                out = self.organelle_modules[name](x, rope, attn_mask)
+            outputs.append(out)
+        return self.gate(tuple(outputs), organelle_mask)
+
+
+# ═══════════════════════════════════════════════════════════════════
+# SymbioBlock and SymbioGPT
+# ═══════════════════════════════════════════════════════════════════
+
+
+class SymbioBlock(nn.Module):
+    def __init__(self, config: SymbioConfig, layer_organelles=None):
+        super().__init__()
+        d = config.d_model
+        if layer_organelles is not None:
+            from dataclasses import replace
+            layer_config = replace(config, organelles=layer_organelles)
+        else:
+            layer_config = config
+        self.ln1 = RMSNorm(d)
+        self.seq_mixer = SymbioSequenceMixer(layer_config)
+        self.skip1 = SkipGate()
+        self.ln2 = RMSNorm(d)
+        self.ffn = SwiGLU(d, config.ffn_mult)
+        self.skip2 = SkipGate()
+
+    def forward(self, x, rope, attn_mask=None, organelle_mask=None):
+        x = x + self.skip1(self.seq_mixer(self.ln1(x), rope, attn_mask, organelle_mask))
+        x = x + self.skip2(self.ffn(self.ln2(x)))
+        return x
+
+
+class SymbioGPT(nn.Module):
+    def __init__(self, config: SymbioConfig):
+        super().__init__()
+        self.config = config
+        self.tok_emb = nn.Embedding(config.vocab_size, config.d_model)
+        self.rope = RotaryEmbedding(config.head_dim, config.context_length)
+        blocks = []
+        for i in range(config.n_layers):
+            layer_org = None
+            if config.per_layer_organelles is not None:
+                layer_org = config.per_layer_organelles[i]
+            blocks.append(SymbioBlock(config, layer_org))
+        self.blocks = nn.ModuleList(blocks)
+        self.ln_f = RMSNorm(config.d_model)
+        if config.weight_tying:
+            self.head = None
+        else:
+            self.head = nn.Linear(config.d_model, config.vocab_size, bias=False)
+
+    def forward(self, input_ids, organelle_mask=None):
+        B, T = input_ids.shape
+        x = self.tok_emb(input_ids)
+        attn_mask = torch.triu(
+            torch.full((T, T), float("-inf"), device=x.device, dtype=x.dtype), diagonal=1
+        )
+        for block in self.blocks:
+            x = block(x, self.rope, attn_mask, organelle_mask)
+        x = self.ln_f(x)
+        if self.head is not None:
+            return self.head(x)
+        return F.linear(x, self.tok_emb.weight)

vocab_fusion_model.py

@@ -0,0 +1,62 @@
+"""VocabFusion-SymbioGPT — SymbioGPT body with donor vocabulary embedding.
+
+Takes a large donor vocab (GPTNeoX-style, ~50k tokens) and projects through
+a junction layer (Linear + LayerNorm) into the 320-dim SymbioGPT body.
+Output projects back to donor dim and is weight-tied to the donor embedding.
+"""
+import math
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from symbio_model import (
+    SymbioConfig,
+    SymbioBlock,
+    RotaryEmbedding,
+    RMSNorm,
+)
+
+
+class VocabFusionSymbioGPT(nn.Module):
+    def __init__(self, config: SymbioConfig, donor_vocab_size: int = 50304, donor_dim: int = 128):
+        super().__init__()
+        self.config = config
+        self.donor_vocab_size = donor_vocab_size
+        self.donor_dim = donor_dim
+
+        # Donor embedding: large vocab -> donor_dim
+        self.donor_embedding = nn.Embedding(donor_vocab_size, donor_dim)
+
+        # Junction: donor_dim -> d_model (Linear + LayerNorm)
+        self.junction = nn.Sequential(
+            nn.Linear(donor_dim, config.d_model, bias=False),
+            nn.LayerNorm(config.d_model),
+        )
+
+        # Standard SymbioGPT body
+        self.rope = RotaryEmbedding(config.head_dim, config.context_length)
+        self.blocks = nn.ModuleList(
+            [SymbioBlock(config) for _ in range(config.n_layers)]
+        )
+        self.ln_f = RMSNorm(config.d_model)
+
+        # Output projection: d_model -> donor_dim (weight-tied with donor_embedding)
+        self.output_proj = nn.Linear(config.d_model, donor_dim, bias=False)
+
+    def forward(self, input_ids, organelle_mask=None):
+        B, T = input_ids.shape
+        x = self.donor_embedding(input_ids)
+        x = self.junction(x)
+
+        attn_mask = torch.triu(
+            torch.full((T, T), float("-inf"), device=x.device, dtype=x.dtype),
+            diagonal=1,
+        )
+        for block in self.blocks:
+            x = block(x, self.rope, attn_mask, organelle_mask)
+        x = self.ln_f(x)
+
+        x = self.output_proj(x)
+        logits = F.linear(x, self.donor_embedding.weight)
+        return logits