Upload folder using huggingface_hub

README.md

@@ -0,0 +1,2 @@
+Self-contained SigLIP2-base Robotics VLA policy for task 23.
+The adapter loads only the bundled local backbone and action head.

action_head.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30acb580f4bb5273216286eecb0ad380fb52a037c14e4a0df36988969a51f932
+size 2878413

backbone/config.json

@@ -0,0 +1,16 @@
+{
+  "architectures": [
+    "Siglip2Model"
+  ],
+  "initializer_factor": 1.0,
+  "model_type": "siglip2",
+  "text_config": {
+    "model_type": "siglip2_text_model",
+    "vocab_size": 256000
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0.dev0",
+  "vision_config": {
+    "model_type": "siglip2_vision_model"
+  }
+}

backbone/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac5f28bbdf92c0c1696ccbd3ce716426049cd67ad8045b66d0d938b0f9c8bbec
+size 1500985224

backbone/preprocessor_config.json

@@ -0,0 +1,22 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Siglip2ImageProcessorFast",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "max_num_patches": 256,
+  "patch_size": 16,
+  "processor_class": "Siglip2Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098
+}

flock_robotics_adapter.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+import hashlib
+import json
+import math
+import re
+from pathlib import Path
+import numpy as np
+import torch
+import torch.nn as nn
+
+class SigLIPActionHead(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.chunk = int(config["chunk"])
+        self.proprio = nn.Sequential(nn.Linear(32, 128), nn.LayerNorm(128), nn.SiLU())
+        self.text = nn.Sequential(nn.Linear(128, 128), nn.LayerNorm(128), nn.SiLU())
+        self.task = nn.Embedding(int(config["task_count"]), 32)
+        self.difficulty = nn.Embedding(int(config["difficulty_count"]), 16)
+        self.head = nn.Sequential(
+            nn.Linear(768 + 128 + 128 + 32 + 16, 512),
+            nn.LayerNorm(512), nn.SiLU(), nn.Dropout(0.10),
+            nn.Linear(512, 256), nn.SiLU(), nn.Linear(256, self.chunk * 7)
+        )
+    def forward(self, features, proprio, text, task_id, difficulty_id):
+        fused = torch.cat([
+            features.float(), self.proprio(proprio.float()), self.text(text.float()),
+            self.task(task_id.clamp(0, self.task.num_embeddings - 1)),
+            self.difficulty(difficulty_id.clamp(0, self.difficulty.num_embeddings - 1)),
+        ], dim=-1)
+        return torch.tanh(self.head(fused)).reshape(-1, self.chunk, 7)
+
+def _text_vector(text, dim):
+    result = np.zeros(dim, dtype=np.float32)
+    for token in re.findall(r"[a-z0-9_]+", str(text).lower()):
+        value = int.from_bytes(hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest(), "little")
+        result[value % dim] += 1.0 if value & 1 else -1.0
+    norm = float(np.linalg.norm(result))
+    return result / norm if norm else result
+
+def _proprio(obs, config):
+    value = np.asarray(obs.get("proprio", np.zeros(25, dtype=np.float32)), dtype=np.float32).reshape(-1)
+    if value.size == 21:
+        value = np.pad(value, (0, 4))
+    if value.size != 25:
+        raise ValueError(f"proprio must have 25 values, got {value.size}")
+    step = float(obs.get("step", 0))
+    horizon = float(obs.get("horizon", 320) or 320)
+    value = np.concatenate([value, np.asarray([step / max(horizon, 1.0)], dtype=np.float32)])
+    value = np.pad(value, (0, max(0, int(config["proprio_dim"]) - value.size)))
+    value = value[: int(config["proprio_dim"])]
+    mean = np.asarray(config["proprio_mean"], dtype=np.float32)
+    std = np.asarray(config["proprio_std"], dtype=np.float32)
+    return ((value - mean) / np.maximum(std, 1e-4)).astype(np.float32)
+
+class Policy:
+    def __init__(self, backbone, processor, head, config, device):
+        self.backbone, self.processor = backbone, processor
+        self.head, self.config, self.device = head, config, device
+        self.chunk = int(config["chunk"])
+        self.last_step, self.last_action = None, np.zeros(7, dtype=np.float32)
+        self.chunks = {}
+        self.task_to_id, self.difficulty_to_id = config["task_to_id"], config["difficulty_to_id"]
+    @torch.inference_mode()
+    def act(self, obs):
+        image = np.asarray(obs.get("image", np.zeros((224, 224, 3), dtype=np.uint8)))
+        if image.ndim == 2:
+            image = np.repeat(image[..., None], 3, axis=-1)
+        if image.shape[-1] == 4:
+            image = image[..., :3]
+        if image.shape[-1] != 3:
+            raise ValueError("image must have 3 or 4 channels")
+        step = int(obs.get("step", 0))
+        if self.last_step is None or step == 0 or step != self.last_step + 1:
+            self.chunks = {}
+        if step == self.last_step:
+            return self.last_action.copy()
+        inputs = self.processor(images=image, return_tensors="pt")
+        inputs = {key: value.to(self.device) for key, value in inputs.items()}
+        result = self.backbone.get_image_features(**inputs)
+        features = result.pooler_output if hasattr(result, "pooler_output") else result.last_hidden_state.mean(dim=1)
+        task = str(obs.get("task", ""))
+        difficulty = str(obs.get("difficulty", "") or "")
+        text = f"task {task} difficulty {difficulty} instruction {obs.get('instruction', '')}"
+        raw = self.head(
+            features, torch.from_numpy(_proprio(obs, self.config)).unsqueeze(0).to(self.device),
+            torch.from_numpy(_text_vector(text, 128)).unsqueeze(0).to(self.device),
+            torch.tensor([self.task_to_id.get(task, len(self.task_to_id))], device=self.device),
+            torch.tensor([self.difficulty_to_id.get(difficulty, len(self.difficulty_to_id))], device=self.device),
+        )[0].float().cpu().numpy()
+        self.chunks[step] = raw
+        candidates, weights = [], []
+        for start, chunk in list(self.chunks.items()):
+            offset = step - start
+            if 0 <= offset < self.chunk:
+                candidates.append(chunk[offset])
+                weights.append(math.exp(-0.55 * offset))
+            else:
+                del self.chunks[start]
+        action = np.average(np.asarray(candidates), axis=0, weights=np.asarray(weights))
+        action = np.clip(action, -1.0, 1.0).astype(np.float32)
+        if action[6] > 0.12:
+            action[6] = 1.0
+        elif action[6] < -0.12:
+            action[6] = -1.0
+        self.last_step, self.last_action = step, action.copy()
+        return action
+
+def load_policy(model_dir: str, device: str, dtype: str):
+    from transformers import AutoImageProcessor, AutoModel
+    root = Path(model_dir)
+    config = json.loads((root / "vla_config.json").read_text())
+    selected = torch.device(device if str(device).startswith("cuda") and torch.cuda.is_available() else "cpu")
+    processor = AutoImageProcessor.from_pretrained(root / "backbone", local_files_only=True)
+    backbone = AutoModel.from_pretrained(root / "backbone", local_files_only=True).to(selected).eval()
+    head = SigLIPActionHead(config).to(selected)
+    state = torch.load(root / "action_head.pt", map_location=selected, weights_only=True)
+    head.load_state_dict(state["state_dict"], strict=True)
+    head.eval()
+    return Policy(backbone, processor, head, config, selected)

vla_config.json

@@ -0,0 +1,110 @@
+{
+  "schema_version": "task23_siglip2_self_contained_v1",
+  "model_type": "siglip2_frozen_vision_action_chunk",
+  "backbone": "google/siglip2-base-patch16-naflex",
+  "chunk": 8,
+  "feature_dim": 768,
+  "text_dim": 128,
+  "proprio_dim": 32,
+  "task_count": 14,
+  "difficulty_count": 5,
+  "proprio_mean": [
+    -0.01592460460960865,
+    0.7428314685821533,
+    0.010429546236991882,
+    -1.932236671447754,
+    0.024123214185237885,
+    2.8178389072418213,
+    0.7557806372642517,
+    0.02269606851041317,
+    0.05282147601246834,
+    0.00876456405967474,
+    0.07294361293315887,
+    -0.021329032257199287,
+    -0.015278438106179237,
+    0.04549413174390793,
+    0.07919428497552872,
+    -0.06642947345972061,
+    0.9300205707550049,
+    0.9961889982223511,
+    -0.004495386034250259,
+    0.0756562203168869,
+    -0.005136882420629263,
+    0.03150780126452446,
+    -0.03062213398516178,
+    0.0001331703970208764,
+    3.20150975312572e-05,
+    0.4979763627052307,
+    0.0,
+    0.0,
+    0.0,
+    0.0,
+    0.0,
+    0.0
+  ],
+  "proprio_std": [
+    0.1833081692457199,
+    0.2757302522659302,
+    0.0830760970711708,
+    0.40317803621292114,
+    0.1892702728509903,
+    0.21985791623592377,
+    0.3677351772785187,
+    0.09155969321727753,
+    0.24703823029994965,
+    0.07063839584589005,
+    0.27435028553009033,
+    0.12254507839679718,
+    0.17527462542057037,
+    0.2087806612253189,
+    0.09155305474996567,
+    0.16622483730316162,
+    0.07347064465284348,
+    0.004743508528918028,
+    0.020507004112005234,
+    0.03240346908569336,
+    0.01718319021165371,
+    0.00835738331079483,
+    0.009069605730473995,
+    0.014785450883209705,
+    0.012382089160382748,
+    0.2886733412742615,
+    1.0,
+    1.0,
+    1.0,
+    1.0,
+    1.0,
+    1.0
+  ],
+  "task_to_id": {
+    "lift_cube": 0,
+    "pick_place_can": 1,
+    "pick_place_milk": 2,
+    "pick_place_bread": 3,
+    "pick_place_cereal": 4,
+    "pick_place_clutter": 5,
+    "stack_blocks": 6,
+    "open_door": 7,
+    "nut_assembly": 8,
+    "nut_assembly_square": 9,
+    "nut_assembly_round": 10,
+    "wipe_table": 11,
+    "tool_hang": 12
+  },
+  "difficulty_to_id": {
+    "low": 0,
+    "medium": 1,
+    "hard": 2,
+    "very_high": 3
+  },
+  "local_files_only": true,
+  "adapter_inputs": [
+    "image",
+    "instruction",
+    "proprio",
+    "task",
+    "step",
+    "difficulty",
+    "horizon"
+  ]
+}