Anonymous supplementary artifact release

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,45 @@
+---
+language:
+- en
+library_name: lerobot
+base_model: allenai/MolmoAct2
+datasets:
+- Spa-Bench/spa-bench-training-teleoperation-1200
+pipeline_tag: robotics
+tags:
+- robotics
+- vision-language-action
+- so-101
+- spa-bench
+---
+
+# MolmoAct2 — Spa-Bench Step 76,596
+
+This checkpoint is released as an anonymous supplementary artifact for a paper under double-blind review. It was evaluated on a physical SO-101 robot in Spa-Bench.
+
+## Model details
+
+| Field | Value |
+| --- | --- |
+| Checkpoint | End of epoch 12; step 76,596 |
+| Inputs | Middle RGB, wrist RGB, six joint positions, and a text instruction |
+| Outputs | Six absolute joint-position targets |
+| Action horizon | 30 |
+| Adaptation | VLM and continuous-action expert updated; token embeddings frozen |
+| Optimizer | AdamW; zero weight decay; component-specific learning rates |
+
+Training-data documentation: [`spa-bench-training-teleoperation-1200`](https://huggingface.co/datasets/Spa-Bench/spa-bench-training-teleoperation-1200).
+
+## Physical evaluation
+
+87/120 familiar trials, 156/300 withheld-composition trials, 73/120 matched withheld trials, and 107/120 matched direct-manipulation controls. These are physical rollout results, not simulation metrics.
+
+Rollouts: [`spa-bench-eval-rollouts-molmoact2-full`](https://huggingface.co/datasets/Spa-Bench/spa-bench-eval-rollouts-molmoact2-full).
+
+## Limitations and safety
+
+Results apply to this checkpoint, SO-101 embodiment, workspace, cameras, objects, and physical protocol. Robot policies can move hardware unexpectedly. Use conservative limits, an accessible emergency stop, a clear workspace, and direct supervision. Do not deploy unattended or in safety-critical settings.
+
+## Double-blind release note
+
+Author, institution, source-repository, and archival citation details are intentionally omitted during review. They will be restored in the archival release.

config.json

@@ -0,0 +1,134 @@
+{
+  "type": "molmoact2",
+  "n_obs_steps": 1,
+  "input_features": {
+    "observation.images.middle": {
+      "type": "VISUAL",
+      "shape": [
+        3,
+        480,
+        640
+      ]
+    },
+    "observation.images.wrist": {
+      "type": "VISUAL",
+      "shape": [
+        3,
+        480,
+        640
+      ]
+    },
+    "observation.state": {
+      "type": "STATE",
+      "shape": [
+        6
+      ]
+    }
+  },
+  "output_features": {
+    "action": {
+      "type": "ACTION",
+      "shape": [
+        6
+      ]
+    }
+  },
+  "device": "cuda",
+  "use_amp": false,
+  "use_peft": false,
+  "push_to_hub": true,
+  "repo_id": "Spa-Bench/spa-bench-model-molmoact2-step-076596",
+  "private": null,
+  "tags": null,
+  "license": null,
+  "pretrained_path": null,
+  "pretrained_revision": null,
+  "checkpoint_path": null,
+  "checkpoint_revision": "e432d85f6e039edca44afb93c262f3084ab72a9c",
+  "checkpoint_force_download": false,
+  "chunk_size": 30,
+  "n_action_steps": 30,
+  "action_mode": "continuous",
+  "inference_action_mode": "continuous",
+  "discrete_action_tokenizer": "allenai/MolmoAct2-FAST-Tokenizer",
+  "discrete_generation_max_steps": null,
+  "norm_tag": null,
+  "setup_type": "single so100/so101 robotic arm in molmoact2",
+  "control_mode": "absolute joint pose",
+  "image_keys": [
+    "observation.images.middle",
+    "observation.images.wrist"
+  ],
+  "normalize_language": true,
+  "add_setup_tokens": true,
+  "add_control_tokens": true,
+  "normalize_gripper": true,
+  "num_state_tokens": 256,
+  "max_sequence_length": null,
+  "expected_max_action_dim": 32,
+  "num_flow_timesteps": 8,
+  "flow_matching_cutoff": 1.0,
+  "flow_matching_time_offset": 0.001,
+  "flow_matching_time_scale": 0.999,
+  "flow_matching_beta_alpha": 1.0,
+  "flow_matching_beta_beta": 1.5,
+  "num_inference_steps": null,
+  "mask_action_dim_padding": true,
+  "enable_inference_cuda_graph": true,
+  "per_episode_seed": false,
+  "eval_seed": null,
+  "rtc_config": null,
+  "joint_signs": null,
+  "joint_offsets": null,
+  "enable_lora_vlm": false,
+  "lora_rank": 64,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "lora_bias": "none",
+  "enable_lora_action_expert": false,
+  "enable_knowledge_insulation": false,
+  "freeze_embedding": true,
+  "train_action_expert_only": false,
+  "gradient_checkpointing": true,
+  "model_dtype": "bfloat16",
+  "softmax_auxiliary_loss": true,
+  "softmax_auxiliary_loss_scale": 0.0001,
+  "discrete_loss_token_weighting": "root_subsegments_root_tokens",
+  "optimizer_lr": 1e-05,
+  "optimizer_vit_lr": 5e-06,
+  "optimizer_connector_lr": 5e-06,
+  "optimizer_action_expert_lr": 5e-05,
+  "optimizer_betas": [
+    0.9,
+    0.95
+  ],
+  "optimizer_eps": 1e-06,
+  "optimizer_weight_decay": 0.0,
+  "optimizer_grad_clip_norm": 1.0,
+  "scheduler_warmup_steps": 200,
+  "scheduler_decay_steps": 102128,
+  "scheduler_decay_lr": 1e-06,
+  "normalization_mapping": {
+    "ACTION": "MIN_MAX",
+    "STATE": "MIN_MAX",
+    "VISUAL": "IDENTITY"
+  },
+  "dataset_feature_names": {
+    "action": [
+      "shoulder_pan.pos",
+      "shoulder_lift.pos",
+      "elbow_flex.pos",
+      "wrist_flex.pos",
+      "wrist_roll.pos",
+      "gripper.pos"
+    ],
+    "observation.state": [
+      "shoulder_pan.pos",
+      "shoulder_lift.pos",
+      "elbow_flex.pos",
+      "wrist_flex.pos",
+      "wrist_roll.pos",
+      "gripper.pos"
+    ]
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c317201ece3a428f77ee6ef8491a7ef2e87f5a80421462ec987b718a0f8eda5
+size 10884573720

policy_postprocessor.json

@@ -0,0 +1,43 @@
+{
+  "name": "policy_postprocessor",
+  "steps": [
+    {
+      "registry_name": "molmoact2_clamp_action",
+      "config": {}
+    },
+    {
+      "registry_name": "molmoact2_masked_unnormalizer",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              6
+            ]
+          }
+        },
+        "norm_map": {
+          "ACTION": "MIN_MAX",
+          "STATE": "MIN_MAX",
+          "VISUAL": "IDENTITY"
+        }
+      },
+      "state_file": "policy_postprocessor_step_1_molmoact2_masked_unnormalizer.safetensors"
+    },
+    {
+      "registry_name": "molmoact2_action_frame_transform",
+      "config": {
+        "joint_signs": null,
+        "joint_offsets": null
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cpu",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_postprocessor_step_1_molmoact2_masked_unnormalizer.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c4bf7312bfd01fdf46829c55b4287d52135f17732e26b839ba8f21337ddd14d
+size 10576

policy_preprocessor.json

@@ -0,0 +1,100 @@
+{
+  "name": "policy_preprocessor",
+  "steps": [
+    {
+      "registry_name": "rename_observations_processor",
+      "config": {
+        "rename_map": {}
+      }
+    },
+    {
+      "registry_name": "to_batch_processor",
+      "config": {}
+    },
+    {
+      "registry_name": "molmoact2_state_frame_transform",
+      "config": {
+        "joint_signs": null,
+        "joint_offsets": null
+      }
+    },
+    {
+      "registry_name": "molmoact2_masked_normalizer",
+      "config": {
+        "eps": 1e-08,
+        "features": {
+          "observation.images.middle": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              480,
+              640
+            ]
+          },
+          "observation.images.wrist": {
+            "type": "VISUAL",
+            "shape": [
+              3,
+              480,
+              640
+            ]
+          },
+          "observation.state": {
+            "type": "STATE",
+            "shape": [
+              6
+            ]
+          },
+          "action": {
+            "type": "ACTION",
+            "shape": [
+              6
+            ]
+          }
+        },
+        "norm_map": {
+          "ACTION": "MIN_MAX",
+          "STATE": "MIN_MAX",
+          "VISUAL": "IDENTITY"
+        }
+      },
+      "state_file": "policy_preprocessor_step_3_molmoact2_masked_normalizer.safetensors"
+    },
+    {
+      "registry_name": "molmoact2_clamp_normalized",
+      "config": {}
+    },
+    {
+      "registry_name": "molmoact2_pack_inputs",
+      "config": {
+        "checkpoint_path": "allenai/MolmoAct2",
+        "checkpoint_revision": "e432d85f6e039edca44afb93c262f3084ab72a9c",
+        "checkpoint_force_download": false,
+        "action_mode": "continuous",
+        "discrete_action_tokenizer": "allenai/MolmoAct2-FAST-Tokenizer",
+        "image_keys": [
+          "observation.images.middle",
+          "observation.images.wrist"
+        ],
+        "allow_image_key_fallback": false,
+        "setup_type": "single so100/so101 robotic arm in molmoact2",
+        "control_mode": "absolute joint pose",
+        "normalize_language": true,
+        "add_setup_tokens": true,
+        "add_control_tokens": true,
+        "num_state_tokens": 256,
+        "max_sequence_length": null,
+        "chunk_size": 30,
+        "max_action_dim": 32,
+        "env_action_dim": 6
+      }
+    },
+    {
+      "registry_name": "device_processor",
+      "config": {
+        "device": "cuda",
+        "float_dtype": null
+      }
+    }
+  ]
+}

policy_preprocessor_step_3_molmoact2_masked_normalizer.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db826b8cfa885ca0696cbff3083231df6f1a1bc409e692447571c0a79c8ee61d
+size 10584

train_config.json

@@ -0,0 +1,287 @@
+{
+  "dataset": {
+    "repo_id": "Spa-Bench/spa-bench-training-teleoperation-1200",
+    "root": null,
+    "episodes": null,
+    "image_transforms": {
+      "enable": false,
+      "max_num_transforms": 3,
+      "random_order": false,
+      "tfs": {
+        "brightness": {
+          "weight": 1.0,
+          "type": "ColorJitter",
+          "kwargs": {
+            "brightness": [
+              0.8,
+              1.2
+            ]
+          }
+        },
+        "contrast": {
+          "weight": 1.0,
+          "type": "ColorJitter",
+          "kwargs": {
+            "contrast": [
+              0.8,
+              1.2
+            ]
+          }
+        },
+        "saturation": {
+          "weight": 1.0,
+          "type": "ColorJitter",
+          "kwargs": {
+            "saturation": [
+              0.5,
+              1.5
+            ]
+          }
+        },
+        "hue": {
+          "weight": 1.0,
+          "type": "ColorJitter",
+          "kwargs": {
+            "hue": [
+              -0.05,
+              0.05
+            ]
+          }
+        },
+        "sharpness": {
+          "weight": 1.0,
+          "type": "SharpnessJitter",
+          "kwargs": {
+            "sharpness": [
+              0.5,
+              1.5
+            ]
+          }
+        },
+        "affine": {
+          "weight": 1.0,
+          "type": "RandomAffine",
+          "kwargs": {
+            "degrees": [
+              -5.0,
+              5.0
+            ],
+            "translate": [
+              0.05,
+              0.05
+            ]
+          }
+        }
+      }
+    },
+    "revision": null,
+    "use_imagenet_stats": true,
+    "video_backend": "pyav",
+    "return_uint8": false,
+    "depth_output_unit": "mm",
+    "streaming": false,
+    "eval_split": 0.0
+  },
+  "env": null,
+  "policy": {
+    "type": "molmoact2",
+    "n_obs_steps": 1,
+    "input_features": {
+      "observation.images.middle": {
+        "type": "VISUAL",
+        "shape": [
+          3,
+          480,
+          640
+        ]
+      },
+      "observation.images.wrist": {
+        "type": "VISUAL",
+        "shape": [
+          3,
+          480,
+          640
+        ]
+      },
+      "observation.state": {
+        "type": "STATE",
+        "shape": [
+          6
+        ]
+      }
+    },
+    "output_features": {
+      "action": {
+        "type": "ACTION",
+        "shape": [
+          6
+        ]
+      }
+    },
+    "device": "cuda",
+    "use_amp": false,
+    "use_peft": false,
+    "push_to_hub": true,
+    "repo_id": "Spa-Bench/spa-bench-model-molmoact2-step-076596",
+    "private": null,
+    "tags": null,
+    "license": null,
+    "pretrained_path": null,
+    "pretrained_revision": null,
+    "checkpoint_path": null,
+    "checkpoint_revision": "e432d85f6e039edca44afb93c262f3084ab72a9c",
+    "checkpoint_force_download": false,
+    "chunk_size": 30,
+    "n_action_steps": 30,
+    "action_mode": "continuous",
+    "inference_action_mode": "continuous",
+    "discrete_action_tokenizer": "allenai/MolmoAct2-FAST-Tokenizer",
+    "discrete_generation_max_steps": null,
+    "norm_tag": null,
+    "setup_type": "single so100/so101 robotic arm in molmoact2",
+    "control_mode": "absolute joint pose",
+    "image_keys": [
+      "observation.images.middle",
+      "observation.images.wrist"
+    ],
+    "normalize_language": true,
+    "add_setup_tokens": true,
+    "add_control_tokens": true,
+    "normalize_gripper": true,
+    "num_state_tokens": 256,
+    "max_sequence_length": null,
+    "expected_max_action_dim": 32,
+    "num_flow_timesteps": 8,
+    "flow_matching_cutoff": 1.0,
+    "flow_matching_time_offset": 0.001,
+    "flow_matching_time_scale": 0.999,
+    "flow_matching_beta_alpha": 1.0,
+    "flow_matching_beta_beta": 1.5,
+    "num_inference_steps": null,
+    "mask_action_dim_padding": true,
+    "enable_inference_cuda_graph": true,
+    "per_episode_seed": false,
+    "eval_seed": null,
+    "rtc_config": null,
+    "joint_signs": null,
+    "joint_offsets": null,
+    "enable_lora_vlm": false,
+    "lora_rank": 64,
+    "lora_alpha": 16,
+    "lora_dropout": 0.05,
+    "lora_bias": "none",
+    "enable_lora_action_expert": false,
+    "enable_knowledge_insulation": false,
+    "freeze_embedding": true,
+    "train_action_expert_only": false,
+    "gradient_checkpointing": true,
+    "model_dtype": "bfloat16",
+    "softmax_auxiliary_loss": true,
+    "softmax_auxiliary_loss_scale": 0.0001,
+    "discrete_loss_token_weighting": "root_subsegments_root_tokens",
+    "optimizer_lr": 1e-05,
+    "optimizer_vit_lr": 5e-06,
+    "optimizer_connector_lr": 5e-06,
+    "optimizer_action_expert_lr": 5e-05,
+    "optimizer_betas": [
+      0.9,
+      0.95
+    ],
+    "optimizer_eps": 1e-06,
+    "optimizer_weight_decay": 0.0,
+    "optimizer_grad_clip_norm": 1.0,
+    "scheduler_warmup_steps": 200,
+    "scheduler_decay_steps": 102128,
+    "scheduler_decay_lr": 1e-06,
+    "normalization_mapping": {
+      "ACTION": "MIN_MAX",
+      "STATE": "MIN_MAX",
+      "VISUAL": "IDENTITY"
+    },
+    "dataset_feature_names": {
+      "action": [
+        "shoulder_pan.pos",
+        "shoulder_lift.pos",
+        "elbow_flex.pos",
+        "wrist_flex.pos",
+        "wrist_roll.pos",
+        "gripper.pos"
+      ],
+      "observation.state": [
+        "shoulder_pan.pos",
+        "shoulder_lift.pos",
+        "elbow_flex.pos",
+        "wrist_flex.pos",
+        "wrist_roll.pos",
+        "gripper.pos"
+      ]
+    }
+  },
+  "reward_model": null,
+  "output_dir": "./outputs/spa-bench-model-molmoact2-step-076596",
+  "job_name": "spa-bench-model-molmoact2-step-076596",
+  "resume": true,
+  "seed": 1000,
+  "cudnn_deterministic": false,
+  "num_workers": 8,
+  "batch_size": 24,
+  "prefetch_factor": 4,
+  "persistent_workers": true,
+  "steps": 102128,
+  "env_eval_freq": 0,
+  "log_freq": 100,
+  "eval_steps": 0,
+  "max_eval_samples": 0,
+  "tolerance_s": 0.0001,
+  "save_checkpoint": true,
+  "save_freq": 6383,
+  "use_policy_training_preset": true,
+  "optimizer": {
+    "type": "adamw",
+    "lr": 1e-05,
+    "weight_decay": 0.0,
+    "grad_clip_norm": 1.0,
+    "betas": [
+      0.9,
+      0.95
+    ],
+    "eps": 1e-06
+  },
+  "scheduler": {
+    "type": "cosine_decay_with_warmup",
+    "num_warmup_steps": 200,
+    "num_decay_steps": 102128,
+    "peak_lr": 1e-05,
+    "decay_lr": 1e-06
+  },
+  "eval": {
+    "n_episodes": 50,
+    "batch_size": 50,
+    "use_async_envs": true,
+    "recording": false,
+    "recording_repo_id": null,
+    "recording_private": false
+  },
+  "wandb": {
+    "enable": false,
+    "disable_artifact": true,
+    "project": "spa-bench",
+    "entity": null,
+    "notes": null,
+    "run_id": null,
+    "mode": "disabled",
+    "add_tags": true
+  },
+  "peft": null,
+  "job": {
+    "target": null,
+    "image": "huggingface/lerobot-gpu:latest",
+    "timeout": "2d",
+    "detach": false,
+    "tags": []
+  },
+  "save_checkpoint_to_hub": false,
+  "sample_weighting": null,
+  "rename_map": {},
+  "checkpoint_path": null
+}