DPP 6-point finetune, human keypoints, 100k steps

100000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec2ca143c351221e01a0dda370c0ccaf0a2c26789269e761e208ce71160b84b6
+size 338666816

README.md

@@ -0,0 +1,84 @@
+---
+license: other
+tags:
+  - robotics
+  - manipulation
+  - dexterous-manipulation
+  - dex-point-policy
+  - openarm
+  - inspire-rh56f1
+---
+
+# DPP 6-point — `human` keypoints (100k)
+
+Dexterous Point Policy (DPP) finetuned on 80 anyh2r pick-and-place episodes, using
+**HaWoR human** as the source of the 6 hand keypoints.
+
+> One of three models that differ **only** in how the 6 keypoints
+> (wrist + 5 fingertips) were derived. Everything else — object point clouds,
+> per-finger contact, camera pose, language, validity mask, episode set,
+> hyper-parameters, pretrained init — is byte-identical across the three.
+
+| repo | keypoint source |
+|---|---|
+| [`dpp-6pt-anyh2r-fk-100k`](https://huggingface.co/RyanL22/dpp-6pt-anyh2r-fk-100k) | IDM wristIK robot state → MuJoCo FK **(ours)** |
+| [`dpp-6pt-anyh2r-human-100k`](https://huggingface.co/RyanL22/dpp-6pt-anyh2r-human-100k) | HaWoR human hand (baseline) |
+| [`dpp-6pt-anyh2r-retgt-100k`](https://huggingface.co/RyanL22/dpp-6pt-anyh2r-retgt-100k) | dex-retargeting → Inspire RH56F1 (baseline) |
+
+**This model:** HaWoR human hand keypoints, used as-is (baseline)
+
+## Files
+
+| file | what |
+|---|---|
+| `100000.pt` | final snapshot (323 MB) — `state_dict`s + optimizer + normalisation stats |
+| `train_config.yaml` / `train_overrides.yaml` | the exact resolved Hydra config of the run |
+| `train_log.csv` | per-100-step training log |
+
+## Training
+
+Code: [`beomjun02/dex-point-policy`](https://github.com/beomjun02/dex-point-policy) @ `e972ba5`
+(private). Init: `vitra_6points_final/snapshot/100000.pt` (VITRA video pretrain, 6points,
+no contact head — the head is added at finetune and loaded with `strict=False`).
+
+```
+agent=dpp  suite.action_mode=6points  suite.history_len=1  num_queries=16
+dataloader.bc_dataset.normalize=false          # raw metres, matches the pretrain run
+dataloader.bc_dataset.hand=right
+dataloader.bc_dataset.max_num_objects=4
+teacher_forcing=true
+use_contact_head=true  contact_loss_coef=1.0  contact_detach=true
+uniform_contact_pos_weight=true                # scalar pos_weight 4.38
+geom_drop_p=0  contact_drop_p=0  robot_noise_std=0
+batch_size=64  lr=1e-4  steps=100000  finetune=true
+```
+
+1× GPU, ~55 min. Final training loss (mean over steps 95k–100k): **0.1083**.
+
+> Training losses are **not** comparable across the three models — each fits a
+> different target sequence. Use rollout success, not this number.
+
+## Data
+
+80 episodes (ball 22 / bottle 20 / box 22 / doll 16), 22,454 frames @ 20 fps, 95.6% kept.
+Built from `dpp_ego3d_v1/openarm_inspire` (pt512 pack) for objects / contact / camera /
+language, intersected with the anyh2r wristIK synthetic set
+(`idm_v4form_rh56f1_vla_stereo-depth-wristIK_20260916`) so all three variants cover
+exactly the same clips.
+
+- Frame: `ego_right_optical_frame` (x right, y down, z forward), metres, static camera.
+- Keypoints: MANO-21 indices `[0, 4, 8, 12, 16, 20]` = wrist, thumb, index, middle, ring, little tip.
+- Objects: 128 points per slot, 2 slots (task object + white container).
+- Contact: per-finger binary, thresholded at 0.3 on the HaWoR-tactile probability.
+
+Verification of the FK path: our forward kinematics reproduces ego3d's own
+`robot/*/hands21.npz` to **0.00 mm** (dohyeon calibrated ego extrinsic, rig in MJCF body
+`pitch_motor_rotor_2`). Mean disagreement between the human and FK keypoints on the same
+frames: wrist 36 mm, thumb 42, index 43, middle 46, ring 47, little 41 — note the wrist
+points are different physical landmarks (MANO wrist joint vs RH56F1 palm site), so a
+steady offset there is geometry, not error.
+
+## License
+
+Inherits the `dex-point-policy` / NVIDIA Source Code License terms of the upstream
+`point_bridge` project. Research use.

train_config.yaml

@@ -0,0 +1,130 @@
+root_dir: ${oc.env:DPP_ROOT,${hydra:runtime.cwd}}
+data_dir: ${oc.env:DPP_DATA_DIR,./data}
+seed: 42
+device: cuda
+save_video: false
+use_tb: false
+batch_size: 64
+use_wandb: false
+wandb_project: dex-point-policy
+wandb_entity: beomjun-alinlab
+num_demos_per_task: 1000000
+use_ego_state: true
+use_language: true
+use_world_model: false
+world_model_loss_coef: 0.1
+world_model_detach: false
+teacher_forcing: true
+use_contact_head: true
+contact_loss_coef: 1.0
+contact_detach: true
+use_contact_ar: true
+geom_drop_p: 0.0
+contact_drop_p: 0.0
+robot_noise_std: 0.0
+eval: false
+finetune: true
+task_names: null
+experiment: dpp_ft_human
+experiment_label: deterministic
+task_filter: null
+action_chunking: true
+num_queries: 16
+temporal_agg_strategy: exponential_average
+expert_dataset: ${dataloader.bc_dataset}
+load_bc: true
+bc_weight: /sjw_alinlab/home/seungyong/data/dpp6/dpp_pt_100k.pt
+agent:
+  _target_: agent.dpp.BCAgent
+  obs_shape: ???
+  action_shape: ???
+  device: ${device}
+  lr: 0.0001
+  hidden_dim: ${suite.hidden_dim}
+  stddev_schedule: 0.1
+  use_tb: ${use_tb}
+  use_language: ${use_language}
+  use_ego_state: ${use_ego_state}
+  history_len: ${suite.history_len}
+  eval_history_len: ${suite.eval_history_len}
+  action_chunking: ${action_chunking}
+  num_queries: ${num_queries}
+  temporal_agg_strategy: ${temporal_agg_strategy}
+  max_episode_len: ${suite.task_make_fn.max_episode_len}
+  film: true
+  action_mode: ${suite.action_mode}
+  robot_points_key: ${suite.robot_points_key}
+  object_points_key: ${suite.object_points_key}
+  num_points_per_obj: ${suite.num_points_per_obj}
+  teacher_forcing: ${teacher_forcing}
+  use_world_model: ${use_world_model}
+  world_model_loss_coef: ${world_model_loss_coef}
+  world_model_detach: ${world_model_detach}
+  use_contact_head: ${use_contact_head}
+  contact_loss_coef: ${contact_loss_coef}
+  contact_detach: ${contact_detach}
+  use_contact_ar: ${use_contact_ar}
+  geom_drop_p: ${geom_drop_p}
+  contact_drop_p: ${contact_drop_p}
+  robot_noise_std: ${robot_noise_std}
+suite:
+  suite: vitra
+  name: vitra
+  action_mode: 6points
+  num_robot_points: 21
+  num_points_per_obj: 128
+  robot_points_key: robot_tracks
+  object_points_key: object_tracks
+  hidden_dim: 512
+  action_repeat: 1
+  num_train_steps: 100000
+  log_every_steps: 100
+  save_every_steps: 5000
+  history_len: 1
+  eval_every_steps: 200000
+  num_eval_episodes: 5
+  eval_history_len: 5
+  save_snapshot: true
+  task_make_fn:
+    _target_: dex_point_policy.suite.vitra.make
+    task_names: []
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    max_episode_len: ???
+    max_state_dim: ???
+    max_num_objects: 4
+    eval: ${eval}
+    num_robot_points: ${suite.num_robot_points}
+    num_points_per_obj: ${suite.num_points_per_obj}
+    robot_points_key: ${suite.robot_points_key}
+    object_points_key: ${suite.object_points_key}
+    action_mode: ${suite.action_mode}
+    temporal_agg_strategy: ${temporal_agg_strategy}
+dataloader:
+  bc_dataset:
+    _target_: dex_point_policy.read_data.vitra.BCDataset
+    path: ${data_dir}
+    suffix: null
+    task_names: ${task_names}
+    num_demos_per_task: ${num_demos_per_task}
+    history_len: ${suite.history_len}
+    action_chunking: ${action_chunking}
+    num_queries: ${num_queries}
+    num_robot_points: ${suite.num_robot_points}
+    num_points_per_obj: ${suite.num_points_per_obj}
+    robot_points_key: ${suite.robot_points_key}
+    object_points_key: ${suite.object_points_key}
+    act_subsample: 1
+    obs_subsample: 1
+    action_mode: ${suite.action_mode}
+    task_indices: null
+    noise_object_points: false
+    noise_std: 0.01
+    hdf5_path: /sjw_alinlab/home/seungyong/data/dpp6/ft_human.h5
+    hand: right
+    normalize: false
+    task_filter: ${task_filter}
+    max_num_objects: 4
+    compute_contact_pos_weight: true
+    uniform_contact_pos_weight: true
+    use_world_model: ${use_world_model}

train_overrides.yaml

@@ -0,0 +1,28 @@
+- experiment=dpp_ft_human
+- agent=dpp
+- use_wandb=false
+- use_tb=false
+- suite.action_mode=6points
+- suite.history_len=1
+- dataloader.bc_dataset.normalize=false
+- dataloader.bc_dataset.hdf5_path=/sjw_alinlab/home/seungyong/data/dpp6/ft_human.h5
+- dataloader.bc_dataset.hand=right
+- dataloader.bc_dataset.max_num_objects=4
+- dataloader.bc_dataset.act_subsample=1
+- dataloader.bc_dataset.noise_object_points=false
+- dataloader.bc_dataset.uniform_contact_pos_weight=true
+- teacher_forcing=true
+- use_contact_head=true
+- contact_loss_coef=1.0
+- contact_detach=true
+- geom_drop_p=0.0
+- contact_drop_p=0.0
+- robot_noise_std=0.0
+- num_queries=16
+- batch_size=64
+- agent.lr=1e-4
+- suite.num_train_steps=100000
+- suite.save_every_steps=5000
+- load_bc=true
+- bc_weight=/sjw_alinlab/home/seungyong/data/dpp6/dpp_pt_100k.pt
+- finetune=true