vam_cross_world2action_config.json

{
  "schema_version": 1,
  "base_config": "configs/mimicvideo/experiments/so101_level4_widowx_texture_2cam_hstack_v2w.json",
  "paths": {
    "action_smoke_export_root": "datasets/mimicvideo/so101_level4_widowx_texture_2cam_hstack_widowx_teleop_recording_frame_v1_action_smoke_ep000000",
    "action_export_root": "datasets/mimicvideo/so101_level4_widowx_texture_2cam_hstack_widowx_teleop_recording_frame_v1_action",
    "action_checkpoint_receipt": "checkpoints/experiments/w2a_so101_level4_widowx_texture_2cam_hstack_action_iter2374_videolora_iter400_widowx_teleop_recording_frame_v1/VAM_CROSS_MIMICVIDEO_ACTION_CHECKPOINT.json",
    "action_input_receipt": "runs/vam_cross/world2action/w2a_so101_level4_widowx_texture_2cam_hstack_action_iter2374_videolora_iter400_widowx_teleop_recording_frame_v1/VAM_CROSS_ACTION_INPUTS.json",
    "action_statistics_receipt": "runs/vam_cross/world2action/w2a_so101_level4_widowx_texture_2cam_hstack_action_iter2374_videolora_iter400_widowx_teleop_recording_frame_v1/VAM_CROSS_ACTION_STATISTICS.json"
  },
  "action_contract": {
    "coordinate_frame": "widowx_reference_base/teleop_aligned_tool",
    "pose_source": "observation.ee_pose",
    "unused_command_pose": "action.ee_pose",
    "pose_source_semantics": "future_achieved_pose",
    "pose_target_representation": "relative_to_current_achieved_pose",
    "pose_rotation_representation": "rotation_6d",
    "gripper_source": "action.gripper_position",
    "gripper_target_representation": "absolute_normalized_0_1",
    "source_fps": 30,
    "target_fps": 5,
    "observation_horizon": 1,
    "action_horizon": 15,
    "observation_dimensions": 10,
    "action_dimensions": 10,
    "action_vector_order": [
      "relative_ee_xyz_3",
      "relative_ee_rotation_6d_6",
      "absolute_gripper_1"
    ],
    "tool_frame_id": "widowx_teleop_recording_frame_v1",
    "tool_frame_definition_sha256": "37cdea9413fef920e6bd14f79596514761c6fdb5c7ee0fee3450d7f91f27c813",
    "tool_frame_definition": {
      "schema_version": 1,
      "recording_frame_id": "widowx_teleop_recording_frame_v1",
      "reference_robot": "widowx250",
      "robot_name": "so101",
      "transform_semantics": "T_widowxbase_recordingtool=inv(T_world_widowxbase)@T_world_robotbase@T_robotbase_raw@T_raw_recordingtool",
      "world_from_reference_base": [
        [
          1.0,
          0.0,
          0.0,
          0.1
        ],
        [
          0.0,
          1.0,
          0.0,
          0.0
        ],
        [
          0.0,
          0.0,
          1.0,
          0.42
        ],
        [
          0.0,
          0.0,
          0.0,
          1.0
        ]
      ],
      "world_from_robot_base": [
        [
          1.0,
          0.0,
          0.0,
          0.15
        ],
        [
          0.0,
          1.0,
          0.0,
          0.0
        ],
        [
          0.0,
          0.0,
          1.0,
          0.42
        ],
        [
          0.0,
          0.0,
          0.0,
          1.0
        ]
      ],
      "reference_from_robot_base": [
        [
          1.0,
          0.0,
          0.0,
          0.04999999999999999
        ],
        [
          0.0,
          1.0,
          0.0,
          0.0
        ],
        [
          0.0,
          0.0,
          1.0,
          0.0
        ],
        [
          0.0,
          0.0,
          0.0,
          1.0
        ]
      ],
      "tool_frame_definition": {
        "schema_version": 1,
        "canonical_tool_frame_id": "widowx_tool_canonical_v1",
        "canonical_reference_robot": "widowx250",
        "robot_name": "so101",
        "axis_semantics": [
          "forward",
          "left_finger",
          "palm_up"
        ],
        "raw_forward_axis": [
          1.0,
          0.0,
          0.0
        ],
        "raw_left_axis": [
          0.0,
          0.0,
          1.0
        ],
        "raw_palm_up_axis": [
          0.0,
          -1.0,
          0.0
        ],
        "raw_from_canonical": [
          [
            1.0,
            0.0,
            0.0
          ],
          [
            0.0,
            0.0,
            -1.0
          ],
          [
            0.0,
            1.0,
            0.0
          ]
        ],
        "robot_xml_sha256": "37e01af73a28743167321f584af79f88f2d725eb8cc6ce35001360ef1fe93268",
        "evidence": [
          "gripperframe raw +X follows the gripper body -Z fingertip direction",
          "moving jaw lies on gripperframe raw +Z relative to the fixed jaw",
          "single-jaw geometry uses the moving-jaw side as the positive lateral axis"
        ]
      },
      "tool_frame_definition_sha256": "728ae1e2c2f0603cfedf3794b32f3707f05a8297e0739ed92e1264babc0f0783",
      "reference_tool_frame_definition_sha256": "f3aef5e783001a5718e4a0c42020b06d137fb429c04e544a67934dac7decf4ba",
      "recording_raw_from_canonical": [
        [
          0.9999801549651472,
          0.000586648295754,
          -0.0062726007092316
        ],
        [
          0.0062925278659326,
          -0.141402145912918,
          0.9899322387033764
        ],
        [
          -0.000306217139993,
          -0.9899520639783521,
          -0.1414030312831526
        ]
      ],
      "jaw_symmetry_correction": [
        [
          0.9999801549651472,
          0.000586648295754,
          -0.0062726007092316
        ],
        [
          -0.000306217139993,
          -0.9899520639783521,
          -0.1414030312831526
        ],
        [
          -0.0062925278659326,
          0.141402145912918,
          -0.9899322387033764
        ]
      ],
      "paired_teleop_alignment": "same_so101_target_orientation_fitted_constant_gauge_removed"
    }
  },
  "language_embedding": {
    "reuse_video_embeddings": true,
    "source_tensor_key": "encoded_text",
    "output_tensor_key": "language_embedding",
    "max_length": 512,
    "embedding_dim": 1024,
    "storage_dtype": "float32"
  },
  "checkpoints": {
    "repo_id": "jonpai/mimic-video",
    "revision": "f28339034831e3c2374be075e622e1ff38ebe0f8",
    "bridge_action_decoder": "action_decoder/w2a_bridge_v2w_bridge_lora_rank256_lr1.778e-04_bsz64_iter_000070043_fused_lr1.000e-04_layer20_bsz256_iter_000014112.pt",
    "bridge_action_decoder_bytes": 998172132,
    "bridge_action_decoder_sha256": "ab36a48096f13e6b5fcf6c48f9d33a9bf902a20e3e37f47f4815920bf3ea6559",
    "video_lora_selection": "latest_complete_after_video_job"
  },
  "published_video_lora": {
    "delivery": "standalone_hub",
    "repo_id": "dreamdifferent/vam-cross-level4-so101-widowx-texture-video-lora-iter-400",
    "revision": "d66032413191d12b18eb5c97be9f649db3ffcc26",
    "path": "checkpoints/model/iter_000000400.pt",
    "path_in_repo": "checkpoints/model/iter_000000400.pt",
    "local_path": "selected_video_loras/dreamdifferent--vam-cross-level4-so101-widowx-texture-video-lora-iter-400-d660324/checkpoints/model/iter_000000400.pt",
    "iteration": 400,
    "source_iteration": 400,
    "bytes": 741588124,
    "sha256": "ab7741672f83baefc5a4bd56a4ba867eba87c9ad5eb118ba60037b1dac3b4f82",
    "manifest_path": "vam_cross_video_lora_manifest.json",
    "receipt_path": "selected_video_loras/dreamdifferent--vam-cross-level4-so101-widowx-texture-video-lora-iter-400-d660324/VAM_CROSS_SELECTED_VIDEO_LORA.json",
    "kind": "video2world_lora"
  },
  "training": {
    "stage": "world2action",
    "initialization": "pretrained_action_decoder",
    "data_config": "bridge",
    "experiment": "w2a_so101_level4_widowx_texture_2cam_hstack_action_iter2374_videolora_iter400_widowx_teleop_recording_frame_v1",
    "job_project": "vam_cross",
    "job_group": "world2action",
    "dataset_seed": 42,
    "num_val_episodes": 10,
    "statistics_num_workers": 0,
    "statistics_batch_size": 4,
    "xattn_layer": 20,
    "learning_rate": 0.0001,
    "run_validation": false,
    "wandb_mode": "offline",
    "profiles": {
      "smoke": {
        "run_name": "w2a_so101_level4_widowx_texture_2cam_hstack_action_iter2374_videolora_iter400_widowx_teleop_recording_frame_v1_smoke",
        "gpus": 1,
        "gpu_type": "a100_80gb",
        "per_gpu_batch_size": 1,
        "gradient_accumulation": 8,
        "num_workers": 0,
        "max_iterations": 10,
        "save_every": 10,
        "log_every": 1,
        "walltime": "04:00:00"
      },
      "full": {
        "run_name": "w2a_so101_level4_widowx_texture_2cam_hstack_action_iter2374_videolora_iter400_widowx_teleop_recording_frame_v1_full",
        "gpus": 2,
        "gpu_type": "a100_80gb",
        "per_gpu_batch_size": 2,
        "gradient_accumulation": 8,
        "num_workers": 1,
        "effective_batch_size": 32,
        "max_iterations": 1800,
        "save_every": 100,
        "log_every": 10,
        "keep_latest_only": false,
        "require_smoke_receipt": false,
        "walltime": "24:00:00"
      }
    }
  },
  "storage": {
    "expected_raw_rgb_bytes": 8409600000,
    "recommended_free_space_gb": 20
  },
  "action_frame_initialization": {
    "schema_version": 1,
    "source_tool_frame_id": "native_grip_site_v1",
    "target_tool_frame_id": "widowx_teleop_recording_frame_v1",
    "reference_robot": "widowx250",
    "numeric_transform": "identity",
    "source_dataset_repo_id": "dreamdifferent/vam-cross-target-widowx250-native",
    "reason": "widowx250 raw grip_site defines the canonical reference"
  },
  "initial_action_decoder": {
    "delivery": "standalone_hub",
    "repo_id": "dreamdifferent/vam-cross-target-widowx250-native-2cam-action-decoder",
    "revision": "93750cccda01620e3c028477e4c49bc5c996a68d",
    "path": "checkpoints/model/iter_000002374.pt",
    "path_in_repo": "checkpoints/model/iter_000002374.pt",
    "local_path": "initial_action_decoders/dreamdifferent--vam-cross-target-widowx250-native-2cam-action-decoder-93750cc/checkpoints/model/iter_000002374.pt",
    "iteration": 2374,
    "source_iteration": 2374,
    "bytes": 998172132,
    "sha256": "3a9a75686164351468ef9b99098ae9152a1f07082b869e1ad03c9e1e1cf6885c",
    "manifest_path": "vam_cross_action_decoder_manifest.json",
    "receipt_path": "initial_action_decoders/dreamdifferent--vam-cross-target-widowx250-native-2cam-action-decoder-93750cc/VAM_CROSS_INITIAL_ACTION_DECODER.json",
    "training_dataset_repo_id": "dreamdifferent/vam-cross-target-widowx250-native",
    "training_robot_name": "widowx250",
    "kind": "pretrained_world2action_decoder"
  }
}