admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-19 07:51:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Enable MolmoAct2 rollout on SO-100/101 with calibration correction (#3879)

Browse Source 
* fix(rollout): improve visual feature mismatch error with --rename_map hint

* feat(policies): add joint frame transform and hardware deployment docs for MolmoAct2

Add MolmoAct2StateFrameTransformStep and MolmoAct2ActionFrameTransformStep
processor steps for cross-calibration compatibility on SO-100/101. Add
joint_signs and joint_offsets config fields. Add hardware deployment section
to molmoact2.mdx with camera naming convention, joint frame correction, and
safety guidance.

* chore(docs): address PR comment

* fix: address reviewer comments
This commit is contained in:
Khalil Meftah
2026-06-29 18:52:59 +02:00
committed by GitHub
parent 18eee1b477
commit 2f2b567951
5 changed files with 208 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/policies/molmoact2/test_molmoact2.py
+35
View File
@@ -44,10 +44,12 @@ from lerobot.policies.molmoact2.modeling_molmoact2 import (
_combine_rollout_seeds,
)
from lerobot.policies.molmoact2.processor_molmoact2 import (
MolmoAct2ActionFrameTransformStep,
MolmoAct2ClampNormalizedProcessorStep,
MolmoAct2MaskedNormalizerProcessorStep,
MolmoAct2MaskedUnnormalizerProcessorStep,
MolmoAct2PackInputsProcessorStep,
MolmoAct2StateFrameTransformStep,
_add_gripper_masks_to_stats,
_build_discrete_state_string,
_normalize_question_text,
@@ -926,6 +928,39 @@ def test_question_normalization_matches_release_prompt_style():
)
def test_joint_frame_transform_round_trip():
signs = [1.0, -1.0, 1.0, 1.0, 1.0, 1.0]
offsets = [0.0, 90.0, 90.0, 0.0, 0.0, 0.0]
original_state = torch.tensor([[10.0, -90.0, -120.0, 30.0, 0.0, -45.0]])
state_step = MolmoAct2StateFrameTransformStep(joint_signs=signs, joint_offsets=offsets)
action_step = MolmoAct2ActionFrameTransformStep(joint_signs=signs, joint_offsets=offsets)
transition = {
TransitionKey.OBSERVATION: {OBS_STATE: original_state.clone()},
}
transformed = state_step(transition)
model_state = transformed[TransitionKey.OBSERVATION][OBS_STATE]
action_transition = {TransitionKey.ACTION: model_state.clone()}
recovered = action_step(action_transition)
recovered_state = recovered[TransitionKey.ACTION]
assert torch.allclose(recovered_state, original_state)
def test_joint_frame_transform_noop_when_none():
state_step = MolmoAct2StateFrameTransformStep(joint_signs=None, joint_offsets=None)
action_step = MolmoAct2ActionFrameTransformStep(joint_signs=None, joint_offsets=None)
state = torch.tensor([[10.0, -90.0, -120.0]])
state_transition = {TransitionKey.OBSERVATION: {OBS_STATE: state}}
assert state_step(state_transition) is state_transition
action_transition = {TransitionKey.ACTION: state}
assert action_step(action_transition) is action_transition
def test_action_padding_marks_only_real_dimensions():
step = object.__new__(MolmoAct2PackInputsProcessorStep)
step.max_action_dim = 32