diff --git a/domain_id.npy b/domain_id.npy new file mode 100644 index 000000000..8d15fe27e Binary files /dev/null and b/domain_id.npy differ diff --git a/eval.sh b/eval.sh index 318ed3fbc..040d9c25e 100644 --- a/eval.sh +++ b/eval.sh @@ -1,5 +1,5 @@ lerobot-eval \ - --policy.path="/raid/jade/models/xvla-lib" \ + --policy.path="/raid/jade/models/xvla-libero-new_migrated2" \ --env.type=libero \ --env.task=libero_spatial \ --env.control_mode=absolute \ diff --git a/image_input.npy b/image_input.npy new file mode 100644 index 000000000..693e960b9 Binary files /dev/null and b/image_input.npy differ diff --git a/image_mask.npy b/image_mask.npy new file mode 100644 index 000000000..393c94c71 Binary files /dev/null and b/image_mask.npy differ diff --git a/input_ids.npy b/input_ids.npy new file mode 100644 index 000000000..2f5ed41da Binary files /dev/null and b/input_ids.npy differ diff --git a/proprio.npy b/proprio.npy new file mode 100644 index 000000000..f7cf870d1 Binary files /dev/null and b/proprio.npy differ diff --git a/src/lerobot/envs/libero.py b/src/lerobot/envs/libero.py index 0e182524c..c68fdaabd 100644 --- a/src/lerobot/envs/libero.py +++ b/src/lerobot/envs/libero.py @@ -244,7 +244,7 @@ class LiberoEnv(gym.Env): eef_quat = raw_obs.get("robot0_eef_quat") # rotation matrix from controller - eef_mat = self._env.robots[0].controller.ee_ori_mat if eef_pos is not None else None + eef_mat = self._env.robots[0].controller.ee_ori_mat gripper_qpos = raw_obs.get("robot0_gripper_qpos") gripper_qvel = raw_obs.get("robot0_gripper_qvel") joint_pos = raw_obs.get("robot0_joint_pos") diff --git a/src/lerobot/policies/xvla/__init__.py b/src/lerobot/policies/xvla/__init__.py new file mode 100644 index 000000000..b3ab92844 --- /dev/null +++ b/src/lerobot/policies/xvla/__init__.py @@ -0,0 +1,6 @@ +from lerobot.policies.xvla.processor_xvla import ( + XVLAAddDomainIdProcessorStep, + XVLAImageScaleProcessorStep, + XVLARotation6DToAxisAngleProcessorStep, + make_xvla_pre_post_processors, +) \ No newline at end of file diff --git a/src/lerobot/policies/xvla/compare.py b/src/lerobot/policies/xvla/compare.py new file mode 100644 index 000000000..3e191e61c --- /dev/null +++ b/src/lerobot/policies/xvla/compare.py @@ -0,0 +1,43 @@ +import numpy as np +import os + +# --- CHANGE THESE TWO PATHS --- +dir_a = "/home/jade_choghari/robot/lerobot" +dir_b = "/home/jade_choghari/robot/robot2/lerobot" + +# keys = ["input_ids", "image_input", "image_mask", "domain_id", "proprio"] +keys = ["domain_id"] + +def load_np(path, key): + file_path = os.path.join(path, f"{key}.npy") + if not os.path.exists(file_path): + raise FileNotFoundError(f"Missing file: {file_path}") + return np.load(file_path) + +for k in keys: + print(f"\n===== {k} =====") + a = load_np(dir_a, k) + b = load_np(dir_b, k) + print(a) + print(b) + print("Shapes:", a.shape, b.shape) + same_shape = a.shape == b.shape + print("Same shape:", same_shape) + + if not same_shape: + continue + + # Allclose for numeric, array_equal for boolean + if a.dtype == bool or b.dtype == bool: + equal = np.array_equal(a, b) + print("Equal:", equal) + + # For boolean arrays, diff = xor + diff = np.sum(a ^ b) + print("Num differing elements:", diff) + else: + close = np.allclose(a, b, atol=1e-6, rtol=1e-6) + print("Allclose:", close) + + diff = np.max(np.abs(a - b)) + print("Max difference:", diff) diff --git a/src/lerobot/policies/xvla/processor_xvla.py b/src/lerobot/policies/xvla/processor_xvla.py index 77dbc7cfa..e11f8953c 100644 --- a/src/lerobot/policies/xvla/processor_xvla.py +++ b/src/lerobot/policies/xvla/processor_xvla.py @@ -62,6 +62,7 @@ def make_xvla_pre_post_processors( padding_side=config.tokenizer_padding_side, ), DeviceProcessorStep(device=config.device), + XVLAAddDomainIdProcessorStep(), NormalizerProcessorStep( features=features, norm_map=config.normalization_mapping, stats=dataset_stats ), @@ -276,7 +277,7 @@ class XVLAAddDomainIdProcessorStep(ProcessorStep): device: Device to place the domain_id tensor on (default: "cuda") """ - domain_id: int = 3 + domain_id: int = 0 device: str = "cuda" def __call__(self, transition: EnvTransition) -> EnvTransition: @@ -293,7 +294,7 @@ class XVLAAddDomainIdProcessorStep(ProcessorStep): if isinstance(v, torch.Tensor): batch_size = v.shape[0] break - + # Add domain_id tensor comp["domain_id"] = torch.tensor([int(self.domain_id)] * batch_size, dtype=torch.long).to(self.device) @@ -387,7 +388,7 @@ def make_xvla_libero_pre_post_processors( """ pre_processor_steps: list[ProcessorStep] = [] post_processor_steps: list[ProcessorStep] = [] - pre_processor_steps.extend([LiberoProcessorStep(), XVLAImageScaleProcessorStep(), XVLAAddDomainIdProcessorStep()]) + pre_processor_steps.extend([LiberoProcessorStep(), XVLAAddDomainIdProcessorStep()]) post_processor_steps.extend([XVLARotation6DToAxisAngleProcessorStep()]) return ( PolicyProcessorPipeline[dict[str, Any], dict[str, Any]]( diff --git a/src/lerobot/processor/normalize_processor.py b/src/lerobot/processor/normalize_processor.py index d2e4e5405..ae799edd9 100644 --- a/src/lerobot/processor/normalize_processor.py +++ b/src/lerobot/processor/normalize_processor.py @@ -338,7 +338,7 @@ class _NormalizationMixin: return (tensor * std + mean) * 255.0 # Normalize - return (tensor / 255.0 - mean) / std + return (tensor - mean) / std stats = self._tensor_stats[key] if norm_mode == NormalizationMode.MEAN_STD: diff --git a/src/lerobot/scripts/lerobot_eval.py b/src/lerobot/scripts/lerobot_eval.py index fed13501f..81fa55218 100644 --- a/src/lerobot/scripts/lerobot_eval.py +++ b/src/lerobot/scripts/lerobot_eval.py @@ -167,10 +167,8 @@ def rollout( # Infer "task" from attributes of environments. # TODO: works with SyncVectorEnv but not AsyncVectorEnv observation = add_envs_task(env, observation) - # Apply environment-specific preprocessing (e.g., LiberoProcessorStep for LIBERO) observation = env_preprocessor(observation) - observation = preprocessor(observation) with torch.inference_mode(): action = policy.select_action(observation) @@ -178,6 +176,7 @@ def rollout( action_transition = {"action": action} action_transition = env_postprocessor(action_transition) action = action_transition["action"] + # Convert to CPU / numpy. action_numpy: np.ndarray = action.to("cpu").numpy() assert action_numpy.ndim == 2, "Action dimensions should be (batch, action_dim)"