From ddb26b7189aa9dfa1a5e75de44700f834295a715 Mon Sep 17 00:00:00 2001 From: Jade Date: Mon, 30 Jun 2025 13:11:16 -0400 Subject: [PATCH 1/5] add multi --- lerobot/common/datasets/lerobot_dataset.py | 367 ++++++++++--- lerobot/common/datasets/utils_must.py | 581 +++++++++++++++++++++ tests/datasets/test_datasets.py | 46 +- 3 files changed, 915 insertions(+), 79 deletions(-) create mode 100644 lerobot/common/datasets/utils_must.py diff --git a/lerobot/common/datasets/lerobot_dataset.py b/lerobot/common/datasets/lerobot_dataset.py index 425e9f7e7..47794493e 100644 --- a/lerobot/common/datasets/lerobot_dataset.py +++ b/lerobot/common/datasets/lerobot_dataset.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import contextlib +import copy import logging import shutil from pathlib import Path @@ -73,6 +74,28 @@ from lerobot.common.datasets.video_utils import ( get_video_info, ) +# mustafa stuff here +from lerobot.common.datasets.utils_must import ( + reshape_features_to_max_dim, + keep_datasets_with_valid_fps, + keep_datasets_with_the_same_features_per_robot_type, + aggregate_stats_per_robot_type, + create_padded_features, + pad_tensor, + map_dict_keys, + ROBOT_TYPE_KEYS_MAPPING, + OBS_IMAGE, + OBS_IMAGE_2, + OBS_IMAGE_3, + TASKS_KEYS_MAPPING, +) +from lerobot.common.constants import ( + ACTION, + OBS_ENV_STATE, + OBS_STATE, + +) + CODEBASE_VERSION = "v2.1" @@ -83,6 +106,7 @@ class LeRobotDatasetMetadata: root: str | Path | None = None, revision: str | None = None, force_cache_sync: bool = False, + feature_keys_mapping: dict[str, str] | None = None, ): self.repo_id = repo_id self.revision = revision if revision else CODEBASE_VERSION @@ -99,6 +123,14 @@ class LeRobotDatasetMetadata: (self.root / "meta").mkdir(exist_ok=True, parents=True) self.pull_from_repo(allow_patterns="meta/") self.load_metadata() + # added by mshukor + self.feature_keys_mapping = feature_keys_mapping.get(repo_id, None) if feature_keys_mapping else None + self.inverse_feature_keys_mapping = ( + {v: k for k, v in self.feature_keys_mapping.items() if v} if self.feature_keys_mapping else {} + ) + self.info["features"] = map_dict_keys( + self.info["features"], feature_keys_mapping=self.feature_keys_mapping + ) def load_metadata(self): self.info = load_info(self.root) @@ -177,7 +209,15 @@ class LeRobotDatasetMetadata: @property def video_keys(self) -> list[str]: """Keys to access visual modalities stored as videos.""" - return [key for key, ft in self.features.items() if ft["dtype"] == "video"] + # changed + keys = [] + for key, ft in self.features.items(): + key_ = ( + self.inverse_feature_keys_mapping.get(key, key) if self.inverse_feature_keys_mapping else key + ) + if ft["dtype"] == "video": + keys.append(key_) + return keys @property def camera_keys(self) -> list[str]: @@ -342,6 +382,19 @@ class LeRobotDataset(torch.utils.data.Dataset): force_cache_sync: bool = False, download_videos: bool = True, video_backend: str | None = None, + + # new thing by M + feature_keys_mapping: dict[str, str] | None = None, + max_action_dim: int = None, + max_state_dim: int = None, + max_num_images: int = None, + max_image_dim: int = None, + training_features: list | None = None, + discard_first_n_frames: int = 0, + discard_first_idle_frames: bool = False, + motion_threshold: float = 5e-2, + motion_window_size: int = 10, + motion_buffer: int = 3, ): """ 2 modes are available for instantiating this class, depending on 2 different use cases: @@ -455,15 +508,31 @@ class LeRobotDataset(torch.utils.data.Dataset): self.video_backend = video_backend if video_backend else get_safe_default_codec() self.delta_indices = None + # by mshukor + self.training_features = training_features + self.discard_first_n_frames = discard_first_n_frames + self.discard_first_idle_frames = discard_first_idle_frames + self.motion_threshold = motion_threshold + self.motion_window_size = motion_window_size + self.motion_buffer = motion_buffer + # Unused attributes self.image_writer = None self.episode_buffer = None self.root.mkdir(exist_ok=True, parents=True) + # more mshukor + self.feature_keys_mapping = feature_keys_mapping.get(repo_id, None) if feature_keys_mapping else None + self.inverse_feature_keys_mapping = ( + {v: k for k, v in self.feature_keys_mapping.items() if v} if self.feature_keys_mapping else {} + ) + # Load metadata + # TODO: change self.meta = LeRobotDatasetMetadata( - self.repo_id, self.root, self.revision, force_cache_sync=force_cache_sync + self.repo_id, self.root, self.revision, force_cache_sync=force_cache_sync, + feature_keys_mapping=feature_keys_mapping, ) if self.episodes is not None and self.meta._version >= packaging.version.parse("v2.1"): episodes_stats = [self.meta.episodes_stats[ep_idx] for ep_idx in self.episodes] @@ -482,17 +551,62 @@ class LeRobotDataset(torch.utils.data.Dataset): self.episode_data_index = get_episode_data_index(self.meta.episodes, self.episodes) + # mustafa code + if self.discard_first_n_frames > 0: + print("Discarding first n frames:", self.discard_first_n_frames) + self.subset_frame_ids = [] + for ep_idx in range(self.num_episodes): + from_ = self.episode_data_index["from"][ep_idx] + to_ = self.episode_data_index["to"][ep_idx] + # TODO implement advanced strategy + self.subset_frame_ids += [frame_idx for frame_idx in range(from_ + int(self.fps*self.discard_first_n_frames), to_)] + elif self.discard_first_idle_frames: + print(f"Discarding first idle frames: motion_threshold={self.motion_threshold}, motion_window_size={self.motion_window_size}, motion_buffer={self.motion_buffer}") + self.robot_states = torch.stack(self.hf_dataset[OBS_ROBOT]).numpy() # shape: [T, D] + self.subset_frame_ids = [] + for ep_idx in range(self.num_episodes): + from_ = self.episode_data_index["from"][ep_idx] + to_ = self.episode_data_index["to"][ep_idx] + ep_states = self.robot_states[from_:to_] + velocities = np.linalg.norm(np.diff(ep_states, axis=0), axis=1) + velocities = np.concatenate([[0.0], velocities]) + start_idx = find_start_of_motion(velocities, self.motion_window_size, self.motion_threshold, self.motion_buffer) + self.subset_frame_ids += list(range(from_ + start_idx, to_)) + # Check timestamps - timestamps = torch.stack(self.hf_dataset["timestamp"]).numpy() - episode_indices = torch.stack(self.hf_dataset["episode_index"]).numpy() - ep_data_index_np = {k: t.numpy() for k, t in self.episode_data_index.items()} - check_timestamps_sync(timestamps, episode_indices, ep_data_index_np, self.fps, self.tolerance_s) + # commented TODO: check why + # timestamps = torch.stack(self.hf_dataset["timestamp"]).numpy() + # episode_indices = torch.stack(self.hf_dataset["episode_index"]).numpy() + # ep_data_index_np = {k: t.numpy() for k, t in self.episode_data_index.items()} + # check_timestamps_sync(timestamps, episode_indices, ep_data_index_np, self.fps, self.tolerance_s) # Setup delta_indices if self.delta_timestamps is not None: - check_delta_timestamps(self.delta_timestamps, self.fps, self.tolerance_s) + # TODO: check why commented + # check_delta_timestamps(self.delta_timestamps, self.fps, self.tolerance_s) self.delta_indices = get_delta_indices(self.delta_timestamps, self.fps) + # Mustafa + self.meta.info["features"] = map_dict_keys( + self.meta.info["features"], feature_keys_mapping=self.feature_keys_mapping, training_features=self.training_features + ) + self.keys_to_max_dim = { + ACTION: max_action_dim, + OBS_ENV_STATE: max_state_dim, + OBS_STATE: max_state_dim, + OBS_IMAGE: max_image_dim, + OBS_IMAGE_2: max_image_dim, + OBS_IMAGE_3: max_image_dim, + } + self.meta.info["features"] = reshape_features_to_max_dim( + self.meta.info["features"], reshape_dim=-1, keys_to_max_dim=self.keys_to_max_dim + ) + self.meta.stats = map_dict_keys(self.meta.stats, feature_keys_mapping=self.feature_keys_mapping, training_features=self.training_features) + self.robot_type = self.meta.info.get("robot_type", "") + # Override tasks + print(TASKS_KEYS_MAPPING.get(self.repo_id, self.meta.tasks), "previous", self.meta.tasks) + self.meta.tasks = TASKS_KEYS_MAPPING.get(self.repo_id, self.meta.tasks) + def push_to_hub( self, branch: str | None = None, @@ -647,6 +761,7 @@ class LeRobotDataset(torch.utils.data.Dataset): key: [max(ep_start.item(), min(ep_end.item() - 1, idx + delta)) for delta in delta_idx] for key, delta_idx in self.delta_indices.items() } + # FIXME(mshukor): what if we train on multiple datasets with different features padding = { # Pad values outside of current episode range f"{key}_is_pad": torch.BoolTensor( [(idx + delta < ep_start.item()) | (idx + delta >= ep_end.item()) for delta in delta_idx] @@ -670,12 +785,18 @@ class LeRobotDataset(torch.utils.data.Dataset): return query_timestamps + # TODO: changed by mustafa def _query_hf_dataset(self, query_indices: dict[str, list[int]]) -> dict: - return { - key: torch.stack(self.hf_dataset.select(q_idx)[key]) - for key, q_idx in query_indices.items() - if key not in self.meta.video_keys - } + queries = {} + for key, q_idx in query_indices.items(): + if key not in self.meta.video_keys and self.inverse_feature_keys_mapping.get(key, key) not in self.meta.video_keys: + key_ = ( + self.inverse_feature_keys_mapping.get(key, key) + if self.inverse_feature_keys_mapping + else key + ) + queries[key] = torch.stack(self.hf_dataset.select(q_idx)[key_]) + return queries def _query_videos(self, query_timestamps: dict[str, list[float]], ep_idx: int) -> dict[str, torch.Tensor]: """Note: When using data workers (e.g. DataLoader with num_workers>0), do not call this function @@ -699,8 +820,12 @@ class LeRobotDataset(torch.utils.data.Dataset): def __len__(self): return self.num_frames + # changed by mshukor def __getitem__(self, idx) -> dict: + if self.discard_first_n_frames > 0 or self.discard_first_idle_frames: + idx = self.subset_frame_ids[idx] item = self.hf_dataset[idx] + item = map_dict_keys(item, feature_keys_mapping=self.feature_keys_mapping) ep_idx = item["episode_index"].item() query_indices = None @@ -717,15 +842,25 @@ class LeRobotDataset(torch.utils.data.Dataset): video_frames = self._query_videos(query_timestamps, ep_idx) item = {**video_frames, **item} - if self.image_transforms is not None: - image_keys = self.meta.camera_keys - for cam in image_keys: - item[cam] = self.image_transforms(item[cam]) - # Add task as a string task_idx = item["task_index"].item() - item["task"] = self.meta.tasks[task_idx] - + try: + item["task"] = self.meta.tasks[task_idx] + except: + print(self.meta.tasks, task_idx, self.repo_id) + if "robot_type" not in item: + item["robot_type"] = self.robot_type + item = map_dict_keys(item, feature_keys_mapping=self.feature_keys_mapping, training_features=self.training_features) + # Add padded features + # item = self._add_padded_features(item, self.training_features) + if self.image_transforms is not None: + for cam in item: + if cam in self.meta.camera_keys or ("image" in cam and "is_pad" not in cam): + item[cam] = self.image_transforms(item[cam]) + # Map pad keys + # print(item.keys(), "before") + # item = map_dict_pad_keys(item, feature_keys_mapping=self.feature_keys_mapping, training_features=self.training_features) + # print(item.keys()) return item def __repr__(self): @@ -1022,54 +1157,161 @@ class MultiLeRobotDataset(torch.utils.data.Dataset): tolerances_s: dict | None = None, download_videos: bool = True, video_backend: str | None = None, + + # add + sampling_weights: list[float] | None = None, + feature_keys_mapping: dict[str, dict[str, str]] | None = None, + max_action_dim: int = None, + max_state_dim: int = None, + max_num_images: int = None, + max_image_dim: int = None, + train_on_all_features: bool = False, + training_features: list | None = None, + discard_first_n_frames: int = 0, + min_fps: int = 1, + max_fps: int = 100, + discard_first_idle_frames: bool = False, + motion_threshold: float = 0.05, + motion_window_size: int = 10, + motion_buffer: int = 3, ): super().__init__() self.repo_ids = repo_ids self.root = Path(root) if root else HF_LEROBOT_HOME - self.tolerances_s = tolerances_s if tolerances_s else dict.fromkeys(repo_ids, 0.0001) + self.tolerances_s = tolerances_s if tolerances_s else {repo_id: 1e-4 for repo_id in repo_ids} # Construct the underlying datasets passing everything but `transform` and `delta_timestamps` which # are handled by this class. - self._datasets = [ - LeRobotDataset( - repo_id, - root=self.root / repo_id, - episodes=episodes[repo_id] if episodes else None, - image_transforms=image_transforms, - delta_timestamps=delta_timestamps, - tolerance_s=self.tolerances_s[repo_id], - download_videos=download_videos, - video_backend=video_backend, - ) - for repo_id in repo_ids - ] + _datasets = [] + datasets_repo_ids = [] + self.sampling_weights = [] + self.training_features = training_features + + sampling_weights = sampling_weights if sampling_weights is not None else [1] * len(repo_ids) + assert len(sampling_weights) == len(repo_ids), ( + "The number of sampling weights must match the number of datasets. " + f"Got {len(sampling_weights)} weights for {len(repo_ids)} datasets." + ) + for i, repo_id in enumerate(repo_ids): + try: + # delta_timestamps = resolve_delta_timestamps(cfg.policy, ds_meta) + _datasets.append( + LeRobotDataset( + repo_id, + root=self.root / repo_id, + episodes=episodes.get(repo_id, None) if episodes else None, + image_transforms=image_transforms, + delta_timestamps = delta_timestamps.get(repo_id, None) if delta_timestamps else None, + tolerance_s=self.tolerances_s[repo_id], + download_videos=download_videos, + video_backend=video_backend, + feature_keys_mapping=feature_keys_mapping, + training_features=training_features, + discard_first_n_frames=discard_first_n_frames, + discard_first_idle_frames=discard_first_idle_frames, + motion_threshold=motion_threshold, + motion_window_size=motion_window_size, + motion_buffer=motion_buffer, + ) + ) + datasets_repo_ids.append(repo_id) + self.sampling_weights.append(float(sampling_weights[i])) + except Exception as e: + print(f"Failed to load dataset: {repo_id} due to Exception: {e}") + print( + f"Finish loading {len(_datasets)} datasets, with sampling weights: {self.sampling_weights} corresponding to: {datasets_repo_ids}" + ) # Disable any data keys that are not common across all of the datasets. Note: we may relax this # restriction in future iterations of this class. For now, this is necessary at least for being able # to use PyTorch's default DataLoader collate function. + # FIXME(mshukor): apply mapping to unify used keys + self.train_on_all_features = train_on_all_features self.disabled_features = set() - intersection_features = set(self._datasets[0].features) - for ds in self._datasets: - intersection_features.intersection_update(ds.features) - if len(intersection_features) == 0: - raise RuntimeError( - "Multiple datasets were provided but they had no keys common to all of them. " - "The multi-dataset functionality currently only keeps common keys." - ) - for repo_id, ds in zip(self.repo_ids, self._datasets, strict=True): - extra_keys = set(ds.features).difference(intersection_features) - logging.warning( - f"keys {extra_keys} of {repo_id} were disabled as they are not contained in all the " - "other datasets." - ) - self.disabled_features.update(extra_keys) + if not self.train_on_all_features: + intersection_features = set(_datasets[0].features) + for ds in _datasets: + intersection_features.intersection_update(ds.features) + if len(intersection_features) == 0: + raise RuntimeError( + "Multiple datasets were provided but they had no keys common to all of them. " + "The multi-dataset functionality currently only keeps common keys." + ) + for repo_id, ds in zip(repo_ids, _datasets, strict=True): + extra_keys = set(ds.features).difference(intersection_features) + logging.warning( + f"keys {extra_keys} of {repo_id} were disabled as they are not contained in all the " + "other datasets." + ) + self.disabled_features.update(extra_keys) + union_features = {} + for ds in _datasets: + for k, v in ds.features.items(): + if k not in self.disabled_features: + union_features[k] = v + + if len(union_features) == 0: + raise RuntimeError("Multiple datasets were provided, but no features were found.") self.image_transforms = image_transforms - self.delta_timestamps = delta_timestamps - # TODO(rcadene, aliberts): We should not perform this aggregation for datasets - # with multiple robots of different ranges. Instead we should have one normalization - # per robot. - self.stats = aggregate_stats([dataset.meta.stats for dataset in self._datasets]) + self.delta_timestamps = ( + delta_timestamps.get(repo_id, None) if delta_timestamps else None + ) # delta_timestamps # FIXME(mshukor): last repo? + # self.stats = aggregate_stats(self._datasets) # FIXME(mshukor): stats should be computed per robot type and then the robot type should be passed as input to the model + for ds in _datasets: + ds.meta.info["robot_type"] = ROBOT_TYPE_KEYS_MAPPING.get(ds.repo_id, ds.meta.info["robot_type"]) + ds.robot_type = ds.meta.info["robot_type"] + # In case datasets with the same robot_type have different features + _datasets = keep_datasets_with_valid_fps(_datasets, min_fps=min_fps, max_fps=max_fps) + self._datasets, datasets_maks = keep_datasets_with_the_same_features_per_robot_type(_datasets) + self.sampling_weights = [self.sampling_weights[i] for i in range(len(_datasets)) if datasets_maks[i]] + self.repo_ids = [repo_ids[i] for i in range(len(_datasets)) if datasets_maks[i]] + self.datasets_repo_ids = [datasets_repo_ids[i] for i in range(len(_datasets)) if datasets_maks[i]] + # Compute cumulative sizes for fast indexing + self.cumulative_sizes = np.array( + [0] + list(torch.cumsum(torch.tensor([len(d) for d in self._datasets]), dim=0)) + ) + self.sampling_weights = np.array(self.sampling_weights, dtype=np.float32) + self.stats = aggregate_stats_per_robot_type(self._datasets) + self.meta = copy.deepcopy(self._datasets[0].meta) # FIXME(mshukor): aggregate meta from all datasets + self.meta.info = { + repo_id: ds.meta.info for repo_id, ds in zip(self.repo_ids, self._datasets, strict=False) + } + # self.meta.info["features"] = self._datasets[0].meta.info["features"] # Assume all datasets have the same features + # FIXME(mshukor): pad based on types in case we have more than one state? + self.keys_to_max_dim = { + ACTION: max_action_dim, + OBS_ENV_STATE: max_state_dim, + OBS_STATE: max_state_dim, + OBS_IMAGE: max_image_dim, + OBS_IMAGE_2: max_image_dim, + OBS_IMAGE_3: max_image_dim, + } + # self.meta.info["features"] = reshape_features_to_max_dim(self._datasets[0].meta.info["features"], reshape_dim=-1, keys_to_max_dim=self.keys_to_max_dim) + self.meta.info["features"] = reshape_features_to_max_dim( + union_features, reshape_dim=-1, keys_to_max_dim=self.keys_to_max_dim + ) + # reshape stats + for robot_type_, stats_ in self.stats.items(): + for feat_key, feat_stats in stats_.items(): + if feat_key in [ACTION, OBS_ENV_STATE, OBS_STATE]: + for k, v in feat_stats.items(): + if k in ["min", "mean"]: + pad_value = 0 + elif k in ["max", "std"]: + pad_value = 1 + else: + continue + self.stats[robot_type_][feat_key][k] = pad_tensor(v, max_size=self.keys_to_max_dim.get(feat_key, -1), pad_dim=-1, pad_value=pad_value) + self.meta.stats = self.stats + # self.meta.info["features"] = aggregate_features(self._datasets) + self.meta.tasks = { + repo_id: ds.meta.tasks for repo_id, ds in zip(self.repo_ids, self._datasets, strict=False) + } + self.meta.episodes = { + repo_id: ds.meta.episodes for repo_id, ds in zip(self.repo_ids, self._datasets, strict=False) + } + self.robot_types = [ds.meta.info["robot_type"] for ds in self._datasets] @property def repo_id_to_index(self): """Return a mapping from dataset repo_id to a dataset index automatically created by this class. @@ -1156,23 +1398,14 @@ class MultiLeRobotDataset(torch.utils.data.Dataset): def __getitem__(self, idx: int) -> dict[str, torch.Tensor]: if idx >= len(self): raise IndexError(f"Index {idx} out of bounds.") - # Determine which dataset to get an item from based on the index. - start_idx = 0 - dataset_idx = 0 - for dataset in self._datasets: - if idx >= start_idx + dataset.num_frames: - start_idx += dataset.num_frames - dataset_idx += 1 - continue - break - else: - raise AssertionError("We expect the loop to break out as long as the index is within bounds.") - item = self._datasets[dataset_idx][idx - start_idx] + dataset_idx = np.searchsorted(self.cumulative_sizes, idx, side="right").item() - 1 + local_idx = (idx - self.cumulative_sizes[dataset_idx]).item() + item = self._datasets[dataset_idx][local_idx] item["dataset_index"] = torch.tensor(dataset_idx) - for data_key in self.disabled_features: + item = create_padded_features(item, self.meta.info["features"]) + for data_key in self.disabled_features: # FIXME(mshukor): not in getitem? if data_key in item: del item[data_key] - return item def __repr__(self): diff --git a/lerobot/common/datasets/utils_must.py b/lerobot/common/datasets/utils_must.py new file mode 100644 index 000000000..147150d40 --- /dev/null +++ b/lerobot/common/datasets/utils_must.py @@ -0,0 +1,581 @@ +""" +Utils function by Mustafa to refactor +""" +import torch +import numpy as np +from lerobot.common.datasets.compute_stats import ( + aggregate_stats +) +from collections import defaultdict +OBS_IMAGE = "observation.image" +OBS_IMAGE_2 = "observation.image2" +OBS_IMAGE_3 = "observation.image3" + +def reshape_features_to_max_dim(features: dict, reshape_dim: int = -1, keys_to_max_dim: dict = {}) -> dict: + """Reshape features to have a maximum dimension of `max_dim`.""" + reshaped_features = {} + for key in features: + if key in keys_to_max_dim and keys_to_max_dim[key] is not None: + reshaped_features[key] = features[key] + shape = list(features[key]["shape"]) + if any([k in key for k in [OBS_IMAGE, OBS_IMAGE_2, OBS_IMAGE_3]]): # Assume square images + shape[-3] = keys_to_max_dim[key] + shape[-2] = keys_to_max_dim[key] + else: + shape[reshape_dim] = keys_to_max_dim[key] + reshaped_features[key]["shape"] = tuple(shape) + else: + reshaped_features[key] = features[key] + return reshaped_features + +def keep_datasets_with_valid_fps( + ls_datasets: list, min_fps: int = 1, max_fps: int = 100 +) -> list: + print(f"Keeping datasets with fps between {min_fps} and {max_fps}. Considering {len(ls_datasets)} datasets.") + for ds in ls_datasets: + if ds.fps < min_fps or ds.fps > max_fps: + print(f"Dataset {ds} has invalid fps: {ds.fps}. Removing it.") + ls_datasets.remove(ds) + print(f"Keeping {len(ls_datasets)} datasets with valid fps.") + return ls_datasets + +def keep_datasets_with_the_same_features_per_robot_type( + ls_datasets: list +) -> list: + """ + Filters datasets to only keep those with consistent feature shapes per robot type. + + Args: + ls_datasets (List): List of datasets, each with a `meta.info['robot_type']` + and `meta.episodes_stats` dictionary. + + Returns: + List: Filtered list of datasets with consistent feature shapes. + """ + robot_types = {ds.meta.info["robot_type"] for ds in ls_datasets} + datasets_to_remove = set() + + for robot_type in robot_types: + # Collect all stats dicts for this robot type + stats_list = [ + ep_stats + for ds in ls_datasets if ds.meta.info["robot_type"] == robot_type + for ep_stats in ds.meta.episodes_stats.values() + ] + if not stats_list: + continue + + # Determine the most common shape for each key + all_keys = {key for stats in stats_list for key in stats} + for ds in ls_datasets: + if ds.meta.info["robot_type"] != robot_type: + continue + for key in all_keys: + shape_counter = defaultdict(int) + + for stats in stats_list: + value = stats.get(key) + if value and "mean" in value and isinstance(value["mean"], (torch.Tensor, np.ndarray)): # FIXME(mshukor): check all stats; min, mean, max + shape_counter[value["mean"].shape] += 1 + if not shape_counter: + continue + + # Identify the most frequent shape + main_shape = max(shape_counter, key=shape_counter.get) + # Flag datasets that don't match the main shape + # for ds in ls_datasets: + first_ep_stats = next(iter(ds.meta.episodes_stats.values()), None) + if not first_ep_stats: + continue + value = first_ep_stats.get(key) + if value and "mean" in value and isinstance(value["mean"], (torch.Tensor, np.ndarray)) and value["mean"].shape != main_shape: + datasets_to_remove.add(ds) + break + + # Filter out inconsistent datasets + datasets_maks = [ds not in datasets_to_remove for ds in ls_datasets] + filtered_datasets = [ds for ds in ls_datasets if ds not in datasets_to_remove] + print(f"Keeping {len(filtered_datasets)} datasets. Removed {len(datasets_to_remove)} inconsistent ones. Inconsistent datasets:\n{datasets_to_remove}") + return filtered_datasets, datasets_maks + + + +def aggregate_stats_per_robot_type(ls_datasets) -> dict[str, dict[str, torch.Tensor]]: + """Aggregate stats of multiple LeRobot datasets into multiple set of stats per robot type. + + The final stats will have the union of all data keys from each of the datasets. + + The final stats will have the union of all data keys from each of the datasets. For instance: + - new_max = max(max_dataset_0, max_dataset_1, ...) + - new_min = min(min_dataset_0, min_dataset_1, ...) + - new_mean = (mean of all data) + - new_std = (std of all data) + """ + + robot_types = {ds.meta.info["robot_type"] for ds in ls_datasets} + stats = {robot_type: {} for robot_type in robot_types} + for robot_type in robot_types: + robot_type_datasets = [] + for ds in ls_datasets: + if ds.meta.info["robot_type"] == robot_type: + robot_type_datasets.extend(list(ds.meta.episodes_stats.values())) + # robot_type_datasets = [list(ds.episodes_stats.values()) for ds in ls_datasets if ds.meta.info["robot_type"] == robot_type] + stat = aggregate_stats(robot_type_datasets) + stats[robot_type] = stat + return stats + +def str_to_torch_dtype(dtype_str): + """Convert a dtype string to a torch dtype.""" + mapping = { + "float32": torch.float32, + "int64": torch.int64, + "int16": torch.int16, + "bool": torch.bool, + "video": torch.float32, # Assuming video is stored as uint8 images + } + return mapping.get(dtype_str, torch.float32) # Default to float32 + +def create_padded_features(item: dict, features: dict = {}): + for key, ft in features.items(): + if any([k in key for k in ["cam", "effort", "absolute"]]): # FIXME(mshukor): temporary hack + continue + shape = ft["shape"] + if len(shape) == 3: # images to torch format (C, H, W) + shape = (shape[2], shape[0], shape[1]) + if len(shape) == 1 and shape[0] == 1: # ft with shape are actually tensor(ele) + shape = [] + if key not in item: + dtype = str_to_torch_dtype(ft["dtype"]) + item[key] = torch.zeros(shape, dtype=dtype) + item[f"{key}_padding_mask"] = torch.tensor(0, dtype=torch.int64) + if "image" in key: # FIXME(mshukor): support other observations + item[f"{key}_is_pad"] = torch.BoolTensor([False]) + else: + item[f"{key}_padding_mask"] = torch.tensor(1, dtype=torch.int64) + return item + +ROBOT_TYPE_KEYS_MAPPING = { + "lerobot/stanford_hydra_dataset": "static_single_arm", + "lerobot/iamlab_cmu_pickup_insert": "static_single_arm", + "lerobot/berkeley_fanuc_manipulation": "static_single_arm", + "lerobot/toto": "static_single_arm", + "lerobot/roboturk": "static_single_arm", + "lerobot/jaco_play": "static_single_arm", + "lerobot/taco_play": "static_single_arm_7statedim", +} + +def pad_tensor( + tensor: torch.Tensor, max_size: int, pad_dim: int = -1, pad_value: float = 0.0 +) -> torch.Tensor: + is_numpy = isinstance(tensor, np.ndarray) + if is_numpy: + tensor = torch.tensor(tensor) + pad = max_size - tensor.shape[pad_dim] + if pad > 0: + pad_sizes = (0, pad) # pad right + tensor = torch.nn.functional.pad(tensor, pad_sizes, value=pad_value) + return tensor.numpy() if is_numpy else tensor + +def map_dict_keys(item: dict, feature_keys_mapping: dict, training_features: list = None, pad_key: str = "is_pad") -> dict: + """Maps feature keys from the dataset to the keys used in the model.""" + if feature_keys_mapping is None: + return item + features = {} + for key in item: + if key in feature_keys_mapping: + if feature_keys_mapping[key] is not None: + if training_features is None or feature_keys_mapping[key] in training_features: + features[feature_keys_mapping[key]] = item[key] + else: + if training_features is None or key in training_features or pad_key in key: + features[key] = item[key] + return features + + +TASKS_KEYS_MAPPING = { + "pranavsaroha/so100_legos4": {0: "Pick up the LEGO block and place it in the bowl of the same color as the LEGO block."}, + "pranavsaroha/so100_onelego2": {0: "Pick up the green LEGO block and place it in the green bowl."}, + "jpata/so100_pick_place_tangerine": {0: "Pick up the tangerine and place it."}, + "pranavsaroha/so100_onelego3": {0: "Pick up the green LEGO block and place it in the green bowl."}, + "pranavsaroha/so100_carrot_2": {0: "Pick up a carrot and put it in the bin."}, + "pranavsaroha/so100_carrot_5": {0: "Pick up a carrot and put it in the bin."}, + "pandaRQ/pick_med_1": {0: "Pick up the object and place it in the box."}, + "HITHY/so100_strawberry": {0: "Grasp a strawberry and put it in the bin."}, + "vladfatu/so100_above": {0: "Pick up red object and place it in the box."}, + "koenvanwijk/orange50-1": {0: "Pick up the orange object and but it in the LEGO box. "}, + "koenvanwijk/orange50-variation-2": {0: "Pick up the orange object and but it in the LEGO box. "}, + "FeiYjf/new_GtoR": {0: "Move along the line on the paper from start to end."}, + "CSCSXX/pick_place_cube_1.18": {0: "Pick up the cube and place it in the box."}, + "vladfatu/so100_office": {0: "Pick up the red object and place it in the box."}, + "dragon-95/so100_sorting": {0: "Pick up the object from box A and place it in box B."}, + "dragon-95/so100_sorting_1": {0: "Pick up the object from box A and place it in box B."}, + "nbaron99/so100_pick_and_place4": {0: "Pick up the triangular object and place it on a green sticker."}, + "Beegbrain/pick_place_green_block": {0: "Pick up the green block and place in the red cup."}, + "Ityl/so100_recording2": {0: "Pick up the red cube and place it on top of the blue cube."}, + "dragon-95/so100_sorting_2": {0: "Pick up the object from box A and place it in box B."}, + "dragon-95/so100_sorting_3": {0: "Pick up the object from box A and place it in box B."}, + "aractingi/push_cube_offline_data": {0: "Push the green cube to the yellow sticker."}, + "HITHY/so100_peach3": {0: "Grasp a peach and put it in the bin."}, + "HITHY/so100_peach4": {0: "Grasp a peach and put it on the plate."}, + "shreyasgite/so100_legocube_50": {0: "Grasp a lego block and put it in the bin."}, + "shreyasgite/so100_base_env": {0: "Grasp a lego block and put it in the bin."}, + "triton7777/so100_dataset_mix": { + 0: "Pick up the black tape and place it inside the white tape roll.", + 1: "Pick up the gift miniatures and place them in the black box.", + 2: "Sort the mixed objects into their appropriate categories.", + 3: "Place the pens into the pen holder.", + 4: "Place pens, bottles, and any suitable items into the pen holder, as appropriate.", + 5: "Place the oranges into the yellow basket.", + 6: "Stack the plates and place the cup on top." + }, + "Deason11/Open_the_drawer_to_place_items": {0: "Put the objects in the open drawer."}, + "Deason11/PLACE_TAPE_PUSH_DRAWER": {0: "Place the tape in the drawer and close it. "}, + "NONHUMAN-RESEARCH/SOARM100_TASK_VENDA": {0: "Pick up the object and place it in the box."}, + "mikechambers/block_cup_14": {0: "Grasp a block and put it in a cup."}, + "samsam0510/tooth_extraction_3": {0: "Extract the tooth and put it somewhere."}, + "samsam0510/tooth_extraction_4": {0: "Extarct the molar and put it somewhere."}, + "samsam0510/cube_reorientation_2": {0: "Rotate the object so it aligns with the silhouette on the table."}, + "samsam0510/cube_reorientation_4": {0: "Rotate the object so it aligns with respect to the line on the table."}, + "samsam0510/glove_reorientation_1": {0: "Rotate the glove so the bottom part aligns with the line on the table."}, + "DorayakiLin/so100_pick_charger_on_tissue": {0: "Pick up the charger and put it on the white tissue."}, + "zijian2022/noticehuman3": {0: "Notice human."}, + "liuhuanjim013/so100_th": {0: "Grasp a lego figure and put it in the box."}, + "Bartm3/tape_to_bin": {0: "Grasp a tape and put it in the bin."}, + + # Community dataset v2 + "Chojins/chess_game_009_white": { + 0: "Move the blue chess pieces to the highlighted squares." + }, + "1g0rrr/sam_openpi03": { + 0: "Pick up the cube and place it in the box." + }, + "sihyun77/suho_3_17_1": { + 0: "Grasp a lego block and put it in the bin." + }, + "sihyun77/sihyun_3_17_2": { + 0: "Grasp a lego block and put it in the bin." + }, + "sihyun77/suho_3_17_3": { + 0: "Grasp a lego block and put it in the bin." + }, + "sihyun77/sihyun_3_17_5": { + 0: "Grasp a lego block and put it in the bin." + }, + "Odog16/so100_cube_drop_pick_v1": { + 0: "Pick up the orange cube, release it, and then pick it up again." + }, + "sihyun77/sihyun_main_2": { + 0: "Grasp a lego block and put it in the bin." + }, + "sihyun77/suho_main_2": { + 0: "Grasp a lego block and put it in the bin." + }, + "Bartm3/dice2": { + 0: "Grasp a dice and put it in the bin." + }, + "sihyun77/sihyun_main_3": { + 0: "Grasp a lego block and put it in the bin." + }, + "Loki0929/so100_duck": { + 0: "Grasp red, green, yellow ducks and put them in the box." + }, + "pietroom/holdthis": { + 0: "Hold the object steadily without releasing it." + }, + "pietroom/actualeasytask": { + 0: "Grasp the marker and put it in the plastic box." + }, + "Beegbrain/pick_lemon_and_drop_in_bowl": { + 0: "Pick the yellow lemon and drop it in the red bowl." + }, + "Beegbrain/sweep_tissue_cube": { + 0: "Sweep the red cubes to the right with the tissue." + }, + "zijian2022/321": { + 0: "Grasp a lego block and put it in the bin." + }, + "1g0rrr/sam_openpi_solder1": { + 0: "Bring contact to the pad on the board." + }, + "1g0rrr/sam_openpi_solder2": { + 0: "Bring contact to the pad on the board." + }, + "gxy1111/so100_pick_place": { + 0: "Grasp a toy panda and put it in the cup." + }, + "Odog16/so100_cube_stacking_v1": { + 0: "Stack the cubes in the following order from bottom to top: black, blue, then orange." + }, + "sihyun77/mond_1": { + 0: "Grasp a lego block and put it in the bin." + }, + "andlyu/so100_indoor_1": { + 0: "Locate and grasp the blueberry." + }, + "andlyu/so100_indoor_3": { + 0: "Locate and grasp the blueberry." + }, + "frk2/so100large": { + 0: "Pick up roll of tape and put it in the bin." + }, + "lirislab/sweep_tissue_cube": { + 0: "Sweep the red cubes to the right with the tissue bag." + }, + "lirislab/lemon_into_bowl": { + 0: "Pick the yellow lemon and drop it in the red bowl" + }, + "lirislab/red_cube_into_green_lego_block": { + 0: "Put the red cube on top of the yellow cube." + }, + "lirislab/red_cube_into_blue_cube": { + 0: "Put the red cube on top of the blue cube." + }, + "00ri/so100_battery": { + 0: "Grasp a battery and put it in the bin." + }, + "frk2/so100largediffcam": { + 0: "Pick up roll of tape and put it in the bin" + }, + "FsqZ/so100_1": { + 0: "Put the yellow cube inside the purple box." + }, + "ZGGZZG/so100_drop0": { + 0: "Grasp a ball and put it in the hole." + }, + "Chojins/chess_game_000_white_red": { + 0: "Move the red chess pieces to the highlighted squares." + }, + "smanni/train_so100_fluffy_box": { + 0: "Grasp a small object and place it in the box." + }, + "ganker5/so100_push_20250328": { + 0: "Grasp a lego block and put it in the bin." + }, + "ganker5/so100_dataline_0328": { + 0: "Grasp a lego block and put it in the bin." + }, + "ganker5/so100_color_0328": { + 0: "Grasp a lego block and put it in the bin." + }, + "CrazyYhang/A1234-B-C_mvA2B": { + 0: "Move the top disk from the left column to the middle column." + }, + "RasmusP/so100_Orange2Green": { + 0: "Grasp the orange block and drop it in the box." + }, + "sixpigs1/so100_pick_cube_in_box": { + 0: "Pick up the red cube and put it in the box." + }, + "ganker5/so100_push_20250331": { + 0: "Grasp a lego block and put it in the bin." + }, + "ganker5/so100_dataline_20250331": { + 0: "Grasp a lego block and put it in the bin." + }, + "lirislab/put_caps_into_teabox": { + 0: "Pick the coffee capsule and put it into the top drawer of the teabox" + }, + "lirislab/close_top_drawer_teabox": { + 0: "Close the top drawer of the teabox" + }, + "lirislab/open_top_drawer_teabox": { + 0: "Open the top drawer of the teabox" + }, + "lirislab/unfold_bottom_right": { + 0: "Unfold the bag from bottom right corner" + }, + "lirislab/push_cup_target": { + 0: "Push the red cup to the pink target" + }, + "lirislab/put_banana_bowl": { + 0: "Put the banana into the red bowl" + }, + "Chojins/chess_game_001_blue_stereo": { + 0: "Move the blue chess pieces to the highlighted squares" + }, + "Chojins/chess_game_001_red_stereo": { + 0: "Move the red chess pieces to the highlighted squares" + }, + "ganker5/so100_toy_20250402": { + 0: "Grasp a lego block and put it in the bin." + }, + "Gano007/so100_medic": { + 0: "Grasp a medic box and put it in the bin." + }, + "00ri/so100_battery_bin_center": { + 0: "Grasp a battery and put it in the bin." + }, + "paszea/so100_whale_2": { + 0: "Grasp a whale and put it in the plate." + }, + "lirislab/fold_bottom_right": { + 0: "Fold the bag from the bottom right corner." + }, + "lirislab/put_coffee_cap_teabox": { + 0: "Put the coffee capsule into the top drawer of the teabox." + }, + "therarelab/so100_pick_place_2": { + 0: "Pick a plaster roll and place it to the blue sticker." + }, + "paszea/so100_whale_3": { + 0: "Grasp a whale and put it in the plate." + }, + "paszea/so100_whale_4": { + 0: "Grasp a whale and put it in the plate." + }, + "paszea/so100_lego": { + 0: "Grasp a lego and put it in the basket." + }, + "LemonadeDai/so100_coca": { + 0: "Grasp the Coca-Cola can and orient it upright with the top facing up." + }, + "zijian2022/backgrounda": { + 0: "Grasp a lego block and put it in the bin." + }, + "zijian2022/backgroundb": { + 0: "Grasp a lego block and put it in the bin." + }, + "356c/so100_nut_sort_1": { + 0: "Pick up the steel nuts and sort them by color." + }, + "Mwuqiu/so100_0408_muti": { + 0: "Grasp a yellow duck and put it in the box." + }, + "aimihat/so100_tape": { + 0: "Pick up the tape and put it in the bowl." + }, + "lirislab/so100_demo": { + 0: "Put the banana into the red bowl." + }, + "356c/so100_duck_reposition_1": { + 0: "Grasp the tool and use it to move the duck to the indicated position." + }, + "zijian2022/sort1": { + 0: "Grasp a box and sort it by color: place grey boxes on the left and black boxes on the right." + }, + "weiye11/so100_410_zwy": { + 0: "Pick up the cube and place it on the black circle." + }, + "VoicAndrei/so100_banana_to_plate_only": { + 0: "Pick up the banana and place it on the plate." + }, + "sixpigs1/so100_stack_cube_error": { + 0: "Pick up the red cube and stack it on the green cube with position offset when grasping.", + 1: "Pick up the red cube and stack it on the green cube with gripper error when grasping.", + 2: "Pick up the red cube and stack it on the green cube with position offset when stacking.", + 3: "Pick up the red cube and stack it on the green cube without errors", + }, + "isadev/bougies3": { + 0: "Grab the candle wick by the aluminium plate and place it in the box." + }, + "zijian2022/close3": { + 0: "Grasp a lego block and put it in the bin." + }, + "bensprenger/left_arm_yellow_brick_in_box_v0": { + 0: "Grasp the yellow lego block and put it in the box." + }, + "bensprenger/left_arm_yellow_brick_in_box_with_purple_noise_v0": { + 0: "Grasp a yellow lego block and put it in the bin." + }, + "roboticshack/team16-can-stacking": { + 0: "Grasp the flipped cup and stack it on top of the midpoint between the two other cups to create a tower" + }, + "bensprenger/right_arm_p_brick_in_box_with_y_noise_v0": { + 0: "Grasp the purple lego block and put it in the box." + }, + "pierfabre/pig2": { + 0: "Pick the pig and place it to the right." + }, + "zijian2022/insert2": { + 0: "Grasp a lego block and put it in the bin." + }, + "roboticshack/team-7-right-arm-grasp-tape": { + 0: "Grasp the tape and put it in the box." + }, + "pierfabre/pig3": { + 0: "Pick the pig and place it to the right." + }, + "Jiangeng/so100_413": { + 0: "Pick up the cube and place it on top of the black circle." + }, + "roboticshack/team9-pick_cube_place_static_plate": { + 0: "Pick up the green cube and place on orange plate." + }, + "AndrejOrsula/lerobot_double_ball_stacking_random": { + 0: "Stack the balls on top of each other." + }, + "roboticshack/left-arm-grasp-lego-brick": { + 0: "Grasp the lego brick and put it in the box." + }, + "roboticshack/team-7-left-arm-grasp-motor": { + 0: "Grasp the black motor and put it in the box." + }, + "pierfabre/cow2": { + 0: "Pick the cow and place it to the right." + }, + "pierfabre/sheep": { + 0: "Pick the sheep and place it to the right." + }, + "roboticshack/team9-pick_chicken_place_plate": { + 0: "Pick up the chicken and place on orange plate" + }, + "roboticshack/team13-two-balls-stacking": { + 0: "Stack the balls on top of each other." + }, + "tkc79/so100_lego_box_1": { + 0: "Grasp a lego block and put it in the box." + }, + "pierfabre/rabbit": { + 0: "Pick the rabbit and place it to the right.", + 1: "Pick the rabbit and put it to the right" + }, + "roboticshack/team13-three-balls-stacking": { + 0: "Stack the balls on top of each other." + }, + "pierfabre/horse": { + 0: "Pick the horse and place it to the right." + }, + "pierfabre/chicken": { + 0: "Pick the chicken and place it to the right." + }, + "roboticshack/team16-water-pouring": { + 0: "Pouring water from one cup to another cup" + }, + "ad330/cubePlace": { + 0: "Grasp white cube and place it in the bowl." + }, + "paszea/so100_lego_2cam": { + 0: "Grap lego blocks and put them in the plate." + }, + "bensprenger/chess_game_001_blue_stereo": { + 0: "Move the blue chess pieces to the highlighted squares." + }, + "Mohamedal/put_banana": { + 0: "Put the banana in the red bowl." + }, + "tkc79/so100_lego_box_2": { + 0: "Grasp a lego block and put it in the box." + }, + "samanthalhy/so100_herding_1": { + 0: "Grasp a green tool and herd all the particles to the grey bin." + }, + "jlesein/TestBoulon7": { + 0: "Pick up the bolt and put it on the plate." + }, + # V3 with VLM (Qwen-VL-2.5-instruct) annotation + "satvikahuja/mixer_on_off_new_1": {0: "Press the button on the blender."}, + "andy309/so100_0314_fold_cloths": {0: "fold the cloths, use two cameras, two arms."}, + "jchun/so100_pickplace_small_20250323_120056": {0: "Grasp items from white bowl and place in black tray"}, + "Ofiroz91/so_100_cube2bowl": {0: "placing cube inside a red bawl"}, + "ZCM5115/so100_1210": {0: "picks up the USB cable."}, + "francescocrivelli/carrot_eating": {0: "pick up carrot and bring to mouth"}, + "ZCM5115/so100_2Arm3cameras_movebox": {0: "Pick up the white box from the table."}, + "pranavsaroha/so100_carrot_1": {0: "pick a carrot and put it in the bin"}, + "pranavsaroha/so100_carrot_3": {0: "pick a carrot and put it in the bin"}, + "maximilienroberti/so100_lego_red_box": {0: "Placing the red Lego in the red box bin."}, "pranavsaroha/so100_squishy": {0: "pick a squishy and put it in the bin"}, "rabhishek100/so100_train_dataset": {0: "picks tape and places it in a cup."}, "pranavsaroha/so100_squishy100": {0: "pick a squishy and put it in the bin"}, "pandaRQ/pickmed": {0: "Place the green block on the table."}, "swarajgosavi/act_kikobot_pusht_real": {0: "picks up the red block."}, "pranavsaroha/so100_squishy2colors_1": {0: "pick the squishies and put them in the bins with their corresponding colors"}, "Chojins/chess_game_001_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "jmrog/so100_sweet_pick": {0: "Pick up the candy and place it in the bowl."}, "Chojins/chess_game_002_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "pranavsaroha/so100_squishy2colors_2_new": {0: "pick the squishies and put them in the bins with their corresponding colors"}, "Chojins/chess_game_003_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "Chojins/chess_game_004_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "Chojins/chess_game_005_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "Chojins/chess_game_006_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "Chojins/chess_game_007_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "koenvanwijk/blue52": {0: "places blue block on red LEGO piece."}, "jlitch/so100multicam7": {0: "pick up brick and put in bin"}, "vladfatu/so100_ds": {0: "Pick up the cube and place it in the box."}, "Chojins/chess_game_000_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "satvikahuja/orange_mixer_1": {0: "pick orange and place in mixer"}, "satvikahuja/mixer_on_off": {0: "switch the mixer on or off"}, "satvikahuja/orange_pick_place_new1": {0: "Pick up the orange and place it in the bowl."}, "satvikahuja/mixer_on_off_new": {0: "Adjust the s position."}, "FeiYjf/Makalu_push": {0: "Pick up the blue cube."}, "chmadran/so100_dataset04": {0: "picks the blue block and places it in the red cup."}, "FeiYjf/Maklu_dataset": {0: "Pick up the blue cube and place it on the paper."}, "FeiYjf/new_Dataset": {0: "Pick up the blue cube."}, "satvikahuja/mixer_on_off_new_4": {0: "Place the lid on the blender."}, "CSCSXX/pick_place_cube_1.17": {0: "Pick up the red block and place it in the box."}, "liyitenga/so100_pick_taffy3": {0: "Place the eraser in the container."}, "liyitenga/so100_pick_taffy6": {0: "Pick up the toy and place it in the purple cup."}, "yuz1wan/so100_pickplace": {0: "Pick the pink block and place it in the paper cup."}, "liyitenga/so100_pick_taffy7": {0: "Pick up the toy and place it in the box."}, "swarajgosavi/act_kikobot_block_real": {0: "Pick up the blue cube and place it in the box."}, "SeanLMH/so100_picknplace_v2": {0: "picks up blue cube and places it in yellow box."}, "DimiSch/so100_50ep_2": {0: "Place the yellow object in the bowl."}, "DimiSch/so100_50ep_3": {0: "Pick the yellow button from the table."}, "SeanLMH/so100_picknplace": {0: "Pick up the blue block and place it in the yellow box."}, "nbaron99/so100_pick_and_place2": {0: "picks up the white object."}, "chmadran/so100_dataset08": {0: "places blue block on paper."}, "Ityl/so100_recording1": {0: "Putting the red square onto the yellow piece"}, "ad330/so100_box_pickPlace": {0: "places jar in box."}, "carpit680/giraffe_task": {0: "Grasp a block and put it in the bin."}, "carpit680/giraffe_sock_demo_1": {0: "Grasp a sock off the floor."}, "DimiSch/so100_terra_50_2": {0: "Grasp a lego block and put it in the bin."}, "aractingi/push_cube_offline_data_cropped_resized": {0: "Push the green cube to the yellow sticker"}, "FeiYjf/Test_NNNN": {0: "Pick up the purple cube and move it to the right."}, "HITHY/so100_peach": {0: "Grasp a peach and put it in the bin."}, "zaringleb/so100_cube_4_binary": {0: "Grasp a lego block and put it in the bin."}, "FeiYjf/Grab_Pieces": {0: "places the black object on the table."}, "hegdearyandev/so100_eraser_cup_v1": {0: "picks up the red object."}, "jbraumann/so100_1902": {0: "picks up the yellow ball."}, "zaringleb/so100_cube_5_linear": {0: "Grasp a lego block and put it in the bin."}, "samsam0510/tape_insert_1": {0: "Grasp a red tape and put it on the box."}, "samsam0510/tape_insert_2": {0: "Grasp a red tape and put it in the yellow tape."}, "pengjunkun/so100_push_to_hole": {0: "Push the T into the hole."}, "Deason11/Random_Kitchen": {0: "Pick up the cup and place it on the table."}, "Loki0929/so100_100": {0: "Grasp a rubber duck and put it in the box."}, "speedyyoshi/so100_grasp_pink_block": {0: "Grasp a lego block and put it in the bin."}, "lirislab/green_lego_block_into_mug": {0: "pick the green block and place it in the red cup"}, "kevin510/lerobot-cat-toy-placement": {0: "Grasp the cat toy and put it in the cup."}, "NONHUMAN-RESEARCH/SOARM100_TASK_VENDA_BOX": {0: "Move the cube to the right side of the table."}, "zijian2022/noticehuman5": {0: "picks up the box."}, "zijian2022/noticehuman70": {0: "Stop movement when human encounter testbed.", 1: "Stop movement when human encounter testbed w/ trigger."}, "Bartm3/tape_to_bin": {0: "Grasp a tape and put it in the bin."}, "Pi-robot/barbecue_flip": {0: "Pick up the orange cone and place it on the table."}, "Pi-robot/barbecue_put": {0: "Pick up the stick and place it in the grill."}, "sshh11/so100_orange_50ep_1": {0: "Grasp an orange object and put it in the bin."}, "sshh11/so100_orange_50ep_2": {0: "Grasp an orange object and put it in the bin."}, "DorayakiLin/so100_pick_cube_in_box": {0: "Pick up the red cube and put it in the box."}, "Bartm3/tape_to_bin2": {0: "Grasp a tape and put it in the bin."}, "andy309/so100_0311_1152": {0: "Grasp and put it in the bin."}, "sihyun77/suho_so100": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/si_so100": {0: "Grasp a lego block and put it in the bin."}, "shreyasgite/so100_base_left": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/suho_red": {0: "Grasp a lego block and put it in the bin."}, "liuhuanjim013/so100_block": {0: "Grasp a lego block and put it in the bin."}, "joaoocruz00/so100_makeitD1": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/suho_angel": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/sihyun_king": {0: "Grasp a lego block and put it in the bin."}, "acrampette/third_arm_01": {0: "Pick up the circuit board from the table."}, "Winster/so100_cube": {0: "Grasp a lego block and put it in the bin."}, "1g0rrr/sam_openpi03": {0: "Grasp a blue cube and put it in the gray box."}, "thedevansh/mar16_1336": {0: "Grasp a lego block and put it in the bin."}, "hkphoooey/throw_stuffie": {0: "Grab stuffed animal and throw it on the dot."}, "acrampette/third_arm_02": {0: "Pick up the tie and place it in the box."}, "kumarhans/so100_tape_task": {0: "Grasp a roll of tape and put it over the candle case."}, "Odog16/so100_tea_towel_folding_v1": {0: "Fold tea towel into quarters"}, "pietroom/first_task_short": {0: "Pick up the marker from the box."}, "zijian2022/c0": {0: "Grasp a lego block and put it in the bin based on color.", 1: "Grasp a lego block and put it in the bin."}, "1g0rrr/sam_openpi_solder1": {0: "bring contact to the pad on the board."}, "1g0rrr/sam_openpi_solder2": {0: "bring contact to the pad on the board."}, "bnarin/so100_tic_tac_toe_we_do_it_live": {0: "move tic tac toe as player 2."}, "chmadran/so100_home_dataset": {0: "Grasp a lego block and put it in the bin."}, "baladhurgesh97/so100_final_picking_3": {0: "Grasp a carrot, plastic bottle and put it in respective bins."}, "zaringleb/so100_cube_6_2d": {0: "Grasp a lego block and put it in the bin."}, "ZGGZZG/so100_drop1": {0: "Grasp a cube and put it in the right place."}, "abhisb/so100_51_ep": {0: "Pick up the cube and place it in the box."}, "allenchienxxx/so100Test": {0: "Grasp a lego block and put it in the bin."}, "lizi178119985/so100_jia": {0: "Grasp a lego block and put it in the bin."}, "andrewcole712/so100_tape_bin_place": {0: "Place the tape in the wooden box."}, "Gano007/so100_doliprane": {0: "Grasp a medic box and put it in the bin."}, "XXRRSSRR/so100_v3_num_episodes_50": {0: "Grasp a box and put it in the side."}, "Gano007/so100_gano": {0: "Grasp a box and put it in the bin."}, "paszea/so100_whale_grab": {0: "Grasp a whale and put it in the plate."}, "Clementppr/lerobot_pick_and_place_dataset_world_model": {0: "Grasp a fruit and put it in the cup."}, "RasmusP/so100_dataset50ep": {0: "Grasp a square block and put it in the box."}, "Gano007/so100_second": {0: "Grasp a yellow box and put it in the bin."}, "zaringleb/so100_cude_linear_and_2d_comb": {0: "Grasp a lego block and put it in the bin."}, "zijian2022/digitalfix3": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/mond_13": {0: "Grasp a lego block and put it in the bin."}, "356c/so100_rope_reposition_1": {0: "Grasp rope and reposition."}, "paszea/so100_lego_mix": {0: "Grasp lego blocks and put them in the plate."}, "jiajun001/eraser00_2": {0: "picks tissue paper from box."}, "VoicAndrei/so100_banana_to_plate_rebel_full": {0: "Pick up the banana and place it on the place"}, "isadev/bougies1": {0: "Put the candles in the box."}, "sixpigs1/so100_pick_cube_in_box_error": {0: "Pick up the red cube and put it in the box with position offset when grasping.", 1: "Pick up the red cube and put it in the box with gripper error when grasping.", 2: "Pick up the red cube and put it in the box with position offset when releasing.", 3: "Pick up the red cube and put it in the box without errors."}, "sixpigs1/so100_push_cube_error": {0: "Push the blue cube to the red and white target with position offset when reaching.", 1: "Push the blue cube to the red and white target with position offset when pushing.", 2: "Push the blue cube to the red and white target with gripper error when pushing.", 3: "Push the blue cube to the red and white target without errors."}, "sixpigs1/so100_pull_cube_error": {0: "Pull the yellow cube to the red and white target with position offset when reaching.", 1: "Pull the yellow cube to the red and white target with position offset when pulling.", 2: "Pull the yellow cube to the red and white target with gripper error when pulling.", 3: "Pull the yellow cube to the red and white target without errors."}, "isadev/bougies2": {0: "grab the candle wick and place it in the tray."}, "therarelab/med_dis_rare_6": {0: "places green object in box."}, "sixpigs1/so100_pull_cube_by_tool_error": {0: "Pick up the L-shaped tool and pull the purple cube by the tool with position offset when grasping.", 1: "Pick up the L-shaped tool and pull the purple cube by the tool with rotation offset when grasping.", 2: "Pick up the L-shaped tool and pull the purple cube by the tool with gripper error when grasping.", 3: "Pick up the L-shaped tool and pull the purple cube by the tool with position offset when lowering.", 4: "Pick up the L-shaped tool and pull the purple cube by the tool with rotation offset when lowering.", 5: "Pick up the L-shaped tool and pull the purple cube by the tool with position offset when pulling.", 6: "Pick up the L-shaped tool and pull the purple cube by the tool with gripper error when pulling.", 7: "Pick up the L-shaped tool and pull the purple cube by the tool without errors."}, "sixpigs1/so100_insert_cylinder_error": {0: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with position offset when grasping.", 1: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with gripper error when grasping.", 2: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with rotation offset when uprighting.", 3: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with position offset when inserting.", 4: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with choice error when inserting.", 5: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf without errors."}, "lirislab/guess_who_no_cond": {0: "Place the card in the slot."}, "lirislab/guess_who_lighting": {0: "Pick up the card from the shelf."}, "nguyen-v/so100_press_red_button": {0: "The places the cube in the box."}, "nguyen-v/so100_bimanual_grab_lemon_put_in_box2": {0: "Grab the lemon with the black arm, then give it to the green arm, then place the lemon in the cardboard box with the green arm."}, "nguyen-v/press_red_button_new": {0: "Press the red button with the black arm"}, "nguyen-v/so100_rotate_red_button": {0: "Rotate the red button clockwise with the black arm"}, "roboticshack/team10-red-block": {0: "Pick a red lego block and move it to the right."}, "Cidoyi/so100_all_notes_1": {0: "Connect the cable to the device."}, "roboticshack/team11_pianobot": {0: "Point at the keyboard."}, "roboticshack/team2-guess_who_so100": {0: "Pick up the card from the shelf."}, "roboticshack/team2-guess_who_so100_light": {0: "Place the card in the slot."}, "roboticshack/team2-guess_who_less_ligth": {0: "Pick up the card and place it in the slot."}, "jiajun001/eraser00_3": {0: "Pick up the white object from the table."}, + "Setchii/so100_grab_ball": {0: "Grasp a ball and put it on a goblet."}, + # V4 with VLM annotation + 'ctbfl/sort_battery': {0: 'put the battery into battery_box'}, 'lerobot/aloha_static_screw_driver': {0: 'Pick up the screwdriver with the right arm, hand it over to the left arm then place it into the cup.'}, 'lerobot/aloha_static_candy': {0: 'Pick up the candy and unwrap it.'}, 'lerobot/aloha_mobile_wipe_wine': {0: 'Pick up the wet cloth on the faucet and use it to clean the spilled wine on the table and underneath the glass.'}, 'lerobot/aloha_static_coffee': {0: "Place the coffee capsule inside the capsule container, then place the cup onto the center of the cup tray, then push the 'Hot Water' and 'Travel Mug' buttons."}, 'lerobot/aloha_static_towel': {0: 'Pick up a piece of paper towel and place it on the spilled liquid.'}, 'lerobot/aloha_static_vinh_cup': {0: 'Pick up the platic cup with the right arm, then pop its lid open with the left arm.'}, 'lerobot/aloha_static_vinh_cup_left': {0: 'Pick up the platic cup with the left arm, then pop its lid open with the right arm.'}, 'lerobot/aloha_static_ziploc_slide': {0: 'Slide open the ziploc bag.'}, 'lerobot/aloha_static_coffee_new': {0: 'Place the coffee capsule inside the capsule container, then place the cup onto the center of the cup tray.'}, 'lerobot/aloha_static_cups_open': {0: 'Pick up the plastic cup and open its lid.'}, 'lerobot/aloha_static_pro_pencil': {0: 'Pick up the pencil with the right arm, hand it over to the left arm then place it back onto the table.'}, 'lerobot/aloha_mobile_wash_pan': {0: 'Pick up the pan, rinse it in the sink and then place it in the drying rack.'}, 'lerobot/aloha_mobile_cabinet': {0: 'Open the top cabinet, store the pot inside it then close the cabinet.'}, 'lerobot/aloha_mobile_chair': {0: 'Push the chairs in front of the desk to place them against it.'}, 'lerobot/aloha_mobile_elevator': {0: 'Take the elevator to the 1st floor.'}, 'aliberts/koch_tutorial': {0: 'Pick the Lego block and drop it in the box on the right.'}, 'underctrl/single-block_multi-color_pick-up_50': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/single-block_blue-color_pick-up_80': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/mutli-stacked-block_mutli-color_pick-up_80': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/single-stacked-block_two-color_pick-up_80': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/single-stacked-block_mutli-color_pick-up_80': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/handcamera_single_blue': {0: 'Pick the Lego block and drop it in the box on the right.'}, 'cmcgartoll/cube_color_organizer': {0: 'Organize blue cube', 1: 'Organize red cube', 2: 'Organize yellow cube'}, 'T-K-233/koch_k1_pour_shot': {0: 'Place the glass on the table.'}, 'Beegbrain/stack_2_cubes': {0: 'picks up the red block.'}, 'seeingrain/pick_place_lego': {0: 'Place the cube in the basket.'}, 'seeingrain/pick_place_lego_wider_range_richard': {0: 'Pick up the blue cube and place it in the basket.'}, 'seeingrain/pick_place_lego_wider_range_dang': {0: 'Pick up the cube and place it in the basket.'}, 'seeingrain/pick_place_lego_wider_range_dong': {0: 'Pick up the blue object and place it in the basket.'}, 'seeingrain/pick_lego_to_hand': {0: 'places blue object on table.'}, 'seeingrain/pick_place_pink_lego': {0: 'Pick up the red cube and place it in the basket.'}, 'seeingrain/pick_place_pink_lego_few_samples': {0: 'pick_place_pink_lego_few_samples '}, 'seeingrain/one_shot_learning_18episodes': {0: 'Pick up the red block and place it in the basket.'}, 'helper2424/hil-serl-push-circle-classifier': {0: 'Push small circle object to the correct position'}, 'seeingrain/lego_3cameras': {0: 'Pick up the red block and place it in the basket.'}, 'Lugenbott/koch_1225': {0: 'Pick up the blue block and place it in the red box.'}, 'twerdster/koch_training_red': {0: 'Pick up the red block.'}, 'dboemer/koch_50-samples': {0: 'Pick up the red block and place it on top of the yellow box.'}, 'seeingrain/241228_pick_place_2cams': {0: 'Place the cube in the basket.'}, 'Eyas/grab_pink_lighter_10_per_loc': {0: 'Pick up the pink object from the table.'}, 'Eyas/grab_bouillon': {0: 'picks up the box and places it in the box.'}, 'twerdster/koch_new_training_red': {0: 'move red cube into cellotape circle'}, 'andabi/shoes_easy': {0: 'picks up the shoe.'}, 'andabi/D2': {0: 'Pick up the shoe and place it on the table.'}, 'Beegbrain/oc_stack_cubes': {0: 'stack the red cube on the blue cube'}, 'abougdira/cube_target': {0: 'put_the cube on the yellow target'}, 'andabi/D3': {0: 'picks up the shoe.'}, 'andabi/D4': {0: 'picks up the shoe.'}, 'andabi/D5': {0: 'places shoe on table.'}, 'andabi/D6': {0: 'picks up the shoe.'}, 'andabi/D7': {0: 'picks up the shoe.'}, 'jainamit/koch_realcube3': {0: 'pick up the cube real with keyboard input'}, 'jainamit/koch_pickcube': {0: 'Pick up the blue cube and place it in the box.'}, 'andabi/D8': {0: 'picks up the shoe.'}, 'andabi/D9': {0: 'The picks up the paper and places it on the table.'}, 'andabi/D10': {0: 'picks up the shoe.'}, 'andabi/D11': {0: 'The picks up the paper and places it on the table.'}, 'andabi/D12': {0: 'The places the cube in the box.'}, 'andabi/D13': {0: 'picks up shoes.'}, 'andabi/D14': {0: 'picks up shoes.'}, 'andabi/D15': {0: 'picks up the shoe.'}, 'rgarreta/koch_pick_place_lego': {0: 'Pick the Lego block and drop it in the box on the right.'}, 'shin1107/koch_train_block': {0: 'Grasp a block and put it in the hole.'}, 'andabi/D16': {0: 'picks up the shoe.'}, 'TrossenRoboticsCommunity/aloha_fold_tshirt': {0: 'Fold the t-shirt.'}, 'rgarreta/koch_pick_place_lego_v2': {0: 'Pick the Lego block and drop it in the box on the right.'}, '1g0rrr/screw1': {0: 'Grasp a lego block and put it in the bin.'}, 'ncavallo/moss_train_grasp': {0: 'Grasp a lego block and put it in the bin.'}, 'andabi/D17': {0: 'picks up the shoe.'}, 'ma3oun/rpi_squares_1': {0: 'Raspberry Pi 5 squares recording 1'}, 'shin1107/koch_move_block_with_some_shapes': {0: 'Grasp a block and put it in the hole with some shapes.'}, 'jannick-st/classifier': {0: 'Move the blue object to the right side of the table.'}, 'rgarreta/koch_pick_place_lego_v3': {0: 'Pick the Lego block and drop it in the box on the right. Top and wrist cameras.'}, 'TrossenRoboticsCommunity/aloha_stationary_logo_assembly': {0: 'Assemble the Trossen Robotics Logo.'}, 'rgarreta/koch_pick_place_lego_v6': {0: 'Grasp a lego block and put it in the bin.'}, 'Beegbrain/put_green_into_blue_bin': {0: 'Put the green cube into the blue bin'}, 'Beegbrain/put_screwdriver_box': {0: 'Put the screwdriver into the box'}, 'Beegbrain/align_three_pens': {0: 'picks up a pen.'}, 'Beegbrain/stack_green_on_blue_cube': {0: 'Stack the blue cube on top of the green cube'}, 'Beegbrain/align_cubes_green_blue': {0: 'Put the green cubes on the left and the blue cube on the right'}, 'IPEC-COMMUNITY/ucsd_kitchen_dataset_lerobot': {0: 'Turn on the faucet', 1: 'Put the bowl inside the kitchen cabinet', 2: 'Open the oven door', 3: 'Place the teapot on the stove', 4: 'Put the white box into the sink', 5: 'Open the carbinet door', 6: 'Put the green box into the sink', 7: 'Put the canned spam into the sink'}, 'dkdltu1111/omx-bottle1': {0: 'Grasp a lego block and put it in the bin.'}, 'rgarreta/koch_pick_place_lego_v7': {0: 'Grasp a lego block and put it in the bin.'}, 'rgarreta/koch_pick_place_lego_v8': {0: 'Grasp a lego block and put it in the bin.'}, 'IPEC-COMMUNITY/berkeley_mvp_lerobot': {0: 'push wooden cube', 1: 'pick detergent from the sink', 2: 'reach red block', 3: 'pick yellow cube', 4: 'close fridge door', 5: 'pick fruit'}, 'BlobDieKatze/GrabBlocks': {0: 'Grasp a lego block and put it in the bin.'}, 'Yuanzhu/koch_bimanual_grasp_0': {0: 'places the yellow block on the mousepad.'}, 'Yuanzhu/koch_bimanual_grasp_3': {0: 'picks up a yellow block.'}, 'takuzennn/aloha-pick100': {0: 'arm pick pen and put it into the cup'}, 'abbyoneill/pusht': {0: 'Grasp a lego block and put it in the bin.'}, 'ncavallo/moss_train_grasp_new': {0: 'Grasp a lego block and put it in the bin.'}, 'mlfu7/pi0_conversion_no_pad_video': {0: 'pickplace deer greybowl', 1: 'stack red on green', 2: 'stack orange cup to yellow cup', 3: 'put orange cup into yellow cup', 4: 'put push red pen to blue pen', 5: 'put tiger to black bowl', 6: 'put potato in bot to black bowl', 7: 'pick up green triangle', 8: 'put push blue pen to red pen', 9: 'close drawer', 10: 'put push green block to red', 11: 'pickup potato', 12: 'open drawer', 13: 'put closing tongs', 14: 'poke block', 15: 'put push blue cube', 16: 'poke tiger', 17: 'pick red cube into black bowl', 18: 'pick blue cube stack on wood block', 19: 'pick blue cube into grey bowl', 20: 'put red ball in black bowl', 21: 'pick green triangle into pink bowl', 22: 'pick red ball into pink bowl', 23: 'poke green triangle', 24: 'poke grey bowl', 25: 'put blue cube pink bowl', 26: 'put red cube into black bowl', 27: 'put deer into gray bowl', 28: 'put red ball into pink bowl', 29: 'put green triangle into pink bowl', 30: 'put pour from yellow cup into black bowl', 31: 'put pour blue cup into pink bowl', 32: 'put brown cube into gray bowl'}, 'pepijn223/lekiwi_pen': {0: 'Fold the jeans.'}, 'TrossenRoboticsCommunity/aloha_baseline_dataset': {0: 'Pick up a blue block and put it in a green bowl. Baseline dataset for testing'}, 'KeWangRobotics/piper_rl_1': {0: 'Pick up the cube and place it in the box.'}, 'nduque/cam_setup2': {0: 'Grasp a green block and put it in the bin.'}, 'KeWangRobotics/piper_rl_1_cropped_resized': {0: 'picks up the cube.'}, 'abbyoneill/new_dataset_pick_place': {0: 'Grasp a lego block and put it in the bin.'}, 'abbyoneill/thurs1120pickplace': {0: 'Grasp a lego block and put it in the bin.'}, 'abbyoneill/data_w_mug': {0: 'Grasp a lego block and put it in the bin.'}, 'pepijn223/lekiwi_drive_in_circle': {0: 'Pick up the red object and place it on the table.'}, 'pepijn223/lekiwi_block_cleanup2': {0: 'Put red block in black box'}, 'KeWangRobotics/piper_rl_2': {0: 'Move the cube to the right side of the table.'}, 'KeWangRobotics/piper_rl_2_cropped_resized': {0: 'Move the block to the right side of the table.'}, 'hannesill/koch_pnp_simple_50': {0: 'Grasp a small block with a specific orientation and put it in the bin with a specific position and orientation.'}, 'KeWangRobotics/piper_rl_3': {0: 'Pick up the wooden block.'}, 'KeWangRobotics/piper_rl_3_cropped_resized': {0: 'Place the block in the box.'}, 'hannesill/koch_pnp_2_blocks_2_bins_200': {0: 'Grasp the blue block first and put it in the first bin that has a specific position and orientation. Then grasp the white block and put it in the second bin that has a specific position and orientation.'}, 'ellen2imagine/pusht_green1': {0: 'Place the green block in the box.'}, 'imatrixlee/koch_place': {0: 'Pick up the white object and place it on the table.'}, 'ellen2imagine/pusht_green_same_init2': {0: 'Place the green block in the correct position.'}, 'KeWangRobotics/piper_rl_4': {0: 'Move the block slightly.'}, 'KeWangRobotics/piper_rl_4_cropped_resized': {0: 'picks the wooden block.'}, 'lalalala0620/koch_blue_paper_tape': {0: 'Grasp a blue paper tape and put it in the bin.'}, 'nduque/act_50_ep': {0: 'Grasp a green block and put it in the bin.'}, 'nduque/act_50_ep2': {0: 'Grasp a green block and put it in the bin.'}, 'HWJ658970/fat_fish': {0: 'Grasp a fat fish toy and put it in the bin.'}, 'Beegbrain/put_red_triangle_green_rect': {0: 'Put the red triangle on top of the green rectangle'}, 'ncavallo/moss_train_gc_block': {0: 'Grasp a lego block and put it in the bin.'}, 'takuzennn/square3': {0: 'Pick the cube from the table.'}, 'HWJ658970/lego_50': {0: 'Grasp a yellow lego block and put it in the bin.'}, 'Deason11/mobile_manipulator_0319': {0: 'Grasp a lego block and put it in the bin.'}, 'Gongsta/grasp_duck_in_cup': {0: 'Grasp the rubber duck and put it in the cup.'}, 'nduque/robustness_e2': {0: 'Grasp a green dice and put it in the bin.'}, 'ibru/bobo_trash_collector': {0: 'Bobo Trash collect and place it in a bin'}, 'Beegbrain/moss_open_drawer_teabox': {0: 'Open the top drawer of the teabox'}, 'Beegbrain/moss_put_cube_teabox': {0: 'Put the green cube in the top drawer of the teabox'}, 'Beegbrain/moss_close_drawer_teabox': {0: 'Close the top drawer of the teabox'}, 'Beegbrain/moss_stack_cubes': {0: 'Stack the green cube on top of the blue cube'}, 'HWJ658970/lego_50_camera_change': {0: 'Grasp a yellow lego block and put it in the bin.'}, 'HWJ658970/lego_100_class': {0: 'Separate yellow and white Lego blocks and place them into the bin.'}, 'nduque/robustness_e3': {0: 'Grasp a green dice and put it in the bin.'}, 'nimitvasavat/Gr00t_lerobot': {0: 'Place the cereal box on the shelf.'}, 'Deason11/mobile_manipulator_0326': {0: 'Grasp a lego block and put it in the bin.', 1: 'mobile_lekiwi.'}, 'KeWangRobotics/piper_push_cube_gamepad_1': {0: 'push the cube to the black area'}, 'KeWangRobotics/piper_push_cube_gamepad_1_cropped_resized': {0: 'push the cube to the black area'}, 'jannick-st/push-cube-classifier_cropped_resized': {0: 'Close the cabinet door.'}, 'nimitvasavat/Gr00t_lerobotV2': {0: 'Place the chocolate chip cookie dough box on the table.'}, 'Zhaoting123/koch_cleanDesk_': {0: 'Grasp a card and use it to clean the desk'}, 'arclabmit/koch_gear_and_bin': {0: 'Pick the gear and place it in the bin.'}, 'Allen-488/koch_dataset_50': {0: 'Grasp a block and put it in the bin.'}, 'dop0/koch_pick_terminal': {0: 'Pick up the terminal and place on the cover.'}, 'nduque/robustness_e4': {0: 'Grasp a green dice and put it in the bin.'}, 'arclabmit/Koch_twoarms': {0: 'official two arms recordings10'}, 'nimitvasavat/Gr00t_lerobot_state_action': {0: 'Place the chocolate chip cookie dough box on the table.'}, 'zliu157/i3r': {0: 'Grasp a lego block and put it in the bin.'}, 'hangwu/koch_pick_terminal': {0: 'Pick up the terminal and place on the cover.'}, 'nduque/robustness_e5': {0: 'Grasp a green dice and put it in the bin.'}, 'zliu157/i3r2': {0: 'Grasp a i3r logo and put it in the bin.'}, 'HuaihaiLyu/groceries': {0: 'Pick the brown long bread and Egg yolk pasry into package'}, 'zliu157/i3r3': {0: 'Grasp a i3r logo and put it in the bin.'}, 'hangwu/piper_pick_terminal_and_place': {0: 'Grasp a terminal and put it on the black box.'}, 'hangwu/piper_pick_terminal_2': {0: 'Grasp the white terminal and put it on the green lid.'}, 'engineer0002/pepper': {0: 'Place the bottle on the table.'}, 'theo-michel/lekiwi_v2': {0: 'Pick up the can on the ground'}, 'theo-michel/lekiwi_v5': {0: 'Pick up the can on the ground'}, 'roboticshack/sandee-kiwiv10': {0: 'Place the bottle on the table.'}, 'ibru/bob_jetson': {0: 'Drive forward pickup the object and put it in the red box and drive back.'}, 'ibru/bobo_jetson': {0: 'Drive forward pickup the object and put it in the red box and drive back.', 1: 'Driver forward'}, 'zliu157/i3r5': {0: 'Grasp a i3r logo and put it in the bin.'}, 'Dongkkka/cable_pick_and_place2': {0: 'Put a black charging cable in a black bowl and put a red charging cable in a green bowl'}, + + +} diff --git a/tests/datasets/test_datasets.py b/tests/datasets/test_datasets.py index b4fca77c1..4bd0baccb 100644 --- a/tests/datasets/test_datasets.py +++ b/tests/datasets/test_datasets.py @@ -396,35 +396,57 @@ def test_factory(env_name, repo_id, policy_name): # TODO(alexander-soare): If you're hunting for savings on testing time, this takes about 5 seconds. @pytest.mark.skip("TODO after fix multidataset") def test_multidataset_frames(): - """Check that all dataset frames are incorporated.""" - # Note: use the image variants of the dataset to make the test approx 3x faster. - # Note: We really do need three repo_ids here as at some point this caught an issue with the chaining - # logic that wouldn't be caught with two repo IDs. + """Check that all dataset frames are incorporated and aligned correctly.""" repo_ids = [ "lerobot/aloha_sim_insertion_human_image", "lerobot/aloha_sim_transfer_cube_human_image", "lerobot/aloha_sim_insertion_scripted_image", ] + + # dummy padding dimensions (simulate training setup) + MAX_ACTION_DIM = 14 + MAX_STATE_DIM = 30 + MAX_NUM_IMAGES = 3 + MAX_IMAGE_DIM = 224 + sub_datasets = [LeRobotDataset(repo_id) for repo_id in repo_ids] - dataset = MultiLeRobotDataset(repo_ids) + dataset = MultiLeRobotDataset( + repo_ids, + max_action_dim=MAX_ACTION_DIM, + max_state_dim=MAX_STATE_DIM, + max_num_images=MAX_NUM_IMAGES, + max_image_dim=MAX_IMAGE_DIM, + ) + assert len(dataset) == sum(len(d) for d in sub_datasets) assert dataset.num_frames == sum(d.num_frames for d in sub_datasets) assert dataset.num_episodes == sum(d.num_episodes for d in sub_datasets) - # Run through all items of the LeRobotDatasets in parallel with the items of the MultiLerobotDataset and - # check they match. expected_dataset_indices = [] for i, sub_dataset in enumerate(sub_datasets): expected_dataset_indices.extend([i] * len(sub_dataset)) - for expected_dataset_index, sub_dataset_item, dataset_item in zip( + for expected_dataset_index, sub_item, multi_item in zip( expected_dataset_indices, chain(*sub_datasets), dataset, strict=True ): - dataset_index = dataset_item.pop("dataset_index") + dataset_index = multi_item.pop("dataset_index") assert dataset_index == expected_dataset_index - assert sub_dataset_item.keys() == dataset_item.keys() - for k in sub_dataset_item: - assert torch.equal(sub_dataset_item[k], dataset_item[k]) + + # we ignore padding_mask and dataset_index keys in multi_item + extra_keys = {k for k in multi_item if "padding_mask" in k} + filtered_multi_keys = set(multi_item.keys()) - extra_keys + assert set(sub_item.keys()) == filtered_multi_keys, f"mismatch in keys" + + for k in sub_item: + if k not in multi_item: + continue + v1, v2 = sub_item[k], multi_item[k] + if isinstance(v1, torch.Tensor) and isinstance(v2, torch.Tensor): + assert torch.equal(v1, v2), f"tensor mismatch on key: {k}" + else: + assert v1 == v2, f"value mismatch on key: {k}" + + # TODO(aliberts): Move to more appropriate location From a9251e612f967bb53eaeb70a96e8b884e50a5747 Mon Sep 17 00:00:00 2001 From: Jade Date: Sat, 5 Jul 2025 13:08:25 -0400 Subject: [PATCH 2/5] cleanup/encapsulation --- lerobot/common/datasets/lerobot_dataset.py | 211 +++++++++++++-------- tests/datasets/test_datasets.py | 3 +- 2 files changed, 133 insertions(+), 81 deletions(-) diff --git a/lerobot/common/datasets/lerobot_dataset.py b/lerobot/common/datasets/lerobot_dataset.py index 47794493e..7b777e0d7 100644 --- a/lerobot/common/datasets/lerobot_dataset.py +++ b/lerobot/common/datasets/lerobot_dataset.py @@ -1139,6 +1139,105 @@ class LeRobotDataset(torch.utils.data.Dataset): obj.video_backend = video_backend if video_backend is not None else get_safe_default_codec() return obj +class MultiLeRobotDatasetMeta: + def __init__( + self, + datasets: list[LeRobotDataset], + repo_ids: list[str], + keys_to_max_dim: dict[str, int], + train_on_all_features: bool = False, + ): + self.repo_ids = repo_ids + self.keys_to_max_dim = keys_to_max_dim + self.train_on_all_features = train_on_all_features + self.robot_types = [ds.meta.info["robot_type"] for ds in datasets] + + # assign robot_type if missing + for ds in datasets: + ds.meta.info["robot_type"] = ROBOT_TYPE_KEYS_MAPPING.get(ds.repo_id, ds.meta.info["robot_type"]) + ds.robot_type = ds.meta.info["robot_type"] + + # step 1: compute disabled features + self.disabled_features = set() + if not self.train_on_all_features: + intersection = set(datasets[0].features) + for ds in datasets: + intersection.intersection_update(ds.features) + if not intersection: + raise RuntimeError("No common features across datasets.") + for repo_id, ds in zip(repo_ids, datasets): + extra = set(ds.features) - intersection + logging.warning(f"Disabling {extra} for repo {repo_id}") + self.disabled_features.update(extra) + + # step 2: build union_features excluding disabled + self.union_features = {} + for ds in datasets: + for k, v in ds.features.items(): + if k not in self.disabled_features: + self.union_features[k] = v + + # step 3: reshape feature schema + self.features = reshape_features_to_max_dim( + self.union_features, reshape_dim=-1, keys_to_max_dim=self.keys_to_max_dim + ) + + # step 4: aggregate stats + self.stats = aggregate_stats_per_robot_type(datasets) + for robot_type_, stats_ in self.stats.items(): + for feat_key, feat_stats in stats_.items(): + if feat_key in [ACTION, OBS_ENV_STATE, OBS_STATE]: + for k, v in feat_stats.items(): + pad_value = 0 if k in ["min", "mean"] else 1 + self.stats[robot_type_][feat_key][k] = pad_tensor( + v, max_size=self.keys_to_max_dim.get(feat_key, -1), pad_dim=-1, pad_value=pad_value + ) + + # step 5: episodes & tasks + self.episodes = { + repo_id: ds.meta.episodes for repo_id, ds in zip(repo_ids, datasets) + } + self.tasks = { + repo_id: ds.meta.tasks for repo_id, ds in zip(repo_ids, datasets) + } + self.info = { + repo_id: ds.meta.info for repo_id, ds in zip(repo_ids, datasets) + } + +class MultiLeRobotDatasetCleaner: + def __init__( + self, + datasets: list[LeRobotDataset], + repo_ids: list[str], + sampling_weights: list[float], + datasets_repo_ids: list[str], + min_fps: int = 1, + max_fps: int = 100, + ): + self.original_datasets = datasets + self.original_repo_ids = repo_ids + self.original_weights = sampling_weights + self.original_datasets_repo_ids = datasets_repo_ids + + # step 1: remove datasets with invalid fps + valid_fps_datasets = keep_datasets_with_valid_fps(datasets, min_fps=min_fps, max_fps=max_fps) + + # step 2: keep datasets with same features per robot type + consistent_datasets, keep_mask = keep_datasets_with_the_same_features_per_robot_type(valid_fps_datasets) + + self.cleaned_datasets = consistent_datasets + self.keep_mask = keep_mask + self.cleaned_weights = [sampling_weights[i] for i in range(len(valid_fps_datasets)) if keep_mask[i]] + self.cleaned_repo_ids = [repo_ids[i] for i in range(len(valid_fps_datasets)) if keep_mask[i]] + self.cleaned_datasets_repo_ids = [ + datasets_repo_ids[i] for i in range(len(valid_fps_datasets)) if keep_mask[i] + ] + + self.cumulative_sizes = np.array( + [0] + list(torch.cumsum(torch.tensor([len(d) for d in consistent_datasets]), dim=0)) + ) + self.cleaned_weights = np.array(self.cleaned_weights, dtype=np.float32) + class MultiLeRobotDataset(torch.utils.data.Dataset): """A dataset consisting of multiple underlying `LeRobotDataset`s. @@ -1225,93 +1324,47 @@ class MultiLeRobotDataset(torch.utils.data.Dataset): # restriction in future iterations of this class. For now, this is necessary at least for being able # to use PyTorch's default DataLoader collate function. # FIXME(mshukor): apply mapping to unify used keys - self.train_on_all_features = train_on_all_features - self.disabled_features = set() - if not self.train_on_all_features: - intersection_features = set(_datasets[0].features) - for ds in _datasets: - intersection_features.intersection_update(ds.features) - if len(intersection_features) == 0: - raise RuntimeError( - "Multiple datasets were provided but they had no keys common to all of them. " - "The multi-dataset functionality currently only keeps common keys." - ) - for repo_id, ds in zip(repo_ids, _datasets, strict=True): - extra_keys = set(ds.features).difference(intersection_features) - logging.warning( - f"keys {extra_keys} of {repo_id} were disabled as they are not contained in all the " - "other datasets." - ) - self.disabled_features.update(extra_keys) - union_features = {} - for ds in _datasets: - for k, v in ds.features.items(): - if k not in self.disabled_features: - union_features[k] = v - - if len(union_features) == 0: - raise RuntimeError("Multiple datasets were provided, but no features were found.") - + # FIXME(mshukor): pad based on types in case we have more than one state? self.image_transforms = image_transforms self.delta_timestamps = ( delta_timestamps.get(repo_id, None) if delta_timestamps else None ) # delta_timestamps # FIXME(mshukor): last repo? - # self.stats = aggregate_stats(self._datasets) # FIXME(mshukor): stats should be computed per robot type and then the robot type should be passed as input to the model - for ds in _datasets: - ds.meta.info["robot_type"] = ROBOT_TYPE_KEYS_MAPPING.get(ds.repo_id, ds.meta.info["robot_type"]) - ds.robot_type = ds.meta.info["robot_type"] # In case datasets with the same robot_type have different features - _datasets = keep_datasets_with_valid_fps(_datasets, min_fps=min_fps, max_fps=max_fps) - self._datasets, datasets_maks = keep_datasets_with_the_same_features_per_robot_type(_datasets) - self.sampling_weights = [self.sampling_weights[i] for i in range(len(_datasets)) if datasets_maks[i]] - self.repo_ids = [repo_ids[i] for i in range(len(_datasets)) if datasets_maks[i]] - self.datasets_repo_ids = [datasets_repo_ids[i] for i in range(len(_datasets)) if datasets_maks[i]] - # Compute cumulative sizes for fast indexing - self.cumulative_sizes = np.array( - [0] + list(torch.cumsum(torch.tensor([len(d) for d in self._datasets]), dim=0)) + cleaner = MultiLeRobotDatasetCleaner( + datasets=_datasets, + repo_ids=repo_ids, + sampling_weights=self.sampling_weights, + datasets_repo_ids=datasets_repo_ids, + min_fps=min_fps, + max_fps=max_fps, ) - self.sampling_weights = np.array(self.sampling_weights, dtype=np.float32) - self.stats = aggregate_stats_per_robot_type(self._datasets) - self.meta = copy.deepcopy(self._datasets[0].meta) # FIXME(mshukor): aggregate meta from all datasets - self.meta.info = { - repo_id: ds.meta.info for repo_id, ds in zip(self.repo_ids, self._datasets, strict=False) - } + self._datasets = cleaner.cleaned_datasets + self.sampling_weights = cleaner.cleaned_weights + self.repo_ids = cleaner.cleaned_repo_ids + self.datasets_repo_ids = cleaner.cleaned_datasets_repo_ids + self.cumulative_sizes = cleaner.cumulative_sizes + # self.meta = copy.deepcopy(self._datasets[0].meta) # FIXME(mshukor): aggregate meta from all datasets + # self.meta.info = { + # repo_id: ds.meta.info for repo_id, ds in zip(self.repo_ids, self._datasets, strict=False) + # } # self.meta.info["features"] = self._datasets[0].meta.info["features"] # Assume all datasets have the same features - # FIXME(mshukor): pad based on types in case we have more than one state? - self.keys_to_max_dim = { - ACTION: max_action_dim, - OBS_ENV_STATE: max_state_dim, - OBS_STATE: max_state_dim, - OBS_IMAGE: max_image_dim, - OBS_IMAGE_2: max_image_dim, - OBS_IMAGE_3: max_image_dim, - } - # self.meta.info["features"] = reshape_features_to_max_dim(self._datasets[0].meta.info["features"], reshape_dim=-1, keys_to_max_dim=self.keys_to_max_dim) - self.meta.info["features"] = reshape_features_to_max_dim( - union_features, reshape_dim=-1, keys_to_max_dim=self.keys_to_max_dim + self.meta = MultiLeRobotDatasetMeta( + datasets=self._datasets, + repo_ids=self.repo_ids, + keys_to_max_dim={ + ACTION: max_action_dim, + OBS_ENV_STATE: max_state_dim, + OBS_STATE: max_state_dim, + OBS_IMAGE: max_image_dim, + OBS_IMAGE_2: max_image_dim, + OBS_IMAGE_3: max_image_dim, + }, + train_on_all_features=train_on_all_features, ) - # reshape stats - for robot_type_, stats_ in self.stats.items(): - for feat_key, feat_stats in stats_.items(): - if feat_key in [ACTION, OBS_ENV_STATE, OBS_STATE]: - for k, v in feat_stats.items(): - if k in ["min", "mean"]: - pad_value = 0 - elif k in ["max", "std"]: - pad_value = 1 - else: - continue - self.stats[robot_type_][feat_key][k] = pad_tensor(v, max_size=self.keys_to_max_dim.get(feat_key, -1), pad_dim=-1, pad_value=pad_value) + self.disabled_features = self.meta.disabled_features + self.stats = self.meta.stats + - self.meta.stats = self.stats - # self.meta.info["features"] = aggregate_features(self._datasets) - self.meta.tasks = { - repo_id: ds.meta.tasks for repo_id, ds in zip(self.repo_ids, self._datasets, strict=False) - } - self.meta.episodes = { - repo_id: ds.meta.episodes for repo_id, ds in zip(self.repo_ids, self._datasets, strict=False) - } - self.robot_types = [ds.meta.info["robot_type"] for ds in self._datasets] @property def repo_id_to_index(self): """Return a mapping from dataset repo_id to a dataset index automatically created by this class. @@ -1402,7 +1455,7 @@ class MultiLeRobotDataset(torch.utils.data.Dataset): local_idx = (idx - self.cumulative_sizes[dataset_idx]).item() item = self._datasets[dataset_idx][local_idx] item["dataset_index"] = torch.tensor(dataset_idx) - item = create_padded_features(item, self.meta.info["features"]) + item = create_padded_features(item, self.meta.features) for data_key in self.disabled_features: # FIXME(mshukor): not in getitem? if data_key in item: del item[data_key] diff --git a/tests/datasets/test_datasets.py b/tests/datasets/test_datasets.py index 4bd0baccb..4a8af1cb7 100644 --- a/tests/datasets/test_datasets.py +++ b/tests/datasets/test_datasets.py @@ -394,7 +394,7 @@ def test_factory(env_name, repo_id, policy_name): # TODO(alexander-soare): If you're hunting for savings on testing time, this takes about 5 seconds. -@pytest.mark.skip("TODO after fix multidataset") +# @pytest.mark.skip("TODO after fix multidataset") def test_multidataset_frames(): """Check that all dataset frames are incorporated and aligned correctly.""" repo_ids = [ @@ -421,7 +421,6 @@ def test_multidataset_frames(): assert len(dataset) == sum(len(d) for d in sub_datasets) assert dataset.num_frames == sum(d.num_frames for d in sub_datasets) assert dataset.num_episodes == sum(d.num_episodes for d in sub_datasets) - expected_dataset_indices = [] for i, sub_dataset in enumerate(sub_datasets): expected_dataset_indices.extend([i] * len(sub_dataset)) From f47fd7aabf9455ad6b69fa3ce229d30d76150263 Mon Sep 17 00:00:00 2001 From: Jade Date: Mon, 7 Jul 2025 09:22:48 -0400 Subject: [PATCH 3/5] add hf hub integration --- lerobot/common/datasets/utils_must.py | 399 +--- lerobot/configs/mappings/features.yaml | 2387 ++++++++++++++++++++++++ lerobot/configs/mappings/tasks.yaml | 1012 ++++++++++ 3 files changed, 3412 insertions(+), 386 deletions(-) create mode 100644 lerobot/configs/mappings/features.yaml create mode 100644 lerobot/configs/mappings/tasks.yaml diff --git a/lerobot/common/datasets/utils_must.py b/lerobot/common/datasets/utils_must.py index 147150d40..93927f6c9 100644 --- a/lerobot/common/datasets/utils_must.py +++ b/lerobot/common/datasets/utils_must.py @@ -191,391 +191,18 @@ def map_dict_keys(item: dict, feature_keys_mapping: dict, training_features: lis features[key] = item[key] return features +import yaml +import requests +def load_yaml_mapping(name: str) -> dict: + """ + Loads a YAML mapping from a Hugging Face repo. + Example: name='features' → https://huggingface.co/jadechoghari/smolvla-keys/resolve/main/features.yaml + """ + url = f"https://huggingface.co/jadechoghari/smolvla-keys/resolve/main/{name}.yaml" + response = requests.get(url) + response.raise_for_status() # raise if the download fails -TASKS_KEYS_MAPPING = { - "pranavsaroha/so100_legos4": {0: "Pick up the LEGO block and place it in the bowl of the same color as the LEGO block."}, - "pranavsaroha/so100_onelego2": {0: "Pick up the green LEGO block and place it in the green bowl."}, - "jpata/so100_pick_place_tangerine": {0: "Pick up the tangerine and place it."}, - "pranavsaroha/so100_onelego3": {0: "Pick up the green LEGO block and place it in the green bowl."}, - "pranavsaroha/so100_carrot_2": {0: "Pick up a carrot and put it in the bin."}, - "pranavsaroha/so100_carrot_5": {0: "Pick up a carrot and put it in the bin."}, - "pandaRQ/pick_med_1": {0: "Pick up the object and place it in the box."}, - "HITHY/so100_strawberry": {0: "Grasp a strawberry and put it in the bin."}, - "vladfatu/so100_above": {0: "Pick up red object and place it in the box."}, - "koenvanwijk/orange50-1": {0: "Pick up the orange object and but it in the LEGO box. "}, - "koenvanwijk/orange50-variation-2": {0: "Pick up the orange object and but it in the LEGO box. "}, - "FeiYjf/new_GtoR": {0: "Move along the line on the paper from start to end."}, - "CSCSXX/pick_place_cube_1.18": {0: "Pick up the cube and place it in the box."}, - "vladfatu/so100_office": {0: "Pick up the red object and place it in the box."}, - "dragon-95/so100_sorting": {0: "Pick up the object from box A and place it in box B."}, - "dragon-95/so100_sorting_1": {0: "Pick up the object from box A and place it in box B."}, - "nbaron99/so100_pick_and_place4": {0: "Pick up the triangular object and place it on a green sticker."}, - "Beegbrain/pick_place_green_block": {0: "Pick up the green block and place in the red cup."}, - "Ityl/so100_recording2": {0: "Pick up the red cube and place it on top of the blue cube."}, - "dragon-95/so100_sorting_2": {0: "Pick up the object from box A and place it in box B."}, - "dragon-95/so100_sorting_3": {0: "Pick up the object from box A and place it in box B."}, - "aractingi/push_cube_offline_data": {0: "Push the green cube to the yellow sticker."}, - "HITHY/so100_peach3": {0: "Grasp a peach and put it in the bin."}, - "HITHY/so100_peach4": {0: "Grasp a peach and put it on the plate."}, - "shreyasgite/so100_legocube_50": {0: "Grasp a lego block and put it in the bin."}, - "shreyasgite/so100_base_env": {0: "Grasp a lego block and put it in the bin."}, - "triton7777/so100_dataset_mix": { - 0: "Pick up the black tape and place it inside the white tape roll.", - 1: "Pick up the gift miniatures and place them in the black box.", - 2: "Sort the mixed objects into their appropriate categories.", - 3: "Place the pens into the pen holder.", - 4: "Place pens, bottles, and any suitable items into the pen holder, as appropriate.", - 5: "Place the oranges into the yellow basket.", - 6: "Stack the plates and place the cup on top." - }, - "Deason11/Open_the_drawer_to_place_items": {0: "Put the objects in the open drawer."}, - "Deason11/PLACE_TAPE_PUSH_DRAWER": {0: "Place the tape in the drawer and close it. "}, - "NONHUMAN-RESEARCH/SOARM100_TASK_VENDA": {0: "Pick up the object and place it in the box."}, - "mikechambers/block_cup_14": {0: "Grasp a block and put it in a cup."}, - "samsam0510/tooth_extraction_3": {0: "Extract the tooth and put it somewhere."}, - "samsam0510/tooth_extraction_4": {0: "Extarct the molar and put it somewhere."}, - "samsam0510/cube_reorientation_2": {0: "Rotate the object so it aligns with the silhouette on the table."}, - "samsam0510/cube_reorientation_4": {0: "Rotate the object so it aligns with respect to the line on the table."}, - "samsam0510/glove_reorientation_1": {0: "Rotate the glove so the bottom part aligns with the line on the table."}, - "DorayakiLin/so100_pick_charger_on_tissue": {0: "Pick up the charger and put it on the white tissue."}, - "zijian2022/noticehuman3": {0: "Notice human."}, - "liuhuanjim013/so100_th": {0: "Grasp a lego figure and put it in the box."}, - "Bartm3/tape_to_bin": {0: "Grasp a tape and put it in the bin."}, + return yaml.safe_load(response.text) - # Community dataset v2 - "Chojins/chess_game_009_white": { - 0: "Move the blue chess pieces to the highlighted squares." - }, - "1g0rrr/sam_openpi03": { - 0: "Pick up the cube and place it in the box." - }, - "sihyun77/suho_3_17_1": { - 0: "Grasp a lego block and put it in the bin." - }, - "sihyun77/sihyun_3_17_2": { - 0: "Grasp a lego block and put it in the bin." - }, - "sihyun77/suho_3_17_3": { - 0: "Grasp a lego block and put it in the bin." - }, - "sihyun77/sihyun_3_17_5": { - 0: "Grasp a lego block and put it in the bin." - }, - "Odog16/so100_cube_drop_pick_v1": { - 0: "Pick up the orange cube, release it, and then pick it up again." - }, - "sihyun77/sihyun_main_2": { - 0: "Grasp a lego block and put it in the bin." - }, - "sihyun77/suho_main_2": { - 0: "Grasp a lego block and put it in the bin." - }, - "Bartm3/dice2": { - 0: "Grasp a dice and put it in the bin." - }, - "sihyun77/sihyun_main_3": { - 0: "Grasp a lego block and put it in the bin." - }, - "Loki0929/so100_duck": { - 0: "Grasp red, green, yellow ducks and put them in the box." - }, - "pietroom/holdthis": { - 0: "Hold the object steadily without releasing it." - }, - "pietroom/actualeasytask": { - 0: "Grasp the marker and put it in the plastic box." - }, - "Beegbrain/pick_lemon_and_drop_in_bowl": { - 0: "Pick the yellow lemon and drop it in the red bowl." - }, - "Beegbrain/sweep_tissue_cube": { - 0: "Sweep the red cubes to the right with the tissue." - }, - "zijian2022/321": { - 0: "Grasp a lego block and put it in the bin." - }, - "1g0rrr/sam_openpi_solder1": { - 0: "Bring contact to the pad on the board." - }, - "1g0rrr/sam_openpi_solder2": { - 0: "Bring contact to the pad on the board." - }, - "gxy1111/so100_pick_place": { - 0: "Grasp a toy panda and put it in the cup." - }, - "Odog16/so100_cube_stacking_v1": { - 0: "Stack the cubes in the following order from bottom to top: black, blue, then orange." - }, - "sihyun77/mond_1": { - 0: "Grasp a lego block and put it in the bin." - }, - "andlyu/so100_indoor_1": { - 0: "Locate and grasp the blueberry." - }, - "andlyu/so100_indoor_3": { - 0: "Locate and grasp the blueberry." - }, - "frk2/so100large": { - 0: "Pick up roll of tape and put it in the bin." - }, - "lirislab/sweep_tissue_cube": { - 0: "Sweep the red cubes to the right with the tissue bag." - }, - "lirislab/lemon_into_bowl": { - 0: "Pick the yellow lemon and drop it in the red bowl" - }, - "lirislab/red_cube_into_green_lego_block": { - 0: "Put the red cube on top of the yellow cube." - }, - "lirislab/red_cube_into_blue_cube": { - 0: "Put the red cube on top of the blue cube." - }, - "00ri/so100_battery": { - 0: "Grasp a battery and put it in the bin." - }, - "frk2/so100largediffcam": { - 0: "Pick up roll of tape and put it in the bin" - }, - "FsqZ/so100_1": { - 0: "Put the yellow cube inside the purple box." - }, - "ZGGZZG/so100_drop0": { - 0: "Grasp a ball and put it in the hole." - }, - "Chojins/chess_game_000_white_red": { - 0: "Move the red chess pieces to the highlighted squares." - }, - "smanni/train_so100_fluffy_box": { - 0: "Grasp a small object and place it in the box." - }, - "ganker5/so100_push_20250328": { - 0: "Grasp a lego block and put it in the bin." - }, - "ganker5/so100_dataline_0328": { - 0: "Grasp a lego block and put it in the bin." - }, - "ganker5/so100_color_0328": { - 0: "Grasp a lego block and put it in the bin." - }, - "CrazyYhang/A1234-B-C_mvA2B": { - 0: "Move the top disk from the left column to the middle column." - }, - "RasmusP/so100_Orange2Green": { - 0: "Grasp the orange block and drop it in the box." - }, - "sixpigs1/so100_pick_cube_in_box": { - 0: "Pick up the red cube and put it in the box." - }, - "ganker5/so100_push_20250331": { - 0: "Grasp a lego block and put it in the bin." - }, - "ganker5/so100_dataline_20250331": { - 0: "Grasp a lego block and put it in the bin." - }, - "lirislab/put_caps_into_teabox": { - 0: "Pick the coffee capsule and put it into the top drawer of the teabox" - }, - "lirislab/close_top_drawer_teabox": { - 0: "Close the top drawer of the teabox" - }, - "lirislab/open_top_drawer_teabox": { - 0: "Open the top drawer of the teabox" - }, - "lirislab/unfold_bottom_right": { - 0: "Unfold the bag from bottom right corner" - }, - "lirislab/push_cup_target": { - 0: "Push the red cup to the pink target" - }, - "lirislab/put_banana_bowl": { - 0: "Put the banana into the red bowl" - }, - "Chojins/chess_game_001_blue_stereo": { - 0: "Move the blue chess pieces to the highlighted squares" - }, - "Chojins/chess_game_001_red_stereo": { - 0: "Move the red chess pieces to the highlighted squares" - }, - "ganker5/so100_toy_20250402": { - 0: "Grasp a lego block and put it in the bin." - }, - "Gano007/so100_medic": { - 0: "Grasp a medic box and put it in the bin." - }, - "00ri/so100_battery_bin_center": { - 0: "Grasp a battery and put it in the bin." - }, - "paszea/so100_whale_2": { - 0: "Grasp a whale and put it in the plate." - }, - "lirislab/fold_bottom_right": { - 0: "Fold the bag from the bottom right corner." - }, - "lirislab/put_coffee_cap_teabox": { - 0: "Put the coffee capsule into the top drawer of the teabox." - }, - "therarelab/so100_pick_place_2": { - 0: "Pick a plaster roll and place it to the blue sticker." - }, - "paszea/so100_whale_3": { - 0: "Grasp a whale and put it in the plate." - }, - "paszea/so100_whale_4": { - 0: "Grasp a whale and put it in the plate." - }, - "paszea/so100_lego": { - 0: "Grasp a lego and put it in the basket." - }, - "LemonadeDai/so100_coca": { - 0: "Grasp the Coca-Cola can and orient it upright with the top facing up." - }, - "zijian2022/backgrounda": { - 0: "Grasp a lego block and put it in the bin." - }, - "zijian2022/backgroundb": { - 0: "Grasp a lego block and put it in the bin." - }, - "356c/so100_nut_sort_1": { - 0: "Pick up the steel nuts and sort them by color." - }, - "Mwuqiu/so100_0408_muti": { - 0: "Grasp a yellow duck and put it in the box." - }, - "aimihat/so100_tape": { - 0: "Pick up the tape and put it in the bowl." - }, - "lirislab/so100_demo": { - 0: "Put the banana into the red bowl." - }, - "356c/so100_duck_reposition_1": { - 0: "Grasp the tool and use it to move the duck to the indicated position." - }, - "zijian2022/sort1": { - 0: "Grasp a box and sort it by color: place grey boxes on the left and black boxes on the right." - }, - "weiye11/so100_410_zwy": { - 0: "Pick up the cube and place it on the black circle." - }, - "VoicAndrei/so100_banana_to_plate_only": { - 0: "Pick up the banana and place it on the plate." - }, - "sixpigs1/so100_stack_cube_error": { - 0: "Pick up the red cube and stack it on the green cube with position offset when grasping.", - 1: "Pick up the red cube and stack it on the green cube with gripper error when grasping.", - 2: "Pick up the red cube and stack it on the green cube with position offset when stacking.", - 3: "Pick up the red cube and stack it on the green cube without errors", - }, - "isadev/bougies3": { - 0: "Grab the candle wick by the aluminium plate and place it in the box." - }, - "zijian2022/close3": { - 0: "Grasp a lego block and put it in the bin." - }, - "bensprenger/left_arm_yellow_brick_in_box_v0": { - 0: "Grasp the yellow lego block and put it in the box." - }, - "bensprenger/left_arm_yellow_brick_in_box_with_purple_noise_v0": { - 0: "Grasp a yellow lego block and put it in the bin." - }, - "roboticshack/team16-can-stacking": { - 0: "Grasp the flipped cup and stack it on top of the midpoint between the two other cups to create a tower" - }, - "bensprenger/right_arm_p_brick_in_box_with_y_noise_v0": { - 0: "Grasp the purple lego block and put it in the box." - }, - "pierfabre/pig2": { - 0: "Pick the pig and place it to the right." - }, - "zijian2022/insert2": { - 0: "Grasp a lego block and put it in the bin." - }, - "roboticshack/team-7-right-arm-grasp-tape": { - 0: "Grasp the tape and put it in the box." - }, - "pierfabre/pig3": { - 0: "Pick the pig and place it to the right." - }, - "Jiangeng/so100_413": { - 0: "Pick up the cube and place it on top of the black circle." - }, - "roboticshack/team9-pick_cube_place_static_plate": { - 0: "Pick up the green cube and place on orange plate." - }, - "AndrejOrsula/lerobot_double_ball_stacking_random": { - 0: "Stack the balls on top of each other." - }, - "roboticshack/left-arm-grasp-lego-brick": { - 0: "Grasp the lego brick and put it in the box." - }, - "roboticshack/team-7-left-arm-grasp-motor": { - 0: "Grasp the black motor and put it in the box." - }, - "pierfabre/cow2": { - 0: "Pick the cow and place it to the right." - }, - "pierfabre/sheep": { - 0: "Pick the sheep and place it to the right." - }, - "roboticshack/team9-pick_chicken_place_plate": { - 0: "Pick up the chicken and place on orange plate" - }, - "roboticshack/team13-two-balls-stacking": { - 0: "Stack the balls on top of each other." - }, - "tkc79/so100_lego_box_1": { - 0: "Grasp a lego block and put it in the box." - }, - "pierfabre/rabbit": { - 0: "Pick the rabbit and place it to the right.", - 1: "Pick the rabbit and put it to the right" - }, - "roboticshack/team13-three-balls-stacking": { - 0: "Stack the balls on top of each other." - }, - "pierfabre/horse": { - 0: "Pick the horse and place it to the right." - }, - "pierfabre/chicken": { - 0: "Pick the chicken and place it to the right." - }, - "roboticshack/team16-water-pouring": { - 0: "Pouring water from one cup to another cup" - }, - "ad330/cubePlace": { - 0: "Grasp white cube and place it in the bowl." - }, - "paszea/so100_lego_2cam": { - 0: "Grap lego blocks and put them in the plate." - }, - "bensprenger/chess_game_001_blue_stereo": { - 0: "Move the blue chess pieces to the highlighted squares." - }, - "Mohamedal/put_banana": { - 0: "Put the banana in the red bowl." - }, - "tkc79/so100_lego_box_2": { - 0: "Grasp a lego block and put it in the box." - }, - "samanthalhy/so100_herding_1": { - 0: "Grasp a green tool and herd all the particles to the grey bin." - }, - "jlesein/TestBoulon7": { - 0: "Pick up the bolt and put it on the plate." - }, - # V3 with VLM (Qwen-VL-2.5-instruct) annotation - "satvikahuja/mixer_on_off_new_1": {0: "Press the button on the blender."}, - "andy309/so100_0314_fold_cloths": {0: "fold the cloths, use two cameras, two arms."}, - "jchun/so100_pickplace_small_20250323_120056": {0: "Grasp items from white bowl and place in black tray"}, - "Ofiroz91/so_100_cube2bowl": {0: "placing cube inside a red bawl"}, - "ZCM5115/so100_1210": {0: "picks up the USB cable."}, - "francescocrivelli/carrot_eating": {0: "pick up carrot and bring to mouth"}, - "ZCM5115/so100_2Arm3cameras_movebox": {0: "Pick up the white box from the table."}, - "pranavsaroha/so100_carrot_1": {0: "pick a carrot and put it in the bin"}, - "pranavsaroha/so100_carrot_3": {0: "pick a carrot and put it in the bin"}, - "maximilienroberti/so100_lego_red_box": {0: "Placing the red Lego in the red box bin."}, "pranavsaroha/so100_squishy": {0: "pick a squishy and put it in the bin"}, "rabhishek100/so100_train_dataset": {0: "picks tape and places it in a cup."}, "pranavsaroha/so100_squishy100": {0: "pick a squishy and put it in the bin"}, "pandaRQ/pickmed": {0: "Place the green block on the table."}, "swarajgosavi/act_kikobot_pusht_real": {0: "picks up the red block."}, "pranavsaroha/so100_squishy2colors_1": {0: "pick the squishies and put them in the bins with their corresponding colors"}, "Chojins/chess_game_001_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "jmrog/so100_sweet_pick": {0: "Pick up the candy and place it in the bowl."}, "Chojins/chess_game_002_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "pranavsaroha/so100_squishy2colors_2_new": {0: "pick the squishies and put them in the bins with their corresponding colors"}, "Chojins/chess_game_003_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "Chojins/chess_game_004_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "Chojins/chess_game_005_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "Chojins/chess_game_006_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "Chojins/chess_game_007_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "koenvanwijk/blue52": {0: "places blue block on red LEGO piece."}, "jlitch/so100multicam7": {0: "pick up brick and put in bin"}, "vladfatu/so100_ds": {0: "Pick up the cube and place it in the box."}, "Chojins/chess_game_000_white": {0: "Move the blue chess pieces as indicated by the highlighted squares"}, "satvikahuja/orange_mixer_1": {0: "pick orange and place in mixer"}, "satvikahuja/mixer_on_off": {0: "switch the mixer on or off"}, "satvikahuja/orange_pick_place_new1": {0: "Pick up the orange and place it in the bowl."}, "satvikahuja/mixer_on_off_new": {0: "Adjust the s position."}, "FeiYjf/Makalu_push": {0: "Pick up the blue cube."}, "chmadran/so100_dataset04": {0: "picks the blue block and places it in the red cup."}, "FeiYjf/Maklu_dataset": {0: "Pick up the blue cube and place it on the paper."}, "FeiYjf/new_Dataset": {0: "Pick up the blue cube."}, "satvikahuja/mixer_on_off_new_4": {0: "Place the lid on the blender."}, "CSCSXX/pick_place_cube_1.17": {0: "Pick up the red block and place it in the box."}, "liyitenga/so100_pick_taffy3": {0: "Place the eraser in the container."}, "liyitenga/so100_pick_taffy6": {0: "Pick up the toy and place it in the purple cup."}, "yuz1wan/so100_pickplace": {0: "Pick the pink block and place it in the paper cup."}, "liyitenga/so100_pick_taffy7": {0: "Pick up the toy and place it in the box."}, "swarajgosavi/act_kikobot_block_real": {0: "Pick up the blue cube and place it in the box."}, "SeanLMH/so100_picknplace_v2": {0: "picks up blue cube and places it in yellow box."}, "DimiSch/so100_50ep_2": {0: "Place the yellow object in the bowl."}, "DimiSch/so100_50ep_3": {0: "Pick the yellow button from the table."}, "SeanLMH/so100_picknplace": {0: "Pick up the blue block and place it in the yellow box."}, "nbaron99/so100_pick_and_place2": {0: "picks up the white object."}, "chmadran/so100_dataset08": {0: "places blue block on paper."}, "Ityl/so100_recording1": {0: "Putting the red square onto the yellow piece"}, "ad330/so100_box_pickPlace": {0: "places jar in box."}, "carpit680/giraffe_task": {0: "Grasp a block and put it in the bin."}, "carpit680/giraffe_sock_demo_1": {0: "Grasp a sock off the floor."}, "DimiSch/so100_terra_50_2": {0: "Grasp a lego block and put it in the bin."}, "aractingi/push_cube_offline_data_cropped_resized": {0: "Push the green cube to the yellow sticker"}, "FeiYjf/Test_NNNN": {0: "Pick up the purple cube and move it to the right."}, "HITHY/so100_peach": {0: "Grasp a peach and put it in the bin."}, "zaringleb/so100_cube_4_binary": {0: "Grasp a lego block and put it in the bin."}, "FeiYjf/Grab_Pieces": {0: "places the black object on the table."}, "hegdearyandev/so100_eraser_cup_v1": {0: "picks up the red object."}, "jbraumann/so100_1902": {0: "picks up the yellow ball."}, "zaringleb/so100_cube_5_linear": {0: "Grasp a lego block and put it in the bin."}, "samsam0510/tape_insert_1": {0: "Grasp a red tape and put it on the box."}, "samsam0510/tape_insert_2": {0: "Grasp a red tape and put it in the yellow tape."}, "pengjunkun/so100_push_to_hole": {0: "Push the T into the hole."}, "Deason11/Random_Kitchen": {0: "Pick up the cup and place it on the table."}, "Loki0929/so100_100": {0: "Grasp a rubber duck and put it in the box."}, "speedyyoshi/so100_grasp_pink_block": {0: "Grasp a lego block and put it in the bin."}, "lirislab/green_lego_block_into_mug": {0: "pick the green block and place it in the red cup"}, "kevin510/lerobot-cat-toy-placement": {0: "Grasp the cat toy and put it in the cup."}, "NONHUMAN-RESEARCH/SOARM100_TASK_VENDA_BOX": {0: "Move the cube to the right side of the table."}, "zijian2022/noticehuman5": {0: "picks up the box."}, "zijian2022/noticehuman70": {0: "Stop movement when human encounter testbed.", 1: "Stop movement when human encounter testbed w/ trigger."}, "Bartm3/tape_to_bin": {0: "Grasp a tape and put it in the bin."}, "Pi-robot/barbecue_flip": {0: "Pick up the orange cone and place it on the table."}, "Pi-robot/barbecue_put": {0: "Pick up the stick and place it in the grill."}, "sshh11/so100_orange_50ep_1": {0: "Grasp an orange object and put it in the bin."}, "sshh11/so100_orange_50ep_2": {0: "Grasp an orange object and put it in the bin."}, "DorayakiLin/so100_pick_cube_in_box": {0: "Pick up the red cube and put it in the box."}, "Bartm3/tape_to_bin2": {0: "Grasp a tape and put it in the bin."}, "andy309/so100_0311_1152": {0: "Grasp and put it in the bin."}, "sihyun77/suho_so100": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/si_so100": {0: "Grasp a lego block and put it in the bin."}, "shreyasgite/so100_base_left": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/suho_red": {0: "Grasp a lego block and put it in the bin."}, "liuhuanjim013/so100_block": {0: "Grasp a lego block and put it in the bin."}, "joaoocruz00/so100_makeitD1": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/suho_angel": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/sihyun_king": {0: "Grasp a lego block and put it in the bin."}, "acrampette/third_arm_01": {0: "Pick up the circuit board from the table."}, "Winster/so100_cube": {0: "Grasp a lego block and put it in the bin."}, "1g0rrr/sam_openpi03": {0: "Grasp a blue cube and put it in the gray box."}, "thedevansh/mar16_1336": {0: "Grasp a lego block and put it in the bin."}, "hkphoooey/throw_stuffie": {0: "Grab stuffed animal and throw it on the dot."}, "acrampette/third_arm_02": {0: "Pick up the tie and place it in the box."}, "kumarhans/so100_tape_task": {0: "Grasp a roll of tape and put it over the candle case."}, "Odog16/so100_tea_towel_folding_v1": {0: "Fold tea towel into quarters"}, "pietroom/first_task_short": {0: "Pick up the marker from the box."}, "zijian2022/c0": {0: "Grasp a lego block and put it in the bin based on color.", 1: "Grasp a lego block and put it in the bin."}, "1g0rrr/sam_openpi_solder1": {0: "bring contact to the pad on the board."}, "1g0rrr/sam_openpi_solder2": {0: "bring contact to the pad on the board."}, "bnarin/so100_tic_tac_toe_we_do_it_live": {0: "move tic tac toe as player 2."}, "chmadran/so100_home_dataset": {0: "Grasp a lego block and put it in the bin."}, "baladhurgesh97/so100_final_picking_3": {0: "Grasp a carrot, plastic bottle and put it in respective bins."}, "zaringleb/so100_cube_6_2d": {0: "Grasp a lego block and put it in the bin."}, "ZGGZZG/so100_drop1": {0: "Grasp a cube and put it in the right place."}, "abhisb/so100_51_ep": {0: "Pick up the cube and place it in the box."}, "allenchienxxx/so100Test": {0: "Grasp a lego block and put it in the bin."}, "lizi178119985/so100_jia": {0: "Grasp a lego block and put it in the bin."}, "andrewcole712/so100_tape_bin_place": {0: "Place the tape in the wooden box."}, "Gano007/so100_doliprane": {0: "Grasp a medic box and put it in the bin."}, "XXRRSSRR/so100_v3_num_episodes_50": {0: "Grasp a box and put it in the side."}, "Gano007/so100_gano": {0: "Grasp a box and put it in the bin."}, "paszea/so100_whale_grab": {0: "Grasp a whale and put it in the plate."}, "Clementppr/lerobot_pick_and_place_dataset_world_model": {0: "Grasp a fruit and put it in the cup."}, "RasmusP/so100_dataset50ep": {0: "Grasp a square block and put it in the box."}, "Gano007/so100_second": {0: "Grasp a yellow box and put it in the bin."}, "zaringleb/so100_cude_linear_and_2d_comb": {0: "Grasp a lego block and put it in the bin."}, "zijian2022/digitalfix3": {0: "Grasp a lego block and put it in the bin."}, "sihyun77/mond_13": {0: "Grasp a lego block and put it in the bin."}, "356c/so100_rope_reposition_1": {0: "Grasp rope and reposition."}, "paszea/so100_lego_mix": {0: "Grasp lego blocks and put them in the plate."}, "jiajun001/eraser00_2": {0: "picks tissue paper from box."}, "VoicAndrei/so100_banana_to_plate_rebel_full": {0: "Pick up the banana and place it on the place"}, "isadev/bougies1": {0: "Put the candles in the box."}, "sixpigs1/so100_pick_cube_in_box_error": {0: "Pick up the red cube and put it in the box with position offset when grasping.", 1: "Pick up the red cube and put it in the box with gripper error when grasping.", 2: "Pick up the red cube and put it in the box with position offset when releasing.", 3: "Pick up the red cube and put it in the box without errors."}, "sixpigs1/so100_push_cube_error": {0: "Push the blue cube to the red and white target with position offset when reaching.", 1: "Push the blue cube to the red and white target with position offset when pushing.", 2: "Push the blue cube to the red and white target with gripper error when pushing.", 3: "Push the blue cube to the red and white target without errors."}, "sixpigs1/so100_pull_cube_error": {0: "Pull the yellow cube to the red and white target with position offset when reaching.", 1: "Pull the yellow cube to the red and white target with position offset when pulling.", 2: "Pull the yellow cube to the red and white target with gripper error when pulling.", 3: "Pull the yellow cube to the red and white target without errors."}, "isadev/bougies2": {0: "grab the candle wick and place it in the tray."}, "therarelab/med_dis_rare_6": {0: "places green object in box."}, "sixpigs1/so100_pull_cube_by_tool_error": {0: "Pick up the L-shaped tool and pull the purple cube by the tool with position offset when grasping.", 1: "Pick up the L-shaped tool and pull the purple cube by the tool with rotation offset when grasping.", 2: "Pick up the L-shaped tool and pull the purple cube by the tool with gripper error when grasping.", 3: "Pick up the L-shaped tool and pull the purple cube by the tool with position offset when lowering.", 4: "Pick up the L-shaped tool and pull the purple cube by the tool with rotation offset when lowering.", 5: "Pick up the L-shaped tool and pull the purple cube by the tool with position offset when pulling.", 6: "Pick up the L-shaped tool and pull the purple cube by the tool with gripper error when pulling.", 7: "Pick up the L-shaped tool and pull the purple cube by the tool without errors."}, "sixpigs1/so100_insert_cylinder_error": {0: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with position offset when grasping.", 1: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with gripper error when grasping.", 2: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with rotation offset when uprighting.", 3: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with position offset when inserting.", 4: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf with choice error when inserting.", 5: "Pick up the cylinder, upright it, and insert it into the middle hole of the shelf without errors."}, "lirislab/guess_who_no_cond": {0: "Place the card in the slot."}, "lirislab/guess_who_lighting": {0: "Pick up the card from the shelf."}, "nguyen-v/so100_press_red_button": {0: "The places the cube in the box."}, "nguyen-v/so100_bimanual_grab_lemon_put_in_box2": {0: "Grab the lemon with the black arm, then give it to the green arm, then place the lemon in the cardboard box with the green arm."}, "nguyen-v/press_red_button_new": {0: "Press the red button with the black arm"}, "nguyen-v/so100_rotate_red_button": {0: "Rotate the red button clockwise with the black arm"}, "roboticshack/team10-red-block": {0: "Pick a red lego block and move it to the right."}, "Cidoyi/so100_all_notes_1": {0: "Connect the cable to the device."}, "roboticshack/team11_pianobot": {0: "Point at the keyboard."}, "roboticshack/team2-guess_who_so100": {0: "Pick up the card from the shelf."}, "roboticshack/team2-guess_who_so100_light": {0: "Place the card in the slot."}, "roboticshack/team2-guess_who_less_ligth": {0: "Pick up the card and place it in the slot."}, "jiajun001/eraser00_3": {0: "Pick up the white object from the table."}, - "Setchii/so100_grab_ball": {0: "Grasp a ball and put it on a goblet."}, - # V4 with VLM annotation - 'ctbfl/sort_battery': {0: 'put the battery into battery_box'}, 'lerobot/aloha_static_screw_driver': {0: 'Pick up the screwdriver with the right arm, hand it over to the left arm then place it into the cup.'}, 'lerobot/aloha_static_candy': {0: 'Pick up the candy and unwrap it.'}, 'lerobot/aloha_mobile_wipe_wine': {0: 'Pick up the wet cloth on the faucet and use it to clean the spilled wine on the table and underneath the glass.'}, 'lerobot/aloha_static_coffee': {0: "Place the coffee capsule inside the capsule container, then place the cup onto the center of the cup tray, then push the 'Hot Water' and 'Travel Mug' buttons."}, 'lerobot/aloha_static_towel': {0: 'Pick up a piece of paper towel and place it on the spilled liquid.'}, 'lerobot/aloha_static_vinh_cup': {0: 'Pick up the platic cup with the right arm, then pop its lid open with the left arm.'}, 'lerobot/aloha_static_vinh_cup_left': {0: 'Pick up the platic cup with the left arm, then pop its lid open with the right arm.'}, 'lerobot/aloha_static_ziploc_slide': {0: 'Slide open the ziploc bag.'}, 'lerobot/aloha_static_coffee_new': {0: 'Place the coffee capsule inside the capsule container, then place the cup onto the center of the cup tray.'}, 'lerobot/aloha_static_cups_open': {0: 'Pick up the plastic cup and open its lid.'}, 'lerobot/aloha_static_pro_pencil': {0: 'Pick up the pencil with the right arm, hand it over to the left arm then place it back onto the table.'}, 'lerobot/aloha_mobile_wash_pan': {0: 'Pick up the pan, rinse it in the sink and then place it in the drying rack.'}, 'lerobot/aloha_mobile_cabinet': {0: 'Open the top cabinet, store the pot inside it then close the cabinet.'}, 'lerobot/aloha_mobile_chair': {0: 'Push the chairs in front of the desk to place them against it.'}, 'lerobot/aloha_mobile_elevator': {0: 'Take the elevator to the 1st floor.'}, 'aliberts/koch_tutorial': {0: 'Pick the Lego block and drop it in the box on the right.'}, 'underctrl/single-block_multi-color_pick-up_50': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/single-block_blue-color_pick-up_80': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/mutli-stacked-block_mutli-color_pick-up_80': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/single-stacked-block_two-color_pick-up_80': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/single-stacked-block_mutli-color_pick-up_80': {0: 'Pick single block of multi-color and drop it in the box on the right.'}, 'underctrl/handcamera_single_blue': {0: 'Pick the Lego block and drop it in the box on the right.'}, 'cmcgartoll/cube_color_organizer': {0: 'Organize blue cube', 1: 'Organize red cube', 2: 'Organize yellow cube'}, 'T-K-233/koch_k1_pour_shot': {0: 'Place the glass on the table.'}, 'Beegbrain/stack_2_cubes': {0: 'picks up the red block.'}, 'seeingrain/pick_place_lego': {0: 'Place the cube in the basket.'}, 'seeingrain/pick_place_lego_wider_range_richard': {0: 'Pick up the blue cube and place it in the basket.'}, 'seeingrain/pick_place_lego_wider_range_dang': {0: 'Pick up the cube and place it in the basket.'}, 'seeingrain/pick_place_lego_wider_range_dong': {0: 'Pick up the blue object and place it in the basket.'}, 'seeingrain/pick_lego_to_hand': {0: 'places blue object on table.'}, 'seeingrain/pick_place_pink_lego': {0: 'Pick up the red cube and place it in the basket.'}, 'seeingrain/pick_place_pink_lego_few_samples': {0: 'pick_place_pink_lego_few_samples '}, 'seeingrain/one_shot_learning_18episodes': {0: 'Pick up the red block and place it in the basket.'}, 'helper2424/hil-serl-push-circle-classifier': {0: 'Push small circle object to the correct position'}, 'seeingrain/lego_3cameras': {0: 'Pick up the red block and place it in the basket.'}, 'Lugenbott/koch_1225': {0: 'Pick up the blue block and place it in the red box.'}, 'twerdster/koch_training_red': {0: 'Pick up the red block.'}, 'dboemer/koch_50-samples': {0: 'Pick up the red block and place it on top of the yellow box.'}, 'seeingrain/241228_pick_place_2cams': {0: 'Place the cube in the basket.'}, 'Eyas/grab_pink_lighter_10_per_loc': {0: 'Pick up the pink object from the table.'}, 'Eyas/grab_bouillon': {0: 'picks up the box and places it in the box.'}, 'twerdster/koch_new_training_red': {0: 'move red cube into cellotape circle'}, 'andabi/shoes_easy': {0: 'picks up the shoe.'}, 'andabi/D2': {0: 'Pick up the shoe and place it on the table.'}, 'Beegbrain/oc_stack_cubes': {0: 'stack the red cube on the blue cube'}, 'abougdira/cube_target': {0: 'put_the cube on the yellow target'}, 'andabi/D3': {0: 'picks up the shoe.'}, 'andabi/D4': {0: 'picks up the shoe.'}, 'andabi/D5': {0: 'places shoe on table.'}, 'andabi/D6': {0: 'picks up the shoe.'}, 'andabi/D7': {0: 'picks up the shoe.'}, 'jainamit/koch_realcube3': {0: 'pick up the cube real with keyboard input'}, 'jainamit/koch_pickcube': {0: 'Pick up the blue cube and place it in the box.'}, 'andabi/D8': {0: 'picks up the shoe.'}, 'andabi/D9': {0: 'The picks up the paper and places it on the table.'}, 'andabi/D10': {0: 'picks up the shoe.'}, 'andabi/D11': {0: 'The picks up the paper and places it on the table.'}, 'andabi/D12': {0: 'The places the cube in the box.'}, 'andabi/D13': {0: 'picks up shoes.'}, 'andabi/D14': {0: 'picks up shoes.'}, 'andabi/D15': {0: 'picks up the shoe.'}, 'rgarreta/koch_pick_place_lego': {0: 'Pick the Lego block and drop it in the box on the right.'}, 'shin1107/koch_train_block': {0: 'Grasp a block and put it in the hole.'}, 'andabi/D16': {0: 'picks up the shoe.'}, 'TrossenRoboticsCommunity/aloha_fold_tshirt': {0: 'Fold the t-shirt.'}, 'rgarreta/koch_pick_place_lego_v2': {0: 'Pick the Lego block and drop it in the box on the right.'}, '1g0rrr/screw1': {0: 'Grasp a lego block and put it in the bin.'}, 'ncavallo/moss_train_grasp': {0: 'Grasp a lego block and put it in the bin.'}, 'andabi/D17': {0: 'picks up the shoe.'}, 'ma3oun/rpi_squares_1': {0: 'Raspberry Pi 5 squares recording 1'}, 'shin1107/koch_move_block_with_some_shapes': {0: 'Grasp a block and put it in the hole with some shapes.'}, 'jannick-st/classifier': {0: 'Move the blue object to the right side of the table.'}, 'rgarreta/koch_pick_place_lego_v3': {0: 'Pick the Lego block and drop it in the box on the right. Top and wrist cameras.'}, 'TrossenRoboticsCommunity/aloha_stationary_logo_assembly': {0: 'Assemble the Trossen Robotics Logo.'}, 'rgarreta/koch_pick_place_lego_v6': {0: 'Grasp a lego block and put it in the bin.'}, 'Beegbrain/put_green_into_blue_bin': {0: 'Put the green cube into the blue bin'}, 'Beegbrain/put_screwdriver_box': {0: 'Put the screwdriver into the box'}, 'Beegbrain/align_three_pens': {0: 'picks up a pen.'}, 'Beegbrain/stack_green_on_blue_cube': {0: 'Stack the blue cube on top of the green cube'}, 'Beegbrain/align_cubes_green_blue': {0: 'Put the green cubes on the left and the blue cube on the right'}, 'IPEC-COMMUNITY/ucsd_kitchen_dataset_lerobot': {0: 'Turn on the faucet', 1: 'Put the bowl inside the kitchen cabinet', 2: 'Open the oven door', 3: 'Place the teapot on the stove', 4: 'Put the white box into the sink', 5: 'Open the carbinet door', 6: 'Put the green box into the sink', 7: 'Put the canned spam into the sink'}, 'dkdltu1111/omx-bottle1': {0: 'Grasp a lego block and put it in the bin.'}, 'rgarreta/koch_pick_place_lego_v7': {0: 'Grasp a lego block and put it in the bin.'}, 'rgarreta/koch_pick_place_lego_v8': {0: 'Grasp a lego block and put it in the bin.'}, 'IPEC-COMMUNITY/berkeley_mvp_lerobot': {0: 'push wooden cube', 1: 'pick detergent from the sink', 2: 'reach red block', 3: 'pick yellow cube', 4: 'close fridge door', 5: 'pick fruit'}, 'BlobDieKatze/GrabBlocks': {0: 'Grasp a lego block and put it in the bin.'}, 'Yuanzhu/koch_bimanual_grasp_0': {0: 'places the yellow block on the mousepad.'}, 'Yuanzhu/koch_bimanual_grasp_3': {0: 'picks up a yellow block.'}, 'takuzennn/aloha-pick100': {0: 'arm pick pen and put it into the cup'}, 'abbyoneill/pusht': {0: 'Grasp a lego block and put it in the bin.'}, 'ncavallo/moss_train_grasp_new': {0: 'Grasp a lego block and put it in the bin.'}, 'mlfu7/pi0_conversion_no_pad_video': {0: 'pickplace deer greybowl', 1: 'stack red on green', 2: 'stack orange cup to yellow cup', 3: 'put orange cup into yellow cup', 4: 'put push red pen to blue pen', 5: 'put tiger to black bowl', 6: 'put potato in bot to black bowl', 7: 'pick up green triangle', 8: 'put push blue pen to red pen', 9: 'close drawer', 10: 'put push green block to red', 11: 'pickup potato', 12: 'open drawer', 13: 'put closing tongs', 14: 'poke block', 15: 'put push blue cube', 16: 'poke tiger', 17: 'pick red cube into black bowl', 18: 'pick blue cube stack on wood block', 19: 'pick blue cube into grey bowl', 20: 'put red ball in black bowl', 21: 'pick green triangle into pink bowl', 22: 'pick red ball into pink bowl', 23: 'poke green triangle', 24: 'poke grey bowl', 25: 'put blue cube pink bowl', 26: 'put red cube into black bowl', 27: 'put deer into gray bowl', 28: 'put red ball into pink bowl', 29: 'put green triangle into pink bowl', 30: 'put pour from yellow cup into black bowl', 31: 'put pour blue cup into pink bowl', 32: 'put brown cube into gray bowl'}, 'pepijn223/lekiwi_pen': {0: 'Fold the jeans.'}, 'TrossenRoboticsCommunity/aloha_baseline_dataset': {0: 'Pick up a blue block and put it in a green bowl. Baseline dataset for testing'}, 'KeWangRobotics/piper_rl_1': {0: 'Pick up the cube and place it in the box.'}, 'nduque/cam_setup2': {0: 'Grasp a green block and put it in the bin.'}, 'KeWangRobotics/piper_rl_1_cropped_resized': {0: 'picks up the cube.'}, 'abbyoneill/new_dataset_pick_place': {0: 'Grasp a lego block and put it in the bin.'}, 'abbyoneill/thurs1120pickplace': {0: 'Grasp a lego block and put it in the bin.'}, 'abbyoneill/data_w_mug': {0: 'Grasp a lego block and put it in the bin.'}, 'pepijn223/lekiwi_drive_in_circle': {0: 'Pick up the red object and place it on the table.'}, 'pepijn223/lekiwi_block_cleanup2': {0: 'Put red block in black box'}, 'KeWangRobotics/piper_rl_2': {0: 'Move the cube to the right side of the table.'}, 'KeWangRobotics/piper_rl_2_cropped_resized': {0: 'Move the block to the right side of the table.'}, 'hannesill/koch_pnp_simple_50': {0: 'Grasp a small block with a specific orientation and put it in the bin with a specific position and orientation.'}, 'KeWangRobotics/piper_rl_3': {0: 'Pick up the wooden block.'}, 'KeWangRobotics/piper_rl_3_cropped_resized': {0: 'Place the block in the box.'}, 'hannesill/koch_pnp_2_blocks_2_bins_200': {0: 'Grasp the blue block first and put it in the first bin that has a specific position and orientation. Then grasp the white block and put it in the second bin that has a specific position and orientation.'}, 'ellen2imagine/pusht_green1': {0: 'Place the green block in the box.'}, 'imatrixlee/koch_place': {0: 'Pick up the white object and place it on the table.'}, 'ellen2imagine/pusht_green_same_init2': {0: 'Place the green block in the correct position.'}, 'KeWangRobotics/piper_rl_4': {0: 'Move the block slightly.'}, 'KeWangRobotics/piper_rl_4_cropped_resized': {0: 'picks the wooden block.'}, 'lalalala0620/koch_blue_paper_tape': {0: 'Grasp a blue paper tape and put it in the bin.'}, 'nduque/act_50_ep': {0: 'Grasp a green block and put it in the bin.'}, 'nduque/act_50_ep2': {0: 'Grasp a green block and put it in the bin.'}, 'HWJ658970/fat_fish': {0: 'Grasp a fat fish toy and put it in the bin.'}, 'Beegbrain/put_red_triangle_green_rect': {0: 'Put the red triangle on top of the green rectangle'}, 'ncavallo/moss_train_gc_block': {0: 'Grasp a lego block and put it in the bin.'}, 'takuzennn/square3': {0: 'Pick the cube from the table.'}, 'HWJ658970/lego_50': {0: 'Grasp a yellow lego block and put it in the bin.'}, 'Deason11/mobile_manipulator_0319': {0: 'Grasp a lego block and put it in the bin.'}, 'Gongsta/grasp_duck_in_cup': {0: 'Grasp the rubber duck and put it in the cup.'}, 'nduque/robustness_e2': {0: 'Grasp a green dice and put it in the bin.'}, 'ibru/bobo_trash_collector': {0: 'Bobo Trash collect and place it in a bin'}, 'Beegbrain/moss_open_drawer_teabox': {0: 'Open the top drawer of the teabox'}, 'Beegbrain/moss_put_cube_teabox': {0: 'Put the green cube in the top drawer of the teabox'}, 'Beegbrain/moss_close_drawer_teabox': {0: 'Close the top drawer of the teabox'}, 'Beegbrain/moss_stack_cubes': {0: 'Stack the green cube on top of the blue cube'}, 'HWJ658970/lego_50_camera_change': {0: 'Grasp a yellow lego block and put it in the bin.'}, 'HWJ658970/lego_100_class': {0: 'Separate yellow and white Lego blocks and place them into the bin.'}, 'nduque/robustness_e3': {0: 'Grasp a green dice and put it in the bin.'}, 'nimitvasavat/Gr00t_lerobot': {0: 'Place the cereal box on the shelf.'}, 'Deason11/mobile_manipulator_0326': {0: 'Grasp a lego block and put it in the bin.', 1: 'mobile_lekiwi.'}, 'KeWangRobotics/piper_push_cube_gamepad_1': {0: 'push the cube to the black area'}, 'KeWangRobotics/piper_push_cube_gamepad_1_cropped_resized': {0: 'push the cube to the black area'}, 'jannick-st/push-cube-classifier_cropped_resized': {0: 'Close the cabinet door.'}, 'nimitvasavat/Gr00t_lerobotV2': {0: 'Place the chocolate chip cookie dough box on the table.'}, 'Zhaoting123/koch_cleanDesk_': {0: 'Grasp a card and use it to clean the desk'}, 'arclabmit/koch_gear_and_bin': {0: 'Pick the gear and place it in the bin.'}, 'Allen-488/koch_dataset_50': {0: 'Grasp a block and put it in the bin.'}, 'dop0/koch_pick_terminal': {0: 'Pick up the terminal and place on the cover.'}, 'nduque/robustness_e4': {0: 'Grasp a green dice and put it in the bin.'}, 'arclabmit/Koch_twoarms': {0: 'official two arms recordings10'}, 'nimitvasavat/Gr00t_lerobot_state_action': {0: 'Place the chocolate chip cookie dough box on the table.'}, 'zliu157/i3r': {0: 'Grasp a lego block and put it in the bin.'}, 'hangwu/koch_pick_terminal': {0: 'Pick up the terminal and place on the cover.'}, 'nduque/robustness_e5': {0: 'Grasp a green dice and put it in the bin.'}, 'zliu157/i3r2': {0: 'Grasp a i3r logo and put it in the bin.'}, 'HuaihaiLyu/groceries': {0: 'Pick the brown long bread and Egg yolk pasry into package'}, 'zliu157/i3r3': {0: 'Grasp a i3r logo and put it in the bin.'}, 'hangwu/piper_pick_terminal_and_place': {0: 'Grasp a terminal and put it on the black box.'}, 'hangwu/piper_pick_terminal_2': {0: 'Grasp the white terminal and put it on the green lid.'}, 'engineer0002/pepper': {0: 'Place the bottle on the table.'}, 'theo-michel/lekiwi_v2': {0: 'Pick up the can on the ground'}, 'theo-michel/lekiwi_v5': {0: 'Pick up the can on the ground'}, 'roboticshack/sandee-kiwiv10': {0: 'Place the bottle on the table.'}, 'ibru/bob_jetson': {0: 'Drive forward pickup the object and put it in the red box and drive back.'}, 'ibru/bobo_jetson': {0: 'Drive forward pickup the object and put it in the red box and drive back.', 1: 'Driver forward'}, 'zliu157/i3r5': {0: 'Grasp a i3r logo and put it in the bin.'}, 'Dongkkka/cable_pick_and_place2': {0: 'Put a black charging cable in a black bowl and put a red charging cable in a green bowl'}, - - -} +# Example usage +TASKS_KEYS_MAPPING = load_yaml_mapping("features") diff --git a/lerobot/configs/mappings/features.yaml b/lerobot/configs/mappings/features.yaml new file mode 100644 index 000000000..816d91acd --- /dev/null +++ b/lerobot/configs/mappings/features.yaml @@ -0,0 +1,2387 @@ +00ri/so100_battery: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +00ri/so100_battery_bin_center: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +0x00raghu/toffee_blue: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +0x00raghu/toffee_blue_2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +0x00raghu/toffee_red: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +0x00raghu/toffee_red_2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +0x00raghu/toffee_red_3__: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +0x00raghu/toffee_to_hand_1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +0x00raghu/toffee_to_hand_2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +1g0rrr/offline_dataset_name2: + observation.images.front: observation.image + observation.images.side: observation.image2 +1g0rrr/offline_dataset_name2_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +1g0rrr/reward_dataset_name2: + observation.images.front: observation.image + observation.images.side: observation.image2 +1g0rrr/reward_dataset_name2_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +1g0rrr/reward_pickplace1: + observation.images.laptop: observation.image +1g0rrr/reward_pickplace1_cropped_resized: + observation.images.laptop: observation.image +1g0rrr/sam_openpi03: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +1g0rrr/sam_openpi_cube_low10: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +1g0rrr/sam_openpi_cube_top10: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +1g0rrr/sam_openpi_solder1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +1g0rrr/sam_openpi_solder2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +1g0rrr/sam_openpi_wire10: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +1g0rrr/screw1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +356c/so100_duck_reposition_1: + observation.images.body: observation.image2 + observation.images.overhead: observation.image + observation.images.side: observation.image3 +356c/so100_nut_sort_1: + observation.images.body: observation.image2 + observation.images.overhead: observation.image + observation.images.side: observation.image3 +356c/so100_rope_reposition_1: + observation.images.body: observation.image2 + observation.images.overhead: observation.image + observation.images.side: observation.image3 +AK51/4090_01: + observation.images.base_cam: observation.image3 + observation.images.left_arm_cam: observation.image2 + observation.images.right_arm_cam: null + observation.images.top_cam: observation.image +AaronNewman/screwdriver_task_batch1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +AaronNewman/screwdriver_task_batch2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +AaronNewman/screwdriver_task_batch3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Allen-488/koch_dataset_50: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 +Anas0711/Gr00t_lerobot_state_action: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +Anas0711/Gr00t_lerobot_state_action_180: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +Anas0711/Gr00t_lerobot_state_action_600: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +Anas0711/Gr00t_lerobot_state_action_60_continuous: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +Anas0711/Gr00t_lerobot_state_action_box_1: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +Anas0711/Gr00t_lerobot_state_action_box_2: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +AndrejOrsula/lerobot_double_ball_stacking_random: + observation.images.realsense: observation.image + observation.images.side: observation.image2 +Ashton3/lerobot-aloha: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Bartm3/dice2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Bartm3/tape_to_bin: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Bartm3/tape_to_bin2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Beegbrain/align_cubes_green_blue: + observation.images.back: observation.image3 + observation.images.side: observation.image +Beegbrain/align_three_pens: + observation.images.back: observation.image3 + observation.images.side: observation.image +Beegbrain/moss_close_drawer_teabox: + observation.images.back: observation.image3 + observation.images.front: observation.image +Beegbrain/moss_open_drawer_teabox: + observation.images.back: observation.image3 + observation.images.front: observation.image +Beegbrain/moss_put_cube_teabox: + observation.images.back: observation.image3 + observation.images.front: observation.image +Beegbrain/moss_stack_cubes: + observation.images.back: observation.image3 + observation.images.front: observation.image +Beegbrain/oc_stack_cubes: + observation.images.realsense: observation.image3 + observation.images.realsense_top: observation.image +Beegbrain/pick_lemon_and_drop_in_bowl: + observation.images.realsense_side: observation.image + observation.images.realsense_top: observation.image2 +Beegbrain/pick_place_green_block: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +Beegbrain/pick_place_green_block_lr: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +Beegbrain/put_green_into_blue_bin: + observation.images.back: observation.image3 + observation.images.side: observation.image +Beegbrain/put_red_triangle_green_rect: + observation.images.back: observation.image3 + observation.images.side: observation.image +Beegbrain/put_screwdriver_box: + observation.images.back: observation.image3 + observation.images.side: observation.image +Beegbrain/so100_put_cube_cup: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +Beegbrain/stack_2_cubes: + observation.images.laptop: observation.image +Beegbrain/stack_green_on_blue_cube: + observation.images.back: observation.image3 + observation.images.side: observation.image +Beegbrain/stack_two_cubes: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +Beegbrain/sweep_tissue_cube: + observation.images.realsense_side: observation.image + observation.images.realsense_top: observation.image2 +BlobDieKatze/GrabBlocks: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +CSCSXX/pick_place_cube_1.17: + observation.images.side: observation.image2 + observation.images.top: observation.image +CSCSXX/pick_place_cube_1.18: + observation.images.side: observation.image2 + observation.images.top: observation.image +Chojins/chess_game_000_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_000_white_red: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_001_blue_stereo: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_001_red_stereo: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_001_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_002_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_003_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_004_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_005_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_006_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_007_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Chojins/chess_game_009_white: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Cidoyi/so100_all_notes: + observation.images.keyboard_camera: observation.image +Cidoyi/so100_all_notes_1: + observation.images.keyboard_camera: observation.image +Cidoyi/so100_all_notes_3: + observation.images.keyboard_camera: observation.image +Clementppr/lerobot_pick_and_place_dataset_world_model: + observation.images.laptop: observation.image +Congying1112/so100_place_blue_bottle_with_single_camera: + observation.images.onhand: observation.image2 +Congying1112/so100_place_blue_bottle_with_two_cameras: + observation.images.onhand: observation.image2 + observation.images.top: observation.image +Congying1112/so100_place_blue_bottle_with_two_cameras2: + observation.images.onhand: observation.image2 + observation.images.top: observation.image +Congying1112/so100_simplepick_with_2_cameras_from_top: + observation.images.onhand: observation.image2 + observation.images.top: observation.image +Congying1112/so100_switch_with_onhand_camera: + observation.images.onhand: observation.image2 +CrazyYhang/A1234-B-C_mvA2B: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Deason11/Open_the_drawer_to_place_items: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Deason11/PLACE_TAPE_PUSH_DRAWER: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Deason11/Random_Kitchen: + observation.images.L_OverheadCamera: observation.image + observation.images.R_OverheadCamera: observation.image3 + observation.images.wrist: observation.image2 +Deason11/mobile_manipulator_0319: + observation.images.L_OverheadCamera: observation.image + observation.images.R_OverheadCamera: observation.image3 + observation.images.wrist: observation.image2 +Deason11/mobile_manipulator_0326: + observation.images.L_OverheadCamera: observation.image + observation.images.R_OverheadCamera: observation.image3 + observation.images.wrist: observation.image2 +DimiSch/so100_50ep_2: + observation.images.realsense: observation.image +DimiSch/so100_50ep_3: + observation.images.realsense: observation.image +DimiSch/so100_terra_50_2: + observation.images.realsense: observation.image +Dongkkka/cable_pick_and_place2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +DorayakiLin/so100_pick_charger_on_tissue: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +DorayakiLin/so100_pick_cube_in_box: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +DorayakiLin/so100_stack_cube: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Eyas/grab_bouillon: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 +Eyas/grab_pink_lighter_10_per_loc: + observation.images.laptop: observation.image +FeiYjf/Grab_Pieces: + observation.images.left: observation.image + observation.images.right: observation.image2 +FeiYjf/Hold_Pieces: + observation.images.left: observation.image + observation.images.right: observation.image2 +FeiYjf/Makalu_push: + observation.images.left: observation.image2 + observation.images.right: observation.image +FeiYjf/Maklu_dataset: + observation.images.left: observation.image + observation.images.right: observation.image2 +FeiYjf/Test_NNNN: + observation.images.left: observation.image + observation.images.right: observation.image2 +FeiYjf/new_Dataset: + observation.images.left: observation.image + observation.images.right: observation.image2 +FeiYjf/new_GtoR: + observation.images.left: observation.image + observation.images.right: observation.image2 +FeiYjf/push_0094: + observation.images.left: observation.image + observation.images.right: observation.image2 +FeiYjf/push_gg: + observation.images.left: observation.image2 + observation.images.right: observation.image +FsqZ/so100_1: + observation.images.side: observation.image +Gano007/so100_doliprane: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Gano007/so100_gano: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Gano007/so100_lolo: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Gano007/so100_medic: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Gano007/so100_second: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Gongsta/grasp_duck_in_cup: + observation.images.laptop: observation.image +HITHY/so100-kiwi: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +HITHY/so100_peach: + observation.images.laptop: observation.image +HITHY/so100_peach1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +HITHY/so100_peach3: + observation.images.laptop: observation.image +HITHY/so100_peach4: + observation.images.laptop: observation.image +HITHY/so100_redstrawberry: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +HITHY/so100_strawberry: + observation.images.laptop: observation.image +HWJ658970/fat_fish: + observation.images.front: observation.image + observation.images.phone: observation.image3 +HWJ658970/lego_100_class: + observation.images.front: observation.image + observation.images.phone: observation.image3 +HWJ658970/lego_50: + observation.images.front: observation.image + observation.images.phone: observation.image3 +HWJ658970/lego_50_camera_change: + observation.images.front: observation.image + observation.images.phone: observation.image3 +HYAIYN/so100_get_orange_10epi: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +HappyPablo/dec3_data2: + observation.images.laptop: null + observation.images.phone: observation.image2 +HuaihaiLyu/groceries: + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_right_wrist: observation.image3 +HuaihaiLyu/stackbasket: + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_right_wrist: observation.image3 +IPEC-COMMUNITY/berkeley_mvp_lerobot: + observation.images.hand_image: observation.image +IPEC-COMMUNITY/libero_10_no_noops_image_lerobot: + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +IPEC-COMMUNITY/ucsd_kitchen_dataset_lerobot: + observation.images.image: observation.image +Ityl/so100_recording1: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +Ityl/so100_recording2: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +JJwuj/koch_static_grasp_0402_v5: + observation.images.E12: observation.image2 + observation.images.E22S: observation.image +Jiafei1224/so100_pa222per: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Jiangeng/so100_413: + observation.images.top: observation.image2 + observation.images.wrist_left: observation.image +KeWangRobotics/piper_push_cube_gamepad_1: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_push_cube_gamepad_1_cropped_resized: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_push_cube_gamepad_2: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_push_cube_gamepad_2_cropped_resized: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_rl_1: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_rl_1_cropped_resized: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_rl_2: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_rl_2_cropped_resized: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_rl_3: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_rl_3_cropped_resized: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_rl_4: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +KeWangRobotics/piper_rl_4_cropped_resized: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +LegrandFrederic/Orange-brick-lower-resolution: + observation.images.main.left: observation.image + observation.images.main.right: observation.image2 + observation.images.secondary_0: observation.image3 +LemonadeDai/so100_coca: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +Loki0929/so100_100: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Loki0929/so100_duck: + observation.images.third: observation.image3 + observation.images.top: observation.image + observation.images.wrist: observation.image2 +Lugenbott/koch_1225: + observation.images.Camera0: observation.image + observation.images.Camera2: observation.image2 +Mohamedal/put_banana: + observation.images.realsense_side: observation.image + observation.images.realsense_top: observation.image2 +MossProphet/so100_square-1-2-3.2: + observation.images.Arm_left: observation.image2 + observation.images.Arm_right: null + observation.images.External: observation.image3 +Mwuqiu/so100_0408: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Mwuqiu/so100_0408_muti: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +NONHUMAN-RESEARCH/SOARM100_TASK_VENDA: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +NONHUMAN-RESEARCH/SOARM100_TASK_VENDA_BOX: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Odog16/so100_cube_drop_pick_v1: + observation.images.workspace: observation.image + observation.images.wrist: observation.image2 +Odog16/so100_cube_stacking_v1: + observation.images.Workspace: observation.image + observation.images.Wrist: observation.image2 +Odog16/so100_tea_towel_folding_v1: + observation.images.workspace: observation.image + observation.images.wrist: observation.image2 +Ofiroz91/so_100_cube2bowl: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +Pi-robot/barbecue_flip: + observation.images.top: observation.image2 + observation.images.wrist: observation.image +Pi-robot/barbecue_put: + observation.images.top: observation.image2 + observation.images.wrist: observation.image +RLWRLD/put_cube_sync: + observation.images.cam_high: observation.image + observation.images.cam_right_wrist: observation.image2 +RasmusP/so100_Orange2Green: + observation.images.phone: observation.image + observation.images.webcam: observation.image2 +RasmusP/so100_dataset50ep: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +RasmusP/so100_dataset50ep_a: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +SeanLMH/so100_picknplace: + observation.images.front: observation.image2 + observation.images.overhead: observation.image +SeanLMH/so100_picknplace_v2: + observation.images.front: observation.image2 + observation.images.overhead: observation.image +Setchii/so100_grab_ball: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +T-K-233/koch_k1_pour_shot: + observation.images.gripper: observation.image2 + observation.images.side: observation.image3 + observation.images.top: observation.image +T1g3rGE/so100_pickplace_small_20250407_171912: + observation.images.webcam: observation.image2 +TrossenRoboticsCommunity/aloha_ai_block: + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +TrossenRoboticsCommunity/aloha_baseline_dataset: + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +TrossenRoboticsCommunity/aloha_fold_tshirt: + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: observation.image + observation.images.cam_right_wrist: observation.image3 +TrossenRoboticsCommunity/aloha_stationary_logo_assembly: + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: observation.image3 + observation.images.cam_right_wrist: null +VoicAndrei/so100_banana_to_plate_only: + observation.images.right: observation.image2 + observation.images.top: observation.image +VoicAndrei/so100_banana_to_plate_rebel_full: + observation.images.right: observation.image2 + observation.images.top: observation.image +Winster/so100_cube: + observation.images.laptop: observation.image +Winster/so100_sim: + observation.images.laptop: observation.image +XXRRSSRR/so100_v3_num_episodes_50: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +Yotofu/so100_sweeper_shoes: + observation.images.end_depth: null + observation.images.end_rgb: observation.image3 + observation.images.front_depth: observation.image2 + observation.images.front_rgb: observation.image +Yuanzhu/koch_bimanual_grasp_0: + observation.images.top_camera: observation.image +Yuanzhu/koch_bimanual_grasp_3: + observation.images.hand_camera: observation.image2 + observation.images.side_camera: observation.image3 + observation.images.top_camera: observation.image +ZCM5115/so100_1210: + observation.images.hikvision: observation.image2 + observation.images.laptop: observation.image +ZCM5115/so100_2Arm3cameras_movebox: + observation.images.left: observation.image2 + observation.images.right: observation.image3 + observation.images.top: observation.image +ZGGZZG/so100_drop0: + observation.images.left: observation.image2 + observation.images.up: observation.image +ZGGZZG/so100_drop1: + observation.images.left: observation.image2 + observation.images.up: observation.image +Zak-Y/so100_grap_dataset: + observation.images.Left_follower: observation.image2 + observation.images.Logic_camera: observation.image + observation.images.Right_follower: observation.image3 +Zak-Y/so100_three_cameras_dataset: + observation.images.Left_follower: observation.image2 + observation.images.Logic_camera: observation.image + observation.images.Right_follower: observation.image2 +Zhaoting123/koch_cleanDesk_: + observation.images.phone: observation.image +abbyoneill/data_w_mug: + observation.images.logitech1: observation.image + observation.images.logitech2: observation.image2 +abbyoneill/new_dataset_pick_place: + observation.images.logitech1: observation.image + observation.images.logitech2: observation.image3 +abbyoneill/pusht: + observation.images.logitech1: observation.image + observation.images.logitech2: observation.image3 +abbyoneill/thurs1120pickplace: + observation.images.logitech1: observation.image + observation.images.logitech2: observation.image3 +abhisb/so100_51_ep: + observation.images.front: observation.image3 + observation.images.mobile: observation.image2 + observation.images.overhead: observation.image +abokinala/sputnik_100_11_pick_place_container: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +abokinala/sputnik_100_12_pick_place_container: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +abokinala/sputnik_100_14_pick_place_container: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +abokinala/sputnik_100_23_pick_place_surface: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +abokinala/sputnik_100_46_custom_tasks: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +abokinala/sputnik_100_54_kitchen_tasks: + observation.images.laptop: null + observation.images.phone: observation.image +abokinala/sputnik_100_55_kitchen_tasks: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +abokinala/sputnik_100_58_kitchen_tasks: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +abokinala/sputnik_100_60_kitchen_tasks: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +abougdira/cube_target: + observation.images.realsense: observation.image3 + observation.images.realsense_top: observation.image +acrampette/third_arm_01: + observation.images.wrist: observation.image2 +acrampette/third_arm_02: + observation.images.wrist: observation.image2 +ad330/cubePlace: + observation.images.phone: observation.image + observation.images.wristCam: observation.image2 +ad330/so100_box_pickPlace: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +aergogo/so100_pick_place: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +agonyxx/koch-aloha: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 + observation.images.phone2: observation.image3 +aimihat/so100_tape: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +airthebear/so100_GL: + observation.images.laptop: observation.image +aliberts/koch_tutorial: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 +allenchienxxx/so100Test: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +andabi/D10: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D11: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D12: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D13: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D14: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D15: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D16: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D17: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D2: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D3: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D4: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D5: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D6: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D7: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D8: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/D9: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/a_shoe_easy_10: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andabi/shoes_easy: + observation.images.bird: observation.image + observation.images.wrist_left: observation.image2 + observation.images.wrist_right: observation.image3 +andlyu/so100_indoor_0: + observation.images.arm_left: observation.image2 + observation.images.arm_right: observation.image3 + observation.images.base_left: null + observation.images.base_right: observation.image +andlyu/so100_indoor_1: + observation.images.arm_left: observation.image3 + observation.images.arm_right: observation.image2 + observation.images.base_left: null + observation.images.base_right: observation.image +andlyu/so100_indoor_10: + observation.images.arm: observation.image + observation.images.base: observation.image3 + observation.images.base_right: null + observation.images.gripper: observation.image2 +andlyu/so100_indoor_2: + observation.images.arm_left: observation.image2 + observation.images.arm_right: observation.image3 + observation.images.base_left: null + observation.images.base_right: observation.image +andlyu/so100_indoor_3: + observation.images.arm_left: observation.image3 + observation.images.arm_right: observation.image2 + observation.images.base_left: null + observation.images.base_right: observation.image +andlyu/so100_indoor_4: + observation.images.arm_left: observation.image2 + observation.images.arm_right: observation.image3 + observation.images.base_left: null + observation.images.base_right: observation.image +andlyu/so100_indoor_val2: + observation.images.arm: observation.image + observation.images.base: observation.image3 + observation.images.base_right: null + observation.images.gripper: observation.image2 +andlyu/so100_indoor_val_0: + observation.images.arm: observation.image + observation.images.base: observation.image3 + observation.images.gripper: observation.image2 +andreasBihlmaier/dual_arm_transfer_2025_02_16: + observation.images.webcam_1: observation.image3 + observation.images.webcam_2: observation.image2 + observation.images.webcam_3: observation.image +andrewcole712/so100_tape_bin_place: + observation.images.phone: observation.image +andy309/so100_0311_1152: + observation.images.back: observation.image2 + observation.images.top: observation.image + observation.images.wrist_right: observation.image3 +andy309/so100_0313_no_wrist_camera: + observation.images.back: observation.image2 + observation.images.top: observation.image +andy309/so100_0313_no_wrist_camera_with_two_arms_cloths: + observation.images.back: observation.image2 + observation.images.top: observation.image +andy309/so100_0314_fold_cloths: + observation.images.back: observation.image2 + observation.images.top: observation.image +aractingi/pick_place_lego_cube: + observation.images.top: observation.image2 + observation.images.wrist: observation.image +aractingi/pick_place_lego_cube_1: + observation.images.top: observation.image2 + observation.images.wrist: observation.image +aractingi/pick_place_lego_cube_cropped_resized: + observation.images.top: observation.image2 + observation.images.wrist: observation.image +aractingi/push_cube_front_side_reward: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_front_side_reward_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_front_side_reward_long: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_front_side_reward_long_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_offline_data: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +aractingi/push_cube_offline_data_cropped_resized: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +aractingi/push_cube_overfit: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_overfit_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_reward: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_reward_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_reward_data: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +aractingi/push_cube_reward_data_cropped_resized: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +aractingi/push_cube_simp_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_square_light_offline_demo: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_square_light_offline_demo_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_square_light_reward: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_square_offline_demo: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_square_offline_demo_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_square_reward_1: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_square_reward_1_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_square_reward_cropped_resized: + observation.images.front: observation.image + observation.images.side: observation.image2 +aractingi/push_cube_to_face_reward: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +aractingi/push_cube_to_face_reward_cropped_resized: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +aractingi/push_green_cube_hf: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +aractingi/push_green_cube_hf_cropped_resized: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +arclabmit/Koch_twoarms: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 +arclabmit/koch_gear_and_bin: + observation.images.nexigo_webcam: observation.image + observation.images.realsense: observation.image3 +astroyat/cube: + observation.images.laptop: observation.image +badwolf256/so100_twin_cam_duck: + observation.images.realsense: observation.image + observation.images.wrist: observation.image2 +badwolf256/so100_twin_cam_duck_v2: + observation.images.realsense: observation.image + observation.images.wrist: observation.image2 +badwolf256/so100_twin_cam_duck_v3: + observation.images.realsense: observation.image + observation.images.wrist: observation.image2 +baladhurgesh97/so100_final_picking_3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +bensprenger/chess_game_001_blue_stereo: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +bensprenger/left_arm_yellow_brick_in_box_v0: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +bensprenger/left_arm_yellow_brick_in_box_with_purple_noise_v0: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +bensprenger/right_arm_p_brick_in_box_with_y_noise_v0: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +bnarin/so100_tic_tac_toe_move_0_0: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +bnarin/so100_tic_tac_toe_move_1_0: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +bnarin/so100_tic_tac_toe_move_2_1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +bnarin/so100_tic_tac_toe_move_4_0: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +bnarin/so100_tic_tac_toe_we_do_it_live: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +cadene/droid_1.0.1: + action.cartesian_position: null + action.cartesian_velocity: null + action.gripper_position: null + action.gripper_velocity: null + action.joint_position: null + action.joint_velocity: null + action.original: null + building: null + camera_extrinsics.exterior_1_left: null + camera_extrinsics.exterior_2_left: null + camera_extrinsics.wrist_left: null + collector_id: null + date: null + discount: null + is_episode_successful: null + language_instruction: null + language_instruction_2: null + language_instruction_3: null + observation.images.exterior_1_left: observation.image + observation.images.exterior_2_left: observation.image3 + observation.images.wrist_left: observation.image2 + observation.state.cartesian_position: null + observation.state.gripper_position: null + observation.state.joint_position: null + task_category: null +carpit680/giraffe_sock_demo_1: + observation.images.webcam: observation.image +carpit680/giraffe_sock_demo_2: + observation.images.webcam: observation.image +carpit680/giraffe_task: + observation.images.webcam: observation.image +chmadran/so100_dataset04: + observation.images.laptop: observation.image +chmadran/so100_dataset08: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +chmadran/so100_home_dataset: + observation.images.laptop: observation.image3 + observation.images.logitech: observation.image + observation.images.phone: observation.image2 +cmcgartoll/cube_color_organizer: + observation.images.claw: observation.image2 + observation.images.front: observation.image + observation.images.phone: observation.image3 +ctbfl/sort_battery: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/so100_task_1: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/so100_task_2: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/so100_task_3: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/so100_task_4: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/so100_v2_task_1: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/so100_v2_task_2: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/so100_v2_task_3: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/so100_v2_task_4: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/svla_so100_task1_v3: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/svla_so100_task4_v3: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/svla_so100_task4_v3_clean: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/svla_so100_task4_v3_multiple: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/svla_so100_task5_v3: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danaaubakirova/svla_so100_task5_v3_clean: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +danielkr452/so100_work6: + observation.images.laptop: null + observation.images.phone: observation.image2 +danmac1/real_real332: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +dboemer/koch_50-samples: + observation.images.side: observation.image3 + observation.images.top: observation.image +dc2ac/so100-t5: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +denghj/dataset_red_tape01: + observation.images.laptop: observation.image +dkdltu1111/omx-bottle1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +dop0/koch_pick_terminal: + observation.images.side: observation.image3 + observation.images.top: observation.image + observation.images.wrist: observation.image2 +doujiangwang/task1_10epi_100000step: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +doujiangwang/task2_10epi_100000step: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +doujiangwang/task4_10epi_100000step: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +doujiangwang/task5_10epi_100000step: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +dragon-95/so100_sorting: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +dragon-95/so100_sorting_1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +dragon-95/so100_sorting_2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +dragon-95/so100_sorting_3: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +dsfsg/bring_bottle: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 + observation.images.phone2: observation.image3 +dsfsg/grasp_pen_and_bottle: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 + observation.images.phone2: observation.image2 +dsfsg/grasp_pens: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 + observation.images.phone2: observation.image3 +duthvik/sputnik_100_17_pick_place_container: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +duthvik/sputnik_100_18_pick_place_container: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_24_pick_place_surface: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +duthvik/sputnik_100_25_pick_place_surface: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +duthvik/sputnik_100_26_pick_place_surface: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +duthvik/sputnik_100_27_pick_place_surface: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +duthvik/sputnik_100_29_pick_place_surface: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_31_pour_liquid: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +duthvik/sputnik_100_32_pour_liquid: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_41_custom_tasks: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_42_custom_tasks: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_43_custom_tasks: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_44_custom_tasks: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_45_custom_tasks: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_51_kitchen_tasks: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_52_kitchen_tasks: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +duthvik/sputnik_100_53_kitchen_tasks: + observation.images.laptop: observation.image + observation.images.side_view: observation.image3 + observation.images.usb_front: observation.image2 +ellen2imagine/pusht_green1: + observation.images.phone: observation.image +ellen2imagine/pusht_green_same_init2: + observation.images.phone: observation.image +engineer0002/pepper: + observation.images.left_wrist: observation.image2 + observation.images.right_wrist: observation.image3 + observation.images.top: observation.image +ethanCSL/your_task_name: + observation.images.front: observation.image + observation.images.phone: observation.image2 +francescocrivelli/carrot_eating: + observation.images.endeffector: observation.image2 + observation.images.workspace: observation.image +francescocrivelli/orange_feeding: + observation.images.endeffector: observation.image2 + observation.images.workspace: observation.image +frk2/so100large: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +frk2/so100largediffcam: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +ganker5/so100_action_20250403: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +ganker5/so100_color_0328: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +ganker5/so100_dataline_0328: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +ganker5/so100_dataline_20250331: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +ganker5/so100_push_20250328: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +ganker5/so100_push_20250331: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +ganker5/so100_toy_20250402: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +gxy1111/so100_pick_place: + observation.images.eye: observation.image + observation.images.wrist: observation.image2 +hangwu/koch_pick_terminal: + observation.images.side: observation.image3 + observation.images.top: observation.image + observation.images.wrist: observation.image2 +hangwu/piper_pick_terminal_2: + observation.images.depth: null + observation.images.realsense_color: observation.image3 + observation.images.side: observation.image2 + observation.images.top: observation.image +hangwu/piper_pick_terminal_and_place: + observation.images.depth: null + observation.images.realsense_color: observation.image3 + observation.images.side: observation.image2 + observation.images.top: observation.image +hannesill/koch_pnp_2_blocks_2_bins_200: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +hannesill/koch_pnp_simple_50: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +hegdearyandev/so100_eraser_cup_v1: + observation.images.phone: observation.image2 + observation.images.webcam-0: observation.image +helper2424/hil-serl-push-circle-classifier: + observation.images.web0: observation.image + observation.images.web1: observation.image3 +hkphoooey/throw_stuffie: + observation.images.phone: observation.image +ibru/bob_jetson: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +ibru/bobo_groot_n1_trash_picker: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +ibru/bobo_jetson: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +ibru/bobo_trash_collector: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +imatrixlee/koch_place: + observation.images.eye: observation.image2 + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +imsyed00/so100_yellowbowl_pickplace_1: + observation.images.laptop: observation.image +isadev/bougies1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +isadev/bougies2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +isadev/bougies3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +jackvial/koch_with_rewards_4: + observation.images.main: observation.image +jadechoghari/genesis-1k: + observation.image.side: observation.image3 + observation.image.top: observation.image + observation.image.wrist: observation.image2 +jadechoghari/svla_so101-sim_task4_v3_multiple_1: + observation.image.side: observation.image3 + observation.image.top: observation.image + observation.image.wrist: observation.image2 +jadechoghari/svla_so101-sim_task4_v3_multiple_2: + observation.image.side: observation.image3 + observation.image.top: observation.image + observation.image.wrist: observation.image2 +jainamit/koch: + observation.images.nexigo: observation.image +jainamit/koch_pickcube: + observation.images.front_camera: observation.image +jainamit/koch_realcube3: + observation.images.front_camera: observation.image +jannick-st/classifier: + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: observation.image3 + observation.images.cam_right_wrist: null +jannick-st/push-cube-classifier_cropped_resized: + observation.images.cam_low: observation.image + observation.images.cam_right_wrist: observation.image2 + observation.images.cam_top: observation.image +jbraumann/so100_1902: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +jchun/so100_pickplace_small_20250322_193929: + observation.images.cv: observation.image2 + observation.images.main: observation.image3 + observation.images.webcam: observation.image +jchun/so100_pickplace_small_20250323_120056: + observation.images.cv: observation.image3 + observation.images.main: observation.image2 + observation.images.webcam: observation.image +jiajun001/eraser00_2: + observation.images.hand: observation.image2 + observation.images.side: observation.image +jiajun001/eraser00_3: + observation.images.front: observation.image + observation.images.hand: observation.image2 + observation.images.side: observation.image3 +jlesein/TestBoulon2: + observation.images.robor: observation.image2 + observation.images.top: observation.image +jlesein/TestBoulon7: + observation.images.robor: observation.image2 + observation.images.top: observation.image +jlitch/so100multicam3: + observation.images.overview2: observation.image + observation.images.wrist: observation.image2 +jlitch/so100multicam6: + observation.images.overview2: observation.image + observation.images.wrist: observation.image2 +jlitch/so100multicam7: + observation.images.overview2: observation.image + observation.images.wrist: observation.image2 +jmrog/so100_sweet_pick: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +joaoocruz00/so100_makeitD1: + observation.images.laptop: observation.image2 + observation.images.realsensergb: observation.image +jpata/so100_pick_place_tangerine: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +kantine/domotic_dishTidyUp_anomaly: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_dishTidyUp_expert: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_groceriesSorting_anomaly: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_groceriesSorting_expert: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_makingCoffee_anomaly: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_makingCoffee_expert: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_pouringCoffee_anomaly: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_pouringCoffee_expert: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_setTheTable_anomaly: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_setTheTable_expert: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_vegetagblesAndFruitsSorting_anomaly: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/domotic_vegetagblesAndFruitsSorting_expert: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/flip_A0: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +kantine/flip_A1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +kantine/flip_A2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +kantine/flip_A3: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +kantine/flip_A4: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +kantine/flip_A5: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +kantine/industrial_robothon_buttons_anomaly: + observation.images.logitech_1: observation.image2 + observation.images.logitech_2: observation.image +kantine/industrial_robothon_buttons_expert: + observation.images.logitech_1: observation.image2 + observation.images.logitech_2: observation.image +kantine/industrial_robothon_hatchAndProbe_anomaly: + observation.images.logitech_1: observation.image2 + observation.images.logitech_2: observation.image +kantine/industrial_robothon_hatchAndProbe_expert: + observation.images.logitech_1: observation.image2 + observation.images.logitech_2: observation.image +kantine/industrial_screws_sorting_anomaly: + observation.images.logitech_1: observation.image2 + observation.images.logitech_2: observation.image +kantine/industrial_screws_sorting_expert: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/industrial_soldering_anomaly: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/industrial_soldering_expert: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kantine/so100_kapla_tower6: + observation.images.logitech_1: observation.image + observation.images.logitech_2: observation.image2 +kevin510/lerobot-cat-toy-placement: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +koenvanwijk/blue2: + observation.images.laptop: observation.image + observation.images.phone: null +koenvanwijk/blue52: + observation.images.laptop: observation.image + observation.images.phone: null +koenvanwijk/orange50-1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +koenvanwijk/orange50-variation-2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +kumarhans/so100_tape_task: + observation.images.laptop1: observation.image + observation.images.laptop2: observation.image2 +lalalala0620/koch_blue_paper_tape: + observation.images.front: observation.image + observation.images.phone: observation.image3 +lerobot/aloha_mobile_cabinet: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_mobile_chair: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_mobile_elevator: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_mobile_shrimp: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_mobile_wash_pan: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_mobile_wipe_wine: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_sim_transfer_cube_human: + observation.images.top: observation.image +lerobot/aloha_static_coffee: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_static_cups_open: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_static_screw_driver: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_static_towel: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_static_vinh_cup: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_static_vinh_cup_left: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/aloha_static_ziploc_slide: + observation.effort: null + observation.images.cam_high: observation.image + observation.images.cam_left_wrist: observation.image2 + observation.images.cam_low: null + observation.images.cam_right_wrist: observation.image3 +lerobot/berkeley_fanuc_manipulation: + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +lerobot/iamlab_cmu_pickup_insert: + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +lerobot/jaco_play: + observation.images.image: observation.image + observation.images.image_wrist: observation.image2 +lerobot/libero_10_image: + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +lerobot/libero_goal_image: + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +lerobot/libero_object_image: + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +lerobot/libero_spatial_image: + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +lerobot/roboturk: + observation.images.front_rgb: observation.image +lerobot/stanford_hydra_dataset: + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +lerobot/taco_play: + observation.images.rgb_gripper: observation.image2 + observation.images.rgb_static: observation.image +lerobot/toto: + observation.images.image: observation.image +lirislab/close_top_drawer_teabox: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/fold_bottom_right: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/green_lego_block_into_mug: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/green_lego_block_into_mug_easy: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/guess_who_lighting: + observation.images.mounted: observation.image2 +lirislab/guess_who_no_cond: + observation.images.mounted: observation.image2 +lirislab/guess_who_so100: + observation.images.mounted: observation.image +lirislab/lemon_into_bowl: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/open_top_drawer_teabox: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/push_cup_target: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/put_banana_bowl: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/put_caps_into_teabox: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/put_coffee_cap_teabox: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/red_cube_into_blue_cube: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/red_cube_into_green_lego_block: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/red_cube_into_mug: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/so100_demo: + observation.images.front: observation.image +lirislab/stack_two_red_cubes: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/sweep_tissue_cube: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +lirislab/unfold_bottom_right: + observation.images.realsense_side: observation.image2 + observation.images.realsense_top: observation.image +liuhuanjim013/so100_block: + observation.images.hand: observation.image2 + observation.images.top: observation.image +liuhuanjim013/so100_th: + observation.images.front: observation.image + observation.images.top: observation.image2 +liuhuanjim013/so100_th_1: + observation.images.hand: observation.image2 + observation.images.top: observation.image +liyitenga/so100_bi_giveme5: + observation.images.center: observation.image + observation.images.left_follower: observation.image2 + observation.images.right_follower: observation.image3 +liyitenga/so100_bi_hello: + observation.images.center: observation.image + observation.images.left_follower: observation.image2 + observation.images.right_follower: observation.image3 +liyitenga/so100_pick_taffy1: + observation.images.left: observation.image +liyitenga/so100_pick_taffy10: + observation.images.gripper: observation.image2 + observation.images.left: observation.image3 + observation.images.top: observation.image +liyitenga/so100_pick_taffy2: + observation.images.gripper: observation.image2 + observation.images.top: observation.image +liyitenga/so100_pick_taffy3: + observation.images.gripper: observation.image2 + observation.images.top: observation.image +liyitenga/so100_pick_taffy4: + observation.images.gripper: observation.image2 + observation.images.top: observation.image +liyitenga/so100_pick_taffy5: + observation.images.gripper: observation.image2 + observation.images.top: observation.image +liyitenga/so100_pick_taffy6: + observation.images.gripper: observation.image2 + observation.images.top: observation.image +liyitenga/so100_pick_taffy7: + observation.images.gripper: observation.image2 + observation.images.left_top: observation.image3 + observation.images.top: observation.image +liyitenga/so100_pick_taffy8: + observation.images.gripper: observation.image2 + observation.images.top: observation.image +lizi178119985/so100_jia: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +lucasngoo/so100_strawberry_grape: + observation.images.webcam: observation.image +luke250305/play_dice_250311.1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +m1b/so100_bluelego: + observation.images.phone: null + observation.images.side: observation.image +m1b/so100_bluelego_updt: + observation.images.phone: observation.image + observation.images.side: observation.image2 +ma3oun/rpi_squares_1: + observation.images.laptop: observation.image +maximilienroberti/so100_lego_red_box: + observation.images.cam_left: observation.image + observation.images.cam_middle: observation.image3 + observation.images.cam_right: observation.image2 +meteorinc/koch_tea: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +mikechambers/block_cup_14: + observation.images.main_cam: observation.image + observation.images.secondary_cam: observation.image2 +mikechambers/block_cup_5: + observation.images.main_cam: observation.image + observation.images.secondary_cam: observation.image2 +mlfu7/pi0_conversion_no_pad_video: + exterior_image_1_left: observation.image + wrist_image_left: observation.image2 +nbaron99/so100_pick_and_place2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +nbaron99/so100_pick_and_place4: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +ncavallo/moss_train_gc_block: + observation.images.front: observation.image + observation.images.top: observation.image2 +ncavallo/moss_train_grasp: + observation.images.front: observation.image + observation.images.top: observation.image2 +ncavallo/moss_train_grasp_new: + observation.images.front: observation.image + observation.images.top: observation.image2 +nduque/act_50_ep: + observation.images.above: observation.image3 + observation.images.front: observation.image +nduque/act_50_ep2: + observation.images.above: observation.image3 + observation.images.front: observation.image +nduque/cam_setup2: + observation.images.above: observation.image3 + observation.images.front: observation.image +nduque/robustness_e1: + observation.images.above: observation.image3 + observation.images.front: observation.image +nduque/robustness_e2: + observation.images.above: observation.image3 + observation.images.front: observation.image +nduque/robustness_e3: + observation.images.above: observation.image3 + observation.images.front: observation.image +nduque/robustness_e4: + observation.images.above: observation.image3 + observation.images.front: observation.image +nduque/robustness_e5: + observation.images.above: observation.image3 + observation.images.front: observation.image +nguyen-v/press_red_button_new: + observation.images.left: observation.image2 + observation.images.right: observation.image3 + observation.images.top: observation.image +nguyen-v/so100_bimanual_grab_lemon_put_in_box2: + observation.images.left: observation.image2 + observation.images.right: observation.image3 + observation.images.top: observation.image +nguyen-v/so100_press_red_button: + observation.images.back: observation.image3 + observation.images.left: observation.image2 + observation.images.right: null + observation.images.top: observation.image +nguyen-v/so100_rotate_red_button: + observation.images.left: observation.image2 + observation.images.right: observation.image3 + observation.images.top: observation.image +nimitvasavat/Gr00t_lerobot: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.back_image: observation.image3 + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +nimitvasavat/Gr00t_lerobotV2: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +nimitvasavat/Gr00t_lerobot_state_action: + absolute_action: null + annotation.human.action.task_description: null + annotation.human.validity: null + next.reward: null + observation.images.image: observation.image + observation.images.wrist_image: observation.image2 +pandaRQ/pick_med_1: + observation.images.laptop: observation.image + observation.images.laptop1: observation.image2 + observation.images.laptop2: observation.image3 +pandaRQ/pickmed: + observation.images.laptop: observation.image + observation.images.laptop1: observation.image2 + observation.images.laptop2: observation.image3 +paszea/so100_lego: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +paszea/so100_lego_2cam: + observation.images.front: observation.image + observation.images.top: observation.image2 +paszea/so100_lego_mix: + observation.images.phone: observation.image +paszea/so100_whale: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +paszea/so100_whale_2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +paszea/so100_whale_3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +paszea/so100_whale_4: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +paszea/so100_whale_grab: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pengjunkun/so100_push_to_hole: + observation.images.laptop: observation.image +pepijn223/lekiwi_block_cleanup2: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +pepijn223/lekiwi_drive_in_circle: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +pepijn223/lekiwi_drive_in_circle_recover: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +pepijn223/lekiwi_pen: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +pepijn223/yellow_lego_in_box1: + observation.images.phone: observation.image +phospho-ai/OrangeBrick3Cameras: + observation.images.main.left: observation.image3 + observation.images.main.right: observation.image2 + observation.images.secondary_0: observation.image +physical-intelligence/libero: + actions: action + image: observation.image + state: observation.state + wrist_image: observation.image2 +pierfabre/chicken: + observation.images.robot: observation.image2 + observation.images.webcam: observation.image +pierfabre/cow: + observation.images.robot: observation.image2 + observation.images.webcam: observation.image +pierfabre/cow2: + observation.images.robot: observation.image2 + observation.images.webcam: observation.image +pierfabre/horse: + observation.images.robot: observation.image2 + observation.images.webcam: observation.image +pierfabre/pig2: + observation.images.robot: observation.image2 + observation.images.webcam: observation.image +pierfabre/pig3: + observation.images.robot: observation.image2 + observation.images.webcam: observation.image +pierfabre/rabbit: + observation.images.robot: observation.image2 + observation.images.webcam: observation.image +pierfabre/sheep: + observation.images.robot: observation.image2 + observation.images.webcam: observation.image +pietroom/actualeasytask: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pietroom/first_task_short: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pietroom/holdthis: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pietroom/second_task: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_carrot_1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_carrot_2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_carrot_3: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_carrot_4: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_carrot_5: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_legos4: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_onelego2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_onelego3: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_squishy: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_squishy100: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_squishy2colors: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +pranavsaroha/so100_squishy2colors_1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +pranavsaroha/so100_squishy2colors_2_new: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +rabhishek100/so100_train_dataset: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +rgarreta/koch_pick_place_lego: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +rgarreta/koch_pick_place_lego_v2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +rgarreta/koch_pick_place_lego_v3: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +rgarreta/koch_pick_place_lego_v6: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +rgarreta/koch_pick_place_lego_v7: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +rgarreta/koch_pick_place_lego_v8: + observation.images.lateral: observation.image3 + observation.images.top: observation.image + observation.images.wrist: observation.image2 +roboticshack/left-arm-grasp-lego-brick: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +roboticshack/sandee-kiwiv10: + observation.images.front: observation.image3 + observation.images.wrist: observation.image2 +roboticshack/team-7-left-arm-grasp-motor: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +roboticshack/team-7-right-arm-grasp-tape: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +roboticshack/team10-red-block: + observation.images.phone: observation.image2 + observation.images.top: observation.image +roboticshack/team11_pianobot: + observation.images.keyboard_camera: observation.image +roboticshack/team13-three-balls-stacking: + observation.images.realsense: observation.image + observation.images.side: observation.image2 +roboticshack/team13-two-balls-stacking: + observation.images.realsense: observation.image + observation.images.side: observation.image2 +roboticshack/team16-can-stacking: + observation.images.head: observation.image + observation.images.wrist: observation.image2 +roboticshack/team16-water-pouring: + observation.images.head: observation.image + observation.images.wrist: observation.image2 +roboticshack/team2-guess_who_less_ligth: + observation.images.mounted: observation.image2 +roboticshack/team2-guess_who_so100: + observation.images.mounted: observation.image +roboticshack/team2-guess_who_so100_edge_case: + observation.images.mounted: observation.image2 +roboticshack/team2-guess_who_so100_light: + observation.images.mounted: observation.image2 +roboticshack/team9-pick_chicken_place_plate: + observation.images.static_left: observation.image + observation.images.static_right: observation.image2 +roboticshack/team9-pick_cube_place_static_plate: + observation.images.static_left: observation.image + observation.images.static_right: observation.image2 +roboticshack/team_5-QuiEstCe_everyBox: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +samanthalhy/so100_herding_1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +samsam0510/cube_reorientation_2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +samsam0510/cube_reorientation_4: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +samsam0510/glove_reorientation_1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +samsam0510/mj_data_temp: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +samsam0510/tape_insert_1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +samsam0510/tape_insert_2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +samsam0510/tooth_extraction_3: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +samsam0510/tooth_extraction_4: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +satvikahuja/mixer_on_off: + observation.images.Lwebcam: observation.image + observation.images.laptop: observation.image2 + observation.images.macwebcam: null + observation.images.phone: null +satvikahuja/mixer_on_off_new: + observation.images.Lwebcam: observation.image + observation.images.laptop: observation.image2 + observation.images.macwebcam: null + observation.images.phone: null +satvikahuja/mixer_on_off_new_1: + observation.images.Lwebcam: observation.image + observation.images.laptop: observation.image2 + observation.images.macwebcam: observation.image3 + observation.images.phone: null +satvikahuja/mixer_on_off_new_4: + observation.images.Lwebcam: observation.image + observation.images.laptop: observation.image2 + observation.images.macwebcam: null + observation.images.phone: null +satvikahuja/orange_mixer_1: + observation.images.Lwebcam: observation.image + observation.images.laptop: observation.image2 + observation.images.macwebcam: null + observation.images.phone: null +satvikahuja/orange_pick_place_new1: + observation.images.Lwebcam: observation.image + observation.images.laptop: observation.image2 + observation.images.macwebcam: null + observation.images.phone: null +seeingrain/241228_pick_place_2cams: + observation.images.side: observation.image3 + observation.images.top: observation.image +seeingrain/lego_3cameras: + observation.images.side: observation.image3 + observation.images.top: observation.image + observation.images.wrist: observation.image2 +seeingrain/one_shot_learning_18episodes: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +seeingrain/pick_lego_to_hand: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +seeingrain/pick_place_lego: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +seeingrain/pick_place_lego_wider_range_dang: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +seeingrain/pick_place_lego_wider_range_dong: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +seeingrain/pick_place_lego_wider_range_richard: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +seeingrain/pick_place_pink_lego: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +seeingrain/pick_place_pink_lego_few_samples: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +seeingrain/pick_place_red_lego: + observation.images.laptop: observation.image3 + observation.images.phone: observation.image +shin1107/koch_move_block_with_some_positions: + observation.images.front: observation.image + observation.images.top: observation.image3 +shin1107/koch_move_block_with_some_shapes: + observation.images.front: observation.image + observation.images.top: observation.image3 +shin1107/koch_train_block: + observation.images.front: observation.image + observation.images.top: observation.image3 +shreyasgite/so100_base_env: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +shreyasgite/so100_base_left: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +shreyasgite/so100_legocube_50: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +sihyun77/mond_1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/mond_13: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/si_so100: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/sihyun_3_17_1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/sihyun_3_17_2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/sihyun_3_17_5: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/sihyun_king: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/sihyun_main: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/sihyun_main_2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/sihyun_main_3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/suho_3_17_1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/suho_3_17_3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/suho_angel: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/suho_main_2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/suho_red: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/suho_red2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sihyun77/suho_so100: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +sixpigs1/so100_insert_cylinder_error: + observation.images.above: observation.image2 + observation.images.rightfront: observation.image +sixpigs1/so100_pick_cube_in_box: + observation.images.above: observation.image2 + observation.images.rightfront: observation.image +sixpigs1/so100_pick_cube_in_box_error: + observation.images.above: observation.image + observation.images.rightfront: observation.image2 +sixpigs1/so100_pull_cube_by_tool_error: + observation.images.above: observation.image2 + observation.images.rightfront: observation.image +sixpigs1/so100_pull_cube_error: + observation.images.above: observation.image + observation.images.rightfront: observation.image2 +sixpigs1/so100_push_cube_error: + observation.images.above: observation.image + observation.images.rightfront: observation.image2 +sixpigs1/so100_stack_cube_error: + observation.images.above: observation.image2 + observation.images.rightfront: observation.image +smanni/train_so100_fluffy_box: + observation.images.intel_realsense: observation.image +speedyyoshi/so100_grasp_pink_block: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +sshh11/so100_orange_50ep_1: + observation.images.laptop: observation.image +sshh11/so100_orange_50ep_2: + observation.images.laptop: observation.image +swarajgosavi/act_kikobot_block_real: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +swarajgosavi/act_kikobot_pusht_real: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +swarajgosavi/kikobot_pusht_real_v2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +takuzennn/aloha-pick100: + observation.image.camera1: observation.image + observation.image.camera2: observation.image2 + observation.image.camera3: observation.image3 +takuzennn/square3: + observations.images.agentview: observation.image + observations.images.robot0_eye_in_hand: observation.image2 +thedevansh/mar16_1336: + observation.images.laptop: observation.image +theo-michel/lekiwi_v2: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +theo-michel/lekiwi_v5: + observation.images.front: observation.image + observation.images.wrist: observation.image2 +therarelab/med_dis_rare_6: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +therarelab/so100_pick_place: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +therarelab/so100_pick_place_2: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +tkc79/so100_lego_box_1: + observation.images.arm: observation.image2 + observation.images.laptop: observation.image +tkc79/so100_lego_box_2: + observation.images.arm: observation.image2 + observation.images.laptop: observation.image +triton7777/so100_dataset_mix: + observation.images.gripper: observation.image2 + observation.images.s_left: null + observation.images.s_right: observation.image3 + observation.images.top: observation.image +twerdster/koch_new_training_red: + observation.images.iphone: observation.image2 + observation.images.laptop: observation.image3 +twerdster/koch_training_red: + observation.images.rightegocam: observation.image2 + observation.images.rightstereocam: observation.image3 +underctrl/handcamera_single_blue: + observation.images.android: observation.image + observation.images.handcam: observation.image2 + observation.images.webcam: observation.image3 +underctrl/mutli-stacked-block_mutli-color_pick-up_80: + observation.images.phone: observation.image3 + observation.images.webcam: observation.image +underctrl/single-block_blue-color_pick-up_80: + observation.images.phone: observation.image3 + observation.images.webcam: observation.image +underctrl/single-block_multi-color_pick-up_50: + observation.images.phone: observation.image3 + observation.images.webcam: observation.image +underctrl/single-stacked-block_mutli-color_pick-up_80: + observation.images.phone: observation.image3 + observation.images.webcam: observation.image +underctrl/single-stacked-block_two-color_pick-up_80: + observation.images.phone: observation.image3 + observation.images.webcam: observation.image +vaishanthr/toy_pick_place: + observation.images.gipper_cam: observation.image2 + observation.images.webcam: observation.image +vaishanthr/toy_pickplace: + observation.images.gipper_cam: observation.image2 + observation.images.webcam: observation.image +vaishanthr/toy_pickplace_50ep: + observation.images.gipper_cam: observation.image2 + observation.images.webcam: observation.image +vladfatu/so100_above: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +vladfatu/so100_ds: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +vladfatu/so100_office: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +wangjl1512/doll: + observation.images.top: observation.image + observation.images.wrist: observation.image2 +wangjl1512/pour_water: + observation.images.laptop: observation.image +wcode/so100_put_pen_50: + observation.images.hand: observation.image2 + observation.images.top: observation.image +weiye11/so100_410_zwy: + observation.images.top: observation.image + observation.images.wrist_left: observation.image2 +yg-dev/koch_red_pen2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +yskim2025/unitylerobot: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +yuz1wan/so100_button: + observation.images.side: observation.image + observation.images.wrist: observation.image2 +yuz1wan/so100_fold_0227_1: + observation.images.side: observation.image +yuz1wan/so100_fold_0227_2: + observation.images.side: observation.image + observation.images.wrist: observation.image2 +yuz1wan/so100_pick_pink: + observation.images.side: observation.image + observation.images.wrist: observation.image2 +yuz1wan/so100_pick_wahaha: + observation.images.side: observation.image + observation.images.wrist: observation.image2 +yuz1wan/so100_pickplace: + observation.images.side: observation.image + observation.images.wrist: observation.image2 +yuz1wan/so100_pickplace_0223_2: + observation.images.side: observation.image +yuz1wan/so100_pickplace_0223_3: + observation.images.side: observation.image +yuz1wan/so100_pour_cup: + observation.images.side: observation.image + observation.images.wrist: observation.image2 +yuz1wan/so100_pp_pink: + observation.images.side: observation.image + observation.images.wrist: observation.image2 +zaringleb/so100_cube_2: + observation.images.cam_high: observation.image +zaringleb/so100_cube_4_binary: + observation.images.cam_high: observation.image +zaringleb/so100_cube_5_linear: + observation.images.cam_high: observation.image +zaringleb/so100_cube_6_2d: + observation.images.cam_high: observation.image +zaringleb/so100_cude_linear_and_2d_comb: + observation.images.cam_high: observation.image +zijian2022/321: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/assemblyarm2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/backgrounda: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/backgroundb: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/bi1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/bi2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/c0: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/close3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/closer3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/digitalfix: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/digitalfix2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/digitalfix3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/force1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/force2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/force3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/hand1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/insert2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/l10_1: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +zijian2022/l10_5: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +zijian2022/l9: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +zijian2022/llm40: + observation.images.laptop: observation.image2 + observation.images.phone: observation.image +zijian2022/n1_2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/noticehuman1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/noticehuman2: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/noticehuman3: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/noticehuman5: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/noticehuman50: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/noticehuman60: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/noticehuman70: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/so100_318: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/so100_318_1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zijian2022/sort1: + observation.images.laptop: observation.image + observation.images.phone: observation.image2 +zliu157/i3r: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 +zliu157/i3r2: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 +zliu157/i3r3: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 +zliu157/i3r5: + observation.images.laptop: observation.image + observation.images.phone: observation.image3 diff --git a/lerobot/configs/mappings/tasks.yaml b/lerobot/configs/mappings/tasks.yaml new file mode 100644 index 000000000..16d71256e --- /dev/null +++ b/lerobot/configs/mappings/tasks.yaml @@ -0,0 +1,1012 @@ +00ri/so100_battery: + 0: Grasp a battery and put it in the bin. +00ri/so100_battery_bin_center: + 0: Grasp a battery and put it in the bin. +1g0rrr/sam_openpi03: + 0: Grasp a blue cube and put it in the gray box. +1g0rrr/sam_openpi_solder1: + 0: bring contact to the pad on the board. +1g0rrr/sam_openpi_solder2: + 0: bring contact to the pad on the board. +1g0rrr/screw1: + 0: Grasp a lego block and put it in the bin. +356c/so100_duck_reposition_1: + 0: Grasp the tool and use it to move the duck to the indicated position. +356c/so100_nut_sort_1: + 0: Pick up the steel nuts and sort them by color. +356c/so100_rope_reposition_1: + 0: Grasp rope and reposition. +Allen-488/koch_dataset_50: + 0: Grasp a block and put it in the bin. +AndrejOrsula/lerobot_double_ball_stacking_random: + 0: Stack the balls on top of each other. +Bartm3/dice2: + 0: Grasp a dice and put it in the bin. +Bartm3/tape_to_bin: + 0: Grasp a tape and put it in the bin. +Bartm3/tape_to_bin2: + 0: Grasp a tape and put it in the bin. +Beegbrain/align_cubes_green_blue: + 0: Put the green cubes on the left and the blue cube on the right +Beegbrain/align_three_pens: + 0: picks up a pen. +Beegbrain/moss_close_drawer_teabox: + 0: Close the top drawer of the teabox +Beegbrain/moss_open_drawer_teabox: + 0: Open the top drawer of the teabox +Beegbrain/moss_put_cube_teabox: + 0: Put the green cube in the top drawer of the teabox +Beegbrain/moss_stack_cubes: + 0: Stack the green cube on top of the blue cube +Beegbrain/oc_stack_cubes: + 0: stack the red cube on the blue cube +Beegbrain/pick_lemon_and_drop_in_bowl: + 0: Pick the yellow lemon and drop it in the red bowl. +Beegbrain/pick_place_green_block: + 0: Pick up the green block and place in the red cup. +Beegbrain/put_green_into_blue_bin: + 0: Put the green cube into the blue bin +Beegbrain/put_red_triangle_green_rect: + 0: Put the red triangle on top of the green rectangle +Beegbrain/put_screwdriver_box: + 0: Put the screwdriver into the box +Beegbrain/stack_2_cubes: + 0: picks up the red block. +Beegbrain/stack_green_on_blue_cube: + 0: Stack the blue cube on top of the green cube +Beegbrain/sweep_tissue_cube: + 0: Sweep the red cubes to the right with the tissue. +BlobDieKatze/GrabBlocks: + 0: Grasp a lego block and put it in the bin. +CSCSXX/pick_place_cube_1.17: + 0: Pick up the red block and place it in the box. +CSCSXX/pick_place_cube_1.18: + 0: Pick up the cube and place it in the box. +Chojins/chess_game_000_white: + 0: Move the blue chess pieces as indicated by the highlighted squares +Chojins/chess_game_000_white_red: + 0: Move the red chess pieces to the highlighted squares. +Chojins/chess_game_001_blue_stereo: + 0: Move the blue chess pieces to the highlighted squares +Chojins/chess_game_001_red_stereo: + 0: Move the red chess pieces to the highlighted squares +Chojins/chess_game_001_white: + 0: Move the blue chess pieces as indicated by the highlighted squares +Chojins/chess_game_002_white: + 0: Move the blue chess pieces as indicated by the highlighted squares +Chojins/chess_game_003_white: + 0: Move the blue chess pieces as indicated by the highlighted squares +Chojins/chess_game_004_white: + 0: Move the blue chess pieces as indicated by the highlighted squares +Chojins/chess_game_005_white: + 0: Move the blue chess pieces as indicated by the highlighted squares +Chojins/chess_game_006_white: + 0: Move the blue chess pieces as indicated by the highlighted squares +Chojins/chess_game_007_white: + 0: Move the blue chess pieces as indicated by the highlighted squares +Chojins/chess_game_009_white: + 0: Move the blue chess pieces to the highlighted squares. +Cidoyi/so100_all_notes_1: + 0: Connect the cable to the device. +Clementppr/lerobot_pick_and_place_dataset_world_model: + 0: Grasp a fruit and put it in the cup. +CrazyYhang/A1234-B-C_mvA2B: + 0: Move the top disk from the left column to the middle column. +Deason11/Open_the_drawer_to_place_items: + 0: Put the objects in the open drawer. +Deason11/PLACE_TAPE_PUSH_DRAWER: + 0: 'Place the tape in the drawer and close it. ' +Deason11/Random_Kitchen: + 0: Pick up the cup and place it on the table. +Deason11/mobile_manipulator_0319: + 0: Grasp a lego block and put it in the bin. +Deason11/mobile_manipulator_0326: + 0: Grasp a lego block and put it in the bin. + 1: mobile_lekiwi. +DimiSch/so100_50ep_2: + 0: Place the yellow object in the bowl. +DimiSch/so100_50ep_3: + 0: Pick the yellow button from the table. +DimiSch/so100_terra_50_2: + 0: Grasp a lego block and put it in the bin. +Dongkkka/cable_pick_and_place2: + 0: Put a black charging cable in a black bowl and put a red charging cable in a + green bowl +DorayakiLin/so100_pick_charger_on_tissue: + 0: Pick up the charger and put it on the white tissue. +DorayakiLin/so100_pick_cube_in_box: + 0: Pick up the red cube and put it in the box. +Eyas/grab_bouillon: + 0: picks up the box and places it in the box. +Eyas/grab_pink_lighter_10_per_loc: + 0: Pick up the pink object from the table. +FeiYjf/Grab_Pieces: + 0: places the black object on the table. +FeiYjf/Makalu_push: + 0: Pick up the blue cube. +FeiYjf/Maklu_dataset: + 0: Pick up the blue cube and place it on the paper. +FeiYjf/Test_NNNN: + 0: Pick up the purple cube and move it to the right. +FeiYjf/new_Dataset: + 0: Pick up the blue cube. +FeiYjf/new_GtoR: + 0: Move along the line on the paper from start to end. +FsqZ/so100_1: + 0: Put the yellow cube inside the purple box. +Gano007/so100_doliprane: + 0: Grasp a medic box and put it in the bin. +Gano007/so100_gano: + 0: Grasp a box and put it in the bin. +Gano007/so100_medic: + 0: Grasp a medic box and put it in the bin. +Gano007/so100_second: + 0: Grasp a yellow box and put it in the bin. +Gongsta/grasp_duck_in_cup: + 0: Grasp the rubber duck and put it in the cup. +HITHY/so100_peach: + 0: Grasp a peach and put it in the bin. +HITHY/so100_peach3: + 0: Grasp a peach and put it in the bin. +HITHY/so100_peach4: + 0: Grasp a peach and put it on the plate. +HITHY/so100_strawberry: + 0: Grasp a strawberry and put it in the bin. +HWJ658970/fat_fish: + 0: Grasp a fat fish toy and put it in the bin. +HWJ658970/lego_100_class: + 0: Separate yellow and white Lego blocks and place them into the bin. +HWJ658970/lego_50: + 0: Grasp a yellow lego block and put it in the bin. +HWJ658970/lego_50_camera_change: + 0: Grasp a yellow lego block and put it in the bin. +HuaihaiLyu/groceries: + 0: Pick the brown long bread and Egg yolk pasry into package +IPEC-COMMUNITY/berkeley_mvp_lerobot: + 0: push wooden cube + 1: pick detergent from the sink + 2: reach red block + 3: pick yellow cube + 4: close fridge door + 5: pick fruit +IPEC-COMMUNITY/ucsd_kitchen_dataset_lerobot: + 0: Turn on the faucet + 1: Put the bowl inside the kitchen cabinet + 2: Open the oven door + 3: Place the teapot on the stove + 4: Put the white box into the sink + 5: Open the carbinet door + 6: Put the green box into the sink + 7: Put the canned spam into the sink +Ityl/so100_recording1: + 0: Putting the red square onto the yellow piece +Ityl/so100_recording2: + 0: Pick up the red cube and place it on top of the blue cube. +Jiangeng/so100_413: + 0: Pick up the cube and place it on top of the black circle. +KeWangRobotics/piper_push_cube_gamepad_1: + 0: push the cube to the black area +KeWangRobotics/piper_push_cube_gamepad_1_cropped_resized: + 0: push the cube to the black area +KeWangRobotics/piper_rl_1: + 0: Pick up the cube and place it in the box. +KeWangRobotics/piper_rl_1_cropped_resized: + 0: picks up the cube. +KeWangRobotics/piper_rl_2: + 0: Move the cube to the right side of the table. +KeWangRobotics/piper_rl_2_cropped_resized: + 0: Move the block to the right side of the table. +KeWangRobotics/piper_rl_3: + 0: Pick up the wooden block. +KeWangRobotics/piper_rl_3_cropped_resized: + 0: Place the block in the box. +KeWangRobotics/piper_rl_4: + 0: Move the block slightly. +KeWangRobotics/piper_rl_4_cropped_resized: + 0: picks the wooden block. +LemonadeDai/so100_coca: + 0: Grasp the Coca-Cola can and orient it upright with the top facing up. +Loki0929/so100_100: + 0: Grasp a rubber duck and put it in the box. +Loki0929/so100_duck: + 0: Grasp red, green, yellow ducks and put them in the box. +Lugenbott/koch_1225: + 0: Pick up the blue block and place it in the red box. +Mohamedal/put_banana: + 0: Put the banana in the red bowl. +Mwuqiu/so100_0408_muti: + 0: Grasp a yellow duck and put it in the box. +NONHUMAN-RESEARCH/SOARM100_TASK_VENDA: + 0: Pick up the object and place it in the box. +NONHUMAN-RESEARCH/SOARM100_TASK_VENDA_BOX: + 0: Move the cube to the right side of the table. +Odog16/so100_cube_drop_pick_v1: + 0: Pick up the orange cube, release it, and then pick it up again. +Odog16/so100_cube_stacking_v1: + 0: 'Stack the cubes in the following order from bottom to top: black, blue, then + orange.' +Odog16/so100_tea_towel_folding_v1: + 0: Fold tea towel into quarters +Ofiroz91/so_100_cube2bowl: + 0: placing cube inside a red bawl +Pi-robot/barbecue_flip: + 0: Pick up the orange cone and place it on the table. +Pi-robot/barbecue_put: + 0: Pick up the stick and place it in the grill. +RasmusP/so100_Orange2Green: + 0: Grasp the orange block and drop it in the box. +RasmusP/so100_dataset50ep: + 0: Grasp a square block and put it in the box. +SeanLMH/so100_picknplace: + 0: Pick up the blue block and place it in the yellow box. +SeanLMH/so100_picknplace_v2: + 0: picks up blue cube and places it in yellow box. +Setchii/so100_grab_ball: + 0: Grasp a ball and put it on a goblet. +T-K-233/koch_k1_pour_shot: + 0: Place the glass on the table. +TrossenRoboticsCommunity/aloha_baseline_dataset: + 0: Pick up a blue block and put it in a green bowl. Baseline dataset for testing +TrossenRoboticsCommunity/aloha_fold_tshirt: + 0: Fold the t-shirt. +TrossenRoboticsCommunity/aloha_stationary_logo_assembly: + 0: Assemble the Trossen Robotics Logo. +VoicAndrei/so100_banana_to_plate_only: + 0: Pick up the banana and place it on the plate. +VoicAndrei/so100_banana_to_plate_rebel_full: + 0: Pick up the banana and place it on the place +Winster/so100_cube: + 0: Grasp a lego block and put it in the bin. +XXRRSSRR/so100_v3_num_episodes_50: + 0: Grasp a box and put it in the side. +Yuanzhu/koch_bimanual_grasp_0: + 0: places the yellow block on the mousepad. +Yuanzhu/koch_bimanual_grasp_3: + 0: picks up a yellow block. +ZCM5115/so100_1210: + 0: picks up the USB cable. +ZCM5115/so100_2Arm3cameras_movebox: + 0: Pick up the white box from the table. +ZGGZZG/so100_drop0: + 0: Grasp a ball and put it in the hole. +ZGGZZG/so100_drop1: + 0: Grasp a cube and put it in the right place. +Zhaoting123/koch_cleanDesk_: + 0: Grasp a card and use it to clean the desk +abbyoneill/data_w_mug: + 0: Grasp a lego block and put it in the bin. +abbyoneill/new_dataset_pick_place: + 0: Grasp a lego block and put it in the bin. +abbyoneill/pusht: + 0: Grasp a lego block and put it in the bin. +abbyoneill/thurs1120pickplace: + 0: Grasp a lego block and put it in the bin. +abhisb/so100_51_ep: + 0: Pick up the cube and place it in the box. +abougdira/cube_target: + 0: put_the cube on the yellow target +acrampette/third_arm_01: + 0: Pick up the circuit board from the table. +acrampette/third_arm_02: + 0: Pick up the tie and place it in the box. +ad330/cubePlace: + 0: Grasp white cube and place it in the bowl. +ad330/so100_box_pickPlace: + 0: places jar in box. +aimihat/so100_tape: + 0: Pick up the tape and put it in the bowl. +aliberts/koch_tutorial: + 0: Pick the Lego block and drop it in the box on the right. +allenchienxxx/so100Test: + 0: Grasp a lego block and put it in the bin. +andabi/D10: + 0: picks up the shoe. +andabi/D11: + 0: The picks up the paper and places it on the table. +andabi/D12: + 0: The places the cube in the box. +andabi/D13: + 0: picks up shoes. +andabi/D14: + 0: picks up shoes. +andabi/D15: + 0: picks up the shoe. +andabi/D16: + 0: picks up the shoe. +andabi/D17: + 0: picks up the shoe. +andabi/D2: + 0: Pick up the shoe and place it on the table. +andabi/D3: + 0: picks up the shoe. +andabi/D4: + 0: picks up the shoe. +andabi/D5: + 0: places shoe on table. +andabi/D6: + 0: picks up the shoe. +andabi/D7: + 0: picks up the shoe. +andabi/D8: + 0: picks up the shoe. +andabi/D9: + 0: The picks up the paper and places it on the table. +andabi/shoes_easy: + 0: picks up the shoe. +andlyu/so100_indoor_1: + 0: Locate and grasp the blueberry. +andlyu/so100_indoor_3: + 0: Locate and grasp the blueberry. +andrewcole712/so100_tape_bin_place: + 0: Place the tape in the wooden box. +andy309/so100_0311_1152: + 0: Grasp and put it in the bin. +andy309/so100_0314_fold_cloths: + 0: fold the cloths, use two cameras, two arms. +aractingi/push_cube_offline_data: + 0: Push the green cube to the yellow sticker. +aractingi/push_cube_offline_data_cropped_resized: + 0: Push the green cube to the yellow sticker +arclabmit/Koch_twoarms: + 0: official two arms recordings10 +arclabmit/koch_gear_and_bin: + 0: Pick the gear and place it in the bin. +baladhurgesh97/so100_final_picking_3: + 0: Grasp a carrot, plastic bottle and put it in respective bins. +bensprenger/chess_game_001_blue_stereo: + 0: Move the blue chess pieces to the highlighted squares. +bensprenger/left_arm_yellow_brick_in_box_v0: + 0: Grasp the yellow lego block and put it in the box. +bensprenger/left_arm_yellow_brick_in_box_with_purple_noise_v0: + 0: Grasp a yellow lego block and put it in the bin. +bensprenger/right_arm_p_brick_in_box_with_y_noise_v0: + 0: Grasp the purple lego block and put it in the box. +bnarin/so100_tic_tac_toe_we_do_it_live: + 0: move tic tac toe as player 2. +carpit680/giraffe_sock_demo_1: + 0: Grasp a sock off the floor. +carpit680/giraffe_task: + 0: Grasp a block and put it in the bin. +chmadran/so100_dataset04: + 0: picks the blue block and places it in the red cup. +chmadran/so100_dataset08: + 0: places blue block on paper. +chmadran/so100_home_dataset: + 0: Grasp a lego block and put it in the bin. +cmcgartoll/cube_color_organizer: + 0: Organize blue cube + 1: Organize red cube + 2: Organize yellow cube +ctbfl/sort_battery: + 0: put the battery into battery_box +dboemer/koch_50-samples: + 0: Pick up the red block and place it on top of the yellow box. +dkdltu1111/omx-bottle1: + 0: Grasp a lego block and put it in the bin. +dop0/koch_pick_terminal: + 0: Pick up the terminal and place on the cover. +dragon-95/so100_sorting: + 0: Pick up the object from box A and place it in box B. +dragon-95/so100_sorting_1: + 0: Pick up the object from box A and place it in box B. +dragon-95/so100_sorting_2: + 0: Pick up the object from box A and place it in box B. +dragon-95/so100_sorting_3: + 0: Pick up the object from box A and place it in box B. +ellen2imagine/pusht_green1: + 0: Place the green block in the box. +ellen2imagine/pusht_green_same_init2: + 0: Place the green block in the correct position. +engineer0002/pepper: + 0: Place the bottle on the table. +francescocrivelli/carrot_eating: + 0: pick up carrot and bring to mouth +frk2/so100large: + 0: Pick up roll of tape and put it in the bin. +frk2/so100largediffcam: + 0: Pick up roll of tape and put it in the bin +ganker5/so100_color_0328: + 0: Grasp a lego block and put it in the bin. +ganker5/so100_dataline_0328: + 0: Grasp a lego block and put it in the bin. +ganker5/so100_dataline_20250331: + 0: Grasp a lego block and put it in the bin. +ganker5/so100_push_20250328: + 0: Grasp a lego block and put it in the bin. +ganker5/so100_push_20250331: + 0: Grasp a lego block and put it in the bin. +ganker5/so100_toy_20250402: + 0: Grasp a lego block and put it in the bin. +gxy1111/so100_pick_place: + 0: Grasp a toy panda and put it in the cup. +hangwu/koch_pick_terminal: + 0: Pick up the terminal and place on the cover. +hangwu/piper_pick_terminal_2: + 0: Grasp the white terminal and put it on the green lid. +hangwu/piper_pick_terminal_and_place: + 0: Grasp a terminal and put it on the black box. +hannesill/koch_pnp_2_blocks_2_bins_200: + 0: Grasp the blue block first and put it in the first bin that has a specific position + and orientation. Then grasp the white block and put it in the second bin that + has a specific position and orientation. +hannesill/koch_pnp_simple_50: + 0: Grasp a small block with a specific orientation and put it in the bin with a + specific position and orientation. +hegdearyandev/so100_eraser_cup_v1: + 0: picks up the red object. +helper2424/hil-serl-push-circle-classifier: + 0: Push small circle object to the correct position +hkphoooey/throw_stuffie: + 0: Grab stuffed animal and throw it on the dot. +ibru/bob_jetson: + 0: Drive forward pickup the object and put it in the red box and drive back. +ibru/bobo_jetson: + 0: Drive forward pickup the object and put it in the red box and drive back. + 1: Driver forward +ibru/bobo_trash_collector: + 0: Bobo Trash collect and place it in a bin +imatrixlee/koch_place: + 0: Pick up the white object and place it on the table. +isadev/bougies1: + 0: Put the candles in the box. +isadev/bougies2: + 0: grab the candle wick and place it in the tray. +isadev/bougies3: + 0: Grab the candle wick by the aluminium plate and place it in the box. +jainamit/koch_pickcube: + 0: Pick up the blue cube and place it in the box. +jainamit/koch_realcube3: + 0: pick up the cube real with keyboard input +jannick-st/classifier: + 0: Move the blue object to the right side of the table. +jannick-st/push-cube-classifier_cropped_resized: + 0: Close the cabinet door. +jbraumann/so100_1902: + 0: picks up the yellow ball. +jchun/so100_pickplace_small_20250323_120056: + 0: Grasp items from white bowl and place in black tray +jiajun001/eraser00_2: + 0: picks tissue paper from box. +jiajun001/eraser00_3: + 0: Pick up the white object from the table. +jlesein/TestBoulon7: + 0: Pick up the bolt and put it on the plate. +jlitch/so100multicam7: + 0: pick up brick and put in bin +jmrog/so100_sweet_pick: + 0: Pick up the candy and place it in the bowl. +joaoocruz00/so100_makeitD1: + 0: Grasp a lego block and put it in the bin. +jpata/so100_pick_place_tangerine: + 0: Pick up the tangerine and place it. +kevin510/lerobot-cat-toy-placement: + 0: Grasp the cat toy and put it in the cup. +koenvanwijk/blue52: + 0: places blue block on red LEGO piece. +koenvanwijk/orange50-1: + 0: 'Pick up the orange object and but it in the LEGO box. ' +koenvanwijk/orange50-variation-2: + 0: 'Pick up the orange object and but it in the LEGO box. ' +kumarhans/so100_tape_task: + 0: Grasp a roll of tape and put it over the candle case. +lalalala0620/koch_blue_paper_tape: + 0: Grasp a blue paper tape and put it in the bin. +lerobot/aloha_mobile_cabinet: + 0: Open the top cabinet, store the pot inside it then close the cabinet. +lerobot/aloha_mobile_chair: + 0: Push the chairs in front of the desk to place them against it. +lerobot/aloha_mobile_elevator: + 0: Take the elevator to the 1st floor. +lerobot/aloha_mobile_wash_pan: + 0: Pick up the pan, rinse it in the sink and then place it in the drying rack. +lerobot/aloha_mobile_wipe_wine: + 0: Pick up the wet cloth on the faucet and use it to clean the spilled wine on the + table and underneath the glass. +lerobot/aloha_static_candy: + 0: Pick up the candy and unwrap it. +lerobot/aloha_static_coffee: + 0: Place the coffee capsule inside the capsule container, then place the cup onto + the center of the cup tray, then push the 'Hot Water' and 'Travel Mug' buttons. +lerobot/aloha_static_coffee_new: + 0: Place the coffee capsule inside the capsule container, then place the cup onto + the center of the cup tray. +lerobot/aloha_static_cups_open: + 0: Pick up the plastic cup and open its lid. +lerobot/aloha_static_pro_pencil: + 0: Pick up the pencil with the right arm, hand it over to the left arm then place + it back onto the table. +lerobot/aloha_static_screw_driver: + 0: Pick up the screwdriver with the right arm, hand it over to the left arm then + place it into the cup. +lerobot/aloha_static_towel: + 0: Pick up a piece of paper towel and place it on the spilled liquid. +lerobot/aloha_static_vinh_cup: + 0: Pick up the platic cup with the right arm, then pop its lid open with the left + arm. +lerobot/aloha_static_vinh_cup_left: + 0: Pick up the platic cup with the left arm, then pop its lid open with the right + arm. +lerobot/aloha_static_ziploc_slide: + 0: Slide open the ziploc bag. +lirislab/close_top_drawer_teabox: + 0: Close the top drawer of the teabox +lirislab/fold_bottom_right: + 0: Fold the bag from the bottom right corner. +lirislab/green_lego_block_into_mug: + 0: pick the green block and place it in the red cup +lirislab/guess_who_lighting: + 0: Pick up the card from the shelf. +lirislab/guess_who_no_cond: + 0: Place the card in the slot. +lirislab/lemon_into_bowl: + 0: Pick the yellow lemon and drop it in the red bowl +lirislab/open_top_drawer_teabox: + 0: Open the top drawer of the teabox +lirislab/push_cup_target: + 0: Push the red cup to the pink target +lirislab/put_banana_bowl: + 0: Put the banana into the red bowl +lirislab/put_caps_into_teabox: + 0: Pick the coffee capsule and put it into the top drawer of the teabox +lirislab/put_coffee_cap_teabox: + 0: Put the coffee capsule into the top drawer of the teabox. +lirislab/red_cube_into_blue_cube: + 0: Put the red cube on top of the blue cube. +lirislab/red_cube_into_green_lego_block: + 0: Put the red cube on top of the yellow cube. +lirislab/so100_demo: + 0: Put the banana into the red bowl. +lirislab/sweep_tissue_cube: + 0: Sweep the red cubes to the right with the tissue bag. +lirislab/unfold_bottom_right: + 0: Unfold the bag from bottom right corner +liuhuanjim013/so100_block: + 0: Grasp a lego block and put it in the bin. +liuhuanjim013/so100_th: + 0: Grasp a lego figure and put it in the box. +liyitenga/so100_pick_taffy3: + 0: Place the eraser in the container. +liyitenga/so100_pick_taffy6: + 0: Pick up the toy and place it in the purple cup. +liyitenga/so100_pick_taffy7: + 0: Pick up the toy and place it in the box. +lizi178119985/so100_jia: + 0: Grasp a lego block and put it in the bin. +ma3oun/rpi_squares_1: + 0: Raspberry Pi 5 squares recording 1 +maximilienroberti/so100_lego_red_box: + 0: Placing the red Lego in the red box bin. +mikechambers/block_cup_14: + 0: Grasp a block and put it in a cup. +mlfu7/pi0_conversion_no_pad_video: + 0: pickplace deer greybowl + 1: stack red on green + 2: stack orange cup to yellow cup + 3: put orange cup into yellow cup + 4: put push red pen to blue pen + 5: put tiger to black bowl + 6: put potato in bot to black bowl + 7: pick up green triangle + 8: put push blue pen to red pen + 9: close drawer + 10: put push green block to red + 11: pickup potato + 12: open drawer + 13: put closing tongs + 14: poke block + 15: put push blue cube + 16: poke tiger + 17: pick red cube into black bowl + 18: pick blue cube stack on wood block + 19: pick blue cube into grey bowl + 20: put red ball in black bowl + 21: pick green triangle into pink bowl + 22: pick red ball into pink bowl + 23: poke green triangle + 24: poke grey bowl + 25: put blue cube pink bowl + 26: put red cube into black bowl + 27: put deer into gray bowl + 28: put red ball into pink bowl + 29: put green triangle into pink bowl + 30: put pour from yellow cup into black bowl + 31: put pour blue cup into pink bowl + 32: put brown cube into gray bowl +nbaron99/so100_pick_and_place2: + 0: picks up the white object. +nbaron99/so100_pick_and_place4: + 0: Pick up the triangular object and place it on a green sticker. +ncavallo/moss_train_gc_block: + 0: Grasp a lego block and put it in the bin. +ncavallo/moss_train_grasp: + 0: Grasp a lego block and put it in the bin. +ncavallo/moss_train_grasp_new: + 0: Grasp a lego block and put it in the bin. +nduque/act_50_ep: + 0: Grasp a green block and put it in the bin. +nduque/act_50_ep2: + 0: Grasp a green block and put it in the bin. +nduque/cam_setup2: + 0: Grasp a green block and put it in the bin. +nduque/robustness_e2: + 0: Grasp a green dice and put it in the bin. +nduque/robustness_e3: + 0: Grasp a green dice and put it in the bin. +nduque/robustness_e4: + 0: Grasp a green dice and put it in the bin. +nduque/robustness_e5: + 0: Grasp a green dice and put it in the bin. +nguyen-v/press_red_button_new: + 0: Press the red button with the black arm +nguyen-v/so100_bimanual_grab_lemon_put_in_box2: + 0: Grab the lemon with the black arm, then give it to the green arm, then place + the lemon in the cardboard box with the green arm. +nguyen-v/so100_press_red_button: + 0: The places the cube in the box. +nguyen-v/so100_rotate_red_button: + 0: Rotate the red button clockwise with the black arm +nimitvasavat/Gr00t_lerobot: + 0: Place the cereal box on the shelf. +nimitvasavat/Gr00t_lerobotV2: + 0: Place the chocolate chip cookie dough box on the table. +nimitvasavat/Gr00t_lerobot_state_action: + 0: Place the chocolate chip cookie dough box on the table. +pandaRQ/pick_med_1: + 0: Pick up the object and place it in the box. +pandaRQ/pickmed: + 0: Place the green block on the table. +paszea/so100_lego: + 0: Grasp a lego and put it in the basket. +paszea/so100_lego_2cam: + 0: Grap lego blocks and put them in the plate. +paszea/so100_lego_mix: + 0: Grasp lego blocks and put them in the plate. +paszea/so100_whale_2: + 0: Grasp a whale and put it in the plate. +paszea/so100_whale_3: + 0: Grasp a whale and put it in the plate. +paszea/so100_whale_4: + 0: Grasp a whale and put it in the plate. +paszea/so100_whale_grab: + 0: Grasp a whale and put it in the plate. +pengjunkun/so100_push_to_hole: + 0: Push the T into the hole. +pepijn223/lekiwi_block_cleanup2: + 0: Put red block in black box +pepijn223/lekiwi_drive_in_circle: + 0: Pick up the red object and place it on the table. +pepijn223/lekiwi_pen: + 0: Fold the jeans. +pierfabre/chicken: + 0: Pick the chicken and place it to the right. +pierfabre/cow2: + 0: Pick the cow and place it to the right. +pierfabre/horse: + 0: Pick the horse and place it to the right. +pierfabre/pig2: + 0: Pick the pig and place it to the right. +pierfabre/pig3: + 0: Pick the pig and place it to the right. +pierfabre/rabbit: + 0: Pick the rabbit and place it to the right. + 1: Pick the rabbit and put it to the right +pierfabre/sheep: + 0: Pick the sheep and place it to the right. +pietroom/actualeasytask: + 0: Grasp the marker and put it in the plastic box. +pietroom/first_task_short: + 0: Pick up the marker from the box. +pietroom/holdthis: + 0: Hold the object steadily without releasing it. +pranavsaroha/so100_carrot_1: + 0: pick a carrot and put it in the bin +pranavsaroha/so100_carrot_2: + 0: Pick up a carrot and put it in the bin. +pranavsaroha/so100_carrot_3: + 0: pick a carrot and put it in the bin +pranavsaroha/so100_carrot_5: + 0: Pick up a carrot and put it in the bin. +pranavsaroha/so100_legos4: + 0: Pick up the LEGO block and place it in the bowl of the same color as the LEGO + block. +pranavsaroha/so100_onelego2: + 0: Pick up the green LEGO block and place it in the green bowl. +pranavsaroha/so100_onelego3: + 0: Pick up the green LEGO block and place it in the green bowl. +pranavsaroha/so100_squishy: + 0: pick a squishy and put it in the bin +pranavsaroha/so100_squishy100: + 0: pick a squishy and put it in the bin +pranavsaroha/so100_squishy2colors_1: + 0: pick the squishies and put them in the bins with their corresponding colors +pranavsaroha/so100_squishy2colors_2_new: + 0: pick the squishies and put them in the bins with their corresponding colors +rabhishek100/so100_train_dataset: + 0: picks tape and places it in a cup. +rgarreta/koch_pick_place_lego: + 0: Pick the Lego block and drop it in the box on the right. +rgarreta/koch_pick_place_lego_v2: + 0: Pick the Lego block and drop it in the box on the right. +rgarreta/koch_pick_place_lego_v3: + 0: Pick the Lego block and drop it in the box on the right. Top and wrist cameras. +rgarreta/koch_pick_place_lego_v6: + 0: Grasp a lego block and put it in the bin. +rgarreta/koch_pick_place_lego_v7: + 0: Grasp a lego block and put it in the bin. +rgarreta/koch_pick_place_lego_v8: + 0: Grasp a lego block and put it in the bin. +roboticshack/left-arm-grasp-lego-brick: + 0: Grasp the lego brick and put it in the box. +roboticshack/sandee-kiwiv10: + 0: Place the bottle on the table. +roboticshack/team-7-left-arm-grasp-motor: + 0: Grasp the black motor and put it in the box. +roboticshack/team-7-right-arm-grasp-tape: + 0: Grasp the tape and put it in the box. +roboticshack/team10-red-block: + 0: Pick a red lego block and move it to the right. +roboticshack/team11_pianobot: + 0: Point at the keyboard. +roboticshack/team13-three-balls-stacking: + 0: Stack the balls on top of each other. +roboticshack/team13-two-balls-stacking: + 0: Stack the balls on top of each other. +roboticshack/team16-can-stacking: + 0: Grasp the flipped cup and stack it on top of the midpoint between the two other + cups to create a tower +roboticshack/team16-water-pouring: + 0: Pouring water from one cup to another cup +roboticshack/team2-guess_who_less_ligth: + 0: Pick up the card and place it in the slot. +roboticshack/team2-guess_who_so100: + 0: Pick up the card from the shelf. +roboticshack/team2-guess_who_so100_light: + 0: Place the card in the slot. +roboticshack/team9-pick_chicken_place_plate: + 0: Pick up the chicken and place on orange plate +roboticshack/team9-pick_cube_place_static_plate: + 0: Pick up the green cube and place on orange plate. +samanthalhy/so100_herding_1: + 0: Grasp a green tool and herd all the particles to the grey bin. +samsam0510/cube_reorientation_2: + 0: Rotate the object so it aligns with the silhouette on the table. +samsam0510/cube_reorientation_4: + 0: Rotate the object so it aligns with respect to the line on the table. +samsam0510/glove_reorientation_1: + 0: Rotate the glove so the bottom part aligns with the line on the table. +samsam0510/tape_insert_1: + 0: Grasp a red tape and put it on the box. +samsam0510/tape_insert_2: + 0: Grasp a red tape and put it in the yellow tape. +samsam0510/tooth_extraction_3: + 0: Extract the tooth and put it somewhere. +samsam0510/tooth_extraction_4: + 0: Extarct the molar and put it somewhere. +satvikahuja/mixer_on_off: + 0: switch the mixer on or off +satvikahuja/mixer_on_off_new: + 0: Adjust the s position. +satvikahuja/mixer_on_off_new_1: + 0: Press the button on the blender. +satvikahuja/mixer_on_off_new_4: + 0: Place the lid on the blender. +satvikahuja/orange_mixer_1: + 0: pick orange and place in mixer +satvikahuja/orange_pick_place_new1: + 0: Pick up the orange and place it in the bowl. +seeingrain/241228_pick_place_2cams: + 0: Place the cube in the basket. +seeingrain/lego_3cameras: + 0: Pick up the red block and place it in the basket. +seeingrain/one_shot_learning_18episodes: + 0: Pick up the red block and place it in the basket. +seeingrain/pick_lego_to_hand: + 0: places blue object on table. +seeingrain/pick_place_lego: + 0: Place the cube in the basket. +seeingrain/pick_place_lego_wider_range_dang: + 0: Pick up the cube and place it in the basket. +seeingrain/pick_place_lego_wider_range_dong: + 0: Pick up the blue object and place it in the basket. +seeingrain/pick_place_lego_wider_range_richard: + 0: Pick up the blue cube and place it in the basket. +seeingrain/pick_place_pink_lego: + 0: Pick up the red cube and place it in the basket. +seeingrain/pick_place_pink_lego_few_samples: + 0: 'pick_place_pink_lego_few_samples ' +shin1107/koch_move_block_with_some_shapes: + 0: Grasp a block and put it in the hole with some shapes. +shin1107/koch_train_block: + 0: Grasp a block and put it in the hole. +shreyasgite/so100_base_env: + 0: Grasp a lego block and put it in the bin. +shreyasgite/so100_base_left: + 0: Grasp a lego block and put it in the bin. +shreyasgite/so100_legocube_50: + 0: Grasp a lego block and put it in the bin. +sihyun77/mond_1: + 0: Grasp a lego block and put it in the bin. +sihyun77/mond_13: + 0: Grasp a lego block and put it in the bin. +sihyun77/si_so100: + 0: Grasp a lego block and put it in the bin. +sihyun77/sihyun_3_17_2: + 0: Grasp a lego block and put it in the bin. +sihyun77/sihyun_3_17_5: + 0: Grasp a lego block and put it in the bin. +sihyun77/sihyun_king: + 0: Grasp a lego block and put it in the bin. +sihyun77/sihyun_main_2: + 0: Grasp a lego block and put it in the bin. +sihyun77/sihyun_main_3: + 0: Grasp a lego block and put it in the bin. +sihyun77/suho_3_17_1: + 0: Grasp a lego block and put it in the bin. +sihyun77/suho_3_17_3: + 0: Grasp a lego block and put it in the bin. +sihyun77/suho_angel: + 0: Grasp a lego block and put it in the bin. +sihyun77/suho_main_2: + 0: Grasp a lego block and put it in the bin. +sihyun77/suho_red: + 0: Grasp a lego block and put it in the bin. +sihyun77/suho_so100: + 0: Grasp a lego block and put it in the bin. +sixpigs1/so100_insert_cylinder_error: + 0: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf + with position offset when grasping. + 1: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf + with gripper error when grasping. + 2: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf + with rotation offset when uprighting. + 3: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf + with position offset when inserting. + 4: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf + with choice error when inserting. + 5: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf + without errors. +sixpigs1/so100_pick_cube_in_box: + 0: Pick up the red cube and put it in the box. +sixpigs1/so100_pick_cube_in_box_error: + 0: Pick up the red cube and put it in the box with position offset when grasping. + 1: Pick up the red cube and put it in the box with gripper error when grasping. + 2: Pick up the red cube and put it in the box with position offset when releasing. + 3: Pick up the red cube and put it in the box without errors. +sixpigs1/so100_pull_cube_by_tool_error: + 0: Pick up the L-shaped tool and pull the purple cube by the tool with position + offset when grasping. + 1: Pick up the L-shaped tool and pull the purple cube by the tool with rotation + offset when grasping. + 2: Pick up the L-shaped tool and pull the purple cube by the tool with gripper error + when grasping. + 3: Pick up the L-shaped tool and pull the purple cube by the tool with position + offset when lowering. + 4: Pick up the L-shaped tool and pull the purple cube by the tool with rotation + offset when lowering. + 5: Pick up the L-shaped tool and pull the purple cube by the tool with position + offset when pulling. + 6: Pick up the L-shaped tool and pull the purple cube by the tool with gripper error + when pulling. + 7: Pick up the L-shaped tool and pull the purple cube by the tool without errors. +sixpigs1/so100_pull_cube_error: + 0: Pull the yellow cube to the red and white target with position offset when reaching. + 1: Pull the yellow cube to the red and white target with position offset when pulling. + 2: Pull the yellow cube to the red and white target with gripper error when pulling. + 3: Pull the yellow cube to the red and white target without errors. +sixpigs1/so100_push_cube_error: + 0: Push the blue cube to the red and white target with position offset when reaching. + 1: Push the blue cube to the red and white target with position offset when pushing. + 2: Push the blue cube to the red and white target with gripper error when pushing. + 3: Push the blue cube to the red and white target without errors. +sixpigs1/so100_stack_cube_error: + 0: Pick up the red cube and stack it on the green cube with position offset when + grasping. + 1: Pick up the red cube and stack it on the green cube with gripper error when grasping. + 2: Pick up the red cube and stack it on the green cube with position offset when + stacking. + 3: Pick up the red cube and stack it on the green cube without errors +smanni/train_so100_fluffy_box: + 0: Grasp a small object and place it in the box. +speedyyoshi/so100_grasp_pink_block: + 0: Grasp a lego block and put it in the bin. +sshh11/so100_orange_50ep_1: + 0: Grasp an orange object and put it in the bin. +sshh11/so100_orange_50ep_2: + 0: Grasp an orange object and put it in the bin. +swarajgosavi/act_kikobot_block_real: + 0: Pick up the blue cube and place it in the box. +swarajgosavi/act_kikobot_pusht_real: + 0: picks up the red block. +takuzennn/aloha-pick100: + 0: arm pick pen and put it into the cup +takuzennn/square3: + 0: Pick the cube from the table. +thedevansh/mar16_1336: + 0: Grasp a lego block and put it in the bin. +theo-michel/lekiwi_v2: + 0: Pick up the can on the ground +theo-michel/lekiwi_v5: + 0: Pick up the can on the ground +therarelab/med_dis_rare_6: + 0: places green object in box. +therarelab/so100_pick_place_2: + 0: Pick a plaster roll and place it to the blue sticker. +tkc79/so100_lego_box_1: + 0: Grasp a lego block and put it in the box. +tkc79/so100_lego_box_2: + 0: Grasp a lego block and put it in the box. +triton7777/so100_dataset_mix: + 0: Pick up the black tape and place it inside the white tape roll. + 1: Pick up the gift miniatures and place them in the black box. + 2: Sort the mixed objects into their appropriate categories. + 3: Place the pens into the pen holder. + 4: Place pens, bottles, and any suitable items into the pen holder, as appropriate. + 5: Place the oranges into the yellow basket. + 6: Stack the plates and place the cup on top. +twerdster/koch_new_training_red: + 0: move red cube into cellotape circle +twerdster/koch_training_red: + 0: Pick up the red block. +underctrl/handcamera_single_blue: + 0: Pick the Lego block and drop it in the box on the right. +underctrl/mutli-stacked-block_mutli-color_pick-up_80: + 0: Pick single block of multi-color and drop it in the box on the right. +underctrl/single-block_blue-color_pick-up_80: + 0: Pick single block of multi-color and drop it in the box on the right. +underctrl/single-block_multi-color_pick-up_50: + 0: Pick single block of multi-color and drop it in the box on the right. +underctrl/single-stacked-block_mutli-color_pick-up_80: + 0: Pick single block of multi-color and drop it in the box on the right. +underctrl/single-stacked-block_two-color_pick-up_80: + 0: Pick single block of multi-color and drop it in the box on the right. +vladfatu/so100_above: + 0: Pick up red object and place it in the box. +vladfatu/so100_ds: + 0: Pick up the cube and place it in the box. +vladfatu/so100_office: + 0: Pick up the red object and place it in the box. +weiye11/so100_410_zwy: + 0: Pick up the cube and place it on the black circle. +yuz1wan/so100_pickplace: + 0: Pick the pink block and place it in the paper cup. +zaringleb/so100_cube_4_binary: + 0: Grasp a lego block and put it in the bin. +zaringleb/so100_cube_5_linear: + 0: Grasp a lego block and put it in the bin. +zaringleb/so100_cube_6_2d: + 0: Grasp a lego block and put it in the bin. +zaringleb/so100_cude_linear_and_2d_comb: + 0: Grasp a lego block and put it in the bin. +zijian2022/321: + 0: Grasp a lego block and put it in the bin. +zijian2022/backgrounda: + 0: Grasp a lego block and put it in the bin. +zijian2022/backgroundb: + 0: Grasp a lego block and put it in the bin. +zijian2022/c0: + 0: Grasp a lego block and put it in the bin based on color. + 1: Grasp a lego block and put it in the bin. +zijian2022/close3: + 0: Grasp a lego block and put it in the bin. +zijian2022/digitalfix3: + 0: Grasp a lego block and put it in the bin. +zijian2022/insert2: + 0: Grasp a lego block and put it in the bin. +zijian2022/noticehuman3: + 0: Notice human. +zijian2022/noticehuman5: + 0: picks up the box. +zijian2022/noticehuman70: + 0: Stop movement when human encounter testbed. + 1: Stop movement when human encounter testbed w/ trigger. +zijian2022/sort1: + 0: 'Grasp a box and sort it by color: place grey boxes on the left and black boxes + on the right.' +zliu157/i3r: + 0: Grasp a lego block and put it in the bin. +zliu157/i3r2: + 0: Grasp a i3r logo and put it in the bin. +zliu157/i3r3: + 0: Grasp a i3r logo and put it in the bin. +zliu157/i3r5: + 0: Grasp a i3r logo and put it in the bin. From 5fbe4f9987a7c3212aa908cc20d3f43c250898c8 Mon Sep 17 00:00:00 2001 From: Jade Date: Mon, 7 Jul 2025 09:23:22 -0400 Subject: [PATCH 4/5] remove yamls --- lerobot/configs/mappings/features.yaml | 2387 ------------------------ lerobot/configs/mappings/tasks.yaml | 1012 ---------- 2 files changed, 3399 deletions(-) delete mode 100644 lerobot/configs/mappings/features.yaml delete mode 100644 lerobot/configs/mappings/tasks.yaml diff --git a/lerobot/configs/mappings/features.yaml b/lerobot/configs/mappings/features.yaml deleted file mode 100644 index 816d91acd..000000000 --- a/lerobot/configs/mappings/features.yaml +++ /dev/null @@ -1,2387 +0,0 @@ -00ri/so100_battery: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -00ri/so100_battery_bin_center: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -0x00raghu/toffee_blue: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -0x00raghu/toffee_blue_2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -0x00raghu/toffee_red: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -0x00raghu/toffee_red_2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -0x00raghu/toffee_red_3__: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -0x00raghu/toffee_to_hand_1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -0x00raghu/toffee_to_hand_2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -1g0rrr/offline_dataset_name2: - observation.images.front: observation.image - observation.images.side: observation.image2 -1g0rrr/offline_dataset_name2_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -1g0rrr/reward_dataset_name2: - observation.images.front: observation.image - observation.images.side: observation.image2 -1g0rrr/reward_dataset_name2_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -1g0rrr/reward_pickplace1: - observation.images.laptop: observation.image -1g0rrr/reward_pickplace1_cropped_resized: - observation.images.laptop: observation.image -1g0rrr/sam_openpi03: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -1g0rrr/sam_openpi_cube_low10: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -1g0rrr/sam_openpi_cube_top10: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -1g0rrr/sam_openpi_solder1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -1g0rrr/sam_openpi_solder2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -1g0rrr/sam_openpi_wire10: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -1g0rrr/screw1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -356c/so100_duck_reposition_1: - observation.images.body: observation.image2 - observation.images.overhead: observation.image - observation.images.side: observation.image3 -356c/so100_nut_sort_1: - observation.images.body: observation.image2 - observation.images.overhead: observation.image - observation.images.side: observation.image3 -356c/so100_rope_reposition_1: - observation.images.body: observation.image2 - observation.images.overhead: observation.image - observation.images.side: observation.image3 -AK51/4090_01: - observation.images.base_cam: observation.image3 - observation.images.left_arm_cam: observation.image2 - observation.images.right_arm_cam: null - observation.images.top_cam: observation.image -AaronNewman/screwdriver_task_batch1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -AaronNewman/screwdriver_task_batch2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -AaronNewman/screwdriver_task_batch3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Allen-488/koch_dataset_50: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 -Anas0711/Gr00t_lerobot_state_action: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -Anas0711/Gr00t_lerobot_state_action_180: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -Anas0711/Gr00t_lerobot_state_action_600: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -Anas0711/Gr00t_lerobot_state_action_60_continuous: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -Anas0711/Gr00t_lerobot_state_action_box_1: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -Anas0711/Gr00t_lerobot_state_action_box_2: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -AndrejOrsula/lerobot_double_ball_stacking_random: - observation.images.realsense: observation.image - observation.images.side: observation.image2 -Ashton3/lerobot-aloha: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Bartm3/dice2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Bartm3/tape_to_bin: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Bartm3/tape_to_bin2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Beegbrain/align_cubes_green_blue: - observation.images.back: observation.image3 - observation.images.side: observation.image -Beegbrain/align_three_pens: - observation.images.back: observation.image3 - observation.images.side: observation.image -Beegbrain/moss_close_drawer_teabox: - observation.images.back: observation.image3 - observation.images.front: observation.image -Beegbrain/moss_open_drawer_teabox: - observation.images.back: observation.image3 - observation.images.front: observation.image -Beegbrain/moss_put_cube_teabox: - observation.images.back: observation.image3 - observation.images.front: observation.image -Beegbrain/moss_stack_cubes: - observation.images.back: observation.image3 - observation.images.front: observation.image -Beegbrain/oc_stack_cubes: - observation.images.realsense: observation.image3 - observation.images.realsense_top: observation.image -Beegbrain/pick_lemon_and_drop_in_bowl: - observation.images.realsense_side: observation.image - observation.images.realsense_top: observation.image2 -Beegbrain/pick_place_green_block: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -Beegbrain/pick_place_green_block_lr: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -Beegbrain/put_green_into_blue_bin: - observation.images.back: observation.image3 - observation.images.side: observation.image -Beegbrain/put_red_triangle_green_rect: - observation.images.back: observation.image3 - observation.images.side: observation.image -Beegbrain/put_screwdriver_box: - observation.images.back: observation.image3 - observation.images.side: observation.image -Beegbrain/so100_put_cube_cup: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -Beegbrain/stack_2_cubes: - observation.images.laptop: observation.image -Beegbrain/stack_green_on_blue_cube: - observation.images.back: observation.image3 - observation.images.side: observation.image -Beegbrain/stack_two_cubes: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -Beegbrain/sweep_tissue_cube: - observation.images.realsense_side: observation.image - observation.images.realsense_top: observation.image2 -BlobDieKatze/GrabBlocks: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -CSCSXX/pick_place_cube_1.17: - observation.images.side: observation.image2 - observation.images.top: observation.image -CSCSXX/pick_place_cube_1.18: - observation.images.side: observation.image2 - observation.images.top: observation.image -Chojins/chess_game_000_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_000_white_red: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_001_blue_stereo: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_001_red_stereo: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_001_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_002_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_003_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_004_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_005_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_006_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_007_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Chojins/chess_game_009_white: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Cidoyi/so100_all_notes: - observation.images.keyboard_camera: observation.image -Cidoyi/so100_all_notes_1: - observation.images.keyboard_camera: observation.image -Cidoyi/so100_all_notes_3: - observation.images.keyboard_camera: observation.image -Clementppr/lerobot_pick_and_place_dataset_world_model: - observation.images.laptop: observation.image -Congying1112/so100_place_blue_bottle_with_single_camera: - observation.images.onhand: observation.image2 -Congying1112/so100_place_blue_bottle_with_two_cameras: - observation.images.onhand: observation.image2 - observation.images.top: observation.image -Congying1112/so100_place_blue_bottle_with_two_cameras2: - observation.images.onhand: observation.image2 - observation.images.top: observation.image -Congying1112/so100_simplepick_with_2_cameras_from_top: - observation.images.onhand: observation.image2 - observation.images.top: observation.image -Congying1112/so100_switch_with_onhand_camera: - observation.images.onhand: observation.image2 -CrazyYhang/A1234-B-C_mvA2B: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Deason11/Open_the_drawer_to_place_items: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Deason11/PLACE_TAPE_PUSH_DRAWER: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Deason11/Random_Kitchen: - observation.images.L_OverheadCamera: observation.image - observation.images.R_OverheadCamera: observation.image3 - observation.images.wrist: observation.image2 -Deason11/mobile_manipulator_0319: - observation.images.L_OverheadCamera: observation.image - observation.images.R_OverheadCamera: observation.image3 - observation.images.wrist: observation.image2 -Deason11/mobile_manipulator_0326: - observation.images.L_OverheadCamera: observation.image - observation.images.R_OverheadCamera: observation.image3 - observation.images.wrist: observation.image2 -DimiSch/so100_50ep_2: - observation.images.realsense: observation.image -DimiSch/so100_50ep_3: - observation.images.realsense: observation.image -DimiSch/so100_terra_50_2: - observation.images.realsense: observation.image -Dongkkka/cable_pick_and_place2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -DorayakiLin/so100_pick_charger_on_tissue: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -DorayakiLin/so100_pick_cube_in_box: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -DorayakiLin/so100_stack_cube: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Eyas/grab_bouillon: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 -Eyas/grab_pink_lighter_10_per_loc: - observation.images.laptop: observation.image -FeiYjf/Grab_Pieces: - observation.images.left: observation.image - observation.images.right: observation.image2 -FeiYjf/Hold_Pieces: - observation.images.left: observation.image - observation.images.right: observation.image2 -FeiYjf/Makalu_push: - observation.images.left: observation.image2 - observation.images.right: observation.image -FeiYjf/Maklu_dataset: - observation.images.left: observation.image - observation.images.right: observation.image2 -FeiYjf/Test_NNNN: - observation.images.left: observation.image - observation.images.right: observation.image2 -FeiYjf/new_Dataset: - observation.images.left: observation.image - observation.images.right: observation.image2 -FeiYjf/new_GtoR: - observation.images.left: observation.image - observation.images.right: observation.image2 -FeiYjf/push_0094: - observation.images.left: observation.image - observation.images.right: observation.image2 -FeiYjf/push_gg: - observation.images.left: observation.image2 - observation.images.right: observation.image -FsqZ/so100_1: - observation.images.side: observation.image -Gano007/so100_doliprane: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Gano007/so100_gano: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Gano007/so100_lolo: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Gano007/so100_medic: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Gano007/so100_second: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Gongsta/grasp_duck_in_cup: - observation.images.laptop: observation.image -HITHY/so100-kiwi: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -HITHY/so100_peach: - observation.images.laptop: observation.image -HITHY/so100_peach1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -HITHY/so100_peach3: - observation.images.laptop: observation.image -HITHY/so100_peach4: - observation.images.laptop: observation.image -HITHY/so100_redstrawberry: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -HITHY/so100_strawberry: - observation.images.laptop: observation.image -HWJ658970/fat_fish: - observation.images.front: observation.image - observation.images.phone: observation.image3 -HWJ658970/lego_100_class: - observation.images.front: observation.image - observation.images.phone: observation.image3 -HWJ658970/lego_50: - observation.images.front: observation.image - observation.images.phone: observation.image3 -HWJ658970/lego_50_camera_change: - observation.images.front: observation.image - observation.images.phone: observation.image3 -HYAIYN/so100_get_orange_10epi: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -HappyPablo/dec3_data2: - observation.images.laptop: null - observation.images.phone: observation.image2 -HuaihaiLyu/groceries: - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_right_wrist: observation.image3 -HuaihaiLyu/stackbasket: - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_right_wrist: observation.image3 -IPEC-COMMUNITY/berkeley_mvp_lerobot: - observation.images.hand_image: observation.image -IPEC-COMMUNITY/libero_10_no_noops_image_lerobot: - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -IPEC-COMMUNITY/ucsd_kitchen_dataset_lerobot: - observation.images.image: observation.image -Ityl/so100_recording1: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -Ityl/so100_recording2: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -JJwuj/koch_static_grasp_0402_v5: - observation.images.E12: observation.image2 - observation.images.E22S: observation.image -Jiafei1224/so100_pa222per: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Jiangeng/so100_413: - observation.images.top: observation.image2 - observation.images.wrist_left: observation.image -KeWangRobotics/piper_push_cube_gamepad_1: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_push_cube_gamepad_1_cropped_resized: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_push_cube_gamepad_2: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_push_cube_gamepad_2_cropped_resized: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_rl_1: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_rl_1_cropped_resized: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_rl_2: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_rl_2_cropped_resized: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_rl_3: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_rl_3_cropped_resized: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_rl_4: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -KeWangRobotics/piper_rl_4_cropped_resized: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -LegrandFrederic/Orange-brick-lower-resolution: - observation.images.main.left: observation.image - observation.images.main.right: observation.image2 - observation.images.secondary_0: observation.image3 -LemonadeDai/so100_coca: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -Loki0929/so100_100: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Loki0929/so100_duck: - observation.images.third: observation.image3 - observation.images.top: observation.image - observation.images.wrist: observation.image2 -Lugenbott/koch_1225: - observation.images.Camera0: observation.image - observation.images.Camera2: observation.image2 -Mohamedal/put_banana: - observation.images.realsense_side: observation.image - observation.images.realsense_top: observation.image2 -MossProphet/so100_square-1-2-3.2: - observation.images.Arm_left: observation.image2 - observation.images.Arm_right: null - observation.images.External: observation.image3 -Mwuqiu/so100_0408: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Mwuqiu/so100_0408_muti: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -NONHUMAN-RESEARCH/SOARM100_TASK_VENDA: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -NONHUMAN-RESEARCH/SOARM100_TASK_VENDA_BOX: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Odog16/so100_cube_drop_pick_v1: - observation.images.workspace: observation.image - observation.images.wrist: observation.image2 -Odog16/so100_cube_stacking_v1: - observation.images.Workspace: observation.image - observation.images.Wrist: observation.image2 -Odog16/so100_tea_towel_folding_v1: - observation.images.workspace: observation.image - observation.images.wrist: observation.image2 -Ofiroz91/so_100_cube2bowl: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -Pi-robot/barbecue_flip: - observation.images.top: observation.image2 - observation.images.wrist: observation.image -Pi-robot/barbecue_put: - observation.images.top: observation.image2 - observation.images.wrist: observation.image -RLWRLD/put_cube_sync: - observation.images.cam_high: observation.image - observation.images.cam_right_wrist: observation.image2 -RasmusP/so100_Orange2Green: - observation.images.phone: observation.image - observation.images.webcam: observation.image2 -RasmusP/so100_dataset50ep: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -RasmusP/so100_dataset50ep_a: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -SeanLMH/so100_picknplace: - observation.images.front: observation.image2 - observation.images.overhead: observation.image -SeanLMH/so100_picknplace_v2: - observation.images.front: observation.image2 - observation.images.overhead: observation.image -Setchii/so100_grab_ball: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -T-K-233/koch_k1_pour_shot: - observation.images.gripper: observation.image2 - observation.images.side: observation.image3 - observation.images.top: observation.image -T1g3rGE/so100_pickplace_small_20250407_171912: - observation.images.webcam: observation.image2 -TrossenRoboticsCommunity/aloha_ai_block: - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -TrossenRoboticsCommunity/aloha_baseline_dataset: - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -TrossenRoboticsCommunity/aloha_fold_tshirt: - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: observation.image - observation.images.cam_right_wrist: observation.image3 -TrossenRoboticsCommunity/aloha_stationary_logo_assembly: - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: observation.image3 - observation.images.cam_right_wrist: null -VoicAndrei/so100_banana_to_plate_only: - observation.images.right: observation.image2 - observation.images.top: observation.image -VoicAndrei/so100_banana_to_plate_rebel_full: - observation.images.right: observation.image2 - observation.images.top: observation.image -Winster/so100_cube: - observation.images.laptop: observation.image -Winster/so100_sim: - observation.images.laptop: observation.image -XXRRSSRR/so100_v3_num_episodes_50: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -Yotofu/so100_sweeper_shoes: - observation.images.end_depth: null - observation.images.end_rgb: observation.image3 - observation.images.front_depth: observation.image2 - observation.images.front_rgb: observation.image -Yuanzhu/koch_bimanual_grasp_0: - observation.images.top_camera: observation.image -Yuanzhu/koch_bimanual_grasp_3: - observation.images.hand_camera: observation.image2 - observation.images.side_camera: observation.image3 - observation.images.top_camera: observation.image -ZCM5115/so100_1210: - observation.images.hikvision: observation.image2 - observation.images.laptop: observation.image -ZCM5115/so100_2Arm3cameras_movebox: - observation.images.left: observation.image2 - observation.images.right: observation.image3 - observation.images.top: observation.image -ZGGZZG/so100_drop0: - observation.images.left: observation.image2 - observation.images.up: observation.image -ZGGZZG/so100_drop1: - observation.images.left: observation.image2 - observation.images.up: observation.image -Zak-Y/so100_grap_dataset: - observation.images.Left_follower: observation.image2 - observation.images.Logic_camera: observation.image - observation.images.Right_follower: observation.image3 -Zak-Y/so100_three_cameras_dataset: - observation.images.Left_follower: observation.image2 - observation.images.Logic_camera: observation.image - observation.images.Right_follower: observation.image2 -Zhaoting123/koch_cleanDesk_: - observation.images.phone: observation.image -abbyoneill/data_w_mug: - observation.images.logitech1: observation.image - observation.images.logitech2: observation.image2 -abbyoneill/new_dataset_pick_place: - observation.images.logitech1: observation.image - observation.images.logitech2: observation.image3 -abbyoneill/pusht: - observation.images.logitech1: observation.image - observation.images.logitech2: observation.image3 -abbyoneill/thurs1120pickplace: - observation.images.logitech1: observation.image - observation.images.logitech2: observation.image3 -abhisb/so100_51_ep: - observation.images.front: observation.image3 - observation.images.mobile: observation.image2 - observation.images.overhead: observation.image -abokinala/sputnik_100_11_pick_place_container: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -abokinala/sputnik_100_12_pick_place_container: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -abokinala/sputnik_100_14_pick_place_container: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -abokinala/sputnik_100_23_pick_place_surface: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -abokinala/sputnik_100_46_custom_tasks: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -abokinala/sputnik_100_54_kitchen_tasks: - observation.images.laptop: null - observation.images.phone: observation.image -abokinala/sputnik_100_55_kitchen_tasks: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -abokinala/sputnik_100_58_kitchen_tasks: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -abokinala/sputnik_100_60_kitchen_tasks: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -abougdira/cube_target: - observation.images.realsense: observation.image3 - observation.images.realsense_top: observation.image -acrampette/third_arm_01: - observation.images.wrist: observation.image2 -acrampette/third_arm_02: - observation.images.wrist: observation.image2 -ad330/cubePlace: - observation.images.phone: observation.image - observation.images.wristCam: observation.image2 -ad330/so100_box_pickPlace: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -aergogo/so100_pick_place: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -agonyxx/koch-aloha: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 - observation.images.phone2: observation.image3 -aimihat/so100_tape: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -airthebear/so100_GL: - observation.images.laptop: observation.image -aliberts/koch_tutorial: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 -allenchienxxx/so100Test: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -andabi/D10: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D11: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D12: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D13: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D14: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D15: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D16: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D17: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D2: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D3: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D4: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D5: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D6: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D7: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D8: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/D9: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/a_shoe_easy_10: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andabi/shoes_easy: - observation.images.bird: observation.image - observation.images.wrist_left: observation.image2 - observation.images.wrist_right: observation.image3 -andlyu/so100_indoor_0: - observation.images.arm_left: observation.image2 - observation.images.arm_right: observation.image3 - observation.images.base_left: null - observation.images.base_right: observation.image -andlyu/so100_indoor_1: - observation.images.arm_left: observation.image3 - observation.images.arm_right: observation.image2 - observation.images.base_left: null - observation.images.base_right: observation.image -andlyu/so100_indoor_10: - observation.images.arm: observation.image - observation.images.base: observation.image3 - observation.images.base_right: null - observation.images.gripper: observation.image2 -andlyu/so100_indoor_2: - observation.images.arm_left: observation.image2 - observation.images.arm_right: observation.image3 - observation.images.base_left: null - observation.images.base_right: observation.image -andlyu/so100_indoor_3: - observation.images.arm_left: observation.image3 - observation.images.arm_right: observation.image2 - observation.images.base_left: null - observation.images.base_right: observation.image -andlyu/so100_indoor_4: - observation.images.arm_left: observation.image2 - observation.images.arm_right: observation.image3 - observation.images.base_left: null - observation.images.base_right: observation.image -andlyu/so100_indoor_val2: - observation.images.arm: observation.image - observation.images.base: observation.image3 - observation.images.base_right: null - observation.images.gripper: observation.image2 -andlyu/so100_indoor_val_0: - observation.images.arm: observation.image - observation.images.base: observation.image3 - observation.images.gripper: observation.image2 -andreasBihlmaier/dual_arm_transfer_2025_02_16: - observation.images.webcam_1: observation.image3 - observation.images.webcam_2: observation.image2 - observation.images.webcam_3: observation.image -andrewcole712/so100_tape_bin_place: - observation.images.phone: observation.image -andy309/so100_0311_1152: - observation.images.back: observation.image2 - observation.images.top: observation.image - observation.images.wrist_right: observation.image3 -andy309/so100_0313_no_wrist_camera: - observation.images.back: observation.image2 - observation.images.top: observation.image -andy309/so100_0313_no_wrist_camera_with_two_arms_cloths: - observation.images.back: observation.image2 - observation.images.top: observation.image -andy309/so100_0314_fold_cloths: - observation.images.back: observation.image2 - observation.images.top: observation.image -aractingi/pick_place_lego_cube: - observation.images.top: observation.image2 - observation.images.wrist: observation.image -aractingi/pick_place_lego_cube_1: - observation.images.top: observation.image2 - observation.images.wrist: observation.image -aractingi/pick_place_lego_cube_cropped_resized: - observation.images.top: observation.image2 - observation.images.wrist: observation.image -aractingi/push_cube_front_side_reward: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_front_side_reward_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_front_side_reward_long: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_front_side_reward_long_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_offline_data: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -aractingi/push_cube_offline_data_cropped_resized: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -aractingi/push_cube_overfit: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_overfit_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_reward: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_reward_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_reward_data: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -aractingi/push_cube_reward_data_cropped_resized: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -aractingi/push_cube_simp_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_square_light_offline_demo: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_square_light_offline_demo_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_square_light_reward: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_square_offline_demo: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_square_offline_demo_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_square_reward_1: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_square_reward_1_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_square_reward_cropped_resized: - observation.images.front: observation.image - observation.images.side: observation.image2 -aractingi/push_cube_to_face_reward: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -aractingi/push_cube_to_face_reward_cropped_resized: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -aractingi/push_green_cube_hf: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -aractingi/push_green_cube_hf_cropped_resized: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -arclabmit/Koch_twoarms: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 -arclabmit/koch_gear_and_bin: - observation.images.nexigo_webcam: observation.image - observation.images.realsense: observation.image3 -astroyat/cube: - observation.images.laptop: observation.image -badwolf256/so100_twin_cam_duck: - observation.images.realsense: observation.image - observation.images.wrist: observation.image2 -badwolf256/so100_twin_cam_duck_v2: - observation.images.realsense: observation.image - observation.images.wrist: observation.image2 -badwolf256/so100_twin_cam_duck_v3: - observation.images.realsense: observation.image - observation.images.wrist: observation.image2 -baladhurgesh97/so100_final_picking_3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -bensprenger/chess_game_001_blue_stereo: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -bensprenger/left_arm_yellow_brick_in_box_v0: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -bensprenger/left_arm_yellow_brick_in_box_with_purple_noise_v0: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -bensprenger/right_arm_p_brick_in_box_with_y_noise_v0: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -bnarin/so100_tic_tac_toe_move_0_0: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -bnarin/so100_tic_tac_toe_move_1_0: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -bnarin/so100_tic_tac_toe_move_2_1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -bnarin/so100_tic_tac_toe_move_4_0: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -bnarin/so100_tic_tac_toe_we_do_it_live: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -cadene/droid_1.0.1: - action.cartesian_position: null - action.cartesian_velocity: null - action.gripper_position: null - action.gripper_velocity: null - action.joint_position: null - action.joint_velocity: null - action.original: null - building: null - camera_extrinsics.exterior_1_left: null - camera_extrinsics.exterior_2_left: null - camera_extrinsics.wrist_left: null - collector_id: null - date: null - discount: null - is_episode_successful: null - language_instruction: null - language_instruction_2: null - language_instruction_3: null - observation.images.exterior_1_left: observation.image - observation.images.exterior_2_left: observation.image3 - observation.images.wrist_left: observation.image2 - observation.state.cartesian_position: null - observation.state.gripper_position: null - observation.state.joint_position: null - task_category: null -carpit680/giraffe_sock_demo_1: - observation.images.webcam: observation.image -carpit680/giraffe_sock_demo_2: - observation.images.webcam: observation.image -carpit680/giraffe_task: - observation.images.webcam: observation.image -chmadran/so100_dataset04: - observation.images.laptop: observation.image -chmadran/so100_dataset08: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -chmadran/so100_home_dataset: - observation.images.laptop: observation.image3 - observation.images.logitech: observation.image - observation.images.phone: observation.image2 -cmcgartoll/cube_color_organizer: - observation.images.claw: observation.image2 - observation.images.front: observation.image - observation.images.phone: observation.image3 -ctbfl/sort_battery: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/so100_task_1: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/so100_task_2: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/so100_task_3: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/so100_task_4: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/so100_v2_task_1: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/so100_v2_task_2: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/so100_v2_task_3: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/so100_v2_task_4: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/svla_so100_task1_v3: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/svla_so100_task4_v3: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/svla_so100_task4_v3_clean: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/svla_so100_task4_v3_multiple: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/svla_so100_task5_v3: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danaaubakirova/svla_so100_task5_v3_clean: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -danielkr452/so100_work6: - observation.images.laptop: null - observation.images.phone: observation.image2 -danmac1/real_real332: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -dboemer/koch_50-samples: - observation.images.side: observation.image3 - observation.images.top: observation.image -dc2ac/so100-t5: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -denghj/dataset_red_tape01: - observation.images.laptop: observation.image -dkdltu1111/omx-bottle1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -dop0/koch_pick_terminal: - observation.images.side: observation.image3 - observation.images.top: observation.image - observation.images.wrist: observation.image2 -doujiangwang/task1_10epi_100000step: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -doujiangwang/task2_10epi_100000step: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -doujiangwang/task4_10epi_100000step: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -doujiangwang/task5_10epi_100000step: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -dragon-95/so100_sorting: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -dragon-95/so100_sorting_1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -dragon-95/so100_sorting_2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -dragon-95/so100_sorting_3: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -dsfsg/bring_bottle: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 - observation.images.phone2: observation.image3 -dsfsg/grasp_pen_and_bottle: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 - observation.images.phone2: observation.image2 -dsfsg/grasp_pens: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 - observation.images.phone2: observation.image3 -duthvik/sputnik_100_17_pick_place_container: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -duthvik/sputnik_100_18_pick_place_container: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_24_pick_place_surface: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -duthvik/sputnik_100_25_pick_place_surface: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -duthvik/sputnik_100_26_pick_place_surface: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -duthvik/sputnik_100_27_pick_place_surface: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -duthvik/sputnik_100_29_pick_place_surface: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_31_pour_liquid: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -duthvik/sputnik_100_32_pour_liquid: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_41_custom_tasks: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_42_custom_tasks: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_43_custom_tasks: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_44_custom_tasks: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_45_custom_tasks: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_51_kitchen_tasks: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_52_kitchen_tasks: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -duthvik/sputnik_100_53_kitchen_tasks: - observation.images.laptop: observation.image - observation.images.side_view: observation.image3 - observation.images.usb_front: observation.image2 -ellen2imagine/pusht_green1: - observation.images.phone: observation.image -ellen2imagine/pusht_green_same_init2: - observation.images.phone: observation.image -engineer0002/pepper: - observation.images.left_wrist: observation.image2 - observation.images.right_wrist: observation.image3 - observation.images.top: observation.image -ethanCSL/your_task_name: - observation.images.front: observation.image - observation.images.phone: observation.image2 -francescocrivelli/carrot_eating: - observation.images.endeffector: observation.image2 - observation.images.workspace: observation.image -francescocrivelli/orange_feeding: - observation.images.endeffector: observation.image2 - observation.images.workspace: observation.image -frk2/so100large: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -frk2/so100largediffcam: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -ganker5/so100_action_20250403: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -ganker5/so100_color_0328: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -ganker5/so100_dataline_0328: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -ganker5/so100_dataline_20250331: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -ganker5/so100_push_20250328: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -ganker5/so100_push_20250331: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -ganker5/so100_toy_20250402: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -gxy1111/so100_pick_place: - observation.images.eye: observation.image - observation.images.wrist: observation.image2 -hangwu/koch_pick_terminal: - observation.images.side: observation.image3 - observation.images.top: observation.image - observation.images.wrist: observation.image2 -hangwu/piper_pick_terminal_2: - observation.images.depth: null - observation.images.realsense_color: observation.image3 - observation.images.side: observation.image2 - observation.images.top: observation.image -hangwu/piper_pick_terminal_and_place: - observation.images.depth: null - observation.images.realsense_color: observation.image3 - observation.images.side: observation.image2 - observation.images.top: observation.image -hannesill/koch_pnp_2_blocks_2_bins_200: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -hannesill/koch_pnp_simple_50: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -hegdearyandev/so100_eraser_cup_v1: - observation.images.phone: observation.image2 - observation.images.webcam-0: observation.image -helper2424/hil-serl-push-circle-classifier: - observation.images.web0: observation.image - observation.images.web1: observation.image3 -hkphoooey/throw_stuffie: - observation.images.phone: observation.image -ibru/bob_jetson: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -ibru/bobo_groot_n1_trash_picker: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -ibru/bobo_jetson: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -ibru/bobo_trash_collector: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -imatrixlee/koch_place: - observation.images.eye: observation.image2 - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -imsyed00/so100_yellowbowl_pickplace_1: - observation.images.laptop: observation.image -isadev/bougies1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -isadev/bougies2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -isadev/bougies3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -jackvial/koch_with_rewards_4: - observation.images.main: observation.image -jadechoghari/genesis-1k: - observation.image.side: observation.image3 - observation.image.top: observation.image - observation.image.wrist: observation.image2 -jadechoghari/svla_so101-sim_task4_v3_multiple_1: - observation.image.side: observation.image3 - observation.image.top: observation.image - observation.image.wrist: observation.image2 -jadechoghari/svla_so101-sim_task4_v3_multiple_2: - observation.image.side: observation.image3 - observation.image.top: observation.image - observation.image.wrist: observation.image2 -jainamit/koch: - observation.images.nexigo: observation.image -jainamit/koch_pickcube: - observation.images.front_camera: observation.image -jainamit/koch_realcube3: - observation.images.front_camera: observation.image -jannick-st/classifier: - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: observation.image3 - observation.images.cam_right_wrist: null -jannick-st/push-cube-classifier_cropped_resized: - observation.images.cam_low: observation.image - observation.images.cam_right_wrist: observation.image2 - observation.images.cam_top: observation.image -jbraumann/so100_1902: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -jchun/so100_pickplace_small_20250322_193929: - observation.images.cv: observation.image2 - observation.images.main: observation.image3 - observation.images.webcam: observation.image -jchun/so100_pickplace_small_20250323_120056: - observation.images.cv: observation.image3 - observation.images.main: observation.image2 - observation.images.webcam: observation.image -jiajun001/eraser00_2: - observation.images.hand: observation.image2 - observation.images.side: observation.image -jiajun001/eraser00_3: - observation.images.front: observation.image - observation.images.hand: observation.image2 - observation.images.side: observation.image3 -jlesein/TestBoulon2: - observation.images.robor: observation.image2 - observation.images.top: observation.image -jlesein/TestBoulon7: - observation.images.robor: observation.image2 - observation.images.top: observation.image -jlitch/so100multicam3: - observation.images.overview2: observation.image - observation.images.wrist: observation.image2 -jlitch/so100multicam6: - observation.images.overview2: observation.image - observation.images.wrist: observation.image2 -jlitch/so100multicam7: - observation.images.overview2: observation.image - observation.images.wrist: observation.image2 -jmrog/so100_sweet_pick: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -joaoocruz00/so100_makeitD1: - observation.images.laptop: observation.image2 - observation.images.realsensergb: observation.image -jpata/so100_pick_place_tangerine: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -kantine/domotic_dishTidyUp_anomaly: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_dishTidyUp_expert: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_groceriesSorting_anomaly: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_groceriesSorting_expert: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_makingCoffee_anomaly: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_makingCoffee_expert: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_pouringCoffee_anomaly: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_pouringCoffee_expert: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_setTheTable_anomaly: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_setTheTable_expert: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_vegetagblesAndFruitsSorting_anomaly: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/domotic_vegetagblesAndFruitsSorting_expert: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/flip_A0: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -kantine/flip_A1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -kantine/flip_A2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -kantine/flip_A3: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -kantine/flip_A4: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -kantine/flip_A5: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -kantine/industrial_robothon_buttons_anomaly: - observation.images.logitech_1: observation.image2 - observation.images.logitech_2: observation.image -kantine/industrial_robothon_buttons_expert: - observation.images.logitech_1: observation.image2 - observation.images.logitech_2: observation.image -kantine/industrial_robothon_hatchAndProbe_anomaly: - observation.images.logitech_1: observation.image2 - observation.images.logitech_2: observation.image -kantine/industrial_robothon_hatchAndProbe_expert: - observation.images.logitech_1: observation.image2 - observation.images.logitech_2: observation.image -kantine/industrial_screws_sorting_anomaly: - observation.images.logitech_1: observation.image2 - observation.images.logitech_2: observation.image -kantine/industrial_screws_sorting_expert: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/industrial_soldering_anomaly: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/industrial_soldering_expert: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kantine/so100_kapla_tower6: - observation.images.logitech_1: observation.image - observation.images.logitech_2: observation.image2 -kevin510/lerobot-cat-toy-placement: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -koenvanwijk/blue2: - observation.images.laptop: observation.image - observation.images.phone: null -koenvanwijk/blue52: - observation.images.laptop: observation.image - observation.images.phone: null -koenvanwijk/orange50-1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -koenvanwijk/orange50-variation-2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -kumarhans/so100_tape_task: - observation.images.laptop1: observation.image - observation.images.laptop2: observation.image2 -lalalala0620/koch_blue_paper_tape: - observation.images.front: observation.image - observation.images.phone: observation.image3 -lerobot/aloha_mobile_cabinet: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_mobile_chair: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_mobile_elevator: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_mobile_shrimp: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_mobile_wash_pan: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_mobile_wipe_wine: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_sim_transfer_cube_human: - observation.images.top: observation.image -lerobot/aloha_static_coffee: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_static_cups_open: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_static_screw_driver: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_static_towel: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_static_vinh_cup: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_static_vinh_cup_left: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/aloha_static_ziploc_slide: - observation.effort: null - observation.images.cam_high: observation.image - observation.images.cam_left_wrist: observation.image2 - observation.images.cam_low: null - observation.images.cam_right_wrist: observation.image3 -lerobot/berkeley_fanuc_manipulation: - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -lerobot/iamlab_cmu_pickup_insert: - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -lerobot/jaco_play: - observation.images.image: observation.image - observation.images.image_wrist: observation.image2 -lerobot/libero_10_image: - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -lerobot/libero_goal_image: - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -lerobot/libero_object_image: - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -lerobot/libero_spatial_image: - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -lerobot/roboturk: - observation.images.front_rgb: observation.image -lerobot/stanford_hydra_dataset: - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -lerobot/taco_play: - observation.images.rgb_gripper: observation.image2 - observation.images.rgb_static: observation.image -lerobot/toto: - observation.images.image: observation.image -lirislab/close_top_drawer_teabox: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/fold_bottom_right: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/green_lego_block_into_mug: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/green_lego_block_into_mug_easy: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/guess_who_lighting: - observation.images.mounted: observation.image2 -lirislab/guess_who_no_cond: - observation.images.mounted: observation.image2 -lirislab/guess_who_so100: - observation.images.mounted: observation.image -lirislab/lemon_into_bowl: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/open_top_drawer_teabox: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/push_cup_target: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/put_banana_bowl: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/put_caps_into_teabox: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/put_coffee_cap_teabox: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/red_cube_into_blue_cube: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/red_cube_into_green_lego_block: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/red_cube_into_mug: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/so100_demo: - observation.images.front: observation.image -lirislab/stack_two_red_cubes: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/sweep_tissue_cube: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -lirislab/unfold_bottom_right: - observation.images.realsense_side: observation.image2 - observation.images.realsense_top: observation.image -liuhuanjim013/so100_block: - observation.images.hand: observation.image2 - observation.images.top: observation.image -liuhuanjim013/so100_th: - observation.images.front: observation.image - observation.images.top: observation.image2 -liuhuanjim013/so100_th_1: - observation.images.hand: observation.image2 - observation.images.top: observation.image -liyitenga/so100_bi_giveme5: - observation.images.center: observation.image - observation.images.left_follower: observation.image2 - observation.images.right_follower: observation.image3 -liyitenga/so100_bi_hello: - observation.images.center: observation.image - observation.images.left_follower: observation.image2 - observation.images.right_follower: observation.image3 -liyitenga/so100_pick_taffy1: - observation.images.left: observation.image -liyitenga/so100_pick_taffy10: - observation.images.gripper: observation.image2 - observation.images.left: observation.image3 - observation.images.top: observation.image -liyitenga/so100_pick_taffy2: - observation.images.gripper: observation.image2 - observation.images.top: observation.image -liyitenga/so100_pick_taffy3: - observation.images.gripper: observation.image2 - observation.images.top: observation.image -liyitenga/so100_pick_taffy4: - observation.images.gripper: observation.image2 - observation.images.top: observation.image -liyitenga/so100_pick_taffy5: - observation.images.gripper: observation.image2 - observation.images.top: observation.image -liyitenga/so100_pick_taffy6: - observation.images.gripper: observation.image2 - observation.images.top: observation.image -liyitenga/so100_pick_taffy7: - observation.images.gripper: observation.image2 - observation.images.left_top: observation.image3 - observation.images.top: observation.image -liyitenga/so100_pick_taffy8: - observation.images.gripper: observation.image2 - observation.images.top: observation.image -lizi178119985/so100_jia: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -lucasngoo/so100_strawberry_grape: - observation.images.webcam: observation.image -luke250305/play_dice_250311.1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -m1b/so100_bluelego: - observation.images.phone: null - observation.images.side: observation.image -m1b/so100_bluelego_updt: - observation.images.phone: observation.image - observation.images.side: observation.image2 -ma3oun/rpi_squares_1: - observation.images.laptop: observation.image -maximilienroberti/so100_lego_red_box: - observation.images.cam_left: observation.image - observation.images.cam_middle: observation.image3 - observation.images.cam_right: observation.image2 -meteorinc/koch_tea: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -mikechambers/block_cup_14: - observation.images.main_cam: observation.image - observation.images.secondary_cam: observation.image2 -mikechambers/block_cup_5: - observation.images.main_cam: observation.image - observation.images.secondary_cam: observation.image2 -mlfu7/pi0_conversion_no_pad_video: - exterior_image_1_left: observation.image - wrist_image_left: observation.image2 -nbaron99/so100_pick_and_place2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -nbaron99/so100_pick_and_place4: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -ncavallo/moss_train_gc_block: - observation.images.front: observation.image - observation.images.top: observation.image2 -ncavallo/moss_train_grasp: - observation.images.front: observation.image - observation.images.top: observation.image2 -ncavallo/moss_train_grasp_new: - observation.images.front: observation.image - observation.images.top: observation.image2 -nduque/act_50_ep: - observation.images.above: observation.image3 - observation.images.front: observation.image -nduque/act_50_ep2: - observation.images.above: observation.image3 - observation.images.front: observation.image -nduque/cam_setup2: - observation.images.above: observation.image3 - observation.images.front: observation.image -nduque/robustness_e1: - observation.images.above: observation.image3 - observation.images.front: observation.image -nduque/robustness_e2: - observation.images.above: observation.image3 - observation.images.front: observation.image -nduque/robustness_e3: - observation.images.above: observation.image3 - observation.images.front: observation.image -nduque/robustness_e4: - observation.images.above: observation.image3 - observation.images.front: observation.image -nduque/robustness_e5: - observation.images.above: observation.image3 - observation.images.front: observation.image -nguyen-v/press_red_button_new: - observation.images.left: observation.image2 - observation.images.right: observation.image3 - observation.images.top: observation.image -nguyen-v/so100_bimanual_grab_lemon_put_in_box2: - observation.images.left: observation.image2 - observation.images.right: observation.image3 - observation.images.top: observation.image -nguyen-v/so100_press_red_button: - observation.images.back: observation.image3 - observation.images.left: observation.image2 - observation.images.right: null - observation.images.top: observation.image -nguyen-v/so100_rotate_red_button: - observation.images.left: observation.image2 - observation.images.right: observation.image3 - observation.images.top: observation.image -nimitvasavat/Gr00t_lerobot: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.back_image: observation.image3 - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -nimitvasavat/Gr00t_lerobotV2: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -nimitvasavat/Gr00t_lerobot_state_action: - absolute_action: null - annotation.human.action.task_description: null - annotation.human.validity: null - next.reward: null - observation.images.image: observation.image - observation.images.wrist_image: observation.image2 -pandaRQ/pick_med_1: - observation.images.laptop: observation.image - observation.images.laptop1: observation.image2 - observation.images.laptop2: observation.image3 -pandaRQ/pickmed: - observation.images.laptop: observation.image - observation.images.laptop1: observation.image2 - observation.images.laptop2: observation.image3 -paszea/so100_lego: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -paszea/so100_lego_2cam: - observation.images.front: observation.image - observation.images.top: observation.image2 -paszea/so100_lego_mix: - observation.images.phone: observation.image -paszea/so100_whale: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -paszea/so100_whale_2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -paszea/so100_whale_3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -paszea/so100_whale_4: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -paszea/so100_whale_grab: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pengjunkun/so100_push_to_hole: - observation.images.laptop: observation.image -pepijn223/lekiwi_block_cleanup2: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -pepijn223/lekiwi_drive_in_circle: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -pepijn223/lekiwi_drive_in_circle_recover: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -pepijn223/lekiwi_pen: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -pepijn223/yellow_lego_in_box1: - observation.images.phone: observation.image -phospho-ai/OrangeBrick3Cameras: - observation.images.main.left: observation.image3 - observation.images.main.right: observation.image2 - observation.images.secondary_0: observation.image -physical-intelligence/libero: - actions: action - image: observation.image - state: observation.state - wrist_image: observation.image2 -pierfabre/chicken: - observation.images.robot: observation.image2 - observation.images.webcam: observation.image -pierfabre/cow: - observation.images.robot: observation.image2 - observation.images.webcam: observation.image -pierfabre/cow2: - observation.images.robot: observation.image2 - observation.images.webcam: observation.image -pierfabre/horse: - observation.images.robot: observation.image2 - observation.images.webcam: observation.image -pierfabre/pig2: - observation.images.robot: observation.image2 - observation.images.webcam: observation.image -pierfabre/pig3: - observation.images.robot: observation.image2 - observation.images.webcam: observation.image -pierfabre/rabbit: - observation.images.robot: observation.image2 - observation.images.webcam: observation.image -pierfabre/sheep: - observation.images.robot: observation.image2 - observation.images.webcam: observation.image -pietroom/actualeasytask: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pietroom/first_task_short: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pietroom/holdthis: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pietroom/second_task: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_carrot_1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_carrot_2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_carrot_3: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_carrot_4: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_carrot_5: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_legos4: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_onelego2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_onelego3: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_squishy: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_squishy100: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_squishy2colors: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -pranavsaroha/so100_squishy2colors_1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -pranavsaroha/so100_squishy2colors_2_new: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -rabhishek100/so100_train_dataset: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -rgarreta/koch_pick_place_lego: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -rgarreta/koch_pick_place_lego_v2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -rgarreta/koch_pick_place_lego_v3: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -rgarreta/koch_pick_place_lego_v6: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -rgarreta/koch_pick_place_lego_v7: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -rgarreta/koch_pick_place_lego_v8: - observation.images.lateral: observation.image3 - observation.images.top: observation.image - observation.images.wrist: observation.image2 -roboticshack/left-arm-grasp-lego-brick: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -roboticshack/sandee-kiwiv10: - observation.images.front: observation.image3 - observation.images.wrist: observation.image2 -roboticshack/team-7-left-arm-grasp-motor: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -roboticshack/team-7-right-arm-grasp-tape: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -roboticshack/team10-red-block: - observation.images.phone: observation.image2 - observation.images.top: observation.image -roboticshack/team11_pianobot: - observation.images.keyboard_camera: observation.image -roboticshack/team13-three-balls-stacking: - observation.images.realsense: observation.image - observation.images.side: observation.image2 -roboticshack/team13-two-balls-stacking: - observation.images.realsense: observation.image - observation.images.side: observation.image2 -roboticshack/team16-can-stacking: - observation.images.head: observation.image - observation.images.wrist: observation.image2 -roboticshack/team16-water-pouring: - observation.images.head: observation.image - observation.images.wrist: observation.image2 -roboticshack/team2-guess_who_less_ligth: - observation.images.mounted: observation.image2 -roboticshack/team2-guess_who_so100: - observation.images.mounted: observation.image -roboticshack/team2-guess_who_so100_edge_case: - observation.images.mounted: observation.image2 -roboticshack/team2-guess_who_so100_light: - observation.images.mounted: observation.image2 -roboticshack/team9-pick_chicken_place_plate: - observation.images.static_left: observation.image - observation.images.static_right: observation.image2 -roboticshack/team9-pick_cube_place_static_plate: - observation.images.static_left: observation.image - observation.images.static_right: observation.image2 -roboticshack/team_5-QuiEstCe_everyBox: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -samanthalhy/so100_herding_1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -samsam0510/cube_reorientation_2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -samsam0510/cube_reorientation_4: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -samsam0510/glove_reorientation_1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -samsam0510/mj_data_temp: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -samsam0510/tape_insert_1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -samsam0510/tape_insert_2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -samsam0510/tooth_extraction_3: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -samsam0510/tooth_extraction_4: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -satvikahuja/mixer_on_off: - observation.images.Lwebcam: observation.image - observation.images.laptop: observation.image2 - observation.images.macwebcam: null - observation.images.phone: null -satvikahuja/mixer_on_off_new: - observation.images.Lwebcam: observation.image - observation.images.laptop: observation.image2 - observation.images.macwebcam: null - observation.images.phone: null -satvikahuja/mixer_on_off_new_1: - observation.images.Lwebcam: observation.image - observation.images.laptop: observation.image2 - observation.images.macwebcam: observation.image3 - observation.images.phone: null -satvikahuja/mixer_on_off_new_4: - observation.images.Lwebcam: observation.image - observation.images.laptop: observation.image2 - observation.images.macwebcam: null - observation.images.phone: null -satvikahuja/orange_mixer_1: - observation.images.Lwebcam: observation.image - observation.images.laptop: observation.image2 - observation.images.macwebcam: null - observation.images.phone: null -satvikahuja/orange_pick_place_new1: - observation.images.Lwebcam: observation.image - observation.images.laptop: observation.image2 - observation.images.macwebcam: null - observation.images.phone: null -seeingrain/241228_pick_place_2cams: - observation.images.side: observation.image3 - observation.images.top: observation.image -seeingrain/lego_3cameras: - observation.images.side: observation.image3 - observation.images.top: observation.image - observation.images.wrist: observation.image2 -seeingrain/one_shot_learning_18episodes: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -seeingrain/pick_lego_to_hand: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -seeingrain/pick_place_lego: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -seeingrain/pick_place_lego_wider_range_dang: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -seeingrain/pick_place_lego_wider_range_dong: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -seeingrain/pick_place_lego_wider_range_richard: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -seeingrain/pick_place_pink_lego: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -seeingrain/pick_place_pink_lego_few_samples: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -seeingrain/pick_place_red_lego: - observation.images.laptop: observation.image3 - observation.images.phone: observation.image -shin1107/koch_move_block_with_some_positions: - observation.images.front: observation.image - observation.images.top: observation.image3 -shin1107/koch_move_block_with_some_shapes: - observation.images.front: observation.image - observation.images.top: observation.image3 -shin1107/koch_train_block: - observation.images.front: observation.image - observation.images.top: observation.image3 -shreyasgite/so100_base_env: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -shreyasgite/so100_base_left: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -shreyasgite/so100_legocube_50: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -sihyun77/mond_1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/mond_13: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/si_so100: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/sihyun_3_17_1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/sihyun_3_17_2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/sihyun_3_17_5: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/sihyun_king: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/sihyun_main: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/sihyun_main_2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/sihyun_main_3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/suho_3_17_1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/suho_3_17_3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/suho_angel: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/suho_main_2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/suho_red: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/suho_red2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sihyun77/suho_so100: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -sixpigs1/so100_insert_cylinder_error: - observation.images.above: observation.image2 - observation.images.rightfront: observation.image -sixpigs1/so100_pick_cube_in_box: - observation.images.above: observation.image2 - observation.images.rightfront: observation.image -sixpigs1/so100_pick_cube_in_box_error: - observation.images.above: observation.image - observation.images.rightfront: observation.image2 -sixpigs1/so100_pull_cube_by_tool_error: - observation.images.above: observation.image2 - observation.images.rightfront: observation.image -sixpigs1/so100_pull_cube_error: - observation.images.above: observation.image - observation.images.rightfront: observation.image2 -sixpigs1/so100_push_cube_error: - observation.images.above: observation.image - observation.images.rightfront: observation.image2 -sixpigs1/so100_stack_cube_error: - observation.images.above: observation.image2 - observation.images.rightfront: observation.image -smanni/train_so100_fluffy_box: - observation.images.intel_realsense: observation.image -speedyyoshi/so100_grasp_pink_block: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -sshh11/so100_orange_50ep_1: - observation.images.laptop: observation.image -sshh11/so100_orange_50ep_2: - observation.images.laptop: observation.image -swarajgosavi/act_kikobot_block_real: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -swarajgosavi/act_kikobot_pusht_real: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -swarajgosavi/kikobot_pusht_real_v2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -takuzennn/aloha-pick100: - observation.image.camera1: observation.image - observation.image.camera2: observation.image2 - observation.image.camera3: observation.image3 -takuzennn/square3: - observations.images.agentview: observation.image - observations.images.robot0_eye_in_hand: observation.image2 -thedevansh/mar16_1336: - observation.images.laptop: observation.image -theo-michel/lekiwi_v2: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -theo-michel/lekiwi_v5: - observation.images.front: observation.image - observation.images.wrist: observation.image2 -therarelab/med_dis_rare_6: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -therarelab/so100_pick_place: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -therarelab/so100_pick_place_2: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -tkc79/so100_lego_box_1: - observation.images.arm: observation.image2 - observation.images.laptop: observation.image -tkc79/so100_lego_box_2: - observation.images.arm: observation.image2 - observation.images.laptop: observation.image -triton7777/so100_dataset_mix: - observation.images.gripper: observation.image2 - observation.images.s_left: null - observation.images.s_right: observation.image3 - observation.images.top: observation.image -twerdster/koch_new_training_red: - observation.images.iphone: observation.image2 - observation.images.laptop: observation.image3 -twerdster/koch_training_red: - observation.images.rightegocam: observation.image2 - observation.images.rightstereocam: observation.image3 -underctrl/handcamera_single_blue: - observation.images.android: observation.image - observation.images.handcam: observation.image2 - observation.images.webcam: observation.image3 -underctrl/mutli-stacked-block_mutli-color_pick-up_80: - observation.images.phone: observation.image3 - observation.images.webcam: observation.image -underctrl/single-block_blue-color_pick-up_80: - observation.images.phone: observation.image3 - observation.images.webcam: observation.image -underctrl/single-block_multi-color_pick-up_50: - observation.images.phone: observation.image3 - observation.images.webcam: observation.image -underctrl/single-stacked-block_mutli-color_pick-up_80: - observation.images.phone: observation.image3 - observation.images.webcam: observation.image -underctrl/single-stacked-block_two-color_pick-up_80: - observation.images.phone: observation.image3 - observation.images.webcam: observation.image -vaishanthr/toy_pick_place: - observation.images.gipper_cam: observation.image2 - observation.images.webcam: observation.image -vaishanthr/toy_pickplace: - observation.images.gipper_cam: observation.image2 - observation.images.webcam: observation.image -vaishanthr/toy_pickplace_50ep: - observation.images.gipper_cam: observation.image2 - observation.images.webcam: observation.image -vladfatu/so100_above: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -vladfatu/so100_ds: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -vladfatu/so100_office: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -wangjl1512/doll: - observation.images.top: observation.image - observation.images.wrist: observation.image2 -wangjl1512/pour_water: - observation.images.laptop: observation.image -wcode/so100_put_pen_50: - observation.images.hand: observation.image2 - observation.images.top: observation.image -weiye11/so100_410_zwy: - observation.images.top: observation.image - observation.images.wrist_left: observation.image2 -yg-dev/koch_red_pen2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -yskim2025/unitylerobot: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -yuz1wan/so100_button: - observation.images.side: observation.image - observation.images.wrist: observation.image2 -yuz1wan/so100_fold_0227_1: - observation.images.side: observation.image -yuz1wan/so100_fold_0227_2: - observation.images.side: observation.image - observation.images.wrist: observation.image2 -yuz1wan/so100_pick_pink: - observation.images.side: observation.image - observation.images.wrist: observation.image2 -yuz1wan/so100_pick_wahaha: - observation.images.side: observation.image - observation.images.wrist: observation.image2 -yuz1wan/so100_pickplace: - observation.images.side: observation.image - observation.images.wrist: observation.image2 -yuz1wan/so100_pickplace_0223_2: - observation.images.side: observation.image -yuz1wan/so100_pickplace_0223_3: - observation.images.side: observation.image -yuz1wan/so100_pour_cup: - observation.images.side: observation.image - observation.images.wrist: observation.image2 -yuz1wan/so100_pp_pink: - observation.images.side: observation.image - observation.images.wrist: observation.image2 -zaringleb/so100_cube_2: - observation.images.cam_high: observation.image -zaringleb/so100_cube_4_binary: - observation.images.cam_high: observation.image -zaringleb/so100_cube_5_linear: - observation.images.cam_high: observation.image -zaringleb/so100_cube_6_2d: - observation.images.cam_high: observation.image -zaringleb/so100_cude_linear_and_2d_comb: - observation.images.cam_high: observation.image -zijian2022/321: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/assemblyarm2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/backgrounda: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/backgroundb: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/bi1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/bi2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/c0: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/close3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/closer3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/digitalfix: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/digitalfix2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/digitalfix3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/force1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/force2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/force3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/hand1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/insert2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/l10_1: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -zijian2022/l10_5: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -zijian2022/l9: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -zijian2022/llm40: - observation.images.laptop: observation.image2 - observation.images.phone: observation.image -zijian2022/n1_2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/noticehuman1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/noticehuman2: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/noticehuman3: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/noticehuman5: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/noticehuman50: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/noticehuman60: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/noticehuman70: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/so100_318: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/so100_318_1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zijian2022/sort1: - observation.images.laptop: observation.image - observation.images.phone: observation.image2 -zliu157/i3r: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 -zliu157/i3r2: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 -zliu157/i3r3: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 -zliu157/i3r5: - observation.images.laptop: observation.image - observation.images.phone: observation.image3 diff --git a/lerobot/configs/mappings/tasks.yaml b/lerobot/configs/mappings/tasks.yaml deleted file mode 100644 index 16d71256e..000000000 --- a/lerobot/configs/mappings/tasks.yaml +++ /dev/null @@ -1,1012 +0,0 @@ -00ri/so100_battery: - 0: Grasp a battery and put it in the bin. -00ri/so100_battery_bin_center: - 0: Grasp a battery and put it in the bin. -1g0rrr/sam_openpi03: - 0: Grasp a blue cube and put it in the gray box. -1g0rrr/sam_openpi_solder1: - 0: bring contact to the pad on the board. -1g0rrr/sam_openpi_solder2: - 0: bring contact to the pad on the board. -1g0rrr/screw1: - 0: Grasp a lego block and put it in the bin. -356c/so100_duck_reposition_1: - 0: Grasp the tool and use it to move the duck to the indicated position. -356c/so100_nut_sort_1: - 0: Pick up the steel nuts and sort them by color. -356c/so100_rope_reposition_1: - 0: Grasp rope and reposition. -Allen-488/koch_dataset_50: - 0: Grasp a block and put it in the bin. -AndrejOrsula/lerobot_double_ball_stacking_random: - 0: Stack the balls on top of each other. -Bartm3/dice2: - 0: Grasp a dice and put it in the bin. -Bartm3/tape_to_bin: - 0: Grasp a tape and put it in the bin. -Bartm3/tape_to_bin2: - 0: Grasp a tape and put it in the bin. -Beegbrain/align_cubes_green_blue: - 0: Put the green cubes on the left and the blue cube on the right -Beegbrain/align_three_pens: - 0: picks up a pen. -Beegbrain/moss_close_drawer_teabox: - 0: Close the top drawer of the teabox -Beegbrain/moss_open_drawer_teabox: - 0: Open the top drawer of the teabox -Beegbrain/moss_put_cube_teabox: - 0: Put the green cube in the top drawer of the teabox -Beegbrain/moss_stack_cubes: - 0: Stack the green cube on top of the blue cube -Beegbrain/oc_stack_cubes: - 0: stack the red cube on the blue cube -Beegbrain/pick_lemon_and_drop_in_bowl: - 0: Pick the yellow lemon and drop it in the red bowl. -Beegbrain/pick_place_green_block: - 0: Pick up the green block and place in the red cup. -Beegbrain/put_green_into_blue_bin: - 0: Put the green cube into the blue bin -Beegbrain/put_red_triangle_green_rect: - 0: Put the red triangle on top of the green rectangle -Beegbrain/put_screwdriver_box: - 0: Put the screwdriver into the box -Beegbrain/stack_2_cubes: - 0: picks up the red block. -Beegbrain/stack_green_on_blue_cube: - 0: Stack the blue cube on top of the green cube -Beegbrain/sweep_tissue_cube: - 0: Sweep the red cubes to the right with the tissue. -BlobDieKatze/GrabBlocks: - 0: Grasp a lego block and put it in the bin. -CSCSXX/pick_place_cube_1.17: - 0: Pick up the red block and place it in the box. -CSCSXX/pick_place_cube_1.18: - 0: Pick up the cube and place it in the box. -Chojins/chess_game_000_white: - 0: Move the blue chess pieces as indicated by the highlighted squares -Chojins/chess_game_000_white_red: - 0: Move the red chess pieces to the highlighted squares. -Chojins/chess_game_001_blue_stereo: - 0: Move the blue chess pieces to the highlighted squares -Chojins/chess_game_001_red_stereo: - 0: Move the red chess pieces to the highlighted squares -Chojins/chess_game_001_white: - 0: Move the blue chess pieces as indicated by the highlighted squares -Chojins/chess_game_002_white: - 0: Move the blue chess pieces as indicated by the highlighted squares -Chojins/chess_game_003_white: - 0: Move the blue chess pieces as indicated by the highlighted squares -Chojins/chess_game_004_white: - 0: Move the blue chess pieces as indicated by the highlighted squares -Chojins/chess_game_005_white: - 0: Move the blue chess pieces as indicated by the highlighted squares -Chojins/chess_game_006_white: - 0: Move the blue chess pieces as indicated by the highlighted squares -Chojins/chess_game_007_white: - 0: Move the blue chess pieces as indicated by the highlighted squares -Chojins/chess_game_009_white: - 0: Move the blue chess pieces to the highlighted squares. -Cidoyi/so100_all_notes_1: - 0: Connect the cable to the device. -Clementppr/lerobot_pick_and_place_dataset_world_model: - 0: Grasp a fruit and put it in the cup. -CrazyYhang/A1234-B-C_mvA2B: - 0: Move the top disk from the left column to the middle column. -Deason11/Open_the_drawer_to_place_items: - 0: Put the objects in the open drawer. -Deason11/PLACE_TAPE_PUSH_DRAWER: - 0: 'Place the tape in the drawer and close it. ' -Deason11/Random_Kitchen: - 0: Pick up the cup and place it on the table. -Deason11/mobile_manipulator_0319: - 0: Grasp a lego block and put it in the bin. -Deason11/mobile_manipulator_0326: - 0: Grasp a lego block and put it in the bin. - 1: mobile_lekiwi. -DimiSch/so100_50ep_2: - 0: Place the yellow object in the bowl. -DimiSch/so100_50ep_3: - 0: Pick the yellow button from the table. -DimiSch/so100_terra_50_2: - 0: Grasp a lego block and put it in the bin. -Dongkkka/cable_pick_and_place2: - 0: Put a black charging cable in a black bowl and put a red charging cable in a - green bowl -DorayakiLin/so100_pick_charger_on_tissue: - 0: Pick up the charger and put it on the white tissue. -DorayakiLin/so100_pick_cube_in_box: - 0: Pick up the red cube and put it in the box. -Eyas/grab_bouillon: - 0: picks up the box and places it in the box. -Eyas/grab_pink_lighter_10_per_loc: - 0: Pick up the pink object from the table. -FeiYjf/Grab_Pieces: - 0: places the black object on the table. -FeiYjf/Makalu_push: - 0: Pick up the blue cube. -FeiYjf/Maklu_dataset: - 0: Pick up the blue cube and place it on the paper. -FeiYjf/Test_NNNN: - 0: Pick up the purple cube and move it to the right. -FeiYjf/new_Dataset: - 0: Pick up the blue cube. -FeiYjf/new_GtoR: - 0: Move along the line on the paper from start to end. -FsqZ/so100_1: - 0: Put the yellow cube inside the purple box. -Gano007/so100_doliprane: - 0: Grasp a medic box and put it in the bin. -Gano007/so100_gano: - 0: Grasp a box and put it in the bin. -Gano007/so100_medic: - 0: Grasp a medic box and put it in the bin. -Gano007/so100_second: - 0: Grasp a yellow box and put it in the bin. -Gongsta/grasp_duck_in_cup: - 0: Grasp the rubber duck and put it in the cup. -HITHY/so100_peach: - 0: Grasp a peach and put it in the bin. -HITHY/so100_peach3: - 0: Grasp a peach and put it in the bin. -HITHY/so100_peach4: - 0: Grasp a peach and put it on the plate. -HITHY/so100_strawberry: - 0: Grasp a strawberry and put it in the bin. -HWJ658970/fat_fish: - 0: Grasp a fat fish toy and put it in the bin. -HWJ658970/lego_100_class: - 0: Separate yellow and white Lego blocks and place them into the bin. -HWJ658970/lego_50: - 0: Grasp a yellow lego block and put it in the bin. -HWJ658970/lego_50_camera_change: - 0: Grasp a yellow lego block and put it in the bin. -HuaihaiLyu/groceries: - 0: Pick the brown long bread and Egg yolk pasry into package -IPEC-COMMUNITY/berkeley_mvp_lerobot: - 0: push wooden cube - 1: pick detergent from the sink - 2: reach red block - 3: pick yellow cube - 4: close fridge door - 5: pick fruit -IPEC-COMMUNITY/ucsd_kitchen_dataset_lerobot: - 0: Turn on the faucet - 1: Put the bowl inside the kitchen cabinet - 2: Open the oven door - 3: Place the teapot on the stove - 4: Put the white box into the sink - 5: Open the carbinet door - 6: Put the green box into the sink - 7: Put the canned spam into the sink -Ityl/so100_recording1: - 0: Putting the red square onto the yellow piece -Ityl/so100_recording2: - 0: Pick up the red cube and place it on top of the blue cube. -Jiangeng/so100_413: - 0: Pick up the cube and place it on top of the black circle. -KeWangRobotics/piper_push_cube_gamepad_1: - 0: push the cube to the black area -KeWangRobotics/piper_push_cube_gamepad_1_cropped_resized: - 0: push the cube to the black area -KeWangRobotics/piper_rl_1: - 0: Pick up the cube and place it in the box. -KeWangRobotics/piper_rl_1_cropped_resized: - 0: picks up the cube. -KeWangRobotics/piper_rl_2: - 0: Move the cube to the right side of the table. -KeWangRobotics/piper_rl_2_cropped_resized: - 0: Move the block to the right side of the table. -KeWangRobotics/piper_rl_3: - 0: Pick up the wooden block. -KeWangRobotics/piper_rl_3_cropped_resized: - 0: Place the block in the box. -KeWangRobotics/piper_rl_4: - 0: Move the block slightly. -KeWangRobotics/piper_rl_4_cropped_resized: - 0: picks the wooden block. -LemonadeDai/so100_coca: - 0: Grasp the Coca-Cola can and orient it upright with the top facing up. -Loki0929/so100_100: - 0: Grasp a rubber duck and put it in the box. -Loki0929/so100_duck: - 0: Grasp red, green, yellow ducks and put them in the box. -Lugenbott/koch_1225: - 0: Pick up the blue block and place it in the red box. -Mohamedal/put_banana: - 0: Put the banana in the red bowl. -Mwuqiu/so100_0408_muti: - 0: Grasp a yellow duck and put it in the box. -NONHUMAN-RESEARCH/SOARM100_TASK_VENDA: - 0: Pick up the object and place it in the box. -NONHUMAN-RESEARCH/SOARM100_TASK_VENDA_BOX: - 0: Move the cube to the right side of the table. -Odog16/so100_cube_drop_pick_v1: - 0: Pick up the orange cube, release it, and then pick it up again. -Odog16/so100_cube_stacking_v1: - 0: 'Stack the cubes in the following order from bottom to top: black, blue, then - orange.' -Odog16/so100_tea_towel_folding_v1: - 0: Fold tea towel into quarters -Ofiroz91/so_100_cube2bowl: - 0: placing cube inside a red bawl -Pi-robot/barbecue_flip: - 0: Pick up the orange cone and place it on the table. -Pi-robot/barbecue_put: - 0: Pick up the stick and place it in the grill. -RasmusP/so100_Orange2Green: - 0: Grasp the orange block and drop it in the box. -RasmusP/so100_dataset50ep: - 0: Grasp a square block and put it in the box. -SeanLMH/so100_picknplace: - 0: Pick up the blue block and place it in the yellow box. -SeanLMH/so100_picknplace_v2: - 0: picks up blue cube and places it in yellow box. -Setchii/so100_grab_ball: - 0: Grasp a ball and put it on a goblet. -T-K-233/koch_k1_pour_shot: - 0: Place the glass on the table. -TrossenRoboticsCommunity/aloha_baseline_dataset: - 0: Pick up a blue block and put it in a green bowl. Baseline dataset for testing -TrossenRoboticsCommunity/aloha_fold_tshirt: - 0: Fold the t-shirt. -TrossenRoboticsCommunity/aloha_stationary_logo_assembly: - 0: Assemble the Trossen Robotics Logo. -VoicAndrei/so100_banana_to_plate_only: - 0: Pick up the banana and place it on the plate. -VoicAndrei/so100_banana_to_plate_rebel_full: - 0: Pick up the banana and place it on the place -Winster/so100_cube: - 0: Grasp a lego block and put it in the bin. -XXRRSSRR/so100_v3_num_episodes_50: - 0: Grasp a box and put it in the side. -Yuanzhu/koch_bimanual_grasp_0: - 0: places the yellow block on the mousepad. -Yuanzhu/koch_bimanual_grasp_3: - 0: picks up a yellow block. -ZCM5115/so100_1210: - 0: picks up the USB cable. -ZCM5115/so100_2Arm3cameras_movebox: - 0: Pick up the white box from the table. -ZGGZZG/so100_drop0: - 0: Grasp a ball and put it in the hole. -ZGGZZG/so100_drop1: - 0: Grasp a cube and put it in the right place. -Zhaoting123/koch_cleanDesk_: - 0: Grasp a card and use it to clean the desk -abbyoneill/data_w_mug: - 0: Grasp a lego block and put it in the bin. -abbyoneill/new_dataset_pick_place: - 0: Grasp a lego block and put it in the bin. -abbyoneill/pusht: - 0: Grasp a lego block and put it in the bin. -abbyoneill/thurs1120pickplace: - 0: Grasp a lego block and put it in the bin. -abhisb/so100_51_ep: - 0: Pick up the cube and place it in the box. -abougdira/cube_target: - 0: put_the cube on the yellow target -acrampette/third_arm_01: - 0: Pick up the circuit board from the table. -acrampette/third_arm_02: - 0: Pick up the tie and place it in the box. -ad330/cubePlace: - 0: Grasp white cube and place it in the bowl. -ad330/so100_box_pickPlace: - 0: places jar in box. -aimihat/so100_tape: - 0: Pick up the tape and put it in the bowl. -aliberts/koch_tutorial: - 0: Pick the Lego block and drop it in the box on the right. -allenchienxxx/so100Test: - 0: Grasp a lego block and put it in the bin. -andabi/D10: - 0: picks up the shoe. -andabi/D11: - 0: The picks up the paper and places it on the table. -andabi/D12: - 0: The places the cube in the box. -andabi/D13: - 0: picks up shoes. -andabi/D14: - 0: picks up shoes. -andabi/D15: - 0: picks up the shoe. -andabi/D16: - 0: picks up the shoe. -andabi/D17: - 0: picks up the shoe. -andabi/D2: - 0: Pick up the shoe and place it on the table. -andabi/D3: - 0: picks up the shoe. -andabi/D4: - 0: picks up the shoe. -andabi/D5: - 0: places shoe on table. -andabi/D6: - 0: picks up the shoe. -andabi/D7: - 0: picks up the shoe. -andabi/D8: - 0: picks up the shoe. -andabi/D9: - 0: The picks up the paper and places it on the table. -andabi/shoes_easy: - 0: picks up the shoe. -andlyu/so100_indoor_1: - 0: Locate and grasp the blueberry. -andlyu/so100_indoor_3: - 0: Locate and grasp the blueberry. -andrewcole712/so100_tape_bin_place: - 0: Place the tape in the wooden box. -andy309/so100_0311_1152: - 0: Grasp and put it in the bin. -andy309/so100_0314_fold_cloths: - 0: fold the cloths, use two cameras, two arms. -aractingi/push_cube_offline_data: - 0: Push the green cube to the yellow sticker. -aractingi/push_cube_offline_data_cropped_resized: - 0: Push the green cube to the yellow sticker -arclabmit/Koch_twoarms: - 0: official two arms recordings10 -arclabmit/koch_gear_and_bin: - 0: Pick the gear and place it in the bin. -baladhurgesh97/so100_final_picking_3: - 0: Grasp a carrot, plastic bottle and put it in respective bins. -bensprenger/chess_game_001_blue_stereo: - 0: Move the blue chess pieces to the highlighted squares. -bensprenger/left_arm_yellow_brick_in_box_v0: - 0: Grasp the yellow lego block and put it in the box. -bensprenger/left_arm_yellow_brick_in_box_with_purple_noise_v0: - 0: Grasp a yellow lego block and put it in the bin. -bensprenger/right_arm_p_brick_in_box_with_y_noise_v0: - 0: Grasp the purple lego block and put it in the box. -bnarin/so100_tic_tac_toe_we_do_it_live: - 0: move tic tac toe as player 2. -carpit680/giraffe_sock_demo_1: - 0: Grasp a sock off the floor. -carpit680/giraffe_task: - 0: Grasp a block and put it in the bin. -chmadran/so100_dataset04: - 0: picks the blue block and places it in the red cup. -chmadran/so100_dataset08: - 0: places blue block on paper. -chmadran/so100_home_dataset: - 0: Grasp a lego block and put it in the bin. -cmcgartoll/cube_color_organizer: - 0: Organize blue cube - 1: Organize red cube - 2: Organize yellow cube -ctbfl/sort_battery: - 0: put the battery into battery_box -dboemer/koch_50-samples: - 0: Pick up the red block and place it on top of the yellow box. -dkdltu1111/omx-bottle1: - 0: Grasp a lego block and put it in the bin. -dop0/koch_pick_terminal: - 0: Pick up the terminal and place on the cover. -dragon-95/so100_sorting: - 0: Pick up the object from box A and place it in box B. -dragon-95/so100_sorting_1: - 0: Pick up the object from box A and place it in box B. -dragon-95/so100_sorting_2: - 0: Pick up the object from box A and place it in box B. -dragon-95/so100_sorting_3: - 0: Pick up the object from box A and place it in box B. -ellen2imagine/pusht_green1: - 0: Place the green block in the box. -ellen2imagine/pusht_green_same_init2: - 0: Place the green block in the correct position. -engineer0002/pepper: - 0: Place the bottle on the table. -francescocrivelli/carrot_eating: - 0: pick up carrot and bring to mouth -frk2/so100large: - 0: Pick up roll of tape and put it in the bin. -frk2/so100largediffcam: - 0: Pick up roll of tape and put it in the bin -ganker5/so100_color_0328: - 0: Grasp a lego block and put it in the bin. -ganker5/so100_dataline_0328: - 0: Grasp a lego block and put it in the bin. -ganker5/so100_dataline_20250331: - 0: Grasp a lego block and put it in the bin. -ganker5/so100_push_20250328: - 0: Grasp a lego block and put it in the bin. -ganker5/so100_push_20250331: - 0: Grasp a lego block and put it in the bin. -ganker5/so100_toy_20250402: - 0: Grasp a lego block and put it in the bin. -gxy1111/so100_pick_place: - 0: Grasp a toy panda and put it in the cup. -hangwu/koch_pick_terminal: - 0: Pick up the terminal and place on the cover. -hangwu/piper_pick_terminal_2: - 0: Grasp the white terminal and put it on the green lid. -hangwu/piper_pick_terminal_and_place: - 0: Grasp a terminal and put it on the black box. -hannesill/koch_pnp_2_blocks_2_bins_200: - 0: Grasp the blue block first and put it in the first bin that has a specific position - and orientation. Then grasp the white block and put it in the second bin that - has a specific position and orientation. -hannesill/koch_pnp_simple_50: - 0: Grasp a small block with a specific orientation and put it in the bin with a - specific position and orientation. -hegdearyandev/so100_eraser_cup_v1: - 0: picks up the red object. -helper2424/hil-serl-push-circle-classifier: - 0: Push small circle object to the correct position -hkphoooey/throw_stuffie: - 0: Grab stuffed animal and throw it on the dot. -ibru/bob_jetson: - 0: Drive forward pickup the object and put it in the red box and drive back. -ibru/bobo_jetson: - 0: Drive forward pickup the object and put it in the red box and drive back. - 1: Driver forward -ibru/bobo_trash_collector: - 0: Bobo Trash collect and place it in a bin -imatrixlee/koch_place: - 0: Pick up the white object and place it on the table. -isadev/bougies1: - 0: Put the candles in the box. -isadev/bougies2: - 0: grab the candle wick and place it in the tray. -isadev/bougies3: - 0: Grab the candle wick by the aluminium plate and place it in the box. -jainamit/koch_pickcube: - 0: Pick up the blue cube and place it in the box. -jainamit/koch_realcube3: - 0: pick up the cube real with keyboard input -jannick-st/classifier: - 0: Move the blue object to the right side of the table. -jannick-st/push-cube-classifier_cropped_resized: - 0: Close the cabinet door. -jbraumann/so100_1902: - 0: picks up the yellow ball. -jchun/so100_pickplace_small_20250323_120056: - 0: Grasp items from white bowl and place in black tray -jiajun001/eraser00_2: - 0: picks tissue paper from box. -jiajun001/eraser00_3: - 0: Pick up the white object from the table. -jlesein/TestBoulon7: - 0: Pick up the bolt and put it on the plate. -jlitch/so100multicam7: - 0: pick up brick and put in bin -jmrog/so100_sweet_pick: - 0: Pick up the candy and place it in the bowl. -joaoocruz00/so100_makeitD1: - 0: Grasp a lego block and put it in the bin. -jpata/so100_pick_place_tangerine: - 0: Pick up the tangerine and place it. -kevin510/lerobot-cat-toy-placement: - 0: Grasp the cat toy and put it in the cup. -koenvanwijk/blue52: - 0: places blue block on red LEGO piece. -koenvanwijk/orange50-1: - 0: 'Pick up the orange object and but it in the LEGO box. ' -koenvanwijk/orange50-variation-2: - 0: 'Pick up the orange object and but it in the LEGO box. ' -kumarhans/so100_tape_task: - 0: Grasp a roll of tape and put it over the candle case. -lalalala0620/koch_blue_paper_tape: - 0: Grasp a blue paper tape and put it in the bin. -lerobot/aloha_mobile_cabinet: - 0: Open the top cabinet, store the pot inside it then close the cabinet. -lerobot/aloha_mobile_chair: - 0: Push the chairs in front of the desk to place them against it. -lerobot/aloha_mobile_elevator: - 0: Take the elevator to the 1st floor. -lerobot/aloha_mobile_wash_pan: - 0: Pick up the pan, rinse it in the sink and then place it in the drying rack. -lerobot/aloha_mobile_wipe_wine: - 0: Pick up the wet cloth on the faucet and use it to clean the spilled wine on the - table and underneath the glass. -lerobot/aloha_static_candy: - 0: Pick up the candy and unwrap it. -lerobot/aloha_static_coffee: - 0: Place the coffee capsule inside the capsule container, then place the cup onto - the center of the cup tray, then push the 'Hot Water' and 'Travel Mug' buttons. -lerobot/aloha_static_coffee_new: - 0: Place the coffee capsule inside the capsule container, then place the cup onto - the center of the cup tray. -lerobot/aloha_static_cups_open: - 0: Pick up the plastic cup and open its lid. -lerobot/aloha_static_pro_pencil: - 0: Pick up the pencil with the right arm, hand it over to the left arm then place - it back onto the table. -lerobot/aloha_static_screw_driver: - 0: Pick up the screwdriver with the right arm, hand it over to the left arm then - place it into the cup. -lerobot/aloha_static_towel: - 0: Pick up a piece of paper towel and place it on the spilled liquid. -lerobot/aloha_static_vinh_cup: - 0: Pick up the platic cup with the right arm, then pop its lid open with the left - arm. -lerobot/aloha_static_vinh_cup_left: - 0: Pick up the platic cup with the left arm, then pop its lid open with the right - arm. -lerobot/aloha_static_ziploc_slide: - 0: Slide open the ziploc bag. -lirislab/close_top_drawer_teabox: - 0: Close the top drawer of the teabox -lirislab/fold_bottom_right: - 0: Fold the bag from the bottom right corner. -lirislab/green_lego_block_into_mug: - 0: pick the green block and place it in the red cup -lirislab/guess_who_lighting: - 0: Pick up the card from the shelf. -lirislab/guess_who_no_cond: - 0: Place the card in the slot. -lirislab/lemon_into_bowl: - 0: Pick the yellow lemon and drop it in the red bowl -lirislab/open_top_drawer_teabox: - 0: Open the top drawer of the teabox -lirislab/push_cup_target: - 0: Push the red cup to the pink target -lirislab/put_banana_bowl: - 0: Put the banana into the red bowl -lirislab/put_caps_into_teabox: - 0: Pick the coffee capsule and put it into the top drawer of the teabox -lirislab/put_coffee_cap_teabox: - 0: Put the coffee capsule into the top drawer of the teabox. -lirislab/red_cube_into_blue_cube: - 0: Put the red cube on top of the blue cube. -lirislab/red_cube_into_green_lego_block: - 0: Put the red cube on top of the yellow cube. -lirislab/so100_demo: - 0: Put the banana into the red bowl. -lirislab/sweep_tissue_cube: - 0: Sweep the red cubes to the right with the tissue bag. -lirislab/unfold_bottom_right: - 0: Unfold the bag from bottom right corner -liuhuanjim013/so100_block: - 0: Grasp a lego block and put it in the bin. -liuhuanjim013/so100_th: - 0: Grasp a lego figure and put it in the box. -liyitenga/so100_pick_taffy3: - 0: Place the eraser in the container. -liyitenga/so100_pick_taffy6: - 0: Pick up the toy and place it in the purple cup. -liyitenga/so100_pick_taffy7: - 0: Pick up the toy and place it in the box. -lizi178119985/so100_jia: - 0: Grasp a lego block and put it in the bin. -ma3oun/rpi_squares_1: - 0: Raspberry Pi 5 squares recording 1 -maximilienroberti/so100_lego_red_box: - 0: Placing the red Lego in the red box bin. -mikechambers/block_cup_14: - 0: Grasp a block and put it in a cup. -mlfu7/pi0_conversion_no_pad_video: - 0: pickplace deer greybowl - 1: stack red on green - 2: stack orange cup to yellow cup - 3: put orange cup into yellow cup - 4: put push red pen to blue pen - 5: put tiger to black bowl - 6: put potato in bot to black bowl - 7: pick up green triangle - 8: put push blue pen to red pen - 9: close drawer - 10: put push green block to red - 11: pickup potato - 12: open drawer - 13: put closing tongs - 14: poke block - 15: put push blue cube - 16: poke tiger - 17: pick red cube into black bowl - 18: pick blue cube stack on wood block - 19: pick blue cube into grey bowl - 20: put red ball in black bowl - 21: pick green triangle into pink bowl - 22: pick red ball into pink bowl - 23: poke green triangle - 24: poke grey bowl - 25: put blue cube pink bowl - 26: put red cube into black bowl - 27: put deer into gray bowl - 28: put red ball into pink bowl - 29: put green triangle into pink bowl - 30: put pour from yellow cup into black bowl - 31: put pour blue cup into pink bowl - 32: put brown cube into gray bowl -nbaron99/so100_pick_and_place2: - 0: picks up the white object. -nbaron99/so100_pick_and_place4: - 0: Pick up the triangular object and place it on a green sticker. -ncavallo/moss_train_gc_block: - 0: Grasp a lego block and put it in the bin. -ncavallo/moss_train_grasp: - 0: Grasp a lego block and put it in the bin. -ncavallo/moss_train_grasp_new: - 0: Grasp a lego block and put it in the bin. -nduque/act_50_ep: - 0: Grasp a green block and put it in the bin. -nduque/act_50_ep2: - 0: Grasp a green block and put it in the bin. -nduque/cam_setup2: - 0: Grasp a green block and put it in the bin. -nduque/robustness_e2: - 0: Grasp a green dice and put it in the bin. -nduque/robustness_e3: - 0: Grasp a green dice and put it in the bin. -nduque/robustness_e4: - 0: Grasp a green dice and put it in the bin. -nduque/robustness_e5: - 0: Grasp a green dice and put it in the bin. -nguyen-v/press_red_button_new: - 0: Press the red button with the black arm -nguyen-v/so100_bimanual_grab_lemon_put_in_box2: - 0: Grab the lemon with the black arm, then give it to the green arm, then place - the lemon in the cardboard box with the green arm. -nguyen-v/so100_press_red_button: - 0: The places the cube in the box. -nguyen-v/so100_rotate_red_button: - 0: Rotate the red button clockwise with the black arm -nimitvasavat/Gr00t_lerobot: - 0: Place the cereal box on the shelf. -nimitvasavat/Gr00t_lerobotV2: - 0: Place the chocolate chip cookie dough box on the table. -nimitvasavat/Gr00t_lerobot_state_action: - 0: Place the chocolate chip cookie dough box on the table. -pandaRQ/pick_med_1: - 0: Pick up the object and place it in the box. -pandaRQ/pickmed: - 0: Place the green block on the table. -paszea/so100_lego: - 0: Grasp a lego and put it in the basket. -paszea/so100_lego_2cam: - 0: Grap lego blocks and put them in the plate. -paszea/so100_lego_mix: - 0: Grasp lego blocks and put them in the plate. -paszea/so100_whale_2: - 0: Grasp a whale and put it in the plate. -paszea/so100_whale_3: - 0: Grasp a whale and put it in the plate. -paszea/so100_whale_4: - 0: Grasp a whale and put it in the plate. -paszea/so100_whale_grab: - 0: Grasp a whale and put it in the plate. -pengjunkun/so100_push_to_hole: - 0: Push the T into the hole. -pepijn223/lekiwi_block_cleanup2: - 0: Put red block in black box -pepijn223/lekiwi_drive_in_circle: - 0: Pick up the red object and place it on the table. -pepijn223/lekiwi_pen: - 0: Fold the jeans. -pierfabre/chicken: - 0: Pick the chicken and place it to the right. -pierfabre/cow2: - 0: Pick the cow and place it to the right. -pierfabre/horse: - 0: Pick the horse and place it to the right. -pierfabre/pig2: - 0: Pick the pig and place it to the right. -pierfabre/pig3: - 0: Pick the pig and place it to the right. -pierfabre/rabbit: - 0: Pick the rabbit and place it to the right. - 1: Pick the rabbit and put it to the right -pierfabre/sheep: - 0: Pick the sheep and place it to the right. -pietroom/actualeasytask: - 0: Grasp the marker and put it in the plastic box. -pietroom/first_task_short: - 0: Pick up the marker from the box. -pietroom/holdthis: - 0: Hold the object steadily without releasing it. -pranavsaroha/so100_carrot_1: - 0: pick a carrot and put it in the bin -pranavsaroha/so100_carrot_2: - 0: Pick up a carrot and put it in the bin. -pranavsaroha/so100_carrot_3: - 0: pick a carrot and put it in the bin -pranavsaroha/so100_carrot_5: - 0: Pick up a carrot and put it in the bin. -pranavsaroha/so100_legos4: - 0: Pick up the LEGO block and place it in the bowl of the same color as the LEGO - block. -pranavsaroha/so100_onelego2: - 0: Pick up the green LEGO block and place it in the green bowl. -pranavsaroha/so100_onelego3: - 0: Pick up the green LEGO block and place it in the green bowl. -pranavsaroha/so100_squishy: - 0: pick a squishy and put it in the bin -pranavsaroha/so100_squishy100: - 0: pick a squishy and put it in the bin -pranavsaroha/so100_squishy2colors_1: - 0: pick the squishies and put them in the bins with their corresponding colors -pranavsaroha/so100_squishy2colors_2_new: - 0: pick the squishies and put them in the bins with their corresponding colors -rabhishek100/so100_train_dataset: - 0: picks tape and places it in a cup. -rgarreta/koch_pick_place_lego: - 0: Pick the Lego block and drop it in the box on the right. -rgarreta/koch_pick_place_lego_v2: - 0: Pick the Lego block and drop it in the box on the right. -rgarreta/koch_pick_place_lego_v3: - 0: Pick the Lego block and drop it in the box on the right. Top and wrist cameras. -rgarreta/koch_pick_place_lego_v6: - 0: Grasp a lego block and put it in the bin. -rgarreta/koch_pick_place_lego_v7: - 0: Grasp a lego block and put it in the bin. -rgarreta/koch_pick_place_lego_v8: - 0: Grasp a lego block and put it in the bin. -roboticshack/left-arm-grasp-lego-brick: - 0: Grasp the lego brick and put it in the box. -roboticshack/sandee-kiwiv10: - 0: Place the bottle on the table. -roboticshack/team-7-left-arm-grasp-motor: - 0: Grasp the black motor and put it in the box. -roboticshack/team-7-right-arm-grasp-tape: - 0: Grasp the tape and put it in the box. -roboticshack/team10-red-block: - 0: Pick a red lego block and move it to the right. -roboticshack/team11_pianobot: - 0: Point at the keyboard. -roboticshack/team13-three-balls-stacking: - 0: Stack the balls on top of each other. -roboticshack/team13-two-balls-stacking: - 0: Stack the balls on top of each other. -roboticshack/team16-can-stacking: - 0: Grasp the flipped cup and stack it on top of the midpoint between the two other - cups to create a tower -roboticshack/team16-water-pouring: - 0: Pouring water from one cup to another cup -roboticshack/team2-guess_who_less_ligth: - 0: Pick up the card and place it in the slot. -roboticshack/team2-guess_who_so100: - 0: Pick up the card from the shelf. -roboticshack/team2-guess_who_so100_light: - 0: Place the card in the slot. -roboticshack/team9-pick_chicken_place_plate: - 0: Pick up the chicken and place on orange plate -roboticshack/team9-pick_cube_place_static_plate: - 0: Pick up the green cube and place on orange plate. -samanthalhy/so100_herding_1: - 0: Grasp a green tool and herd all the particles to the grey bin. -samsam0510/cube_reorientation_2: - 0: Rotate the object so it aligns with the silhouette on the table. -samsam0510/cube_reorientation_4: - 0: Rotate the object so it aligns with respect to the line on the table. -samsam0510/glove_reorientation_1: - 0: Rotate the glove so the bottom part aligns with the line on the table. -samsam0510/tape_insert_1: - 0: Grasp a red tape and put it on the box. -samsam0510/tape_insert_2: - 0: Grasp a red tape and put it in the yellow tape. -samsam0510/tooth_extraction_3: - 0: Extract the tooth and put it somewhere. -samsam0510/tooth_extraction_4: - 0: Extarct the molar and put it somewhere. -satvikahuja/mixer_on_off: - 0: switch the mixer on or off -satvikahuja/mixer_on_off_new: - 0: Adjust the s position. -satvikahuja/mixer_on_off_new_1: - 0: Press the button on the blender. -satvikahuja/mixer_on_off_new_4: - 0: Place the lid on the blender. -satvikahuja/orange_mixer_1: - 0: pick orange and place in mixer -satvikahuja/orange_pick_place_new1: - 0: Pick up the orange and place it in the bowl. -seeingrain/241228_pick_place_2cams: - 0: Place the cube in the basket. -seeingrain/lego_3cameras: - 0: Pick up the red block and place it in the basket. -seeingrain/one_shot_learning_18episodes: - 0: Pick up the red block and place it in the basket. -seeingrain/pick_lego_to_hand: - 0: places blue object on table. -seeingrain/pick_place_lego: - 0: Place the cube in the basket. -seeingrain/pick_place_lego_wider_range_dang: - 0: Pick up the cube and place it in the basket. -seeingrain/pick_place_lego_wider_range_dong: - 0: Pick up the blue object and place it in the basket. -seeingrain/pick_place_lego_wider_range_richard: - 0: Pick up the blue cube and place it in the basket. -seeingrain/pick_place_pink_lego: - 0: Pick up the red cube and place it in the basket. -seeingrain/pick_place_pink_lego_few_samples: - 0: 'pick_place_pink_lego_few_samples ' -shin1107/koch_move_block_with_some_shapes: - 0: Grasp a block and put it in the hole with some shapes. -shin1107/koch_train_block: - 0: Grasp a block and put it in the hole. -shreyasgite/so100_base_env: - 0: Grasp a lego block and put it in the bin. -shreyasgite/so100_base_left: - 0: Grasp a lego block and put it in the bin. -shreyasgite/so100_legocube_50: - 0: Grasp a lego block and put it in the bin. -sihyun77/mond_1: - 0: Grasp a lego block and put it in the bin. -sihyun77/mond_13: - 0: Grasp a lego block and put it in the bin. -sihyun77/si_so100: - 0: Grasp a lego block and put it in the bin. -sihyun77/sihyun_3_17_2: - 0: Grasp a lego block and put it in the bin. -sihyun77/sihyun_3_17_5: - 0: Grasp a lego block and put it in the bin. -sihyun77/sihyun_king: - 0: Grasp a lego block and put it in the bin. -sihyun77/sihyun_main_2: - 0: Grasp a lego block and put it in the bin. -sihyun77/sihyun_main_3: - 0: Grasp a lego block and put it in the bin. -sihyun77/suho_3_17_1: - 0: Grasp a lego block and put it in the bin. -sihyun77/suho_3_17_3: - 0: Grasp a lego block and put it in the bin. -sihyun77/suho_angel: - 0: Grasp a lego block and put it in the bin. -sihyun77/suho_main_2: - 0: Grasp a lego block and put it in the bin. -sihyun77/suho_red: - 0: Grasp a lego block and put it in the bin. -sihyun77/suho_so100: - 0: Grasp a lego block and put it in the bin. -sixpigs1/so100_insert_cylinder_error: - 0: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf - with position offset when grasping. - 1: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf - with gripper error when grasping. - 2: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf - with rotation offset when uprighting. - 3: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf - with position offset when inserting. - 4: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf - with choice error when inserting. - 5: Pick up the cylinder, upright it, and insert it into the middle hole of the shelf - without errors. -sixpigs1/so100_pick_cube_in_box: - 0: Pick up the red cube and put it in the box. -sixpigs1/so100_pick_cube_in_box_error: - 0: Pick up the red cube and put it in the box with position offset when grasping. - 1: Pick up the red cube and put it in the box with gripper error when grasping. - 2: Pick up the red cube and put it in the box with position offset when releasing. - 3: Pick up the red cube and put it in the box without errors. -sixpigs1/so100_pull_cube_by_tool_error: - 0: Pick up the L-shaped tool and pull the purple cube by the tool with position - offset when grasping. - 1: Pick up the L-shaped tool and pull the purple cube by the tool with rotation - offset when grasping. - 2: Pick up the L-shaped tool and pull the purple cube by the tool with gripper error - when grasping. - 3: Pick up the L-shaped tool and pull the purple cube by the tool with position - offset when lowering. - 4: Pick up the L-shaped tool and pull the purple cube by the tool with rotation - offset when lowering. - 5: Pick up the L-shaped tool and pull the purple cube by the tool with position - offset when pulling. - 6: Pick up the L-shaped tool and pull the purple cube by the tool with gripper error - when pulling. - 7: Pick up the L-shaped tool and pull the purple cube by the tool without errors. -sixpigs1/so100_pull_cube_error: - 0: Pull the yellow cube to the red and white target with position offset when reaching. - 1: Pull the yellow cube to the red and white target with position offset when pulling. - 2: Pull the yellow cube to the red and white target with gripper error when pulling. - 3: Pull the yellow cube to the red and white target without errors. -sixpigs1/so100_push_cube_error: - 0: Push the blue cube to the red and white target with position offset when reaching. - 1: Push the blue cube to the red and white target with position offset when pushing. - 2: Push the blue cube to the red and white target with gripper error when pushing. - 3: Push the blue cube to the red and white target without errors. -sixpigs1/so100_stack_cube_error: - 0: Pick up the red cube and stack it on the green cube with position offset when - grasping. - 1: Pick up the red cube and stack it on the green cube with gripper error when grasping. - 2: Pick up the red cube and stack it on the green cube with position offset when - stacking. - 3: Pick up the red cube and stack it on the green cube without errors -smanni/train_so100_fluffy_box: - 0: Grasp a small object and place it in the box. -speedyyoshi/so100_grasp_pink_block: - 0: Grasp a lego block and put it in the bin. -sshh11/so100_orange_50ep_1: - 0: Grasp an orange object and put it in the bin. -sshh11/so100_orange_50ep_2: - 0: Grasp an orange object and put it in the bin. -swarajgosavi/act_kikobot_block_real: - 0: Pick up the blue cube and place it in the box. -swarajgosavi/act_kikobot_pusht_real: - 0: picks up the red block. -takuzennn/aloha-pick100: - 0: arm pick pen and put it into the cup -takuzennn/square3: - 0: Pick the cube from the table. -thedevansh/mar16_1336: - 0: Grasp a lego block and put it in the bin. -theo-michel/lekiwi_v2: - 0: Pick up the can on the ground -theo-michel/lekiwi_v5: - 0: Pick up the can on the ground -therarelab/med_dis_rare_6: - 0: places green object in box. -therarelab/so100_pick_place_2: - 0: Pick a plaster roll and place it to the blue sticker. -tkc79/so100_lego_box_1: - 0: Grasp a lego block and put it in the box. -tkc79/so100_lego_box_2: - 0: Grasp a lego block and put it in the box. -triton7777/so100_dataset_mix: - 0: Pick up the black tape and place it inside the white tape roll. - 1: Pick up the gift miniatures and place them in the black box. - 2: Sort the mixed objects into their appropriate categories. - 3: Place the pens into the pen holder. - 4: Place pens, bottles, and any suitable items into the pen holder, as appropriate. - 5: Place the oranges into the yellow basket. - 6: Stack the plates and place the cup on top. -twerdster/koch_new_training_red: - 0: move red cube into cellotape circle -twerdster/koch_training_red: - 0: Pick up the red block. -underctrl/handcamera_single_blue: - 0: Pick the Lego block and drop it in the box on the right. -underctrl/mutli-stacked-block_mutli-color_pick-up_80: - 0: Pick single block of multi-color and drop it in the box on the right. -underctrl/single-block_blue-color_pick-up_80: - 0: Pick single block of multi-color and drop it in the box on the right. -underctrl/single-block_multi-color_pick-up_50: - 0: Pick single block of multi-color and drop it in the box on the right. -underctrl/single-stacked-block_mutli-color_pick-up_80: - 0: Pick single block of multi-color and drop it in the box on the right. -underctrl/single-stacked-block_two-color_pick-up_80: - 0: Pick single block of multi-color and drop it in the box on the right. -vladfatu/so100_above: - 0: Pick up red object and place it in the box. -vladfatu/so100_ds: - 0: Pick up the cube and place it in the box. -vladfatu/so100_office: - 0: Pick up the red object and place it in the box. -weiye11/so100_410_zwy: - 0: Pick up the cube and place it on the black circle. -yuz1wan/so100_pickplace: - 0: Pick the pink block and place it in the paper cup. -zaringleb/so100_cube_4_binary: - 0: Grasp a lego block and put it in the bin. -zaringleb/so100_cube_5_linear: - 0: Grasp a lego block and put it in the bin. -zaringleb/so100_cube_6_2d: - 0: Grasp a lego block and put it in the bin. -zaringleb/so100_cude_linear_and_2d_comb: - 0: Grasp a lego block and put it in the bin. -zijian2022/321: - 0: Grasp a lego block and put it in the bin. -zijian2022/backgrounda: - 0: Grasp a lego block and put it in the bin. -zijian2022/backgroundb: - 0: Grasp a lego block and put it in the bin. -zijian2022/c0: - 0: Grasp a lego block and put it in the bin based on color. - 1: Grasp a lego block and put it in the bin. -zijian2022/close3: - 0: Grasp a lego block and put it in the bin. -zijian2022/digitalfix3: - 0: Grasp a lego block and put it in the bin. -zijian2022/insert2: - 0: Grasp a lego block and put it in the bin. -zijian2022/noticehuman3: - 0: Notice human. -zijian2022/noticehuman5: - 0: picks up the box. -zijian2022/noticehuman70: - 0: Stop movement when human encounter testbed. - 1: Stop movement when human encounter testbed w/ trigger. -zijian2022/sort1: - 0: 'Grasp a box and sort it by color: place grey boxes on the left and black boxes - on the right.' -zliu157/i3r: - 0: Grasp a lego block and put it in the bin. -zliu157/i3r2: - 0: Grasp a i3r logo and put it in the bin. -zliu157/i3r3: - 0: Grasp a i3r logo and put it in the bin. -zliu157/i3r5: - 0: Grasp a i3r logo and put it in the bin. From 9779c58b07608d7b5283dee18c17770ba41df9eb Mon Sep 17 00:00:00 2001 From: Jade Date: Tue, 8 Jul 2025 23:57:46 -0400 Subject: [PATCH 5/5] add training support --- lerobot/common/datasets/factory.py | 96 +- lerobot/common/datasets/lerobot_dataset.py | 3 +- lerobot/common/datasets/utils_must.py | 24 +- .../smolvla2/configuration_smolvla2.py | 191 +++ .../policies/smolvla2/modeling_smolvla2.py | 1085 +++++++++++++++++ .../policies/smolvla2/smolvlm_with_expert2.py | 599 +++++++++ 6 files changed, 1995 insertions(+), 3 deletions(-) create mode 100644 lerobot/common/policies/smolvla2/configuration_smolvla2.py create mode 100644 lerobot/common/policies/smolvla2/modeling_smolvla2.py create mode 100644 lerobot/common/policies/smolvla2/smolvlm_with_expert2.py diff --git a/lerobot/common/datasets/factory.py b/lerobot/common/datasets/factory.py index 88d3f767f..e9b05bce1 100644 --- a/lerobot/common/datasets/factory.py +++ b/lerobot/common/datasets/factory.py @@ -32,6 +32,7 @@ IMAGENET_STATS = { "std": [[[0.229]], [[0.224]], [[0.225]]], # (c,1,1) } +from lerobot.common.datasets.utils_must import (EPISODES_DATASET_MAPPING, TRAINING_FEATURES, FEATURE_KEYS_MAPPING) def resolve_delta_timestamps( cfg: PreTrainedConfig, ds_meta: LeRobotDatasetMetadata @@ -66,7 +67,7 @@ def resolve_delta_timestamps( return delta_timestamps -def make_dataset(cfg: TrainPipelineConfig) -> LeRobotDataset | MultiLeRobotDataset: +def make_dataset1(cfg: TrainPipelineConfig) -> LeRobotDataset | MultiLeRobotDataset: """Handles the logic of setting up delta timestamps and image transforms before creating a dataset. Args: @@ -116,3 +117,96 @@ def make_dataset(cfg: TrainPipelineConfig) -> LeRobotDataset | MultiLeRobotDatas dataset.meta.stats[key][stats_type] = torch.tensor(stats, dtype=torch.float32) return dataset + +def make_dataset(cfg: TrainPipelineConfig) -> LeRobotDataset | MultiLeRobotDataset: + """Handles the logic of setting up delta timestamps and image transforms before creating a dataset. + + Args: + cfg (TrainPipelineConfig): A TrainPipelineConfig config which contains a DatasetConfig and a PreTrainedConfig. + + Raises: + NotImplementedError: The MultiLeRobotDataset is currently deactivated. + + Returns: + LeRobotDataset | MultiLeRobotDataset + """ + image_transforms = ( + ImageTransforms(cfg.dataset.image_transforms) if cfg.dataset.image_transforms.enable else None + ) + if "," in cfg.dataset.repo_id: + repo_id = cfg.dataset.repo_id.split(",") + repo_id = [r for r in repo_id if r] + else: + repo_id = cfg.dataset.repo_id + sampling_weights = cfg.dataset.sampling_weights.split(",") if cfg.dataset.sampling_weights else None + feature_keys_mapping = FEATURE_KEYS_MAPPING + if isinstance(repo_id, str): + revision = getattr(cfg.dataset, "revision", None) + ds_meta = LeRobotDatasetMetadata( + cfg.dataset.repo_id, + local_files_only=cfg.dataset.local_files_only, + feature_keys_mapping=feature_keys_mapping, + revision=revision, + ) + delta_timestamps = resolve_delta_timestamps(cfg.policy, ds_meta) + dataset = LeRobotDataset( + cfg.dataset.repo_id, + root=getattr(cfg.dataset, "root", None), + episodes=cfg.dataset.episodes, + delta_timestamps=delta_timestamps, + image_transforms=image_transforms, + revision=revision, + video_backend=cfg.dataset.video_backend, + local_files_only=cfg.dataset.local_files_only, + feature_keys_mapping=feature_keys_mapping, + max_action_dim=cfg.dataset.max_action_dim, + max_state_dim=cfg.dataset.max_state_dim, + max_num_images=cfg.dataset.max_num_images, + max_image_dim=cfg.dataset.max_image_dim, + ) + else: + delta_timestamps = {} + episodes = {} + for i in range(len(repo_id)): + ds_meta = LeRobotDatasetMetadata( + repo_id[i], + local_files_only=cfg.dataset.local_files_only, + feature_keys_mapping=feature_keys_mapping, + ) # FIXME(mshukor): ? + delta_timestamps[repo_id[i]] = resolve_delta_timestamps(cfg.policy, ds_meta) + episodes[repo_id[i]] = EPISODES_DATASET_MAPPING.get(repo_id[i], cfg.dataset.episodes) + training_features = TRAINING_FEATURES.get(cfg.dataset.features_version, None) + dataset = MultiLeRobotDataset( + repo_id, + # TODO(aliberts): add proper support for multi dataset + episodes=episodes, + delta_timestamps=delta_timestamps, + image_transforms=image_transforms, + video_backend=cfg.dataset.video_backend, + local_files_only=cfg.dataset.local_files_only, + sampling_weights=sampling_weights, + feature_keys_mapping=feature_keys_mapping, + max_action_dim=cfg.dataset.max_action_dim, + max_state_dim=cfg.dataset.max_state_dim, + max_num_images=cfg.dataset.max_num_images, + max_image_dim=cfg.dataset.max_image_dim, + train_on_all_features=cfg.dataset.train_on_all_features, + training_features=training_features, + discard_first_n_frames=cfg.dataset.discard_first_n_frames, + min_fps=cfg.dataset.min_fps, + max_fps=cfg.dataset.max_fps, + discard_first_idle_frames=cfg.dataset.discard_first_idle_frames, + motion_threshold=cfg.dataset.motion_threshold, + motion_window_size=cfg.dataset.motion_window_size, + motion_buffer=cfg.dataset.motion_buffer, + ) + logging.info( + "Multiple datasets were provided. Applied the following index mapping to the provided datasets: " + f"{pformat(dataset.repo_id_to_index , indent=2)}" + ) + if cfg.dataset.use_imagenet_stats: + for key in dataset.meta.camera_keys: + for stats_type, stats in IMAGENET_STATS.items(): + dataset.meta.stats[key][stats_type] = torch.tensor(stats, dtype=torch.float32) + + return dataset diff --git a/lerobot/common/datasets/lerobot_dataset.py b/lerobot/common/datasets/lerobot_dataset.py index 7b777e0d7..df13bed72 100644 --- a/lerobot/common/datasets/lerobot_dataset.py +++ b/lerobot/common/datasets/lerobot_dataset.py @@ -83,6 +83,7 @@ from lerobot.common.datasets.utils_must import ( create_padded_features, pad_tensor, map_dict_keys, + find_start_of_motion, ROBOT_TYPE_KEYS_MAPPING, OBS_IMAGE, OBS_IMAGE_2, @@ -562,7 +563,7 @@ class LeRobotDataset(torch.utils.data.Dataset): self.subset_frame_ids += [frame_idx for frame_idx in range(from_ + int(self.fps*self.discard_first_n_frames), to_)] elif self.discard_first_idle_frames: print(f"Discarding first idle frames: motion_threshold={self.motion_threshold}, motion_window_size={self.motion_window_size}, motion_buffer={self.motion_buffer}") - self.robot_states = torch.stack(self.hf_dataset[OBS_ROBOT]).numpy() # shape: [T, D] + self.robot_states = torch.stack(self.hf_dataset[OBS_STATE]).numpy() # shape: [T, D] self.subset_frame_ids = [] for ep_idx in range(self.num_episodes): from_ = self.episode_data_index["from"][ep_idx] diff --git a/lerobot/common/datasets/utils_must.py b/lerobot/common/datasets/utils_must.py index 93927f6c9..040627dfa 100644 --- a/lerobot/common/datasets/utils_must.py +++ b/lerobot/common/datasets/utils_must.py @@ -191,6 +191,13 @@ def map_dict_keys(item: dict, feature_keys_mapping: dict, training_features: lis features[key] = item[key] return features +def find_start_of_motion(velocities, window_size, threshold, motion_buffer): + for t in range(len(velocities) - window_size): + window_mean = velocities[t:t+window_size].mean() + if window_mean > threshold: + return max(0, t - motion_buffer) # include slight context before motion + return 0 + import yaml import requests def load_yaml_mapping(name: str) -> dict: @@ -205,4 +212,19 @@ def load_yaml_mapping(name: str) -> dict: return yaml.safe_load(response.text) # Example usage -TASKS_KEYS_MAPPING = load_yaml_mapping("features") +TASKS_KEYS_MAPPING = load_yaml_mapping("tasks") +FEATURE_KEYS_MAPPING = load_yaml_mapping("features") +EPISODES_DATASET_MAPPING = { + "cadene/droid_1.0.1": list(range(50)), + "danaaubakirova/svla_so100_task5_v3": [0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51], + "danaaubakirova/svla_so100_task4_v3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53], +} +ACTION = "action" +OBS_STATE = "observation.state" +TASK = "task" +ROBOT = "robot_type" +TRAINING_FEATURES = { + 0: [ACTION, OBS_STATE, TASK, ROBOT, OBS_IMAGE], + 1: [ACTION, OBS_STATE, TASK, ROBOT, OBS_IMAGE, OBS_IMAGE_2], + 2: [ACTION, OBS_STATE, TASK, ROBOT, OBS_IMAGE, OBS_IMAGE_2, OBS_IMAGE_3], +} diff --git a/lerobot/common/policies/smolvla2/configuration_smolvla2.py b/lerobot/common/policies/smolvla2/configuration_smolvla2.py new file mode 100644 index 000000000..9b839c62a --- /dev/null +++ b/lerobot/common/policies/smolvla2/configuration_smolvla2.py @@ -0,0 +1,191 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field + +from lerobot.common.optim.optimizers import AdamWConfig +from lerobot.common.optim.schedulers import ( + CosineDecayWithWarmupSchedulerConfig, +) +from lerobot.configs.policies import PreTrainedConfig +from lerobot.configs.types import FeatureType, NormalizationMode, PolicyFeature + + +@dataclass +class PEFTConfig: + r: int = 4 + lora_alpha: int = 16 + lora_dropout: float = 0.1 + target_modules: str = "q_proj,v_proj" + + +@PreTrainedConfig.register_subclass("smolvla2") +@dataclass +class SmolVLA2Config(PreTrainedConfig): + # Input / output structure. + n_obs_steps: int = 1 + chunk_size: int = 50 + n_action_steps: int = 50 + + normalization_mapping: dict[str, NormalizationMode] = field( + default_factory=lambda: { + "VISUAL": NormalizationMode.IDENTITY, + "STATE": NormalizationMode.MEAN_STD, + "ACTION": NormalizationMode.MEAN_STD, + } + ) + + # Shorter state and action vectors will be padded + max_state_dim: int = 32 + max_action_dim: int = 32 + + # Image preprocessing + resize_imgs_with_padding: tuple[int, int] = (512, 512) + + # Add empty images. Used by smolvla_aloha_sim which adds the empty + # left and right wrist cameras in addition to the top camera. + empty_cameras: int = 0 + + # Converts the joint and gripper values from the standard Aloha space to + # the space used by the pi internal runtime which was used to train the base model. + adapt_to_pi_aloha: bool = False + + # Converts joint dimensions to deltas with respect to the current state before passing to the model. + # Gripper dimensions will remain in absolute values. + use_delta_joint_actions_aloha: bool = False + + # Tokenizer + tokenizer_max_length: int = 48 + proj_width: int = 480 + # Decoding + num_steps: int = 10 + + # Attention utils + use_cache: bool = True + + # Finetuning settings + freeze_vision_encoder: bool = True + train_expert_only: bool = False + train_state_proj: bool = True + + # Training presets + optimizer_lr: float = 2.5e-5 # 1e-4 + optimizer_betas: tuple[float, float] = (0.9, 0.95) + optimizer_eps: float = 1e-8 + optimizer_weight_decay: float = 1e-10 + optimizer_grad_clip_norm: float = 10 + optimizer_lr_vlm: float = 0 + + scheduler_warmup_steps: int = 1_000 + scheduler_decay_steps: int = 30_000 + scheduler_decay_lr: float = 2.5e-6 + + vlm_model_name: str = "HuggingFaceTB/SmolVLM2-500M-Video-Instruct" # Select the VLM backbone. + load_vlm_weights: bool = False # Set to True in case of training the expert from scratch. True when init from pretrained SmolVLA weights + checkpoint_path: str = None + peft_method: str = "" + peft_config: PEFTConfig = field(default_factory=PEFTConfig) + peft_target_model: str = "" + add_image_special_tokens: bool = False # Whether to use special image tokens around image features. + + attention_mode: str = "cross_attn" + + prefix_length: int = -1 + + pad_language_to: str = "longest" # "max_length" + + num_expert_layers: int = -1 # Less or equal to 0 is the default where the action expert has the same number of layers of VLM. Otherwise the expert have less layers. + num_vlm_layers: int = 16 + past_obs_keys: str = "image" + add_local_special_image_tokens: bool = False + + reverse_images_order: bool = False + + state_to_prefix: bool = False + + pad_language_to: str = "longest" # "max_length" + causal_action_attention_mask: bool = False + + self_attn_every_n_layers: int = -1 # Number of layers used in the VLM (first num_vlm_layers layers) + # self_attn_every_n_layers: int = 2 # Interleave SA layers each self_attn_every_n_layers + expert_width_multiplier: float = 0.75 # The action expert hidden size (wrt to the VLM) + + min_period: float = 4e-3 # sensitivity range for the timestep used in sine-cosine positional encoding + max_period: float = 4.0 + + robot_type: str = "" + + self_attn_only_actions: bool = False + + causal_attention_on_history: bool = False + + predict_relative_actions: bool = False + relative_actions_mode: str = "first" + + shuffle_camera_positions: bool = False + vlm_img_size: int = -1 + + regression_loss: bool = False + + def __post_init__(self): + super().__post_init__() + + """Input validation (not exhaustive).""" + if self.n_action_steps > self.chunk_size: + raise ValueError( + f"The chunk size is the upper bound for the number of action steps per model invocation. Got " + f"{self.n_action_steps} for `n_action_steps` and {self.chunk_size} for `chunk_size`." + ) + if self.use_delta_joint_actions_aloha: + raise NotImplementedError( + "`use_delta_joint_actions_aloha` is used by smolvla for aloha real models. It is not ported yet in LeRobot." + ) + + def validate_features(self) -> None: + for i in range(self.empty_cameras): + key = f"observation.images.empty_camera_{i}" + empty_camera = PolicyFeature( + type=FeatureType.VISUAL, + shape=(3, 480, 640), + ) + self.input_features[key] = empty_camera + + def get_optimizer_preset(self) -> AdamWConfig: + return AdamWConfig( + lr=self.optimizer_lr, + betas=self.optimizer_betas, + eps=self.optimizer_eps, + weight_decay=self.optimizer_weight_decay, + grad_clip_norm=self.optimizer_grad_clip_norm, + ) + + def get_scheduler_preset(self): + return CosineDecayWithWarmupSchedulerConfig( + peak_lr=self.optimizer_lr, + decay_lr=self.scheduler_decay_lr, + num_warmup_steps=self.scheduler_warmup_steps, + num_decay_steps=self.scheduler_decay_steps, + ) + + @property + def observation_delta_indices(self) -> list: + return [0] + + @property + def action_delta_indices(self) -> list: + return list(range(self.chunk_size)) + + @property + def reward_delta_indices(self) -> None: + return None diff --git a/lerobot/common/policies/smolvla2/modeling_smolvla2.py b/lerobot/common/policies/smolvla2/modeling_smolvla2.py new file mode 100644 index 000000000..3e85ba0ff --- /dev/null +++ b/lerobot/common/policies/smolvla2/modeling_smolvla2.py @@ -0,0 +1,1085 @@ +#!/usr/bin/env python + +# Copyright 2025 HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +SmolVLA: + +[Paper](https://huggingface.co/papers/2506.01844) + +Designed by Hugging Face. + +Install smolvla extra dependencies: +```bash +pip install -e ".[smolvla]" +``` + +Example of finetuning the smolvla pretrained model (`smolvla_base`): +```bash +python lerobot/scripts/train.py \ +--policy.path=lerobot/smolvla_base \ +--dataset.repo_id=danaaubakirova/svla_so100_task1_v3 \ +--batch_size=64 \ +--steps=200000 +``` + +Example of finetuning a smolVLA. SmolVLA is composed of a pretrained VLM, +and an action expert. +```bash +python lerobot/scripts/train.py \ +--policy.type=smolvla \ +--dataset.repo_id=danaaubakirova/svla_so100_task1_v3 \ +--batch_size=64 \ +--steps=200000 +``` + +Example of using the smolvla pretrained model outside LeRobot training framework: +```python +policy = SmolVLAPolicy.from_pretrained("lerobot/smolvla_base") +``` + +""" + +import math +import os +import random +import re +from collections import deque + +import safetensors +import torch +import torch.nn.functional as F # noqa: N812 +from torch import Tensor, nn +from transformers import AutoProcessor + +from lerobot.common.constants import ACTION, OBS_STATE +from lerobot.common.policies.normalize import ( + Normalize, + Unnormalize, +) +from lerobot.common.policies.pretrained import PreTrainedPolicy +from lerobot.common.policies.smolvla.smolvlm_with_expert import SmolVLMWithExpertModel +from lerobot.common.policies.smolvla2.configuration_smolvla2 import SmolVLA2Config +from lerobot.common.policies.utils import ( + populate_queues, +) +from lerobot.common.utils.utils import get_safe_dtype +from lerobot.datasets import IMAGES_ORDER + +# Matches ".soNNN", optionally followed by "-something", up to the "_buffer_" marker +_VARIANT_RE = re.compile(r"\.so\d+(?:-[\w]+)?_buffer_") + + +def canonicalise(k: str) -> str: + """ + Remove dataset-variant markers like '.so100-blue_' or '.so100_' from a + normalisation-buffer key. + """ + return _VARIANT_RE.sub(".buffer_", k) + + +def standardise_state_dict( + checkpoint: dict[str, torch.Tensor], ref_keys: set[str], *, verbose: bool = True +) -> tuple[dict[str, torch.Tensor], list[str]]: + """ + • Re-keys `checkpoint ` so that every entry matches the *reference* key set. + • If several variant keys collapse to the same canonical name we keep the + first one and log the collision. + • Returns the new dict + a list of entries that could not be matched. + """ + out, collisions, unmatched = {}, {}, [] + + for k, v in checkpoint.items(): + canon = canonicalise(k) + if canon in ref_keys: + if canon in out: # duplicate after collapsing + collisions.setdefault(canon, []).append(k) + else: + out[canon] = v + else: + unmatched.append(k) + + if verbose: + for canon, variants in collisions.items(): + print(f"[standardise_state_dict] '{canon}' ← {variants}") + if unmatched: + print(f"[standardise_state_dict] kept {len(unmatched)} unmatched keys") + + out.update({k: checkpoint[k] for k in unmatched}) + return out, unmatched + + +def rename_checkpoint_keys(checkpoint: dict, rename_str: str): + """ + Renames keys in a checkpoint dictionary based on the given rename string. + + Args: + checkpoint (dict): The checkpoint dictionary. + rename_str (str): A string specifying key mappings in the format "old1//new1,old2//new2". + + Returns: + dict: The modified checkpoint with renamed keys. + """ + + rename_dict = dict(pair.split("//") for pair in rename_str.split(",")) + + new_checkpoint = {} + for k, v in checkpoint.items(): + for old_key, new_key in rename_dict.items(): + if old_key in k: + k = k.replace(old_key, new_key) + new_checkpoint[k] = v + return new_checkpoint + + +def load_smolvla( + model: torch.nn.Module, + filename: str | os.PathLike, + *, + device: str = "cpu", + checkpoint_keys_mapping: str = "", +) -> torch.nn.Module: + state_dict = safetensors.torch.load_file(filename, device=device) + + # Optional user-supplied renames (e.g. "model._orig_mod.//model.") + if checkpoint_keys_mapping and "//" in checkpoint_keys_mapping: + state_dict = rename_checkpoint_keys(state_dict, checkpoint_keys_mapping) + + state_dict, _ = standardise_state_dict(state_dict, set(model.state_dict().keys())) + + # HACK(aliberts): to not overwrite normalization parameters as they should come from the dataset + norm_keys = ("normalize_inputs", "normalize_targets", "unnormalize_outputs") + state_dict = {k: v for k, v in state_dict.items() if not k.startswith(norm_keys)} + + missing, unexpected = model.load_state_dict(state_dict, strict=False) + + if not all(key.startswith(norm_keys) for key in missing) or unexpected: + raise RuntimeError( + "SmolVLA %d missing / %d unexpected keys", + len(missing), + len(unexpected), + ) + + return model + + +def create_sinusoidal_pos_embedding( + time: torch.tensor, dimension: int, min_period: float, max_period: float, device="cpu" +) -> Tensor: + """Computes sine-cosine positional embedding vectors for scalar positions.""" + if dimension % 2 != 0: + raise ValueError(f"dimension ({dimension}) must be divisible by 2") + + if time.ndim != 1: + raise ValueError("The time tensor is expected to be of shape `(batch_size, )`.") + + dtype = get_safe_dtype(torch.float64, device.type) + fraction = torch.linspace(0.0, 1.0, dimension // 2, dtype=dtype, device=device) + period = min_period * (max_period / min_period) ** fraction + + # Compute the outer product + scaling_factor = 1.0 / period * 2 * math.pi + sin_input = scaling_factor[None, :] * time[:, None] + pos_emb = torch.cat([torch.sin(sin_input), torch.cos(sin_input)], dim=1) + return pos_emb + + +def sample_beta(alpha, beta, bsize, device): + gamma1 = torch.empty((bsize,), device=device).uniform_(0, 1).pow(1 / alpha) + gamma2 = torch.empty((bsize,), device=device).uniform_(0, 1).pow(1 / beta) + return gamma1 / (gamma1 + gamma2) + + +def make_att_2d_masks(pad_masks, att_masks): + """Copied from big_vision. + + Tokens can attend to valid inputs tokens which have a cumulative mask_ar + smaller or equal to theirs. This way `mask_ar` int[B, N] can be used to + setup several types of attention, for example: + + [[1 1 1 1 1 1]]: pure causal attention. + + [[0 0 0 1 1 1]]: prefix-lm attention. The first 3 tokens can attend between + themselves and the last 3 tokens have a causal attention. The first + entry could also be a 1 without changing behaviour. + + [[1 0 1 0 1 0 0 1 0 0]]: causal attention between 4 blocks. Tokens of a + block can attend all previous blocks and all tokens on the same block. + + Args: + input_mask: bool[B, N] true if its part of the input, false if padding. + mask_ar: int32[B, N] mask that's 1 where previous tokens cannot depend on + it and 0 where it shares the same attention mask as the previous token. + """ + if att_masks.ndim != 2: + raise ValueError(att_masks.ndim) + if pad_masks.ndim != 2: + raise ValueError(pad_masks.ndim) + + cumsum = torch.cumsum(att_masks, dim=1) + att_2d_masks = cumsum[:, None, :] <= cumsum[:, :, None] + pad_2d_masks = pad_masks[:, None, :] * pad_masks[:, :, None] + att_2d_masks = att_2d_masks & pad_2d_masks + return att_2d_masks + + +def resize_with_pad(img, width, height, pad_value=-1): + # assume no-op when width height fits already + if img.ndim != 4: + raise ValueError(f"(b,c,h,w) expected, but {img.shape}") + + cur_height, cur_width = img.shape[2:] + + ratio = max(cur_width / width, cur_height / height) + resized_height = int(cur_height / ratio) + resized_width = int(cur_width / ratio) + resized_img = F.interpolate( + img, size=(resized_height, resized_width), mode="bilinear", align_corners=False + ) + + pad_height = max(0, int(height - resized_height)) + pad_width = max(0, int(width - resized_width)) + + # pad on left and top of image + padded_img = F.pad(resized_img, (pad_width, 0, pad_height, 0), value=pad_value) + return padded_img + + +def pad_vector(vector, new_dim): + """Can be (batch_size x sequence_length x features_dimension) + or (batch_size x features_dimension) + """ + if vector.shape[-1] == new_dim: + return vector + shape = list(vector.shape) + current_dim = shape[-1] + shape[-1] = new_dim + new_vector = torch.zeros(*shape, dtype=vector.dtype, device=vector.device) + new_vector[..., :current_dim] = vector + return new_vector + + +def normalize(x, min_val, max_val): + return (x - min_val) / (max_val - min_val) + + +def unnormalize(x, min_val, max_val): + return x * (max_val - min_val) + min_val + + +def safe_arcsin(value): + # This ensures that the input stays within + # [−1,1] to avoid invalid values for arcsin + return torch.arcsin(torch.clamp(value, -1.0, 1.0)) + + +def aloha_gripper_to_angular(value): + # Aloha transforms the gripper positions into a linear space. The following code + # reverses this transformation to be consistent with smolvla which is pretrained in + # angular space. + # + # These values are coming from the Aloha code: + # PUPPET_GRIPPER_POSITION_OPEN, PUPPET_GRIPPER_POSITION_CLOSED + value = unnormalize(value, min_val=0.01844, max_val=0.05800) + + # This is the inverse of the angular to linear transformation inside the Interbotix code. + def linear_to_radian(linear_position, arm_length, horn_radius): + value = (horn_radius**2 + linear_position**2 - arm_length**2) / (2 * horn_radius * linear_position) + return safe_arcsin(value) + + # The constants are taken from the Interbotix code. + value = linear_to_radian(value, arm_length=0.036, horn_radius=0.022) + + # Normalize to [0, 1]. + # The values 0.4 and 1.5 were measured on an actual Trossen robot. + return normalize(value, min_val=0.4, max_val=1.5) + + +def aloha_gripper_from_angular(value): + # Convert from the gripper position used by smolvla to the gripper position that is used by Aloha. + # Note that the units are still angular but the range is different. + + # The values 0.4 and 1.5 were measured on an actual Trossen robot. + value = unnormalize(value, min_val=0.4, max_val=1.5) + + # These values are coming from the Aloha code: + # PUPPET_GRIPPER_JOINT_OPEN, PUPPET_GRIPPER_JOINT_CLOSE + return normalize(value, min_val=-0.6213, max_val=1.4910) + + +def aloha_gripper_from_angular_inv(value): + # Directly inverts the gripper_from_angular function. + value = unnormalize(value, min_val=-0.6213, max_val=1.4910) + return normalize(value, min_val=0.4, max_val=1.5) + + +class SmolVLA2Policy(PreTrainedPolicy): + """Wrapper class around VLAFlowMatching model to train and run inference within LeRobot.""" + + config_class = SmolVLA2Config + name = "smolvla2" + + def __init__( + self, + config: SmolVLA2Config, + dataset_stats: dict[str, dict[str, Tensor]] | None = None, + ): + """ + Args: + config: Policy configuration class instance or None, in which case the default instantiation of + the configuration class is used. + dataset_stats: Dataset statistics to be used for normalization. If not passed here, it is expected + that they will be passed with a call to `load_state_dict` before the policy is used. + """ + + super().__init__(config) + config.validate_features() + self.config = config + self.normalize_inputs = Normalize(config.input_features, config.normalization_mapping, dataset_stats) + self.normalize_targets = Normalize( + config.output_features, config.normalization_mapping, dataset_stats + ) + self.unnormalize_outputs = Unnormalize( + config.output_features, config.normalization_mapping, dataset_stats + ) + + self.language_tokenizer = AutoProcessor.from_pretrained(self.config.vlm_model_name).tokenizer + self.model = VLAFlowMatching(config) + self.reset() + + def reset(self): + """This should be called whenever the environment is reset.""" + self._queues = { + ACTION: deque(maxlen=self.config.n_action_steps), + } + if self.config.n_obs_steps > 1: + for k in self.config.input_features: + if any([past_obs_key in k for past_obs_key in self.config.past_obs_keys.split(",")]): + self._queues[k] = deque(maxlen=self.config.n_obs_steps) + + # HACK(aliberts, danaaubakirova): we overwrite this classmethod here to fix smolVLA-specific issues + @classmethod + def _load_as_safetensor( + cls, + model: "SmolVLA2Policy", + model_file: str, + map_location: str, + strict: bool, + ): + safetensors.torch.load_model(model, model_file, strict=strict, device=map_location) + return load_smolvla( + model, + model_file, + device=map_location, + checkpoint_keys_mapping="model._orig_mod.//model.", + ) + + def get_optim_params(self) -> dict: + return self.parameters() + + def merge_peft_model_weights(self) -> None: + if "lora" in self.config.peft_method: + self.model.vlm_with_expert.merge_lora_weights() + + @torch.no_grad + def select_action_chunk(self, batch: dict[str, Tensor], noise: Tensor | None = None) -> Tensor: + """Select a single action given environment observations. + + This method wraps `select_actions` in order to return one action at a time for execution in the + environment. It works by managing the actions in a queue and only calling `select_actions` when the + queue is empty. + """ + self.eval() + + if self.config.adapt_to_pi_aloha: + batch[OBS_STATE] = self._pi_aloha_decode_state(batch[OBS_STATE]) + + batch = self.normalize_inputs(batch) + + images, img_masks = self.prepare_images(batch) + state = self.prepare_state(batch) + lang_tokens, lang_masks = self.prepare_language(batch) + + actions = self.model.sample_actions(images, img_masks, lang_tokens, lang_masks, state, noise=noise) + # Unpad actions + original_action_dim = self.config.action_feature.shape[0] + actions = actions[:, :, :original_action_dim] + + actions = self.unnormalize_outputs({"action": actions, "robot_type": batch["robot_type"]})["action"] + + if self.config.adapt_to_pi_aloha: + actions = self._pi_aloha_encode_actions(actions) + + return actions + + @torch.no_grad + def select_action(self, batch: dict[str, Tensor], noise: Tensor | None = None) -> Tensor: + """Select a single action given environment observations. + + This method wraps `select_actions` in order to return one action at a time for execution in the + environment. It works by managing the actions in a queue and only calling `select_actions` when the + queue is empty. + """ + self.eval() + + if self.config.adapt_to_pi_aloha: + batch[OBS_STATE] = self._pi_aloha_decode_state(batch[OBS_STATE]) + + batch = self.normalize_inputs(batch) + + self._queues = populate_queues(self._queues, batch, exclude_keys=[ACTION]) + # Action queue logic for n_action_steps > 1. When the action_queue is depleted, populate it by + # querying the policy. + if len(self._queues[ACTION]) == 0: + for k in batch: + if k in self._queues: + batch[k] = torch.stack(list(self._queues[k]), dim=1) + images, img_masks = self.prepare_images(batch) + state = self.prepare_state(batch) + lang_tokens, lang_masks = self.prepare_language(batch) + + actions = self.model.sample_actions( + images, img_masks, lang_tokens, lang_masks, state, noise=noise + ) + if self.config.predict_relative_actions and actions.ndim == 3: + # If the model predicts relative actions, we need to unpad the actions + # and then convert them to absolute actions. + if self.config.relative_actions_mode == "first": + actions = torch.cat((actions[:, :1], actions[:, 1:] + actions[:, :1]), dim=1) + elif self.config.relative_actions_mode == "state": + actions = actions + state.unsqueeze(1) + else: + actions = torch.cat((actions[:, :1], actions[:, 1:] + actions[:, :-1]), dim=1) + # Unpad actions + + original_action_dim = self.config.action_feature.shape[0] + actions = actions[:, :, :original_action_dim] + + actions = self.unnormalize_outputs({"action": actions})["action"] + + if self.config.adapt_to_pi_aloha: + actions = self._pi_aloha_encode_actions(actions) + + # `self.model.forward` returns a (batch_size, n_action_steps, action_dim) tensor, but the queue + # effectively has shape (n_action_steps, batch_size, *), hence the transpose. + self._queues[ACTION].extend(actions.transpose(0, 1)[: self.config.n_action_steps]) + return self._queues[ACTION].popleft() + + def forward(self, batch: dict[str, Tensor], noise=None, time=None) -> dict[str, Tensor]: + """Do a full training forward pass to compute the loss""" + if self.config.adapt_to_pi_aloha: + batch[OBS_STATE] = self._pi_aloha_decode_state(batch[OBS_STATE]) + batch[ACTION] = self._pi_aloha_encode_actions_inv(batch[ACTION]) + batch = self.normalize_inputs(batch) + batch = self.normalize_targets(batch) + images, img_masks = self.prepare_images(batch) + state = self.prepare_state(batch) + lang_tokens, lang_masks = self.prepare_language(batch) + actions = self.prepare_action(batch, state=state) + actions_is_pad = batch.get("actions_id_pad") + loss_dict = {} + losses = self.model.forward(images, img_masks, lang_tokens, lang_masks, state, actions, noise, time) + loss_dict["losses_after_forward"] = losses.clone() + + if actions_is_pad is not None: + in_episode_bound = ~actions_is_pad + losses = losses * in_episode_bound.unsqueeze(-1) + loss_dict["losses_after_in_ep_bound"] = losses.clone() + + # Remove padding + losses = losses[:, :, : self.config.max_action_dim] + loss_dict["losses_after_rm_padding"] = losses.clone() + + # For backward pass + loss = losses.mean() + # For backward pass + loss_dict["loss"] = loss.item() + return loss, loss_dict + + def prepare_images(self, batch): + """Apply Pi0 preprocessing to the images, like resizing to 224x224 and padding to keep aspect ratio, and + convert pixel range from [0.0, 1.0] to [-1.0, 1.0] as requested by SigLIP. + """ + images = [] + img_masks = [] + present_img_keys = [key for key in self.config.image_features if key in batch] + missing_img_keys = [key for key in self.config.image_features if key not in batch] + + present_img_keys = sorted( + present_img_keys, + key=lambda k: IMAGES_ORDER.get(k, float("inf")), + reverse=self.config.reverse_images_order, + ) + if self.config.shuffle_camera_positions and ACTION in batch: # only during training + present_img_keys = random.sample(present_img_keys, len(present_img_keys)) + if len(present_img_keys) == 0: + raise ValueError( + f"All image features are missing from the batch. At least one expected. (batch: {batch.keys()}) (image_features:{self.config.image_features})" + ) + for i in range(self.num_past_images): + # Preprocess image features present in the batch + for key in present_img_keys: + img = batch[key][:, i, :, :, :] if batch[key].ndim == 5 else batch[key] + if self.config.resize_imgs_with_padding is not None: + img = resize_with_pad(img, *self.config.resize_imgs_with_padding, pad_value=0) + + # Normalize from range [0,1] to [-1,1] as expacted by siglip + img = img * 2.0 - 1.0 + + bsize = img.shape[0] + device = img.device + if f"{key}_padding_mask" in batch: + mask = batch[f"{key}_padding_mask"].bool() + else: + mask = torch.ones(bsize, dtype=torch.bool, device=device) + images.append(img) + img_masks.append(mask) + + # Create image features not present in the batch + # as fully 0 padded images. + for num_empty_cameras in range(len(missing_img_keys)): + if num_empty_cameras >= self.config.empty_cameras: + break + img = torch.ones_like(img) * -1 + mask = torch.zeros_like(mask) + images.append(img) + img_masks.append(mask) + return images, img_masks + + def prepare_language(self, batch) -> tuple[Tensor, Tensor]: + """Tokenize the text input""" + device = batch[OBS_STATE].device + tasks = batch["task"] + if isinstance(tasks, str): + tasks = [tasks] + + if len(tasks) == 1: + tasks = [tasks[0] for _ in range(batch[OBS_STATE].shape[0])] + + tasks = [task if task.endswith("\n") else f"{task}\n" for task in tasks] + + tokenized_prompt = self.language_tokenizer.__call__( + tasks, + padding=self.config.pad_language_to, + padding_side="right", + max_length=self.config.tokenizer_max_length, + return_tensors="pt", + truncation=True, # FIXME(mshukor) + ) + lang_tokens = tokenized_prompt["input_ids"].to(device=device) + lang_masks = tokenized_prompt["attention_mask"].to(device=device, dtype=torch.bool) + + return lang_tokens, lang_masks + + def _pi_aloha_decode_state(self, state): + # Flip the joints. + for motor_idx in [1, 2, 8, 9]: + state[:, motor_idx] *= -1 + # Reverse the gripper transformation that is being applied by the Aloha runtime. + for motor_idx in [6, 13]: + state[:, motor_idx] = aloha_gripper_to_angular(state[:, motor_idx]) + return state + + def _pi_aloha_encode_actions(self, actions): + # Flip the joints. + for motor_idx in [1, 2, 8, 9]: + actions[:, :, motor_idx] *= -1 + # Reverse the gripper transformation that is being applied by the Aloha runtime. + for motor_idx in [6, 13]: + actions[:, :, motor_idx] = aloha_gripper_from_angular(actions[:, :, motor_idx]) + return actions + + def _pi_aloha_encode_actions_inv(self, actions): + # Flip the joints again. + for motor_idx in [1, 2, 8, 9]: + actions[:, :, motor_idx] *= -1 + # Reverse the gripper transformation that is being applied by the Aloha runtime. + for motor_idx in [6, 13]: + actions[:, :, motor_idx] = aloha_gripper_from_angular_inv(actions[:, :, motor_idx]) + return actions + + def prepare_state(self, batch): + """Pad state""" + state = batch[OBS_STATE][:, -1, :] if batch[OBS_STATE].ndim > 2 else batch[OBS_STATE] + state = pad_vector(state, self.config.max_state_dim) + return state + + def prepare_action(self, batch, state=None): + """Pad action""" + actions = pad_vector(batch[ACTION], self.config.max_action_dim) + if self.config.predict_relative_actions and actions.ndim == 3: + if self.config.relative_actions_mode == "first": + actions = torch.cat((actions[:, :1], actions[:, 1:] - actions[:, :1]), dim=1) + elif self.config.relative_actions_mode == "state": + assert batch[ACTION].shape[-1] == batch[OBS_STATE].shape[-1], ( + "Relative action mode 'state' requires the action and state to have the same dimension." + ) + if state.ndim == 2: + state = state.unsqueeze(1) + actions = actions - state + else: + actions = torch.cat((actions[:, :1], actions[:, 1:] - actions[:, :-1]), dim=1) + return actions + + +def pad_tensor(tensor, max_len, pad_value=0): + """ + Efficiently pads a tensor along sequence dimension to match max_len. + + Args: + tensor (torch.Tensor): Shape (B, L, ...) or (B, L). + max_len (int): Fixed sequence length. + pad_value (int/float): Value for padding. + + Returns: + torch.Tensor: Shape (B, max_len, ...) or (B, max_len). + """ + b, d = tensor.shape[:2] + + # Create a padded tensor of max_len and copy the existing values + padded_tensor = torch.full( + (b, max_len, *tensor.shape[2:]), pad_value, dtype=tensor.dtype, device=tensor.device + ) + padded_tensor[:, :d] = tensor # Efficient in-place copy + + return padded_tensor + + +class VLAFlowMatching(nn.Module): + """ + SmolVLA + + [Paper]() + + Designed by Hugging Face. + ┌──────────────────────────────┐ + │ actions │ + │ ▲ │ + │ ┌─────────┐ ┌─|────┐ │ + │ | │────► │ │ │ + │ | │ kv │ │ │ + │ | │────► │Action│ │ + │ | VLM │cache │Expert│ | + │ │ │────► | │ │ + │ │ │ │ │ │ + │ └▲──▲───▲─┘ └───▲──┘ | + │ │ | | │ | + │ | | | noise │ + │ │ │ state │ + │ │ language tokens │ + │ image(s) │ + └──────────────────────────────┘ + """ + + def __init__(self, config): + super().__init__() + self.config = config + + self.vlm_with_expert = SmolVLMWithExpertModel( + model_id=self.config.vlm_model_name, + freeze_vision_encoder=self.config.freeze_vision_encoder, + train_expert_only=self.config.train_expert_only, + attention_implementation=self.config.attention_implementation, + load_vlm_weights=self.config.load_vlm_weights, + attention_mode=self.config.attention_mode, + num_expert_layers=self.config.num_expert_layers, + num_vlm_layers=self.config.num_vlm_layers, + self_attn_every_n_layers=self.config.self_attn_every_n_layers, + expert_width_multiplier=self.config.expert_width_multiplier, + ) + self.vlm_with_expert.configure_peft(config=self.config) + # Projections are float32 + self.state_to_prefix = self.config.state_to_prefix + if self.state_to_prefix: + self.state_proj = nn.Linear( + self.config.max_state_dim, self.vlm_with_expert.config.text_config.hidden_size + ) + else: + self.state_proj = nn.Linear(self.config.max_state_dim, self.vlm_with_expert.expert_hidden_size) + self.action_in_proj = nn.Linear(self.config.max_action_dim, self.vlm_with_expert.expert_hidden_size) + self.action_out_proj = nn.Linear(self.vlm_with_expert.expert_hidden_size, self.config.max_action_dim) + + self.action_time_mlp_in = nn.Linear( + self.vlm_with_expert.expert_hidden_size * 2, self.vlm_with_expert.expert_hidden_size + ) + self.action_time_mlp_out = nn.Linear( + self.vlm_with_expert.expert_hidden_size, self.vlm_with_expert.expert_hidden_size + ) + + self.set_requires_grad() + # SmolVLM2 has: [fake_tok + crop_tok + crop + fake_tok + crop_tok ... + fake_tok + global_tok + global + fake_tok] + [second image] + ... + self.fake_image_token = self.vlm_with_expert.processor.tokenizer.fake_image_token_id + self.global_image_token = self.vlm_with_expert.processor.tokenizer.global_image_token_id + self.global_image_start_token = torch.tensor( + [self.fake_image_token, self.global_image_token], dtype=torch.long + ) + + self.add_image_special_tokens = self.config.add_image_special_tokens + self.image_end_token = torch.tensor([self.fake_image_token], dtype=torch.long) + self.prefix_length = self.config.prefix_length + self.include_past_images = self.config.n_obs_steps > 1 and "image" in self.config.past_obs_keys.split( + "," + ) + self.num_past_images = self.config.n_obs_steps if self.include_past_images else 1 + self.causal_attention_on_history = self.config.causal_attention_on_history + + def set_requires_grad(self): + for params in self.state_proj.parameters(): + params.requires_grad = self.config.train_state_proj + + def sample_noise(self, shape, device): + noise = torch.normal( + mean=0.0, + std=1.0, + size=shape, + dtype=torch.float32, + device=device, + ) + return noise + + def sample_time(self, bsize, device): + time_beta = sample_beta(1.5, 1.0, bsize, device) + time = time_beta * 0.999 + 0.001 + return time.to(dtype=torch.float32, device=device) + + def embed_prefix( + self, + images, + img_masks, + lang_tokens, + lang_masks, + state: torch.Tensor = None, + pointtrackers=None, + pt_masks=None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Embed multiple modalities for vlm processing. + + Simple, extensible approach using list + torch.cat. + Easy to add new information/modalities like point trackers, audio, etc. + + Args: + images: List of image tensors + img_masks: List of image masks + lang_tokens: Language token tensor + lang_masks: Language mask tensor + state: Optional state tensor + pointtrackers: Optional point tracker tensors (future extension) + pt_masks: Optional point tracker masks (future extension) + **kwargs: Additional modalities for future extensions + """ + embs = [] + pad_masks = [] + att_masks = [] + + # Process each modality type + self._add_image_embeddings(images, img_masks, embs, pad_masks, att_masks) + self._add_language_embeddings(lang_tokens, lang_masks, embs, pad_masks, att_masks) + + if state is not None and self.state_to_prefix: + self._add_state_embeddings(state, embs, pad_masks, att_masks) + + # Future extensions - easy to add new modalities + if pointtrackers is not None: + self._add_pointtracker_embeddings(pointtrackers, pt_masks, embs, pad_masks, att_masks) + + # Add more modalities here as needed: + # if audio is not None: + # self._add_audio_embeddings(audio, audio_masks, embs, pad_masks, att_masks) + + # Concatenate all embeddings + embs = torch.cat(embs, dim=1) + pad_masks = torch.cat(pad_masks, dim=1) + att_masks = torch.tensor(att_masks, dtype=torch.bool, device=pad_masks.device) + + # Handle prefix length padding + seq_len = pad_masks.shape[1] + if seq_len < self.prefix_length: + embs = pad_tensor(embs, self.prefix_length, pad_value=0) + pad_masks = pad_tensor(pad_masks, self.prefix_length, pad_value=0) + att_masks = pad_tensor(att_masks, self.prefix_length, pad_value=0) + + # Expand attention masks to batch size + bsize = pad_masks.shape[0] + att_masks = att_masks[None, :].expand(bsize, -1) + + return embs, pad_masks, att_masks + + def _add_image_embeddings(self, images, img_masks, embs, pad_masks, att_masks): + """Add image embeddings with special tokens to the lists.""" + for img, img_mask in zip(images, img_masks, strict=False): + # Add image start tokens if enabled + if self.add_image_special_tokens: + start_emb = ( + self.vlm_with_expert.embed_language_tokens( + self.global_image_start_token.to(device=img.device) + ) + .unsqueeze(0) + .expand(img.shape[0], -1, -1) + ) + + start_mask = torch.ones_like(start_emb[:, :, 0], dtype=torch.bool) + embs.append(start_emb) + pad_masks.append(start_mask) + att_masks += [0] * start_emb.shape[1] + + # Process image embedding + img_emb = self.vlm_with_expert.embed_image(img) + + # Normalize image embeddings + img_emb_dim = img_emb.shape[-1] + img_emb = img_emb * torch.tensor(img_emb_dim**0.5, dtype=img_emb.dtype, device=img_emb.device) + + # Expand mask to match image embedding sequence length + bsize, num_img_embs = img_emb.shape[:2] + expanded_mask = img_mask[:, None].expand(bsize, num_img_embs) + + embs.append(img_emb) + pad_masks.append(expanded_mask) + att_masks += [0] * num_img_embs + + # Add image end tokens if enabled + if self.add_image_special_tokens: + end_emb = ( + self.vlm_with_expert.embed_language_tokens(self.image_end_token.to(device=img.device)) + .unsqueeze(0) + .expand(img.shape[0], -1, -1) + ) + + end_mask = torch.ones_like(end_emb[:, :, 0], dtype=torch.bool) + embs.append(end_emb) + pad_masks.append(end_mask) + att_masks += [0] * end_emb.shape[1] + + def _add_language_embeddings(self, lang_tokens, lang_masks, embs, pad_masks, att_masks): + """Add language embeddings to the lists.""" + lang_emb = self.vlm_with_expert.embed_language_tokens(lang_tokens) + + # Normalize language embeddings + lang_emb_dim = lang_emb.shape[-1] + lang_emb = lang_emb * math.sqrt(lang_emb_dim) + + embs.append(lang_emb) + pad_masks.append(lang_masks) + att_masks += [0] * lang_emb.shape[1] + + def _add_state_embeddings(self, state, embs, pad_masks, att_masks): + """Add state embeddings to the lists.""" + state_emb = self.state_proj(state) + state_emb = state_emb[:, None, :] if state_emb.ndim == 2 else state_emb + + bsize, states_seq_len = state_emb.shape[:2] + state_mask = torch.ones(bsize, states_seq_len, dtype=torch.bool, device=state_emb.device) + + embs.append(state_emb) + pad_masks.append(state_mask) + att_masks += [1] * states_seq_len # State tokens get causal attention + + def _add_pointtracker_embeddings(self, pointtrackers, pt_masks, embs, pad_masks, att_masks): + """Add point tracker embeddings to the lists (future extension).""" + # TODO: Implement point tracker processing + # Example implementation: + # for pt, pt_mask in zip(pointtrackers, pt_masks): + # pt_emb = self.pointtracker_encoder(pt) # Need to add this + # embs.append(pt_emb) + # pad_masks.append(pt_mask) + # att_masks += [0] * pt_emb.shape[1] + pass + + def embed_suffix(self, state, noisy_actions, timestep): + """Embed state, noisy_actions, timestep to prepare for Expert Gemma processing.""" + embs = [] + pad_masks = [] + att_masks = [] + # Embed state + if not self.state_to_prefix: + state_emb = self.state_proj(state) + state_emb = ( + state_emb[:, None, :] if state_emb.ndim == 2 else state_emb + ) # .to(dtype=self.vlm_with_expert.type) + embs.append(state_emb) + bsize = state_emb.shape[0] + dtype = state_emb.dtype + device = state_emb.device + + states_seq_len = state_emb.shape[1] + state_mask = torch.ones(bsize, states_seq_len, dtype=torch.bool, device=device) + pad_masks.append(state_mask) + + # Set attention masks so that image and language inputs do not attend to state or actions + att_masks += [1] + [0] * (states_seq_len - 1) + # Fuse timestep + action information using an MLP + action_emb = self.action_in_proj(noisy_actions) + device = action_emb.device + bsize = action_emb.shape[0] + dtype = action_emb.dtype + # Embed timestep using sine-cosine positional encoding with sensitivity in the range [0, 1] + time_emb = create_sinusoidal_pos_embedding( + timestep, + self.vlm_with_expert.expert_hidden_size, + self.config.min_period, + self.config.max_period, + device=device, + ) + time_emb = time_emb.type(dtype=dtype) + + time_emb = time_emb[:, None, :].expand_as(action_emb) + action_time_emb = torch.cat([action_emb, time_emb], dim=2) + + action_time_emb = self.action_time_mlp_in(action_time_emb) + action_time_emb = F.silu(action_time_emb) # swish == silu + action_time_emb = self.action_time_mlp_out(action_time_emb) + + # Add to input tokens + embs.append(action_time_emb) + + bsize, action_time_dim = action_time_emb.shape[:2] + action_time_mask = torch.ones(bsize, action_time_dim, dtype=torch.bool, device=device) + pad_masks.append(action_time_mask) + + # Set attention masks so that image, language and state inputs do not attend to action tokens + att_masks += [1] * self.config.chunk_size + embs = torch.cat(embs, dim=1) + pad_masks = torch.cat(pad_masks, dim=1) + att_masks = torch.tensor(att_masks, dtype=embs.dtype, device=embs.device) + att_masks = att_masks[None, :].expand(bsize, len(att_masks)) + return embs, pad_masks, att_masks + + def forward( + self, images, img_masks, lang_tokens, lang_masks, state, actions, noise=None, time=None + ) -> Tensor: + """Do a full training forward pass and compute the loss (batch_size x num_steps x num_motors)""" + if noise is None: + noise = self.sample_noise(actions.shape, actions.device) + + if time is None: + time = self.sample_time(actions.shape[0], actions.device) + + time_expanded = time[:, None, None] + x_t = time_expanded * noise + (1 - time_expanded) * actions + u_t = noise - actions + prefix_embs, prefix_pad_masks, prefix_att_masks = self.embed_prefix( + images, img_masks, lang_tokens, lang_masks, state=state + ) + suffix_embs, suffix_pad_masks, suffix_att_masks = self.embed_suffix(x_t, time) + + pad_masks = torch.cat([prefix_pad_masks, suffix_pad_masks], dim=1) + att_masks = torch.cat([prefix_att_masks, suffix_att_masks], dim=1) + + att_2d_masks = make_att_2d_masks(pad_masks, att_masks) + position_ids = torch.cumsum(pad_masks, dim=1) - 1 + (_, suffix_out), _ = self.vlm_with_expert.forward( + attention_mask=att_2d_masks, + position_ids=position_ids, + past_key_values=None, + inputs_embeds=[prefix_embs, suffix_embs], + use_cache=False, + fill_kv_cache=False, + ) + suffix_out = suffix_out[:, -self.config.chunk_size :] + # Original openpi code, upcast attention output + suffix_out = suffix_out.to(dtype=torch.float32) + v_t = self.action_out_proj(suffix_out) + losses = F.mse_loss(u_t, v_t, reduction="none") + return losses + + def sample_actions(self, images, img_masks, lang_tokens, lang_masks, state, noise=None) -> Tensor: + """Do a full inference forward and compute the action (batch_size x num_steps x num_motors)""" + bsize = state.shape[0] + device = state.device + + if noise is None: + actions_shape = (bsize, self.config.chunk_size, self.config.max_action_dim) + noise = self.sample_noise(actions_shape, device) + + prefix_embs, prefix_pad_masks, prefix_att_masks = self.embed_prefix( + images, img_masks, lang_tokens, lang_masks, state=state + ) + prefix_att_2d_masks = make_att_2d_masks(prefix_pad_masks, prefix_att_masks) + prefix_position_ids = torch.cumsum(prefix_pad_masks, dim=1) - 1 + # Compute image and language key value cache + _, past_key_values = self.vlm_with_expert.forward( + attention_mask=prefix_att_2d_masks, + position_ids=prefix_position_ids, + past_key_values=None, + inputs_embeds=[prefix_embs, None], + use_cache=self.config.use_cache, + fill_kv_cache=True, + ) + if self.config.regression_loss: + x_t = torch.zeros_like(noise, dtype=torch.float32, device=device) + expanded_time = torch.zeros(bsize, dtype=torch.float32, device=device) + x_t = self.denoise_step( + state, + prefix_pad_masks, + past_key_values, + x_t, + expanded_time, + ) + else: + dt = -1.0 / self.config.num_steps + dt = torch.tensor(dt, dtype=torch.float32, device=device) + + x_t = noise + time = torch.tensor(1.0, dtype=torch.float32, device=device) + while time >= -dt / 2: + expanded_time = time.expand(bsize) + v_t = self.denoise_step( + state, + prefix_pad_masks, + past_key_values, + x_t, + expanded_time, + ) + # Euler step + x_t += dt * v_t + time += dt + return x_t + + def denoise_step( + self, + state, + prefix_pad_masks, + past_key_values, + x_t, + timestep, + ): + """Apply one denoising step of the noise `x_t` at a given timestep.""" + suffix_embs, suffix_pad_masks, suffix_att_masks = self.embed_suffix(state, x_t, timestep) + + suffix_len = suffix_pad_masks.shape[1] + batch_size = prefix_pad_masks.shape[0] + prefix_len = prefix_pad_masks.shape[1] + prefix_pad_2d_masks = prefix_pad_masks[:, None, :].expand(batch_size, suffix_len, prefix_len) + + suffix_att_2d_masks = make_att_2d_masks(suffix_pad_masks, suffix_att_masks) + + full_att_2d_masks = torch.cat([prefix_pad_2d_masks, suffix_att_2d_masks], dim=2) + prefix_offsets = torch.sum(prefix_pad_masks, dim=-1)[:, None] + position_ids = prefix_offsets + torch.cumsum(suffix_pad_masks, dim=1) - 1 + + outputs_embeds, _ = self.vlm_with_expert.forward( + attention_mask=full_att_2d_masks, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=[None, suffix_embs], + use_cache=self.config.use_cache, + fill_kv_cache=False, + ) + suffix_out = outputs_embeds[1] + suffix_out = suffix_out[:, -self.config.chunk_size :] + suffix_out = suffix_out.to(dtype=torch.float32) + v_t = self.action_out_proj(suffix_out) + return v_t diff --git a/lerobot/common/policies/smolvla2/smolvlm_with_expert2.py b/lerobot/common/policies/smolvla2/smolvlm_with_expert2.py new file mode 100644 index 000000000..f1030cc4b --- /dev/null +++ b/lerobot/common/policies/smolvla2/smolvlm_with_expert2.py @@ -0,0 +1,599 @@ +# Copyright 2025 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from typing import List, Optional + +import torch +from torch import nn +from transformers import ( + AutoConfig, + AutoModel, + AutoModelForImageTextToText, + AutoProcessor, + SmolVLMForConditionalGeneration, +) + + +def apply_rope(x, positions, max_wavelength=10_000): + """ + Applies RoPE positions [B, L] to x [B, L, H, D]. + """ + d_half = x.shape[-1] // 2 + device = x.device + dtype = x.dtype + x = x.to(torch.float32) + + freq_exponents = (2.0 / x.shape[-1]) * torch.arange(d_half, dtype=torch.float32, device=device) + timescale = max_wavelength**freq_exponents + radians = positions[..., None].to(torch.float32) / timescale[None, None, :].to(torch.float32) + + radians = radians[..., None, :] + + sin = torch.sin(radians) # .to(dtype=dtype) + cos = torch.cos(radians) # .to(dtype=dtype) + + x1, x2 = x.split(d_half, dim=-1) + res = torch.empty_like(x) + res[..., :d_half] = x1 * cos - x2 * sin + res[..., d_half:] = x2 * cos + x1 * sin + + return res.to(dtype) + + +def get_intermediate_size(hidden_dim, ffn_dim_multiplier=4, multiple_of=256): + hidden_dim = int(2 * hidden_dim / 3) + hidden_dim = int(ffn_dim_multiplier * hidden_dim) + hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of) + return hidden_dim + + +class SmolVLMWithExpertModel(nn.Module): + def __init__( + self, + model_id: str = "HuggingFaceTB/SmolVLM2-500M-Video-Instruct", + load_vlm_weights: bool = True, + train_expert_only: bool = True, + freeze_vision_encoder: bool = False, + attention_mode: str = "self_attn", + num_expert_layers: int = -1, + num_vlm_layers: int = -1, + self_attn_every_n_layers: int = -1, + expert_width_multiplier: float = 0.5, + ): + super().__init__() + if load_vlm_weights: + print(f"Loading {model_id} weights ...") + self.vlm = AutoModelForImageTextToText.from_pretrained( + model_id, + device_map="auto", + torch_dtype="bfloat16", + low_cpu_mem_usage=True, + ) + config = self.vlm.config + else: + config = AutoConfig.from_pretrained(model_id) + self.vlm = SmolVLMForConditionalGeneration(config=config) + self.processor = AutoProcessor.from_pretrained(model_id) + if num_vlm_layers > 0: + print(f"Reducing the number of VLM layers to {num_vlm_layers} ...") + self.get_vlm_model().text_model.layers = self.get_vlm_model().text_model.layers[:num_vlm_layers] + self.num_vlm_layers = len(self.get_vlm_model().text_model.layers) + self.config = config + # Smaller lm expert + lm_expert_config = copy.deepcopy(config.text_config) + hidden_size = lm_expert_config.hidden_size + lm_expert_config.hidden_size = int(hidden_size * expert_width_multiplier) # hidden_size // 2 + lm_expert_config.intermediate_size = get_intermediate_size(int(hidden_size * expert_width_multiplier)) + lm_expert_config.num_hidden_layers = self.num_vlm_layers + if num_expert_layers > 0: + assert len(self.get_vlm_model().text_model.layers) % num_expert_layers == 0, ( + f"Number of layers in the VLM {len(self.get_vlm_model().text_model.layers)} are not multiple of num_expert_layers {num_expert_layers}" + ) + lm_expert_config.num_hidden_layers = num_expert_layers + self.lm_expert = AutoModel.from_config(lm_expert_config) + + self.num_expert_layers = len(self.lm_expert.layers) + self.self_attn_every_n_layers = self_attn_every_n_layers + if "cross" in attention_mode: + # Reshape qkv projections to have the same input dimension as the vlm + for layer_idx in range(len(self.lm_expert.layers)): + if self.self_attn_every_n_layers > 0 and layer_idx % self.self_attn_every_n_layers == 0: + continue + self.lm_expert.layers[layer_idx].self_attn.k_proj = nn.Linear( + config.text_config.num_key_value_heads * config.text_config.head_dim, + lm_expert_config.num_key_value_heads * lm_expert_config.head_dim, + bias=lm_expert_config.attention_bias, + ) + self.lm_expert.layers[layer_idx].self_attn.v_proj = nn.Linear( + config.text_config.num_key_value_heads * config.text_config.head_dim, + lm_expert_config.num_key_value_heads * lm_expert_config.head_dim, + bias=lm_expert_config.attention_bias, + ) + # Remove unused embed_tokens + self.lm_expert.embed_tokens = None + + self.num_attention_heads = self.config.text_config.num_attention_heads + self.num_key_value_heads = self.config.text_config.num_key_value_heads + + self.freeze_vision_encoder = freeze_vision_encoder + self.train_expert_only = train_expert_only + self.attention_mode = attention_mode + self.expert_hidden_size = lm_expert_config.hidden_size + self.set_requires_grad() + + def configure_peft(self, config): + # return model + self.peft_method = config.peft_method + self.peft_target_model = config.peft_target_model + if "lora" in self.peft_method: + peft_config = config.peft_config + target_modules = peft_config.target_modules + if not isinstance(target_modules, list): + target_modules = target_modules.split(",") + lora_config = LoraConfig( + task_type=TaskType.CAUSAL_LM, # Based on the task type (e.g., language modeling, etc.) + r=peft_config.r, # The rank of the low-rank adaptation + lora_alpha=peft_config.lora_alpha, # Scaling factor + lora_dropout=peft_config.lora_dropout, # Dropout applied to LoRA layers + target_modules=target_modules, # The components where LoRA is applied + exclude_modules=[ + "lm_expert", + "model.lm_expert.model.layers", + ], # FIXME(mshukor): this does not work for now + ) + self.lora_config = lora_config + # Apply LoRA and ensure only LoRA parameters are trainable + if "text" in self.peft_target_model: + self.get_vlm_model().text_model = get_peft_model(self.get_vlm_model().text_model, lora_config) + else: + self.vlm = get_peft_model(self.vlm, lora_config) + # assert config.train_expert_only, "Backbone should be frozen and only lora parameters are " # FIXME(mshukor): handle this here? + for name, param in self.vlm.named_parameters(): + if ( + "lora" in name and "text_model.model.layers.17" not in name + ): # lm_head is not a parameter in most LLMs becasue it's tied to the embedding layer + param.requires_grad = True + else: + param.requires_grad = False + + def merge_lora_weights(self): + """ + Merge LoRA weights into the base model. + """ + if "text" in self.peft_target_model: + self.get_vlm_model().text_model = self.get_vlm_model().text_model.merge_and_unload() + else: + self.vlm = self.vlm.merge_and_unload() + + def get_vlm_model( + self, + ): + if hasattr(self.vlm.model, "model"): # When using peft + return self.vlm.model.model + else: + return self.vlm.model + + def set_requires_grad(self): + if self.freeze_vision_encoder: + self.get_vlm_model().vision_model.eval() + for params in self.get_vlm_model().vision_model.parameters(): + params.requires_grad = False + if self.train_expert_only: + self.vlm.eval() + for params in self.vlm.parameters(): + params.requires_grad = False + else: + # To avoid unused params issue with distributed training + last_layers = [self.num_vlm_layers - 1] + if ( + self.num_vlm_layers != self.num_expert_layers + and self.num_vlm_layers % self.num_expert_layers == 0 + ): + last_layers.append(self.num_vlm_layers - 2) + frozen_layers = [ + "lm_head", + "text_model.model.norm.weight", + ] + for layer in last_layers: + frozen_layers.append(f"text_model.model.layers.{layer}.") + + for name, params in self.vlm.named_parameters(): + if any(k in name for k in frozen_layers): + params.requires_grad = False + # To avoid unused params issue with distributed training + for name, params in self.lm_expert.named_parameters(): + if "lm_head" in name: + params.requires_grad = False + + def train(self, mode: bool = True): + super().train(mode) + + if self.freeze_vision_encoder: + self.get_vlm_model().vision_model.eval() + + if self.train_expert_only: + self.vlm.eval() + + def embed_image(self, image: torch.Tensor): + patch_attention_mask = None + # Get sequence from the vision encoder + image_hidden_states = ( + self.get_vlm_model() + .vision_model( + pixel_values=image.to(dtype=self.get_vlm_model().vision_model.dtype), + patch_attention_mask=patch_attention_mask, + ) + .last_hidden_state + ) + # Modality projection & resampling + image_hidden_states = self.get_vlm_model().connector(image_hidden_states) + return image_hidden_states + + def embed_language_tokens(self, tokens: torch.Tensor): + return self.get_vlm_model().text_model.get_input_embeddings()(tokens) + + def forward_attn_layer( + self, + model_layers, + inputs_embeds, + layer_idx, + position_ids, + attention_mask, + batch_size, + head_dim, + use_cache: bool = True, + fill_kv_cache: bool = True, + past_key_values=None, + ) -> list[torch.Tensor]: + query_states = [] + key_states = [] + value_states = [] + for i, hidden_states in enumerate(inputs_embeds): + layer = model_layers[i][layer_idx] + if hidden_states is None or layer is None: + continue + hidden_states = layer.input_layernorm(hidden_states) + + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, layer.self_attn.head_dim) + + hidden_states = hidden_states.to(dtype=layer.self_attn.q_proj.weight.dtype) + query_state = layer.self_attn.q_proj(hidden_states).view(hidden_shape) + key_state = layer.self_attn.k_proj(hidden_states).view(hidden_shape) + value_state = layer.self_attn.v_proj(hidden_states).view(hidden_shape) + + query_states.append(query_state) + key_states.append(key_state) + value_states.append(value_state) + + # B,L,H,D with L sequence length, H number of heads, D head dim + # concatenate on the number of embeddings/tokens + query_states = torch.cat(query_states, dim=1) + key_states = torch.cat(key_states, dim=1) + value_states = torch.cat(value_states, dim=1) + seq_len = query_states.shape[1] + if seq_len < position_ids.shape[1]: + _position_ids = position_ids[:, :seq_len] + _attention_mask = attention_mask[:, :seq_len, :seq_len] + else: + _position_ids = position_ids + _attention_mask = attention_mask + + attention_mask_ = _attention_mask + position_ids_ = _position_ids + + query_states = apply_rope(query_states, position_ids_) + key_states = apply_rope(key_states, position_ids_) + + if use_cache and past_key_values is None: + past_key_values = {} + + if use_cache: + if fill_kv_cache: + past_key_values[layer_idx] = { + "key_states": key_states, + "value_states": value_states, + } + else: + # TODO here, some optimization can be done - similar to a `StaticCache` we can declare the `max_len` before. + # so we create an empty cache, with just one cuda malloc, and if (in autoregressive case) we reach + # the max len, then we (for instance) double the cache size. This implementation already exists + # in `transformers`. (molbap) + key_states = torch.cat([past_key_values[layer_idx]["key_states"], key_states], dim=1) + value_states = torch.cat([past_key_values[layer_idx]["value_states"], value_states], dim=1) + + attention_interface = self.get_attention_interface() + + att_output = attention_interface( + attention_mask_, batch_size, head_dim, query_states, key_states, value_states + ) + return [att_output], past_key_values + + def forward_cross_attn_layer( + self, + model_layers, + inputs_embeds, + layer_idx, + position_ids, + attention_mask, + batch_size, + head_dim, + use_cache: bool = True, + fill_kv_cache: bool = True, + past_key_values=None, + ) -> list[torch.Tensor]: + attention_interface = self.get_attention_interface() + + att_outputs = [] + assert len(inputs_embeds) == 2 or (use_cache and past_key_values is not None and not fill_kv_cache), ( + f"Both len(inputs_embeds) == {len(inputs_embeds)} and past_key_values is {past_key_values}" + ) + + if len(inputs_embeds) == 2 and not past_key_values: + # Prefix attention + seq_len = inputs_embeds[0].shape[1] + position_id, expert_position_id = position_ids[:, :seq_len], position_ids[:, seq_len:] + prefix_attention_mask = attention_mask[:, :seq_len, :seq_len] + + layer = model_layers[0][layer_idx] + + hidden_states = layer.input_layernorm(inputs_embeds[0]) + + input_shape = hidden_states.shape[:-1] + hidden_shape = (*input_shape, -1, layer.self_attn.head_dim) + + hidden_states = hidden_states.to(dtype=layer.self_attn.q_proj.weight.dtype) + query_state = layer.self_attn.q_proj(hidden_states).view(hidden_shape) + key_state = layer.self_attn.k_proj(hidden_states).view(hidden_shape) + value_states = layer.self_attn.v_proj(hidden_states).view(hidden_shape) + + # B,L,H,D with L sequence length, H number of heads, D head dim + query_states = apply_rope(query_state, position_id) + key_states = apply_rope(key_state, position_id) + + att_output = attention_interface( + prefix_attention_mask, batch_size, head_dim, query_states, key_states, value_states + ) + att_outputs.append(att_output) + else: + expert_position_id = position_ids + + if use_cache and past_key_values is None: + past_key_values = {} + + if use_cache: + if fill_kv_cache: + past_key_values[layer_idx] = { + "key_states": key_states, + "value_states": value_states, + } + else: + # TODO here, some optimization can be done - similar to a `StaticCache` we can declare the `max_len` before. + # so we create an empty cache, with just one cuda malloc, and if (in autoregressive case) we reach + # the max len, then we (for instance) double the cache size. This implementation already exists + # in `transformers`. (molbap) + key_states = past_key_values[layer_idx]["key_states"] + value_states = past_key_values[layer_idx]["value_states"] + + # Expert + expert_layer = model_layers[1][layer_idx] + if expert_layer is not None: + expert_hidden_states = expert_layer.input_layernorm(inputs_embeds[1]) + + expert_input_shape = expert_hidden_states.shape[:-1] + expert_hidden_shape = (*expert_input_shape, -1, expert_layer.self_attn.head_dim) + + expert_hidden_states = expert_hidden_states.to(dtype=expert_layer.self_attn.q_proj.weight.dtype) + expert_query_state = expert_layer.self_attn.q_proj(expert_hidden_states).view(expert_hidden_shape) + + _key_states = key_states.to(dtype=expert_layer.self_attn.k_proj.weight.dtype).view( + *key_states.shape[:2], -1 + ) + expert_key_states = expert_layer.self_attn.k_proj(_key_states).view( + *_key_states.shape[:-1], -1, expert_layer.self_attn.head_dim + ) # k_proj should have same dim as kv + + _value_states = value_states.to(dtype=expert_layer.self_attn.v_proj.weight.dtype).view( + *value_states.shape[:2], -1 + ) + expert_value_states = expert_layer.self_attn.v_proj(_value_states).view( + *_value_states.shape[:-1], -1, expert_layer.self_attn.head_dim + ) + + expert_position_id = ( + expert_position_id - torch.min(expert_position_id, dim=1, keepdim=True).values + ) # start from 0 + expert_attention_mask = attention_mask[ + :, -inputs_embeds[1].shape[1] :, : expert_key_states.shape[1] : + ] # take into account kv + + expert_query_states = apply_rope(expert_query_state, expert_position_id) + + att_output = attention_interface( + expert_attention_mask, + batch_size, + head_dim, + expert_query_states, + expert_key_states, + expert_value_states, + ) + att_outputs.append(att_output) + else: + att_outputs.append(None) + + # att_output = att_output.to(dtype=models[i].dtype) + return att_outputs, past_key_values + + def get_model_layers(self, models: list) -> list: + vlm_layers = [] + expert_layers = [] + multiple_of = self.num_vlm_layers // self.num_expert_layers + for i in range(self.num_vlm_layers): + if multiple_of > 0 and i > 0 and i % multiple_of != 0: + expert_layer = None + else: + expert_layer_index = i // multiple_of if multiple_of > 0 else i + expert_layer = models[1].layers[expert_layer_index] + vlm_layers.append(models[0].layers[i]) + expert_layers.append(expert_layer) + return [vlm_layers, expert_layers] + + def forward( + self, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: List[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + fill_kv_cache: Optional[bool] = None, + ): + models = [self.get_vlm_model().text_model, self.lm_expert] + model_layers = self.get_model_layers(models) + for hidden_states in inputs_embeds: + # TODO this is very inefficient + # dtype is always the same, batch size too (if > 1 len) + # device could be trickier in multi gpu edge cases but that's it + if hidden_states is None: + continue + batch_size = hidden_states.shape[0] + + # RMSNorm + num_layers = self.num_vlm_layers + head_dim = self.vlm.config.text_config.head_dim + for layer_idx in range(num_layers): + if ( + fill_kv_cache + or "cross" not in self.attention_mode + or (self.self_attn_every_n_layers > 0 and layer_idx % self.self_attn_every_n_layers == 0) + ): + att_outputs, past_key_values = self.forward_attn_layer( + model_layers, + inputs_embeds, + layer_idx, + position_ids, + attention_mask, + batch_size, + head_dim, + use_cache=use_cache, + fill_kv_cache=fill_kv_cache, + past_key_values=past_key_values, + ) + else: + att_outputs, past_key_values = self.forward_cross_attn_layer( + model_layers, + inputs_embeds, + layer_idx, + position_ids, + attention_mask, + batch_size, + head_dim, + use_cache=use_cache, + fill_kv_cache=fill_kv_cache, + past_key_values=past_key_values, + ) + outputs_embeds = [] + start = 0 + for i, hidden_states in enumerate(inputs_embeds): + layer = model_layers[i][layer_idx] + att_output = ( + att_outputs[i] if i < len(att_outputs) else att_outputs[0] + ) # in case of self_attn + if hidden_states is not None: + if layer is None: + outputs_embeds.append(hidden_states) + continue + end = start + hidden_states.shape[1] + + if att_output.dtype != layer.self_attn.o_proj.weight.dtype: + att_output = att_output.to(layer.self_attn.o_proj.weight.dtype) + att_out = att_output[:, start:end] + out_emb = layer.self_attn.o_proj(att_out) + + out_emb += hidden_states + after_first_residual = out_emb.clone() + + out_emb = layer.post_attention_layernorm(out_emb) + out_emb = layer.mlp(out_emb) + + out_emb += after_first_residual + + outputs_embeds.append(out_emb) + + start = end if len(att_outputs) == 1 else 0 + else: + outputs_embeds.append(None) + + inputs_embeds = outputs_embeds + + # final norm + outputs_embeds = [] + for i, hidden_states in enumerate(inputs_embeds): + if hidden_states is not None: + out_emb = models[i].norm(hidden_states) + outputs_embeds.append(out_emb) + else: + outputs_embeds.append(None) + return outputs_embeds, past_key_values + + def get_attention_interface(self): + attention_interface = self.eager_attention_forward + return attention_interface + + def eager_attention_forward( + self, attention_mask, batch_size, head_dim, query_states, key_states, value_states + ): + num_att_heads = self.num_attention_heads + num_key_value_heads = self.num_key_value_heads + num_key_value_groups = num_att_heads // num_key_value_heads + + sequence_length = key_states.shape[1] + + key_states = key_states[:, :, :, None, :].expand( + batch_size, sequence_length, num_key_value_heads, num_key_value_groups, head_dim + ) + key_states = key_states.reshape( + batch_size, sequence_length, num_key_value_heads * num_key_value_groups, head_dim + ) + + value_states = value_states[:, :, :, None, :].expand( + batch_size, sequence_length, num_key_value_heads, num_key_value_groups, head_dim + ) + value_states = value_states.reshape( + batch_size, sequence_length, num_key_value_heads * num_key_value_groups, head_dim + ) + + # Attention here is upcasted to float32 to match the original eager implementation. + query_states = query_states.to(dtype=torch.float32) + key_states = key_states.to(dtype=torch.float32) + + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + + att_weights = torch.matmul(query_states, key_states.transpose(2, 3)) + att_weights *= head_dim**-0.5 + + att_weights = att_weights.to(dtype=torch.float32) + big_neg = torch.finfo(att_weights.dtype).min # -2.3819763e38 # See gemma/modules.py + masked_att_weights = torch.where(attention_mask[:, None, :, :], att_weights, big_neg) + probs = nn.functional.softmax(masked_att_weights, dim=-1) + probs = probs.to(dtype=value_states.dtype) + + att_output = torch.matmul(probs, value_states.permute(0, 2, 1, 3)) + + att_output = att_output.permute(0, 2, 1, 3) + # we use -1 because sequence length can change + att_output = att_output.reshape(batch_size, -1, num_key_value_heads * num_key_value_groups * head_dim) + + return att_output