mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-26 03:06:01 +00:00
chore: Merge origin/main into streaming branch
This commit is contained in:
@@ -83,7 +83,7 @@ episode_index=0
|
|||||||
print(f"{dataset[episode_index]['action'].shape=}\n")
|
print(f"{dataset[episode_index]['action'].shape=}\n")
|
||||||
```
|
```
|
||||||
|
|
||||||
Learn more about it in the [LeRobotDataset Documentation](https://huggingface.co/docs/lerobot/lerobot-dataset-v3)
|
Learn more about it in the [LeRobotDataset Documentation](https://huggingface.co/docs/lerobot/lerobot-dataset-v3).
|
||||||
|
|
||||||
## SoTA Models
|
## SoTA Models
|
||||||
|
|
||||||
@@ -109,7 +109,7 @@ lerobot-train \
|
|||||||
| **World Models** | [VLA-JEPA](./docs/source/vla_jepa.mdx), [LingBot-VA](./docs/source/lingbot_va.mdx), [FastWAM](./docs/source/fastwam.mdx) |
|
| **World Models** | [VLA-JEPA](./docs/source/vla_jepa.mdx), [LingBot-VA](./docs/source/lingbot_va.mdx), [FastWAM](./docs/source/fastwam.mdx) |
|
||||||
| **Reward Models** | [SARM](./docs/source/sarm.mdx), [TOPReward](./docs/source/topreward.mdx), [Robometer](./docs/source/robometer.mdx) |
|
| **Reward Models** | [SARM](./docs/source/sarm.mdx), [TOPReward](./docs/source/topreward.mdx), [Robometer](./docs/source/robometer.mdx) |
|
||||||
|
|
||||||
Similarly to the hardware, you can easily implement your own policy & leverage LeRobot's data collection, training, and visualization tools, and share your model to the HF Hub
|
Similarly to the hardware, you can easily implement your own policy & leverage LeRobot's data collection, training, and visualization tools, and share your model to the HF Hub.
|
||||||
|
|
||||||
For detailed policy setup guides, see the [Policy Documentation](https://huggingface.co/docs/lerobot/bring_your_own_policies). For GPU/RAM requirements and expected training time per policy, see the [Compute Hardware Guide](https://huggingface.co/docs/lerobot/hardware_guide).
|
For detailed policy setup guides, see the [Policy Documentation](https://huggingface.co/docs/lerobot/bring_your_own_policies). For GPU/RAM requirements and expected training time per policy, see the [Compute Hardware Guide](https://huggingface.co/docs/lerobot/hardware_guide).
|
||||||
|
|
||||||
@@ -126,7 +126,7 @@ lerobot-eval \
|
|||||||
--eval.n_episodes=10
|
--eval.n_episodes=10
|
||||||
```
|
```
|
||||||
|
|
||||||
Learn how to implement your own simulation environment or benchmark and distribute it from the HF Hub by following the [EnvHub Documentation](https://huggingface.co/docs/lerobot/envhub)
|
Learn how to implement your own simulation environment or benchmark and distribute it from the HF Hub by following the [EnvHub Documentation](https://huggingface.co/docs/lerobot/envhub).
|
||||||
|
|
||||||
## Resources
|
## Resources
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,11 @@
|
|||||||
|
# OMX
|
||||||
|
|
||||||
|
<img
|
||||||
|
src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/lerobot/omx_mainimage.png"
|
||||||
|
alt="OMX"
|
||||||
|
width=600
|
||||||
|
/>
|
||||||
|
|
||||||
## Order and Assemble the parts
|
## Order and Assemble the parts
|
||||||
|
|
||||||
First, assemble the OMX hardware following the official assembly guide.
|
First, assemble the OMX hardware following the official assembly guide.
|
||||||
|
|||||||
@@ -524,8 +524,6 @@ class PI05Pytorch(nn.Module): # see openpi `PI0Pytorch`
|
|||||||
|
|
||||||
def embed_suffix(self, noisy_actions, timestep):
|
def embed_suffix(self, noisy_actions, timestep):
|
||||||
"""Embed noisy_actions, timestep to prepare for Expert Gemma processing."""
|
"""Embed noisy_actions, timestep to prepare for Expert Gemma processing."""
|
||||||
embs = []
|
|
||||||
pad_masks = []
|
|
||||||
att_masks = []
|
att_masks = []
|
||||||
|
|
||||||
# Embed timestep using sine-cosine positional encoding
|
# Embed timestep using sine-cosine positional encoding
|
||||||
@@ -551,23 +549,17 @@ class PI05Pytorch(nn.Module): # see openpi `PI0Pytorch`
|
|||||||
return F.silu(x)
|
return F.silu(x)
|
||||||
|
|
||||||
time_emb = self._apply_checkpoint(time_mlp_func, time_emb)
|
time_emb = self._apply_checkpoint(time_mlp_func, time_emb)
|
||||||
action_time_emb = action_emb
|
|
||||||
adarms_cond = time_emb
|
adarms_cond = time_emb
|
||||||
|
|
||||||
embs.append(action_time_emb)
|
bsize, action_time_dim = action_emb.shape[:2]
|
||||||
bsize, action_time_dim = action_time_emb.shape[:2]
|
pad_masks = torch.ones(bsize, action_time_dim, dtype=torch.bool, device=timestep.device)
|
||||||
action_time_mask = torch.ones(bsize, action_time_dim, dtype=torch.bool, device=timestep.device)
|
|
||||||
pad_masks.append(action_time_mask)
|
|
||||||
|
|
||||||
# Set attention masks so that image, language and state inputs do not attend to action tokens
|
# Set attention masks so that image, language and state inputs do not attend to action tokens
|
||||||
att_masks += [1] + ([0] * (self.config.chunk_size - 1))
|
att_masks += [1] + ([0] * (self.config.chunk_size - 1))
|
||||||
|
att_masks = torch.tensor(att_masks, dtype=action_emb.dtype, device=action_emb.device)
|
||||||
embs = torch.cat(embs, dim=1)
|
|
||||||
pad_masks = torch.cat(pad_masks, dim=1)
|
|
||||||
att_masks = torch.tensor(att_masks, dtype=embs.dtype, device=embs.device)
|
|
||||||
att_masks = att_masks[None, :].expand(bsize, len(att_masks))
|
att_masks = att_masks[None, :].expand(bsize, len(att_masks))
|
||||||
|
|
||||||
return embs, pad_masks, att_masks, adarms_cond
|
return action_emb, pad_masks, att_masks, adarms_cond
|
||||||
|
|
||||||
def forward(self, images, img_masks, tokens, masks, actions, noise, time) -> Tensor:
|
def forward(self, images, img_masks, tokens, masks, actions, noise, time) -> Tensor:
|
||||||
"""Do a full training forward pass and compute the loss."""
|
"""Do a full training forward pass and compute the loss."""
|
||||||
|
|||||||
Reference in New Issue
Block a user