mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-22 17:32:07 +00:00
minor fixes
This commit is contained in:
@@ -14,6 +14,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
import logging
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -42,6 +43,9 @@ else:
|
|||||||
Timesteps = None
|
Timesteps = None
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class TimestepEncoder(nn.Module):
|
class TimestepEncoder(nn.Module):
|
||||||
def __init__(self, embedding_dim, compute_dtype=torch.float32):
|
def __init__(self, embedding_dim, compute_dtype=torch.float32):
|
||||||
require_package("diffusers", extra="groot")
|
require_package("diffusers", extra="groot")
|
||||||
@@ -265,8 +269,8 @@ class DiT(ModelMixin, ConfigMixin):
|
|||||||
self.norm_out = nn.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6)
|
self.norm_out = nn.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6)
|
||||||
self.proj_out_1 = nn.Linear(self.inner_dim, 2 * self.inner_dim)
|
self.proj_out_1 = nn.Linear(self.inner_dim, 2 * self.inner_dim)
|
||||||
self.proj_out_2 = nn.Linear(self.inner_dim, self.config.output_dim)
|
self.proj_out_2 = nn.Linear(self.inner_dim, self.config.output_dim)
|
||||||
print(
|
logger.debug(
|
||||||
"Total number of DiT parameters: ",
|
"Total number of DiT parameters: %d",
|
||||||
sum(p.numel() for p in self.parameters() if p.requires_grad),
|
sum(p.numel() for p in self.parameters() if p.requires_grad),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -426,8 +430,8 @@ class SelfAttentionTransformer(ModelMixin, ConfigMixin):
|
|||||||
for _ in range(self.config.num_layers)
|
for _ in range(self.config.num_layers)
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
print(
|
logger.debug(
|
||||||
"Total number of SelfAttentionTransformer parameters: ",
|
"Total number of SelfAttentionTransformer parameters: %d",
|
||||||
sum(p.numel() for p in self.parameters() if p.requires_grad),
|
sum(p.numel() for p in self.parameters() if p.requires_grad),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -352,12 +352,16 @@ class GrootConfig(PreTrainedConfig):
|
|||||||
# Maximum action dimension. Shorter actions will be zero-padded.
|
# Maximum action dimension. Shorter actions will be zero-padded.
|
||||||
max_action_dim: int = 132
|
max_action_dim: int = 132
|
||||||
|
|
||||||
# Normalization (start with identity, adjust as needed)
|
# GR00T normalizes state/action internally in its processor steps (min/max with
|
||||||
|
# q01/q99 percentiles, per embodiment), and the Qwen3-VL backbone's image processor
|
||||||
|
# handles image normalization. The policy therefore does NOT use LeRobot's
|
||||||
|
# NormalizerProcessorStep/UnnormalizerProcessorStep, so this mapping is intentionally
|
||||||
|
# IDENTITY for every feature and is not consulted by make_groot_pre_post_processors.
|
||||||
normalization_mapping: dict[str, NormalizationMode] = field(
|
normalization_mapping: dict[str, NormalizationMode] = field(
|
||||||
default_factory=lambda: {
|
default_factory=lambda: {
|
||||||
"VISUAL": NormalizationMode.IDENTITY,
|
"VISUAL": NormalizationMode.IDENTITY,
|
||||||
"STATE": NormalizationMode.MEAN_STD,
|
"STATE": NormalizationMode.IDENTITY,
|
||||||
"ACTION": NormalizationMode.MEAN_STD,
|
"ACTION": NormalizationMode.IDENTITY,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -578,11 +582,22 @@ class GrootConfig(PreTrainedConfig):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def action_delta_indices(self) -> list[int]:
|
def action_delta_indices(self) -> list[int]:
|
||||||
"""Return indices for delta actions."""
|
"""Return indices for delta actions.
|
||||||
|
|
||||||
|
The model action horizon is read from the checkpoint's processor_config.json
|
||||||
|
when available; the result is cached (keyed on the inputs that determine it) so
|
||||||
|
repeated access during dataset/training setup does not re-read from disk.
|
||||||
|
"""
|
||||||
|
cache_key = (self.base_model_path, self.embodiment_tag, self.chunk_size)
|
||||||
|
cached = getattr(self, "_action_delta_indices_cache", None)
|
||||||
|
if cached is not None and cached[0] == cache_key:
|
||||||
|
return cached[1]
|
||||||
model_action_horizon = (
|
model_action_horizon = (
|
||||||
infer_groot_n1_7_action_horizon(self.base_model_path, self.embodiment_tag) or 40
|
infer_groot_n1_7_action_horizon(self.base_model_path, self.embodiment_tag) or 40
|
||||||
)
|
)
|
||||||
return list(range(min(self.chunk_size, model_action_horizon)))
|
indices = list(range(min(self.chunk_size, model_action_horizon)))
|
||||||
|
object.__setattr__(self, "_action_delta_indices_cache", (cache_key, indices))
|
||||||
|
return indices
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def reward_delta_indices(self) -> None:
|
def reward_delta_indices(self) -> None:
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ GR00T_N1_7_DEFAULTS: dict[str, Any] = {
|
|||||||
"backbone_embedding_dim": 2048,
|
"backbone_embedding_dim": 2048,
|
||||||
"tune_llm": False,
|
"tune_llm": False,
|
||||||
"tune_visual": False,
|
"tune_visual": False,
|
||||||
"select_layer": 12,
|
"select_layer": 16, # N1.7-3B checkpoint value; real checkpoint loads override this from config.json
|
||||||
"reproject_vision": False,
|
"reproject_vision": False,
|
||||||
"use_flash_attention": True,
|
"use_flash_attention": True,
|
||||||
"load_bf16": False,
|
"load_bf16": False,
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ orchestration are handled by LeRobot's standard training stack.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import builtins
|
import builtins
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
from collections import deque
|
from collections import deque
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -48,6 +49,8 @@ from .configuration_groot import (
|
|||||||
normalize_groot_model_version,
|
normalize_groot_model_version,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
T = TypeVar("T", bound="GrootPolicy")
|
T = TypeVar("T", bound="GrootPolicy")
|
||||||
|
|
||||||
|
|
||||||
@@ -149,9 +152,10 @@ class GrootPolicy(PreTrainedPolicy):
|
|||||||
if config is not None
|
if config is not None
|
||||||
else infer_groot_model_version(str(pretrained_name_or_path)) or GROOT_N1_7
|
else infer_groot_model_version(str(pretrained_name_or_path)) or GROOT_N1_7
|
||||||
)
|
)
|
||||||
print(
|
logger.info(
|
||||||
f"The Groot policy is a wrapper around Nvidia's GR00T {requested_version} model.\n"
|
"The Groot policy wraps NVIDIA's GR00T %s model. Loading pretrained model from: %s",
|
||||||
f"Loading pretrained model from: {pretrained_name_or_path}"
|
requested_version,
|
||||||
|
pretrained_name_or_path,
|
||||||
)
|
)
|
||||||
|
|
||||||
model_id = str(pretrained_name_or_path)
|
model_id = str(pretrained_name_or_path)
|
||||||
@@ -182,7 +186,7 @@ class GrootPolicy(PreTrainedPolicy):
|
|||||||
|
|
||||||
if is_finetuned_checkpoint:
|
if is_finetuned_checkpoint:
|
||||||
# This is a fine-tuned LeRobot checkpoint - use parent class loading
|
# This is a fine-tuned LeRobot checkpoint - use parent class loading
|
||||||
print("Detected fine-tuned LeRobot checkpoint, loading with state dict...")
|
logger.info("Detected fine-tuned LeRobot checkpoint, loading with state dict...")
|
||||||
return super().from_pretrained(
|
return super().from_pretrained(
|
||||||
pretrained_name_or_path=pretrained_name_or_path,
|
pretrained_name_or_path=pretrained_name_or_path,
|
||||||
config=config,
|
config=config,
|
||||||
@@ -198,7 +202,7 @@ class GrootPolicy(PreTrainedPolicy):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# This is a base GR00T model - load it fresh
|
# This is a base GR00T model - load it fresh
|
||||||
print("Detected base GR00T model, loading from HuggingFace...")
|
logger.info("Detected base GR00T model, loading from HuggingFace...")
|
||||||
|
|
||||||
if config is None:
|
if config is None:
|
||||||
model_version = infer_groot_model_version(str(pretrained_name_or_path)) or GROOT_N1_7
|
model_version = infer_groot_model_version(str(pretrained_name_or_path)) or GROOT_N1_7
|
||||||
@@ -409,6 +413,11 @@ class GrootPolicy(PreTrainedPolicy):
|
|||||||
|
|
||||||
# Isaac-GR00T returns a BatchFeature; loss key is typically 'loss'
|
# Isaac-GR00T returns a BatchFeature; loss key is typically 'loss'
|
||||||
loss = outputs.get("loss")
|
loss = outputs.get("loss")
|
||||||
|
if loss is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"GR00T model.forward did not return a 'loss'. Training batches must include "
|
||||||
|
"'action' and 'action_mask'; check the preprocessor output."
|
||||||
|
)
|
||||||
|
|
||||||
loss_dict = {"loss": loss.item()}
|
loss_dict = {"loss": loss.item()}
|
||||||
|
|
||||||
@@ -471,33 +480,21 @@ class GrootPolicy(PreTrainedPolicy):
|
|||||||
# Internal helpers
|
# Internal helpers
|
||||||
# -------------------------
|
# -------------------------
|
||||||
def _handle_flash_attention_compatibility(self) -> None:
|
def _handle_flash_attention_compatibility(self) -> None:
|
||||||
"""Handle Flash Attention compatibility issues by setting environment variables.
|
"""Log Flash Attention availability (diagnostic only).
|
||||||
|
|
||||||
This addresses the common 'undefined symbol' error that occurs when Flash Attention
|
The GR00T N1.7 backbone automatically falls back to SDPA when ``flash_attn`` is
|
||||||
is compiled against a different PyTorch version than what's currently installed.
|
unavailable (see ``Qwen3Backbone``), so this probe only emits a hint; it does not
|
||||||
|
change behaviour or mutate global state.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Set environment variables to handle Flash Attention compatibility
|
|
||||||
# These help with symbol resolution issues
|
|
||||||
os.environ.setdefault("FLASH_ATTENTION_FORCE_BUILD", "0")
|
|
||||||
os.environ.setdefault("FLASH_ATTENTION_SKIP_CUDA_BUILD", "0")
|
|
||||||
|
|
||||||
# Try to import flash_attn and handle failures gracefully
|
|
||||||
try:
|
try:
|
||||||
import flash_attn
|
import flash_attn
|
||||||
|
|
||||||
print(f"[GROOT] Flash Attention version: {flash_attn.__version__}")
|
logger.debug("Flash Attention %s is available.", flash_attn.__version__)
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
print(f"[GROOT] Flash Attention not available: {e}")
|
logger.debug("Flash Attention is not installed; the GR00T backbone will use SDPA.")
|
||||||
print("[GROOT] Will use fallback attention mechanism")
|
except Exception as e: # noqa: BLE001
|
||||||
except Exception as e:
|
logger.warning(
|
||||||
if "undefined symbol" in str(e):
|
"Flash Attention failed to import (%s); the GR00T backbone will use SDPA. If this is "
|
||||||
print(f"[GROOT] Flash Attention compatibility issue detected: {e}")
|
"an 'undefined symbol' error, reinstall a flash-attn build matching your torch version.",
|
||||||
print("[GROOT] This is likely due to PyTorch/Flash Attention version mismatch")
|
e,
|
||||||
print("[GROOT] Consider reinstalling Flash Attention with compatible version:")
|
)
|
||||||
print(" pip uninstall flash-attn")
|
|
||||||
print(" pip install --no-build-isolation flash-attn==2.6.3")
|
|
||||||
print("[GROOT] Continuing with fallback attention mechanism")
|
|
||||||
else:
|
|
||||||
print(f"[GROOT] Flash Attention error: {e}")
|
|
||||||
print("[GROOT] Continuing with fallback attention mechanism")
|
|
||||||
|
|||||||
Reference in New Issue
Block a user