diff --git a/benchmarks/g05_checkpoint_parity.py b/benchmarks/g05_checkpoint_parity.py deleted file mode 100644 index 58775a1c5..000000000 --- a/benchmarks/g05_checkpoint_parity.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. - -"""Compare a converted G0.5 checkpoint with the pinned author implementation. - -This is intentionally an opt-in checkpoint test: it requires an accepted gated -checkpoint, the pinned GalaxeaVLA source checkout, and its CUDA dependencies. -""" - -from __future__ import annotations - -import argparse -import copy -import json -import sys -from pathlib import Path -from typing import Any - -import torch - -from lerobot.configs.policies import PreTrainedConfig -from lerobot.policies.factory import make_pre_post_processors -from lerobot.policies.g05.configuration_g05 import G05Config -from lerobot.policies.g05.modeling_g05 import G05Policy -from lerobot.utils.constants import ACTION, OBS_STATE - - -def _tensor_error(reference: torch.Tensor, actual: torch.Tensor) -> dict[str, float]: - reference = reference.detach().float().cpu() - actual = actual.detach().float().cpu() - if reference.shape != actual.shape: - raise AssertionError(f"shape mismatch: {tuple(reference.shape)} != {tuple(actual.shape)}") - error = (reference - actual).abs().flatten() - return { - "max": error.max().item() if error.numel() else 0.0, - "p99": torch.quantile(error, 0.99).item() if error.numel() else 0.0, - "mean": error.mean().item() if error.numel() else 0.0, - } - - -def _assert_bool_equal(name: str, reference: torch.Tensor, actual: torch.Tensor) -> None: - if not torch.equal(reference.detach().cpu().bool(), actual.detach().cpu().bool()): - raise AssertionError(f"{name} differs") - - -def _move_to(value: Any, device: torch.device) -> Any: - if isinstance(value, torch.Tensor): - return value.to(device) - if isinstance(value, dict): - return {key: _move_to(item, device) for key, item in value.items()} - if isinstance(value, list): - return [_move_to(item, device) for item in value] - if isinstance(value, tuple): - return tuple(_move_to(item, device) for item in value) - return value - - -def _raw_sample(processor: Any, index: int, task: str) -> dict[str, Any]: - images: dict[str, torch.Tensor] = {} - for camera_index, meta in enumerate(processor.shape_meta["images"]): - channels, height, width = meta["raw_shape"] - values = torch.arange(channels * height * width, dtype=torch.int64) - images[meta["key"]] = ( - (values.reshape(channels, height, width) + index * 37 + camera_index * 71) - .remainder(256) - .to(torch.uint8) - ) - - state = {} - for part_index, meta in enumerate(processor.shape_meta["state"]): - width = int(meta["raw_shape"]) - state[meta["key"]] = torch.linspace( - -0.2 + 0.03 * index + 0.01 * part_index, - 0.2 + 0.03 * index + 0.01 * part_index, - width, - ) - - action = {} - horizon = int(processor.action_horizon) - for part_index, meta in enumerate(processor.shape_meta["action"]): - width = int(meta["raw_shape"]) - action[meta["key"]] = torch.linspace( - -0.1 + 0.02 * index + 0.01 * part_index, - 0.1 + 0.02 * index + 0.01 * part_index, - horizon * width, - ).reshape(horizon, width) - - return { - "images": { - key: value.unsqueeze(0).expand(processor.num_obs_steps, -1, -1, -1) - for key, value in images.items() - }, - "state": { - key: value.unsqueeze(0).expand(processor.num_obs_steps, -1) for key, value in state.items() - }, - "action": action, - "action_is_pad": torch.zeros(horizon, dtype=torch.bool), - "state_is_pad": torch.zeros(processor.num_obs_steps, dtype=torch.bool), - "image_is_pad": torch.zeros(processor.num_obs_steps, dtype=torch.bool), - "task": task, - "frequency": 15.0, - "idx": index, - } - - -def _lerobot_input(raw: dict[str, Any], processor: Any, config: G05Config) -> dict[str, Any]: - image_pairs = zip(processor.shape_meta["images"], config.camera_order, strict=True) - return { - OBS_STATE: torch.cat( - [raw["state"][meta["key"]][-1] for meta in processor.shape_meta["state"]], dim=-1 - ), - ACTION: torch.cat([raw["action"][meta["key"]] for meta in processor.shape_meta["action"]], dim=-1), - **{lerobot_key: raw["images"][meta["key"]][-1] for meta, lerobot_key in image_pairs}, - "action_is_pad": raw["action_is_pad"], - "task": raw["task"], - } - - -def _collate_lerobot(samples: list[dict[str, Any]], config: G05Config) -> dict[str, Any]: - result: dict[str, Any] = { - OBS_STATE: torch.cat([sample[OBS_STATE] for sample in samples], dim=0), - ACTION: torch.stack([sample[ACTION] for sample in samples], dim=0), - "task": [ - sample["task"][0] if isinstance(sample["task"], list) else sample["task"] for sample in samples - ], - } - for key in (*config.camera_order, "proprio_dim_is_pad", "action_dim_is_pad"): - result[key] = torch.cat([sample[key] for sample in samples], dim=0) - action_pad = [sample["action_is_pad"] for sample in samples] - result["action_is_pad"] = torch.stack( - [value.squeeze(0) if value.ndim == 2 else value for value in action_pad], dim=0 - ) - return result - - -def _flatten_author_action(action: dict[str, torch.Tensor], processor: Any) -> torch.Tensor: - return torch.cat([action[meta["key"]] for meta in processor.shape_meta["action"]], dim=-1) - - -def run(args: argparse.Namespace) -> dict[str, Any]: - sys.path.insert(0, str(args.author_source / "src")) - from g05.utils.data.data_utils import collate_fn_pad_sequences - from g05.utils.data.normalizer import load_dataset_stats_from_json - from g05.utils.data.processor_utils import build_processors - from omegaconf import OmegaConf - - device = torch.device(args.device) - checkpoint = args.checkpoint.resolve() - if not OmegaConf.has_resolver("oc.load"): - - def _oc_load(path: str, key: str | None = None) -> Any: - loaded = OmegaConf.load(args.author_source / path) - return OmegaConf.select(loaded, key) if key is not None else loaded - - OmegaConf.register_new_resolver( - "oc.load", - _oc_load, - ) - author_cfg = OmegaConf.load(checkpoint / "author_config.yaml") - author_processors = build_processors(author_cfg) - author_processors.set_normalizer_from_stats( - load_dataset_stats_from_json(checkpoint / "g05_dataset_stats.json") - ) - author_processors.eval() - author_processor = author_processors.processors[args.embodiment] - author_processor.action_horizon = int(author_cfg.data.action_size) - - config = PreTrainedConfig.from_pretrained(checkpoint) - if not isinstance(config, G05Config): - raise TypeError(f"Expected G05Config, got {type(config).__name__}") - preprocessor, postprocessor = make_pre_post_processors(config, pretrained_path=checkpoint) - - tasks = [ - " Pick café cup\nverbatim ", - "第二个 task — keep Unicode and whitespace\t", - ][: args.batch_size] - raw_samples = [_raw_sample(author_processor, index, task) for index, task in enumerate(tasks)] - author_samples = [author_processor.preprocess(copy.deepcopy(sample)) for sample in raw_samples] - author_batch = collate_fn_pad_sequences(copy.deepcopy(author_samples)) - lerobot_samples = [ - preprocessor(_lerobot_input(sample, author_processor, config)) for sample in raw_samples - ] - lerobot_batch = _collate_lerobot(lerobot_samples, config) - - policy = G05Policy.from_pretrained( - checkpoint, - local_files_only=True, - strict=True, - ).to(device) - policy.eval() - port_author_batch = policy._prepare_author_batch(lerobot_batch) - - report: dict[str, Any] = { - "batch_size": args.batch_size, - "device": str(device), - "dtype": str(next(policy.parameters()).dtype), - "prompt_exact": all( - left["template"] == right["template"] - and left["command"] == right["command"] - and left["embodiment"] == right["embodiment"] - for left, right in zip(author_batch["samples"], port_author_batch["samples"], strict=True) - ), - } - if not report["prompt_exact"]: - raise AssertionError("author and LeRobot prompt payloads differ") - - image_errors = {} - for (author_key, author_images), (port_key, port_images) in zip( - author_batch["pixel_values"].items(), - port_author_batch["pixel_values"].items(), - strict=True, - ): - image_errors[f"{author_key}->{port_key}"] = _tensor_error(author_images, port_images) - report["images"] = image_errors - - author_proprio = torch.stack([sample["proprio"]["value"] for sample in author_batch["samples"]], dim=0) - port_proprio = torch.stack([sample["proprio"]["value"] for sample in port_author_batch["samples"]], dim=0) - report["proprio"] = _tensor_error(author_proprio, port_proprio) - report["normalized_input_action"] = _tensor_error(author_batch[ACTION], lerobot_batch[ACTION]) - for index, (author_sample, port_sample) in enumerate( - zip(author_batch["samples"], port_author_batch["samples"], strict=True) - ): - _assert_bool_equal( - f"proprio_dim_is_pad[{index}]", - author_sample["proprio"]["proprio_dim_is_pad"], - port_sample["proprio"]["proprio_dim_is_pad"], - ) - _assert_bool_equal( - "action_dim_is_pad", author_batch["action_dim_is_pad"], port_author_batch["action_dim_is_pad"] - ) - report["masks_exact"] = True - - author_ids, author_attention = policy.backend.processor.encode_inference( - copy.deepcopy(author_batch["samples"]), device=device, mode="fm" - ) - port_ids, port_attention = policy.backend.processor.encode_inference( - copy.deepcopy(port_author_batch["samples"]), device=device, mode="fm" - ) - _assert_bool_equal("input_ids", author_ids, port_ids) - _assert_bool_equal("attention_mask", author_attention, port_attention) - report["tokens_exact"] = True - report["token_shape"] = list(author_ids.shape) - - author_cuda = _move_to(copy.deepcopy(author_batch), device) - port_cuda = _move_to(lerobot_batch, device) - torch.manual_seed(args.seed) - with ( - torch.inference_mode(), - torch.autocast( - device_type=device.type, - dtype=torch.bfloat16, - enabled=config.model_weights_to_bf16 and device.type == "cuda", - ), - ): - author_output = policy.backend.predict_action(author_cuda) - torch.manual_seed(args.seed) - with torch.inference_mode(): - port_action = policy.predict_action_chunk(port_cuda) - report["normalized_action"] = _tensor_error(author_output[ACTION], port_action) - - author_post = author_processor.postprocess( - { - ACTION: author_output[ACTION].detach().cpu(), - "proprio": author_cuda["proprio"].detach().cpu(), - "action_dim_is_pad": author_cuda.get("action_dim_is_pad"), - "proprio_dim_is_pad": author_cuda.get("proprio_dim_is_pad"), - } - ) - author_env_action = _flatten_author_action(author_post[ACTION], author_processor) - # Isolate processor parity from small repeated BF16 sampling drift by feeding - # both postprocessors the same normalized author action chunk. - port_env_action = postprocessor(author_output[ACTION]) - report["environment_action"] = _tensor_error(author_env_action, port_env_action) - - if args.compare_training_loss: - policy.train() - torch.manual_seed(args.seed) - with torch.no_grad(): - author_loss, _ = policy.backend(_move_to(copy.deepcopy(author_batch), device)) - torch.manual_seed(args.seed) - with torch.no_grad(): - port_loss, _ = policy(_move_to(lerobot_batch, device)) - report["training_loss"] = { - "author": author_loss.detach().float().item(), - "lerobot": port_loss.detach().float().item(), - "absolute_error": abs(author_loss.detach().float().item() - port_loss.detach().float().item()), - } - - numeric_sections = ( - *report["images"].values(), - report["proprio"], - report["normalized_input_action"], - report["normalized_action"], - report["environment_action"], - ) - report["tolerance"] = args.atol - report["passed"] = all(section["max"] <= args.atol for section in numeric_sections) - if "training_loss" in report: - report["passed"] &= report["training_loss"]["absolute_error"] <= args.atol - return report - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--author-source", type=Path, required=True) - parser.add_argument("--checkpoint", type=Path, required=True) - parser.add_argument("--embodiment", default="libero") - parser.add_argument("--batch-size", type=int, choices=(1, 2), default=1) - parser.add_argument("--device", default="cuda") - parser.add_argument("--seed", type=int, default=1234) - parser.add_argument("--atol", type=float, default=5e-5) - parser.add_argument("--compare-training-loss", action="store_true") - parser.add_argument("--output", type=Path) - args = parser.parse_args() - report = run(args) - payload = json.dumps(report, indent=2, sort_keys=True) - if args.output is not None: - args.output.write_text(f"{payload}\n") - print(payload) - if not report["passed"]: - raise SystemExit(1) - - -if __name__ == "__main__": - main() diff --git a/docs/source/g05.mdx b/docs/source/g05.mdx index 31ab1d579..e002c9250 100644 --- a/docs/source/g05.mdx +++ b/docs/source/g05.mdx @@ -6,6 +6,12 @@ and the optional native chain-of-thought phase as System 2. They are not separat models: the runtime obtains both from one inference call and the action stays conditioned on the same post-reasoning KV state. +Transformers includes the native multimodal Qwen3.5 backbone, vision tower, and +processor. G0.5 is not a stock `Qwen3_5ForConditionalGeneration` checkpoint, +however: it adds the proprioception/action path, action expert, flow-matching +head, ActionCodec, and unified CoT/action decode. The current integration +therefore keeps the pinned G0.5 author package as the model backend. + > [!WARNING] > G0.5 code and checkpoints use the > [G0.5 Community License](https://huggingface.co/OpenGalaxea/G05/blob/main/licenses/LICENSE-G0.5), @@ -22,9 +28,9 @@ conditioned on the same post-reasoning KV state. | `g05-robotwin20` | Continuous flow | two arms 6+gripper → 20D grouped layout | high, left wrist, right wrist | 32 | 8 | stepwise q01/q99 | | `g05-so101` | Flow or AR ActionCodec + native CoT | right arm joints 6 → 20D grouped layout | exterior, optional left + right wrist | 32 | 16 | stepwise q01/q99 | -The converter stores the resolved Hydra model, processor, ActionCodec metadata, -statistics, exact prompt template, source revision, and license with the converted -checkpoint. Loading rejects a different head, horizon, processor mode, or +Each packaged checkpoint stores the resolved model and processor configuration, +ActionCodec metadata, statistics, exact prompt template, source revision, and +license. Loading rejects a different head, horizon, processor mode, or normalization contract. The named `atomic_4` adapter is intentionally separate from LIBERO. Its raw state @@ -52,67 +58,11 @@ git -C GalaxeaVLA checkout b34966f387dd2ae0f003143b81494afd9213e613 export PYTHONPATH="/path/to/GalaxeaVLA/src:${PYTHONPATH}" ``` -## Convert a local checkpoint - -The command never contacts the Hub. Point it at a complete local bundle containing -the checkpoint Hydra config, weights, `dataset_stats.json`, -`action_tokenizer.pt`, and `hf_processor/`. For released bundles, the converter -also discovers the tokenizer and `qwen3_5_2b_base_processor/` in the parent -directory. - -```bash -uv run python -m lerobot.policies.g05.convert_g05_checkpoint \ - --source-dir /path/to/checkpoints/g05-libero \ - --output-dir outputs/g05-libero-lerobot \ - --profile g05-libero \ - --license-file /path/to/GalaxeaVLA/LICENSE-G0.5 -``` - -For the base checkpoint, select both its concrete embodiment and one of its -enabled output heads: - -```bash -uv run python -m lerobot.policies.g05.convert_g05_checkpoint \ - --source-dir /path/to/checkpoints/g05-base \ - --output-dir outputs/g05-base-r1lite-flow-lerobot \ - --profile g05-base \ - --embodiment galaxea_r1lite \ - --action-head flow \ - --license-file /path/to/GalaxeaVLA/LICENSE-G0.5 -``` - -Use `--action-head actioncodec` for the autoregressive action path, -`--embodiment galaxea_r1pro` for R1 Pro, or `--profile g05-robotwin20` for -RoboTwin. The base model's 32-step head contains five history-alignment steps; -postprocessing returns the 27 executable actions. `conversion_report.json` -records every mapped, missing, unexpected, duplicate, and shape-mismatched -tensor; conversion fails when strict required-state validation fails. - -The official RoboTwin config contains a pinned data include. Resolve and package -it from the audited author checkout: - -```bash -uv run python -m lerobot.policies.g05.convert_g05_checkpoint \ - --source-dir /path/to/G05/g05-robotwin20 \ - --output-dir outputs/g05-robotwin20-lerobot \ - --profile g05-robotwin20 \ - --author-source /path/to/GalaxeaVLA \ - --license-file /path/to/GalaxeaVLA/LICENSE-G0.5 -``` - +The LeRobot organization hosts the prepared LIBERO, RoboTwin, and SO-101 +checkpoints privately. Authenticate with `hf auth login` before loading them. SO-100 and SO-101 share the released `so100` embodiment token and six-joint right-arm contract. A missing left-wrist camera is zero-padded exactly as in the -author deployment client: - -```bash -uv run python -m lerobot.policies.g05.convert_g05_checkpoint \ - --source-dir /path/to/G05/g05-so101 \ - --output-dir outputs/g05-so101-lerobot \ - --profile g05-so101 \ - --action-head flow \ - --author-source /path/to/GalaxeaVLA \ - --license-file /path/to/GalaxeaVLA/LICENSE-G0.5 -``` +author deployment client. ## Interactive System 1 and System 2 runtime @@ -120,13 +70,13 @@ System 1 executes the selected ActionCodec or flow chunk directly: ```bash lerobot-rollout \ - --policy.path=outputs/g05-base-lerobot \ + --policy.path=lerobot/g05_so101 \ --language --direct_subtask \ --task="pick up the cup" \ --mode=action ``` -System 2 is available only when the converted checkpoint metadata has +System 2 is available only when the packaged checkpoint metadata has `predict_cot=true`. The adapter forwards the operator task byte-for-byte and returns CoT telemetry and the matching action chunk atomically. It never samples `task_aug` text, launches a second planner, or feeds generated CoT back as a @@ -134,7 +84,7 @@ replacement task. ```bash lerobot-rollout \ - --policy.path=outputs/g05-system2-lerobot \ + --policy.path=lerobot/g05_so101 \ --language \ --task="clear the table" \ --mode=action @@ -165,31 +115,14 @@ the checkpoint's native System 2 CoT telemetry. ## Validation status CPU unit tests cover factory loading, config incompatibilities, prompt pass-through, -LIBERO and `atomic_4` mappings, padding masks, inverse action projection, strict -conversion diagnostics, a finite forward/backward/update, and save/reload parity. -The opt-in author-oracle gate compares the exact prompt tokens, every image/state/action -tensor and mask, seeded flow loss and samples, and de-normalized environment actions -for batch sizes one and two: +LIBERO and `atomic_4` mappings, padding masks, inverse action projection, a finite +forward/backward/update, and save/reload parity: ```bash uv run pytest tests/policies/g05 tests/runtime/test_g05_adapter.py -q uv run ruff check src/lerobot/policies/g05 tests/policies/g05 - -PYTHONPATH=/path/to/GalaxeaVLA/src uv run python benchmarks/g05_checkpoint_parity.py \ - --author-source /path/to/GalaxeaVLA \ - --checkpoint outputs/g05-libero-lerobot \ - --batch-size 1 \ - --compare-training-loss \ - --output outputs/g05-libero-parity-b1.json - -PYTHONPATH=/path/to/GalaxeaVLA/src uv run python benchmarks/g05_checkpoint_parity.py \ - --author-source /path/to/GalaxeaVLA \ - --checkpoint outputs/g05-libero-lerobot \ - --batch-size 2 \ - --output outputs/g05-libero-parity-b2.json ``` -This command still requires licensed checkpoint access and suitable CUDA hardware. A 50-episode LIBERO/RoboTwin success-rate comparison additionally requires the matching simulator, task assets, reset seeds, and author evaluator; no task-level benchmark number is claimed until that separate gate runs. @@ -199,7 +132,7 @@ LeRobot rollout with the author camera names and relative control: ```bash lerobot-eval \ - --policy.path=outputs/g05-libero-lerobot \ + --policy.path=lerobot/g05_libero \ --policy.device=cuda \ --env.type=libero \ --env.task=libero_goal \ diff --git a/src/lerobot/policies/g05/configuration_g05.py b/src/lerobot/policies/g05/configuration_g05.py index ec8b7ce83..e34d6da8c 100644 --- a/src/lerobot/policies/g05/configuration_g05.py +++ b/src/lerobot/policies/g05/configuration_g05.py @@ -155,9 +155,9 @@ _PROFILE_DEFAULTS = { class G05Config(PreTrainedConfig): """LeRobot-side, checkpoint-auditable configuration for G0.5. - ``author_model_config`` is populated by the conversion script from the selected - checkpoint's Hydra config. It is intentionally checkpoint state rather than a - collection of guessed LeRobot defaults. + ``author_model_config`` comes from the packaged checkpoint's resolved Hydra + config. It is intentionally checkpoint state rather than a collection of + guessed LeRobot defaults. """ checkpoint_profile: str = "g05-base" @@ -250,7 +250,7 @@ class G05Config(PreTrainedConfig): if self.runtime_system not in {"system1", "system2"}: raise ValueError("runtime_system must be 'system1' or 'system2'.") if self.runtime_system == "system2" and not self.predict_cot: - raise ValueError("G0.5 System 2 requires predict_cot=True in the converted checkpoint.") + raise ValueError("G0.5 System 2 requires predict_cot=True in the packaged checkpoint.") if not 1 <= self.n_action_steps <= self.chunk_size: raise ValueError("n_action_steps must be between 1 and chunk_size.") if self.action_head == "actioncodec" and not self.discrete_action: diff --git a/src/lerobot/policies/g05/convert_g05_checkpoint.py b/src/lerobot/policies/g05/convert_g05_checkpoint.py deleted file mode 100644 index b99cef3b9..000000000 --- a/src/lerobot/policies/g05/convert_g05_checkpoint.py +++ /dev/null @@ -1,691 +0,0 @@ -#!/usr/bin/env python -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 - -"""Deterministically package a user-authorized G0.5 checkpoint for LeRobot. - -This command never downloads from the gated Hub. The user supplies a local checkpoint -after accepting Galaxea's license. -""" - -from __future__ import annotations - -import argparse -import json -import shutil -from dataclasses import asdict, dataclass, field -from pathlib import Path -from typing import Any - -import torch -import yaml -from safetensors.torch import save_file - -from lerobot.policies.g05.configuration_g05 import ( - G05_CAMERA_PROFILES, - G05_HUB_REVISION, - G05_SOURCE_REVISION, - G05Config, -) -from lerobot.policies.g05.processor_g05 import make_g05_pre_post_processors -from lerobot.utils.constants import ACTION, OBS_STATE - -_EXACT_RENAMES = { - "model.embed_tokens.weight": "model.vlm.input_proj.weight", -} -_PREFIX_RENAMES = ( - ("model.joint_model.mixtures.vlm.", "model.vlm."), - ("model.joint_model.mixtures.action.", "model.action_expert."), - ("model.action_encoder.", "model.action_expert.input_proj."), - ("model.action_decoder.", "model.action_expert.output_proj."), -) -_REQUIRED_PREFIXES = ( - "backend.model.vlm.", - "backend.model.vision_tower.", - "backend.model.action_expert.", -) - - -@dataclass -class ConversionReport: - mapped: dict[str, str] = field(default_factory=dict) - shared_aliases: dict[str, str] = field(default_factory=dict) - missing: list[str] = field(default_factory=list) - unexpected: list[str] = field(default_factory=list) - duplicate: list[str] = field(default_factory=list) - shape_mismatched: dict[str, dict[str, list[int]]] = field(default_factory=dict) - - def fail_if_invalid(self) -> None: - if self.missing or self.unexpected or self.duplicate or self.shape_mismatched: - raise ValueError( - "G0.5 conversion failed strict validation: " - f"missing={len(self.missing)}, unexpected={len(self.unexpected)}, " - f"duplicate={len(self.duplicate)}, shape_mismatched={len(self.shape_mismatched)}" - ) - - -def _mapped_key(key: str) -> str: - key = _EXACT_RENAMES.get(key, key) - for old, new in _PREFIX_RENAMES: - if key.startswith(old): - key = new + key.removeprefix(old) - break - return key if key.startswith("backend.") else f"backend.{key}" - - -def convert_state_dict( - source: dict[str, torch.Tensor], - expected: dict[str, torch.Tensor] | None = None, -) -> tuple[dict[str, torch.Tensor], ConversionReport]: - """Map every tensor exactly once and optionally validate a target state dict.""" - - converted: dict[str, torch.Tensor] = {} - report = ConversionReport() - for old_key in sorted(source): - value = source[old_key] - if not isinstance(value, torch.Tensor): - report.unexpected.append(old_key) - continue - new_key = _mapped_key(old_key) - if new_key in converted: - report.duplicate.append(new_key) - continue - converted[new_key] = value.detach().cpu().contiguous() - report.mapped[old_key] = new_key - - if expected is not None: - report.missing = sorted(set(expected) - set(converted)) - report.unexpected.extend(sorted(set(converted) - set(expected))) - for key in sorted(set(expected) & set(converted)): - if expected[key].shape != converted[key].shape: - report.shape_mismatched[key] = { - "source": list(converted[key].shape), - "expected": list(expected[key].shape), - } - else: - for prefix in _REQUIRED_PREFIXES: - if not any(key.startswith(prefix) for key in converted): - report.missing.append(f"{prefix}*") - return converted, report - - -def save_converted_state_dict(state_dict: dict[str, torch.Tensor], path: Path) -> dict[str, str]: - """Save exact tensor aliases once, matching safetensors' strict model loader.""" - - aliases: dict[str, str] = {} - unique: dict[str, torch.Tensor] = {} - seen: dict[tuple[int, int, tuple[int, ...], tuple[int, ...]], str] = {} - for key in sorted(state_dict): - tensor = state_dict[key] - identity = ( - tensor.untyped_storage().data_ptr(), - tensor.storage_offset(), - tuple(tensor.shape), - tuple(tensor.stride()), - ) - if identity in seen: - aliases[key] = seen[identity] - else: - seen[identity] = key - unique[key] = tensor - save_file(unique, path, metadata=aliases or None) - return aliases - - -def _load_checkpoint(path: Path) -> dict[str, torch.Tensor]: - payload = torch.load(path, map_location="cpu", weights_only=True) - if isinstance(payload, dict) and isinstance(payload.get("model_state_dict"), dict): - payload = payload["model_state_dict"] - if not isinstance(payload, dict): - raise TypeError(f"Expected a state-dict mapping in {path}.") - return payload - - -def _resolve_model_arch(hydra: dict[str, Any], data_override: dict[str, Any] | None) -> dict[str, Any]: - """Resolve the official Hydra snapshot without importing the author package.""" - - from omegaconf import OmegaConf - - def multiply(expression: str) -> int: - factors = [part.strip() for part in str(expression).split("*")] - if not factors or any(not part.isdigit() for part in factors): - raise ValueError(f"Unsupported G0.5 eval expression {expression!r}.") - result = 1 - for factor in factors: - result *= int(factor) - return result - - if not OmegaConf.has_resolver("obs_image_steps"): - OmegaConf.register_new_resolver("obs_image_steps", lambda value: int(value)) - if not OmegaConf.has_resolver("eval"): - OmegaConf.register_new_resolver("eval", multiply) - payload = dict(hydra) - if data_override is not None: - payload["data"] = data_override - resolved = OmegaConf.to_container(OmegaConf.create(payload)["model"]["model_arch"], resolve=True) - if not isinstance(resolved, dict): - raise TypeError("Expected model.model_arch to resolve to a mapping.") - return resolved - - -def _profile_config( - profile: str, - hydra: dict[str, Any], - embodiment: str | None = None, - action_head: str | None = None, - data_override: dict[str, Any] | None = None, -) -> G05Config: - model = hydra.get("model", {}) - data = data_override if data_override is not None else hydra.get("data", {}) - if not isinstance(data, dict): - raise ValueError( - "The checkpoint Hydra config contains an unresolved data include. " - "Pass --author-source so the converter can package the pinned data contract." - ) - arch = _resolve_model_arch(hydra, data_override) - action_codec_metadata = model.get("tokenizer") - if not isinstance(action_codec_metadata, dict): - action_codec_metadata = hydra.get("tokenizer") - if not isinstance(action_codec_metadata, dict): - action_codec_metadata = {} - fixed_embodiment = { - "g05-libero": "libero", - "g05-robotwin20": "robotwin", - "g05-so101": "so100", - }.get(profile) - if fixed_embodiment is not None: - if embodiment is not None and embodiment != fixed_embodiment: - raise ValueError(f"{profile} requires embodiment={fixed_embodiment!r}.") - embodiment = fixed_embodiment - elif profile == "g05-base": - available = sorted((data.get("processors") or {}).keys()) - if embodiment is None: - raise ValueError(f"g05-base requires a concrete --embodiment; choose one of {available}.") - if embodiment not in available: - raise ValueError(f"g05-base embodiment must be one of {available}, got {embodiment!r}.") - else: - raise ValueError(f"Unsupported G0.5 profile {profile!r}.") - assert embodiment is not None - data_processor = (data.get("processors") or {}).get(embodiment, {}) - # Match the author's build_processors merge order: embodiment data first, - # then task-level model.processor overrides. - processor_metadata = {**data_processor, **model.get("processor", {})} - horizon = int(data.get("action_size", arch.get("horizon_steps", 16))) - predict_cot = bool(arch.get("predict_cot", False)) - discrete = bool(arch.get("discrete_action", True)) - continuous = bool(arch.get("continuous_action", False)) - if action_head is None: - action_head = "flow" if continuous else "actioncodec" - if action_head not in {"flow", "actioncodec"}: - raise ValueError("action_head must be 'flow' or 'actioncodec'.") - if action_head == "flow" and not continuous: - raise ValueError("The selected checkpoint does not enable the continuous flow action head.") - if action_head == "actioncodec" and not discrete: - raise ValueError("The selected checkpoint does not enable the autoregressive ActionCodec head.") - if profile not in {"g05-base", "g05-so101"} and action_head != "flow": - raise ValueError(f"The released {profile} checkpoint supports only --action-head flow.") - norm_name = str(processor_metadata.get("norm_default_mode", "")).lower() - checkpoint_normalization = { - "q01/q99": "q01_q99", - "z-score": "z_score", - "z-score-tail": "z_score_tail_mixed", - "identity": "identity", - "dummy": "identity", - }.get(norm_name) - arch = dict(arch) - arch.pop("_target_", None) - n_obs_steps = int(processor_metadata.get("num_obs_steps", data.get("obs_size", 1))) - raw_num_input_images = arch.get("num_input_images", len(G05_CAMERA_PROFILES[embodiment]) * n_obs_steps) - num_input_images = ( - int(raw_num_input_images) - if isinstance(raw_num_input_images, int | float) - else len(G05_CAMERA_PROFILES[embodiment]) * n_obs_steps - ) - if profile == "g05-libero": - return G05Config( - checkpoint_profile=profile, - embodiment="libero", - action_head="flow", - runtime_system="system1", - predict_cot=predict_cot, - discrete_action=discrete, - continuous_action=continuous, - return_continuous_action=True, - chunk_size=horizon, - n_action_steps=10, - normalization_mode="q01_q99", - normalization_clip=(-5.0, 5.0), - use_stepwise_action_norm=True, - libero_gripper_binarize=True, - camera_order=G05_CAMERA_PROFILES["libero"], - num_input_images=num_input_images, - author_model_config=arch, - processor_metadata=processor_metadata, - action_codec_metadata=action_codec_metadata, - author_source_revision=G05_SOURCE_REVISION, - source_checkpoint_revision=G05_HUB_REVISION, - license="other", - tags=["g05", "robotics", "non-commercial"], - ) - if profile == "g05-robotwin20": - return G05Config( - checkpoint_profile=profile, - embodiment="robotwin", - action_head="flow", - runtime_system="system1", - predict_cot=predict_cot, - discrete_action=discrete, - continuous_action=continuous, - return_continuous_action=True, - raw_state_dim=14, - raw_action_dim=14, - chunk_size=horizon, - n_action_steps=min(8, horizon), - normalization_mode="q01_q99", - normalization_clip=(-5.0, 5.0), - use_stepwise_action_norm=True, - camera_order=G05_CAMERA_PROFILES["robotwin"], - num_input_images=num_input_images, - author_model_config=arch, - processor_metadata=processor_metadata, - action_codec_metadata=action_codec_metadata, - author_source_revision=G05_SOURCE_REVISION, - source_checkpoint_revision=G05_HUB_REVISION, - license="other", - tags=["g05", "robotics", "non-commercial"], - ) - if profile == "g05-so101": - shape_meta = processor_metadata.get("shape_meta") or {} - raw_state_dim = sum(int(item["shape"]) for item in shape_meta.get("state", [])) - raw_action_dim = sum(int(item["shape"]) for item in shape_meta.get("action", [])) - action_feature_names = tuple( - f"{item['key']}.{index}" - for item in shape_meta.get("action", []) - for index in range(int(item["shape"])) - ) - return G05Config( - checkpoint_profile=profile, - embodiment="so100", - action_head=action_head, - runtime_system="system2", - predict_cot=predict_cot, - discrete_action=discrete, - continuous_action=continuous, - return_continuous_action=action_head == "flow", - raw_state_dim=raw_state_dim, - raw_action_dim=raw_action_dim, - chunk_size=horizon, - n_action_steps=min(16, horizon), - normalization_mode="q01_q99", - normalization_clip=(-5.0, 5.0), - use_relative_actions=True, - action_feature_names=action_feature_names, - use_stepwise_action_norm=True, - camera_order=G05_CAMERA_PROFILES["so100"], - optional_camera_keys=("observation.images.wrist_left",), - num_input_images=num_input_images, - author_model_config=arch, - processor_metadata=processor_metadata, - action_codec_metadata=action_codec_metadata, - author_source_revision=G05_SOURCE_REVISION, - source_checkpoint_revision=G05_HUB_REVISION, - license="other", - tags=["g05", "robotics", "so101", "non-commercial"], - ) - if checkpoint_normalization != "z_score_tail_mixed": - raise ValueError("The pinned g05-base R1 contract requires z-score-tail normalization.") - exceptions = processor_metadata.get("norm_exception_mode") or {} - exception_modes = {mode for category in exceptions.values() for mode in (category or {}).values()} - if exception_modes - {"q01/q99"}: - raise ValueError(f"Unsupported g05-base normalization exceptions: {exceptions}.") - shape_meta = processor_metadata.get("shape_meta") or {} - raw_state_dim = sum(int(item["shape"]) for item in shape_meta.get("state", [])) - raw_action_dim = sum(int(item["shape"]) for item in shape_meta.get("action", [])) - action_feature_names = tuple( - f"{item['key']}.{index}" - for item in shape_meta.get("action", []) - for index in range(int(item["shape"])) - ) - return G05Config( - checkpoint_profile="g05-base", - embodiment=embodiment, - action_head=action_head, - runtime_system="system2" if predict_cot else "system1", - predict_cot=predict_cot, - discrete_action=discrete, - continuous_action=continuous, - return_continuous_action=action_head == "flow", - policy_action_dim=int(arch.get("action_dim", 27)), - policy_state_dim=int(arch.get("proprio_dim", 27)), - raw_state_dim=raw_state_dim, - raw_action_dim=raw_action_dim, - chunk_size=horizon, - n_action_steps=min(16, horizon), - normalization_mode=checkpoint_normalization, - normalization_clip=(-5.0, 5.0), - use_relative_actions=True, - relative_exclude_joints=("gripper",), - action_feature_names=action_feature_names, - use_stepwise_action_norm=bool(processor_metadata.get("use_stepwise_action_norm", False)), - camera_order=G05_CAMERA_PROFILES[embodiment], - n_obs_steps=int(processor_metadata.get("num_obs_steps", 1)), - num_input_images=num_input_images, - author_model_config=arch, - processor_metadata=processor_metadata, - action_codec_metadata=action_codec_metadata, - author_source_revision=G05_SOURCE_REVISION, - source_checkpoint_revision=G05_HUB_REVISION, - license="other", - tags=["g05", "robotics", "non-commercial"], - ) - - -def _camera_sizes( - processor_metadata: dict[str, Any], camera_order: tuple[str, ...] -) -> dict[str, tuple[int, int]]: - images = (processor_metadata.get("shape_meta") or {}).get("images") or [] - camera_size_config = processor_metadata.get("camera_size_config") or {} - by_lerobot_key = { - item.get("lerobot_key"): tuple(camera_size_config.get(item.get("camera_type"), item["shape"][-2:])) - for item in images - if item.get("lerobot_key") and len(item.get("shape") or ()) >= 3 - } - if by_lerobot_key and all(key in by_lerobot_key for key in camera_order): - return {key: by_lerobot_key[key] for key in camera_order} - if len(images) == len(camera_order): - return { - key: tuple(camera_size_config.get(item.get("camera_type"), item["shape"][-2:])) - for key, item in zip(camera_order, images, strict=True) - } - return {} - - -def _write_model_card(output_dir: Path, config: G05Config) -> None: - system = "System 2 (native CoT + actions)" if config.runtime_system == "system2" else "System 1" - output_dir.joinpath("README.md").write_text( - f"""--- -license: other -license_name: g05-community-license -license_link: https://huggingface.co/OpenGalaxea/G05/blob/main/licenses/LICENSE-G0.5 -library_name: lerobot -tags: -- robotics -- lerobot -- g05 -- non-commercial ---- - -# LeRobot conversion of OpenGalaxea G0.5 - -This is a mechanically converted `{config.checkpoint_profile}` checkpoint for LeRobot. -It uses the `{config.action_head}` action head and exposes {system}. - -The weights, configuration, ActionCodec tokenizer, and processor assets are derivative -materials licensed under the **G0.5 Community License Agreement -(Non-Commercial + Limited Patent License)**, not Apache-2.0. Use is limited to the -purposes allowed by `LICENSE-G0.5`. - -## Modification notice - -LeRobot changed the checkpoint container, tensor key prefixes, portable sidecar paths, -and serialized preprocessing/postprocessing metadata. Model tensor values were not -trained or numerically altered. See `conversion_report.json` for the complete key map -and source revisions. - -This repository does not imply endorsement by Galaxea. -""" - ) - - -def _resolve_data_include(hydra: dict[str, Any], author_source: Path | None) -> dict[str, Any] | None: - if isinstance(hydra.get("data"), dict): - return None - if author_source is None: - return None - candidate = author_source / "configs" / "data" / "robotwin.yaml" - if not candidate.is_file(): - raise FileNotFoundError(f"Could not resolve checkpoint data include from {candidate}.") - payload = yaml.safe_load(candidate.read_text()) - if not isinstance(payload, dict): - raise TypeError(f"Expected a data mapping in {candidate}.") - return payload - - -def convert_dataset_stats(payload: dict[str, Any], config: G05Config) -> dict[str, dict[str, torch.Tensor]]: - """Flatten author per-part stats into LeRobot raw state/action feature stats.""" - - if config.embodiment in payload: - payload = payload[config.embodiment] - shape_meta = config.processor_metadata.get("shape_meta") or {} - if config.normalization_mode == "z_score_tail_mixed": - exceptions = config.processor_metadata.get("norm_exception_mode") or {} - default_mode = str(config.processor_metadata.get("norm_default_mode")) - result: dict[str, dict[str, torch.Tensor]] = {} - for source_category, feature_name in (("state", OBS_STATE), ("action", ACTION)): - category_stats = payload.get(source_category) - component_meta = shape_meta.get(source_category) - if not isinstance(category_stats, dict) or not isinstance(component_meta, list): - raise ValueError( - f"dataset_stats.json and processor shape_meta must define {source_category!r} components." - ) - prefix = ( - "stepwise" if source_category == "action" and config.use_stepwise_action_norm else "global" - ) - collected: dict[str, list[torch.Tensor]] = { - name: [] for name in ("mean", "std", "tail_q01", "tail_q99", "tail_mean", "tail_mask") - } - for component in component_meta: - key = component["key"] - stats = category_stats.get(key) - if not isinstance(stats, dict): - raise ValueError(f"Missing checkpoint statistics for {source_category}.{key}.") - mode = (exceptions.get(source_category) or {}).get(key, default_mode) - q01 = torch.as_tensor(stats[f"{prefix}_q01"], dtype=torch.float32) - q99 = torch.as_tensor(stats[f"{prefix}_q99"], dtype=torch.float32) - mean = torch.as_tensor(stats[f"{prefix}_mean"], dtype=torch.float32) - std = torch.as_tensor(stats[f"{prefix}_std"], dtype=torch.float32) - if mode == "q01/q99": - collected["mean"].append((q01 + q99) / 2) - collected["std"].append((q99 - q01) / 2) - tail_mask = torch.zeros(q01.shape[-1], dtype=torch.bool) - elif mode == "z-score-tail": - collected["mean"].append(mean) - collected["std"].append(std) - tail_mask = torch.ones(q01.shape[-1], dtype=torch.bool) - else: - raise ValueError( - f"Unsupported checkpoint normalization mode {source_category}.{key}={mode!r}." - ) - collected["tail_q01"].append(q01) - collected["tail_q99"].append(q99) - collected["tail_mean"].append(mean) - collected["tail_mask"].append(tail_mask) - result[feature_name] = {name: torch.cat(parts, dim=-1) for name, parts in collected.items()} - return result - - stat_names = ( - ("q01", "q99") - if config.normalization_mode == "q01_q99" - else ("mean", "std") - if config.normalization_mode == "z_score" - else () - ) - result: dict[str, dict[str, torch.Tensor]] = {} - for source_category, feature_name in (("state", OBS_STATE), ("action", ACTION)): - if not stat_names: - continue - category_stats = payload.get(source_category) - component_meta = shape_meta.get(source_category) - if not isinstance(category_stats, dict) or not isinstance(component_meta, list): - raise ValueError( - f"dataset_stats.json and processor shape_meta must define {source_category!r} components." - ) - converted_feature: dict[str, torch.Tensor] = {} - for short_name in stat_names: - source_name = ( - f"stepwise_{short_name}" - if source_category == "action" and config.use_stepwise_action_norm - else f"global_{short_name}" - ) - parts = [] - for component in component_meta: - key = component["key"] - if key not in category_stats or source_name not in category_stats[key]: - raise ValueError(f"Missing checkpoint statistic {source_category}.{key}.{source_name}.") - parts.append(torch.as_tensor(category_stats[key][source_name], dtype=torch.float32)) - converted_feature[short_name] = torch.cat(parts, dim=-1) - expected_width = config.raw_state_dim if feature_name == OBS_STATE else config.raw_action_dim - if converted_feature[stat_names[0]].shape[-1] != expected_width: - raise ValueError( - f"Flattened {feature_name} statistics have width " - f"{converted_feature[stat_names[0]].shape[-1]}, expected {expected_width}." - ) - result[feature_name] = converted_feature - return result - - -def convert_checkpoint( - source_dir: Path, - output_dir: Path, - profile: str, - *, - license_file: Path, - embodiment: str | None = None, - action_head: str | None = None, - author_source: Path | None = None, - expected_state: dict[str, torch.Tensor] | None = None, -) -> ConversionReport: - hydra_path = source_dir / ".hydra" / "config.yaml" - stats_path = source_dir / "dataset_stats.json" - candidates = ( - source_dir / "model.pt", - source_dir / "checkpoints" / "model_state_dict.pt", - ) - checkpoint_path = next((path for path in candidates if path.is_file()), None) - tokenizer_path = next( - ( - path - for path in ( - source_dir / "action_tokenizer.pt", - source_dir / "checkpoints" / "action_tokenizer.pt", - source_dir.parent / "action_tokenizer.pt", - ) - if path.is_file() - ), - source_dir / "action_tokenizer.pt", - ) - processor_path = next( - ( - path - for path in ( - source_dir / "hf_processor", - source_dir / "qwen3_5_2b_base_processor", - source_dir.parent / "qwen3_5_2b_base_processor", - ) - if path.is_dir() - ), - source_dir / "hf_processor", - ) - required = [hydra_path, stats_path, tokenizer_path, processor_path, license_file] - missing_files = [str(path) for path in required if not path.exists()] - if checkpoint_path is None: - missing_files.append("model.pt or checkpoints/model_state_dict.pt") - if missing_files: - raise FileNotFoundError(f"Incomplete G0.5 checkpoint bundle: {missing_files}") - - hydra = yaml.safe_load(hydra_path.read_text()) - data_override = _resolve_data_include(hydra, author_source) - config = _profile_config( - profile, - hydra, - embodiment=embodiment, - action_head=action_head, - data_override=data_override, - ) - camera_sizes = _camera_sizes(config.processor_metadata, config.camera_order) - if camera_sizes: - config.camera_sizes = camera_sizes - config.validate_features() - stats_payload = json.loads(stats_path.read_text()) - lerobot_stats = convert_dataset_stats(stats_payload, config) - converted, report = convert_state_dict(_load_checkpoint(checkpoint_path), expected_state) - report.fail_if_invalid() - - output_dir.mkdir(parents=True, exist_ok=True) - report.shared_aliases = save_converted_state_dict(converted, output_dir / "model.safetensors") - config._save_pretrained(output_dir) - preprocessor, postprocessor = make_g05_pre_post_processors(config, dataset_stats=lerobot_stats) - preprocessor.save_pretrained(output_dir) - postprocessor.save_pretrained(output_dir) - shutil.copy2(stats_path, output_dir / "g05_dataset_stats.json") - shutil.copy2(tokenizer_path, output_dir / "action_tokenizer.pt") - shutil.copytree(processor_path, output_dir / "hf_processor", dirs_exist_ok=True) - packaged_hydra = dict(hydra) - if data_override is not None: - packaged_hydra["data"] = data_override - (output_dir / "author_config.yaml").write_text(yaml.safe_dump(packaged_hydra, sort_keys=False)) - shutil.copy2(license_file, output_dir / "LICENSE-G0.5") - if author_source is not None: - for name in ("LICENSE_QWEN3_5.txt", "THIRD_PARTY_NOTICES.md"): - source = author_source / name - if source.is_file(): - shutil.copy2(source, output_dir / name) - (output_dir / "NOTICE").write_text( - "G0.5 is licensed under the G0.5 Community License Agreement " - "(Non-Commercial + Limited Patent License), not sold, Copyright © 2026 " - "Galaxea. All rights reserved by Galaxea. “Galaxea” and related marks are " - "trademarks of Galaxea or its affiliates.\n\n" - "Modification notice: LeRobot converted the checkpoint container, tensor key " - "prefixes, portable sidecar paths, and processor metadata. Tensor values were " - "not trained or numerically altered.\n" - ) - _write_model_card(output_dir, config) - (output_dir / "conversion_report.json").write_text(json.dumps(asdict(report), indent=2, sort_keys=True)) - return report - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--source-dir", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument( - "--profile", - choices=("g05-base", "g05-libero", "g05-robotwin20", "g05-so101"), - required=True, - ) - parser.add_argument( - "--embodiment", - choices=("galaxea_r1lite", "galaxea_r1pro"), - help="Required concrete embodiment for g05-base.", - ) - parser.add_argument( - "--action-head", - choices=("flow", "actioncodec"), - help="Select one enabled g05-base output head; benchmark checkpoints are flow-only.", - ) - parser.add_argument("--license-file", type=Path, required=True) - parser.add_argument( - "--author-source", - type=Path, - help="Pinned GalaxeaVLA checkout used to resolve official config includes and notices.", - ) - args = parser.parse_args() - report = convert_checkpoint( - args.source_dir, - args.output_dir, - args.profile, - license_file=args.license_file, - embodiment=args.embodiment, - action_head=args.action_head, - author_source=args.author_source, - ) - print(json.dumps(asdict(report), indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/src/lerobot/policies/g05/modeling_g05.py b/src/lerobot/policies/g05/modeling_g05.py index c4cb610b8..2bdf673d7 100644 --- a/src/lerobot/policies/g05/modeling_g05.py +++ b/src/lerobot/policies/g05/modeling_g05.py @@ -31,7 +31,7 @@ from .configuration_g05 import G05_POLICY_PARTS, G05Config def _author_backend(config: G05Config) -> nn.Module: if not config.author_model_config: raise ValueError( - "G0.5 author_model_config is empty. Convert an official checkpoint first, or " + "G0.5 author_model_config is empty. Load a packaged checkpoint, or " "inject a backend explicitly for testing." ) try: @@ -137,7 +137,6 @@ class G05Policy(PreTrainedPolicy): "THIRD_PARTY_NOTICES.md", "NOTICE", "README.md", - "conversion_report.json", ): source = next((root / name for root in roots if (root / name).is_file()), None) if source is not None and source.resolve() != (save_directory / name).resolve(): diff --git a/src/lerobot/policies/g05/processor_g05.py b/src/lerobot/policies/g05/processor_g05.py index e994b1d07..ccac5b60f 100644 --- a/src/lerobot/policies/g05/processor_g05.py +++ b/src/lerobot/policies/g05/processor_g05.py @@ -666,7 +666,7 @@ def make_g05_pre_post_processors( if config.normalization_mode == "checkpoint" and not config.processor_metadata: raise ValueError( - "normalization_mode='checkpoint' requires processor_metadata from the converted checkpoint." + "normalization_mode='checkpoint' requires processor_metadata from the packaged checkpoint." ) mode = _normalization_mode(config) if mode is NormalizationMode.QUANTILES and dataset_stats: diff --git a/tests/policies/g05/test_g05.py b/tests/policies/g05/test_g05.py index 63382e8f6..03a030873 100644 --- a/tests/policies/g05/test_g05.py +++ b/tests/policies/g05/test_g05.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json import os from pathlib import Path @@ -14,14 +13,6 @@ from lerobot.configs.policies import PreTrainedConfig from lerobot.configs.types import FeatureType, PolicyFeature from lerobot.policies.factory import get_policy_class, make_policy_config, make_pre_post_processors from lerobot.policies.g05.configuration_g05 import G05_EMBODIMENT_MAPPINGS, G05Config -from lerobot.policies.g05.convert_g05_checkpoint import ( - _camera_sizes, - _profile_config, - convert_checkpoint, - convert_dataset_stats, - convert_state_dict, - save_converted_state_dict, -) from lerobot.policies.g05.modeling_g05 import G05Policy from lerobot.processor import PolicyProcessorPipeline from lerobot.utils.constants import ACTION, OBS_STATE, POLICY_PREPROCESSOR_DEFAULT_NAME @@ -109,175 +100,43 @@ def test_system2_fm_only_builder_uses_exact_cot_template_without_action_tokens() assert "" in config.prompt_template - # G0.5-base's 32-step head includes n_obs_steps - 1 alignment steps. - assert processed["action_dim_is_pad"].shape == (1, 27) - assert not processed["action_op_mask"].any() - assert restored.shape == (1, 27, 14) - torch.testing.assert_close(restored, raw_action[:, 5:], atol=2e-5, rtol=2e-5) - - preprocessor.save_pretrained(tmp_path) - postprocessor.save_pretrained(tmp_path) - loaded_preprocessor, loaded_postprocessor = make_pre_post_processors(config, pretrained_path=tmp_path) - reloaded = loaded_preprocessor(raw_batch) - reloaded_restored = loaded_postprocessor(reloaded[ACTION]) - torch.testing.assert_close(reloaded[ACTION], processed[ACTION]) - torch.testing.assert_close(reloaded_restored, restored) - - def test_libero_and_atomic4_are_distinct_validated_mappings(): with pytest.raises(ValueError, match="27D"): G05Config( @@ -763,13 +521,13 @@ def test_forward_backward_update_and_save_reload(tmp_path: Path): def test_save_pretrained_copies_required_gated_sidecars_portably(tmp_path: Path): - source = tmp_path / "converted" + source = tmp_path / "checkpoint" processor = source / "hf_processor" processor.mkdir(parents=True) (processor / "tokenizer.json").write_text("{}") tokenizer = source / "action_tokenizer.pt" torch.save({"codec": "ActionCodec"}, tokenizer) - for name in ("LICENSE-G0.5", "NOTICE", "conversion_report.json"): + for name in ("LICENSE-G0.5", "NOTICE"): (source / name).write_text("{}") config = _config( author_model_config={ @@ -804,125 +562,6 @@ def test_tiny_fixed_batch_overfit_reduces_loss(): assert final < initial * 0.25 -def test_conversion_reports_mapping_duplicates_shapes_and_required_prefixes(): - source = { - "model.embed_tokens.weight": torch.zeros(2, 3), - "model.vision_tower.block.weight": torch.ones(2, 2), - "model.action_expert.block.weight": torch.ones(2, 2), - } - converted, report = convert_state_dict(source) - assert "backend.model.vlm.input_proj.weight" in converted - assert report.missing == [] - - expected = { - "backend.model.vlm.input_proj.weight": torch.zeros(3, 3), - "backend.model.vision_tower.block.weight": torch.zeros(2, 2), - "backend.model.action_expert.block.weight": torch.ones(2, 2), - } - _, strict_report = convert_state_dict(source, expected) - assert strict_report.shape_mismatched["backend.model.vlm.input_proj.weight"]["source"] == [2, 3] - - -def test_conversion_records_and_deduplicates_tied_weight_aliases(tmp_path: Path): - tied = torch.zeros(2, 3) - aliases = save_converted_state_dict( - { - "backend.model.vlm.input_proj.weight": tied, - "backend.model.vlm.output_proj.weight": tied, - }, - tmp_path / "model.safetensors", - ) - - assert aliases == {"backend.model.vlm.output_proj.weight": "backend.model.vlm.input_proj.weight"} - - -def test_libero_conversion_packages_model_processors_and_provenance(tmp_path: Path): - source = tmp_path / "author" - output = tmp_path / "lerobot" - (source / ".hydra").mkdir(parents=True) - (source / "hf_processor").mkdir() - (source / "hf_processor" / "tokenizer.json").write_text("{}") - (source / ".hydra" / "config.yaml").write_text( - """ -model: - model_arch: - num_input_images: 2 - horizon_steps: 32 - predict_cot: false - discrete_action: false - continuous_action: true - processor: - use_stepwise_action_norm: true - norm_default_mode: q01/q99 - camera_size_config: - exterior: [256, 256] - wrist_right: [256, 256] -data: - action_size: 32 - processors: - libero: - shape_meta: - state: - - {key: right_ee_pose, shape: 6} - - {key: right_gripper, shape: 1} - action: - - {key: right_ee_pose, shape: 6} - - {key: right_gripper, shape: 1} - images: - - {key: image, camera_type: exterior, lerobot_key: observation.images.image, shape: [3, 224, 224]} - - {key: wrist_image, camera_type: wrist_right, lerobot_key: observation.images.wrist_image, shape: [3, 224, 224]} -tokenizer: - vq_config: {block_wise_autoregressive: false} -""" - ) - action_stats = { - "right_ee_pose": { - "stepwise_q01": torch.zeros(32, 6).tolist(), - "stepwise_q99": torch.ones(32, 6).tolist(), - }, - "right_gripper": { - "stepwise_q01": torch.zeros(32, 1).tolist(), - "stepwise_q99": torch.ones(32, 1).tolist(), - }, - } - state_stats = { - "right_ee_pose": {"global_q01": [0.0] * 6, "global_q99": [1.0] * 6}, - "right_gripper": {"global_q01": [0.0], "global_q99": [1.0]}, - } - (source / "dataset_stats.json").write_text( - json.dumps({"libero": {"state": state_stats, "action": action_stats}}) - ) - torch.save( - { - "model.vlm.block.weight": torch.zeros(2, 2), - "model.vision_tower.block.weight": torch.zeros(2, 2), - "model.action_expert.block.weight": torch.zeros(2, 2), - }, - source / "model.pt", - ) - torch.save({"tokenizer_meta": {"codec": "ActionCodec"}}, source / "action_tokenizer.pt") - license_file = source / "LICENSE-G0.5" - license_file.write_text("test fixture license") - - report = convert_checkpoint(source, output, "g05-libero", license_file=license_file) - - assert len(report.mapped) == 3 - assert (output / "model.safetensors").is_file() - assert (output / "policy_preprocessor.json").is_file() - assert (output / "policy_postprocessor.json").is_file() - assert (output / "conversion_report.json").is_file() - assert (output / "README.md").is_file() - assert "Modification notice" in (output / "NOTICE").read_text() - config = PreTrainedConfig.from_pretrained(output) - assert isinstance(config, G05Config) - assert config.source_checkpoint_revision - assert config.prompt_template.startswith("") - assert config.camera_sizes == { - "observation.images.image": (256, 256), - "observation.images.wrist_image": (256, 256), - } - - @pytest.mark.skipif( not os.environ.get("LEROBOT_G05_CHECKPOINT"), reason="requires an accepted gated OpenGalaxea/G05 checkpoint and author CUDA environment", diff --git a/uv.lock b/uv.lock index 59627efce..d8a5494e5 100644 --- a/uv.lock +++ b/uv.lock @@ -3508,7 +3508,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'transformers-dep'", specifier = ">=5.4.0,<5.6.0" }, { name = "wandb", marker = "extra == 'training'", specifier = ">=0.24.0,<0.28.0" }, ] -provides-extras = ["dataset", "training", "hardware", "viz", "core-scripts", "evaluation", "dataset-viz", "av-dep", "pygame-dep", "placo-dep", "transformers-dep", "grpcio-dep", "accelerate-dep", "can-dep", "peft-dep", "scipy-dep", "diffusers-dep", "qwen-vl-utils-dep", "matplotlib-dep", "pyserial-dep", "deepdiff-dep", "pynput-dep", "pyzmq-dep", "motorbridge-dep", "motorbridge-smart-servo-dep", "timm-dep", "feetech", "dynamixel", "damiao", "robstride", "openarms", "gamepad", "hopejr", "lekiwi", "unitree-g1", "reachy2", "rebot", "kinematics", "intelrealsense", "phone", "diffusion", "wallx", "pi", "molmoact2", "smolvla", "multi-task-dit", "groot", "sarm", "robometer", "topreward", "xvla", "eo1", "fastwam", "evo1", "hilserl", "vla-jepa", "lingbot-va", "async", "peft", "annotations", "dev", "notebook", "test", "video-benchmark", "aloha", "pusht", "libero", "metaworld", "all"] +provides-extras = ["dataset", "training", "hardware", "viz", "core-scripts", "evaluation", "dataset-viz", "av-dep", "pygame-dep", "placo-dep", "transformers-dep", "grpcio-dep", "accelerate-dep", "can-dep", "peft-dep", "scipy-dep", "diffusers-dep", "qwen-vl-utils-dep", "matplotlib-dep", "pyserial-dep", "deepdiff-dep", "pynput-dep", "pyzmq-dep", "motorbridge-dep", "motorbridge-smart-servo-dep", "timm-dep", "feetech", "dynamixel", "damiao", "robstride", "openarms", "gamepad", "hopejr", "lekiwi", "unitree-g1", "reachy2", "rebot", "kinematics", "intelrealsense", "phone", "diffusion", "wallx", "pi", "molmoact2", "smolvla", "multi-task-dit", "groot", "sarm", "robometer", "topreward", "xvla", "eo1", "fastwam", "evo1", "hilserl", "vla-jepa", "lingbot-va", "g05", "async", "peft", "annotations", "dev", "notebook", "test", "video-benchmark", "aloha", "pusht", "libero", "metaworld", "all"] [[package]] name = "librt"