feat(train): resume training from a Hub checkpoint

Allow --config_path to be a Hub repo id when resuming, not only a local path.
The latest checkpoint under checkpoints/<step>/ is downloaded into a fresh local
run dir and resumed from there (optimizer, scheduler, RNG and data order
restored as for a local resume). TrainPipelineConfig.from_pretrained falls back
to the latest checkpoint's train_config.json when a repo has no root config
(an interrupted run that only pushed checkpoints). The download is skipped when
dispatching remotely so the executor (local machine or HF Jobs pod) performs it.

- add find_latest_hub_checkpoint (utils/hub) and resolve_resume_checkpoint
  (common/train_utils), the symmetric download counterpart to
  push_checkpoint_to_hub
- unit tests for both helpers and the from_pretrained fallback
This commit is contained in:
Nicolas Rabault
2026-06-24 10:15:55 +02:00
parent 955b172585
commit 838ab9e234
6 changed files with 265 additions and 28 deletions
+35
View File
@@ -17,6 +17,8 @@
from pathlib import Path
from unittest.mock import MagicMock, Mock, patch
import pytest
from lerobot.common.train_utils import (
get_step_checkpoint_dir,
get_step_identifier,
@@ -188,3 +190,36 @@ def test_push_checkpoint_to_hub_defaults_to_hub_default_visibility(tmp_path, mon
push_checkpoint_to_hub(ckpt, "user/run")
api.create_repo.assert_called_once()
assert api.create_repo.call_args.kwargs["private"] is None
def test_resolve_resume_checkpoint_downloads_latest_and_links(tmp_path, monkeypatch):
from lerobot.common import train_utils
out = tmp_path / "run"
def fake_snapshot_download(repo_id, repo_type, allow_patterns, local_dir):
# Mimic the Hub layout the real download materializes locally.
assert allow_patterns == "checkpoints/020000/*"
(Path(local_dir) / "checkpoints" / "020000" / "pretrained_model").mkdir(parents=True)
return local_dir
monkeypatch.setattr("lerobot.common.train_utils.snapshot_download", fake_snapshot_download)
monkeypatch.setattr(
"lerobot.common.train_utils.find_latest_hub_checkpoint", lambda repo_id: "checkpoints/020000"
)
checkpoint_dir = train_utils.resolve_resume_checkpoint("u/run", out)
assert checkpoint_dir == out / CHECKPOINTS_DIR / "020000"
last = out / CHECKPOINTS_DIR / LAST_CHECKPOINT_LINK
assert last.is_symlink()
# `last` points at the downloaded step dir.
assert (last.parent / last.readlink()).resolve() == checkpoint_dir.resolve()
def test_resolve_resume_checkpoint_raises_without_checkpoints(tmp_path, monkeypatch):
from lerobot.common import train_utils
monkeypatch.setattr("lerobot.common.train_utils.find_latest_hub_checkpoint", lambda repo_id: None)
with pytest.raises(FileNotFoundError, match="No checkpoint"):
train_utils.resolve_resume_checkpoint("u/run", tmp_path / "run")