feat(train): resume training from a Hub checkpoint

Allow --config_path to be a Hub repo id when resuming, not only a local path.
The latest checkpoint under checkpoints/<step>/ is downloaded into a fresh local
run dir and resumed from there (optimizer, scheduler, RNG and data order
restored as for a local resume). TrainPipelineConfig.from_pretrained falls back
to the latest checkpoint's train_config.json when a repo has no root config
(an interrupted run that only pushed checkpoints). The download is skipped when
dispatching remotely so the executor (local machine or HF Jobs pod) performs it.

- add find_latest_hub_checkpoint (utils/hub) and resolve_resume_checkpoint
  (common/train_utils), the symmetric download counterpart to
  push_checkpoint_to_hub
- unit tests for both helpers and the from_pretrained fallback
This commit is contained in:
Nicolas Rabault
2026-06-24 10:15:55 +02:00
parent 6b7cd87a91
commit d90687e534
6 changed files with 265 additions and 28 deletions
+54
View File
@@ -0,0 +1,54 @@
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock
from lerobot.utils.hub import find_latest_hub_checkpoint
def _patch_list_files(monkeypatch, files):
api = MagicMock()
api.list_repo_files.return_value = files
# HfApi is imported into lerobot.utils.hub at module load, so patch it there.
monkeypatch.setattr("lerobot.utils.hub.HfApi", lambda *a, **k: api)
return api
def test_find_latest_hub_checkpoint_picks_highest_step(monkeypatch):
_patch_list_files(
monkeypatch,
[
"README.md",
"checkpoints/000500/pretrained_model/model.safetensors",
"checkpoints/000500/training_state/training_step.json",
"checkpoints/020000/pretrained_model/model.safetensors",
"checkpoints/001000/training_state/training_step.json",
],
)
# Numeric max, not lexicographic — "020000" beats "001000"/"000500".
assert find_latest_hub_checkpoint("u/run") == "checkpoints/020000"
def test_find_latest_hub_checkpoint_ignores_non_step_entries(monkeypatch):
_patch_list_files(
monkeypatch,
["checkpoints/last/pretrained_model/model.safetensors", "config.json"],
)
# "last" (a symlink target name) is not a numeric step → no resolvable checkpoint.
assert find_latest_hub_checkpoint("u/run") is None
def test_find_latest_hub_checkpoint_none_when_no_checkpoints(monkeypatch):
_patch_list_files(monkeypatch, ["config.json", "model.safetensors"])
assert find_latest_hub_checkpoint("u/run") is None
+35
View File
@@ -17,6 +17,8 @@
from pathlib import Path
from unittest.mock import MagicMock, Mock, patch
import pytest
from lerobot.common.train_utils import (
get_step_checkpoint_dir,
get_step_identifier,
@@ -188,3 +190,36 @@ def test_push_checkpoint_to_hub_defaults_to_hub_default_visibility(tmp_path, mon
push_checkpoint_to_hub(ckpt, "user/run")
api.create_repo.assert_called_once()
assert api.create_repo.call_args.kwargs["private"] is None
def test_resolve_resume_checkpoint_downloads_latest_and_links(tmp_path, monkeypatch):
from lerobot.common import train_utils
out = tmp_path / "run"
def fake_snapshot_download(repo_id, repo_type, allow_patterns, local_dir):
# Mimic the Hub layout the real download materializes locally.
assert allow_patterns == "checkpoints/020000/*"
(Path(local_dir) / "checkpoints" / "020000" / "pretrained_model").mkdir(parents=True)
return local_dir
monkeypatch.setattr("lerobot.common.train_utils.snapshot_download", fake_snapshot_download)
monkeypatch.setattr(
"lerobot.common.train_utils.find_latest_hub_checkpoint", lambda repo_id: "checkpoints/020000"
)
checkpoint_dir = train_utils.resolve_resume_checkpoint("u/run", out)
assert checkpoint_dir == out / CHECKPOINTS_DIR / "020000"
last = out / CHECKPOINTS_DIR / LAST_CHECKPOINT_LINK
assert last.is_symlink()
# `last` points at the downloaded step dir.
assert (last.parent / last.readlink()).resolve() == checkpoint_dir.resolve()
def test_resolve_resume_checkpoint_raises_without_checkpoints(tmp_path, monkeypatch):
from lerobot.common import train_utils
monkeypatch.setattr("lerobot.common.train_utils.find_latest_hub_checkpoint", lambda repo_id: None)
with pytest.raises(FileNotFoundError, match="No checkpoint"):
train_utils.resolve_resume_checkpoint("u/run", tmp_path / "run")