mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-13 04:51:59 +00:00
feat(train): resume training from a Hub checkpoint
Allow --config_path to be a Hub repo id when resuming, not only a local path. The latest checkpoint under checkpoints/<step>/ is downloaded into a fresh local run dir and resumed from there (optimizer, scheduler, RNG and data order restored as for a local resume). TrainPipelineConfig.from_pretrained falls back to the latest checkpoint's train_config.json when a repo has no root config (an interrupted run that only pushed checkpoints). The download is skipped when dispatching remotely so the executor (local machine or HF Jobs pod) performs it. - add find_latest_hub_checkpoint (utils/hub) and resolve_resume_checkpoint (common/train_utils), the symmetric download counterpart to push_checkpoint_to_hub - unit tests for both helpers and the from_pretrained fallback
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from lerobot.utils.hub import find_latest_hub_checkpoint
|
||||
|
||||
|
||||
def _patch_list_files(monkeypatch, files):
|
||||
api = MagicMock()
|
||||
api.list_repo_files.return_value = files
|
||||
# HfApi is imported into lerobot.utils.hub at module load, so patch it there.
|
||||
monkeypatch.setattr("lerobot.utils.hub.HfApi", lambda *a, **k: api)
|
||||
return api
|
||||
|
||||
|
||||
def test_find_latest_hub_checkpoint_picks_highest_step(monkeypatch):
|
||||
_patch_list_files(
|
||||
monkeypatch,
|
||||
[
|
||||
"README.md",
|
||||
"checkpoints/000500/pretrained_model/model.safetensors",
|
||||
"checkpoints/000500/training_state/training_step.json",
|
||||
"checkpoints/020000/pretrained_model/model.safetensors",
|
||||
"checkpoints/001000/training_state/training_step.json",
|
||||
],
|
||||
)
|
||||
# Numeric max, not lexicographic — "020000" beats "001000"/"000500".
|
||||
assert find_latest_hub_checkpoint("u/run") == "checkpoints/020000"
|
||||
|
||||
|
||||
def test_find_latest_hub_checkpoint_ignores_non_step_entries(monkeypatch):
|
||||
_patch_list_files(
|
||||
monkeypatch,
|
||||
["checkpoints/last/pretrained_model/model.safetensors", "config.json"],
|
||||
)
|
||||
# "last" (a symlink target name) is not a numeric step → no resolvable checkpoint.
|
||||
assert find_latest_hub_checkpoint("u/run") is None
|
||||
|
||||
|
||||
def test_find_latest_hub_checkpoint_none_when_no_checkpoints(monkeypatch):
|
||||
_patch_list_files(monkeypatch, ["config.json", "model.safetensors"])
|
||||
assert find_latest_hub_checkpoint("u/run") is None
|
||||
@@ -17,6 +17,8 @@
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from lerobot.common.train_utils import (
|
||||
get_step_checkpoint_dir,
|
||||
get_step_identifier,
|
||||
@@ -188,3 +190,36 @@ def test_push_checkpoint_to_hub_defaults_to_hub_default_visibility(tmp_path, mon
|
||||
push_checkpoint_to_hub(ckpt, "user/run")
|
||||
api.create_repo.assert_called_once()
|
||||
assert api.create_repo.call_args.kwargs["private"] is None
|
||||
|
||||
|
||||
def test_resolve_resume_checkpoint_downloads_latest_and_links(tmp_path, monkeypatch):
|
||||
from lerobot.common import train_utils
|
||||
|
||||
out = tmp_path / "run"
|
||||
|
||||
def fake_snapshot_download(repo_id, repo_type, allow_patterns, local_dir):
|
||||
# Mimic the Hub layout the real download materializes locally.
|
||||
assert allow_patterns == "checkpoints/020000/*"
|
||||
(Path(local_dir) / "checkpoints" / "020000" / "pretrained_model").mkdir(parents=True)
|
||||
return local_dir
|
||||
|
||||
monkeypatch.setattr("lerobot.common.train_utils.snapshot_download", fake_snapshot_download)
|
||||
monkeypatch.setattr(
|
||||
"lerobot.common.train_utils.find_latest_hub_checkpoint", lambda repo_id: "checkpoints/020000"
|
||||
)
|
||||
|
||||
checkpoint_dir = train_utils.resolve_resume_checkpoint("u/run", out)
|
||||
|
||||
assert checkpoint_dir == out / CHECKPOINTS_DIR / "020000"
|
||||
last = out / CHECKPOINTS_DIR / LAST_CHECKPOINT_LINK
|
||||
assert last.is_symlink()
|
||||
# `last` points at the downloaded step dir.
|
||||
assert (last.parent / last.readlink()).resolve() == checkpoint_dir.resolve()
|
||||
|
||||
|
||||
def test_resolve_resume_checkpoint_raises_without_checkpoints(tmp_path, monkeypatch):
|
||||
from lerobot.common import train_utils
|
||||
|
||||
monkeypatch.setattr("lerobot.common.train_utils.find_latest_hub_checkpoint", lambda repo_id: None)
|
||||
with pytest.raises(FileNotFoundError, match="No checkpoint"):
|
||||
train_utils.resolve_resume_checkpoint("u/run", tmp_path / "run")
|
||||
|
||||
Reference in New Issue
Block a user