mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-09 11:01:53 +00:00
Nicolas Rabault
607a8a6b68
When --job.target names a GPU flavor, train() dispatches to lerobot.jobs.submit_to_hf instead of training locally: it authenticates, ensures the dataset is on the Hub (pushing a local-only one privately), serializes a pod-compatible train_config.json (strips client-only fields, points at the model repo), submits via HfApi.run_job with HF_TOKEN/WANDB_API_KEY secrets, then streams logs and finishes when the model is pushed. Wires push_checkpoint_to_hub into the training loop behind save_checkpoint_to_hub, and tags jobs/datasets/model with 'lerobot' + --job.tags.
79 lines
2.8 KiB
Python
79 lines
2.8 KiB
Python
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import sys
|
|
from unittest.mock import MagicMock
|
|
|
|
import httpx
|
|
import pytest
|
|
from huggingface_hub.errors import RepositoryNotFoundError
|
|
|
|
from lerobot.jobs.dataset import ensure_dataset_available
|
|
|
|
|
|
def _repo_not_found() -> RepositoryNotFoundError:
|
|
req = httpx.Request("GET", "https://huggingface.co/datasets/test")
|
|
resp = httpx.Response(404, request=req)
|
|
return RepositoryNotFoundError("nope", response=resp)
|
|
|
|
|
|
def _api_with_dataset(exists: bool):
|
|
api = MagicMock()
|
|
if exists:
|
|
api.dataset_info.return_value = object()
|
|
else:
|
|
api.dataset_info.side_effect = _repo_not_found()
|
|
return api
|
|
|
|
|
|
def _make_local_cache(tmp_path, repo_id: str) -> None:
|
|
"""Create the minimal local-cache layout that ensure_dataset_available checks."""
|
|
info = tmp_path / repo_id / "meta" / "info.json"
|
|
info.parent.mkdir(parents=True)
|
|
info.write_text("{}")
|
|
|
|
|
|
# Branch 1: dataset already on Hub → no push, no error (pod downloads by repo_id).
|
|
def test_dataset_already_on_hub_is_noop():
|
|
api = _api_with_dataset(True)
|
|
assert ensure_dataset_available("user/ds", api=api) is None
|
|
api.dataset_info.assert_called_once_with("user/ds")
|
|
|
|
|
|
# Branch 2: not on Hub but present locally → always push privately.
|
|
def test_dataset_local_only_uploads_privately(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HF_LEROBOT_HOME", str(tmp_path))
|
|
_make_local_cache(tmp_path, "user/ds")
|
|
|
|
api = _api_with_dataset(False)
|
|
mock_ds_cls = MagicMock()
|
|
fake_datasets_module = MagicMock()
|
|
fake_datasets_module.LeRobotDataset = mock_ds_cls
|
|
monkeypatch.setitem(sys.modules, "lerobot.datasets", fake_datasets_module)
|
|
|
|
assert ensure_dataset_available("user/ds", api=api, tags=["lerobot", "lelab"]) is None
|
|
|
|
mock_ds_cls.assert_called_once_with("user/ds")
|
|
mock_ds_cls.return_value.push_to_hub.assert_called_once_with(private=True, tags=["lerobot", "lelab"])
|
|
|
|
|
|
# Branch 3: not on Hub, NOT in local cache → RuntimeError "neither".
|
|
def test_dataset_neither_on_hub_nor_local_raises(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HF_LEROBOT_HOME", str(tmp_path))
|
|
# tmp_path is empty — no local cache.
|
|
|
|
api = _api_with_dataset(False)
|
|
with pytest.raises(RuntimeError, match="neither"):
|
|
ensure_dataset_available("user/ds", api=api)
|