mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-24 02:06:15 +00:00
Nicolas Rabault
5cf72ec9d4
Resolve the local dataset cache via lerobot.utils.constants.HF_LEROBOT_HOME instead of re-reading the env var by hand, dropping the os/Path imports. Tests now patch the imported constant and assert on a stable message substring (the previous "neither" match only passed by accident, matching the test name embedded in the pytest tmp_path).
56 lines
2.4 KiB
Python
56 lines
2.4 KiB
Python
# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Make a training dataset reachable from an HF Job pod.
|
|
|
|
The pod can't see the host's ~/.cache/huggingface/lerobot, so the dataset has to
|
|
live on the Hub: the pod downloads it by repo_id at train time (the forwarded
|
|
HF_TOKEN covers private datasets). A dataset already on the Hub is used as-is; a
|
|
local-only dataset is pushed to a PRIVATE repo first (never public).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
from lerobot.utils.constants import HF_LEROBOT_HOME
|
|
|
|
if TYPE_CHECKING:
|
|
from huggingface_hub import HfApi
|
|
|
|
|
|
def ensure_dataset_available(repo_id: str, *, api: HfApi, tags: list[str] | None = None) -> None:
|
|
"""Ensure repo_id resolves on the Hub, pushing a local-only dataset privately first.
|
|
|
|
`tags` are attached to the dataset only when we push it (an already-on-Hub
|
|
dataset is left untouched). Raises RuntimeError if the dataset is neither on
|
|
the Hub nor in the local cache.
|
|
"""
|
|
if api.repo_exists(repo_id, repo_type="dataset"):
|
|
return
|
|
|
|
local_present = (HF_LEROBOT_HOME / repo_id / "meta" / "info.json").is_file()
|
|
if not local_present:
|
|
raise RuntimeError(
|
|
f"Dataset '{repo_id}' is not in the local cache ({HF_LEROBOT_HOME}) and could not be "
|
|
f"reached on the Hub — it may not exist, or be private and inaccessible with your "
|
|
f"token. Record or download it first, or run `hf auth login`."
|
|
)
|
|
|
|
print(f"[dataset] '{repo_id}' is local-only; pushing to a PRIVATE Hub repo...")
|
|
# Lazy import: LeRobotDataset pulls in heavy dataset deps; defer until actually needed.
|
|
from lerobot.datasets import LeRobotDataset
|
|
|
|
LeRobotDataset(repo_id).push_to_hub(private=True, tags=tags)
|
|
print(f"[dataset] '{repo_id}' uploaded (private). The job will download it by repo_id.")
|