admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-24 10:16:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

migration: add extract_dataset.py to pull a sub-dataset into a standalone repo

Browse Source 
download_subfolder gains a repo= arg so it can source from any monorepo; the new
extract_dataset.py scoped-downloads one sub-dataset and re-uploads it at the root
of a new standalone dataset repo.
This commit is contained in:
CarolinePascal
2026-07-17 15:45:29 +02:00
parent a02a0befd9
commit 3ea347a3d9
2 changed files with 59 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
community_v3_migration/run_migration.py
+4 -4
View File
@@ -22,8 +22,8 @@ from fix_dataset import fix_dataset_in_place
SRC_REPO = "HuggingFaceVLA/community_dataset_v3"
def download_subfolder(sub: str, work_dir: str, patterns: list[str] | None = None) -> None:
"""Download only ``SRC_REPO/{sub}/...`` into ``work_dir``.
def download_subfolder(sub: str, work_dir: str, patterns: list[str] | None = None, repo: str = SRC_REPO) -> None:
"""Download only ``{repo}/{sub}/...`` into ``work_dir``.
``snapshot_download`` walks the entire repo tree (``list_repo_tree(recursive=True)``
with no path scope) before applying ``allow_patterns``. On this 791-dataset monorepo
@@ -36,12 +36,12 @@ def download_subfolder(sub: str, work_dir: str, patterns: list[str] | None = Non
from huggingface_hub.hf_api import RepoFile
api = HfApi()
for entry in api.list_repo_tree(SRC_REPO, path_in_repo=sub, repo_type="dataset", recursive=True):
for entry in api.list_repo_tree(repo, path_in_repo=sub, repo_type="dataset", recursive=True):
if not isinstance(entry, RepoFile):
continue
if patterns and not any(fnmatch(entry.path, pat) for pat in patterns):
continue
hf_hub_download(SRC_REPO, filename=entry.path, repo_type="dataset", local_dir=work_dir)
hf_hub_download(repo, filename=entry.path, repo_type="dataset", local_dir=work_dir)
def list_datasets(api: HfApi, repo: str) -> list[str]: