admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-23 09:46:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

migration: add --list-missing to prune_destination.py

Browse Source 
Report datasets present in the manifest but absent from the destination repo,
grouped by their migration action (ERROR/skipped/...), to explain the src-vs-dst
count gap. Also de-duplicate manifest rows by root (resumed runs append).
This commit is contained in:
CarolinePascal
2026-07-17 17:25:59 +02:00
parent 84896aa8c8
commit ba6cf118cf
1 changed files with 15 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
community_v3_migration/prune_destination.py
+15 -2
View File
@@ -69,19 +69,32 @@ def main():
help="Delete datasets labelled SO but not a real 6-DOF SO arm.") help="Delete datasets labelled SO but not a real 6-DOF SO arm.")
ap.add_argument("--yes", action="store_true", ap.add_argument("--yes", action="store_true",
help="Actually delete. Without it, only prints what would be deleted (dry-run).") help="Actually delete. Without it, only prints what would be deleted (dry-run).")
ap.add_argument("--list-missing", action="store_true",
help="Report datasets in the manifest that are NOT present in the destination "
"repo (grouped by their migration action), then exit without deleting.")
args = ap.parse_args() args = ap.parse_args()
if not (args.errored or args.mislabeled_so): if not (args.errored or args.mislabeled_so or args.list_missing):
ap.error("enable at least one filter: --errored and/or --mislabeled-so") ap.error("enable at least one of: --errored, --mislabeled-so, --list-missing")
df = pd.concat([pd.read_csv(p) for p in args.manifest], ignore_index=True) df = pd.concat([pd.read_csv(p) for p in args.manifest], ignore_index=True)
if "root" not in df.columns: if "root" not in df.columns:
ap.error("manifest has no 'root' column; is this a run_migration.py manifest?") ap.error("manifest has no 'root' column; is this a run_migration.py manifest?")
df = df.drop_duplicates(subset="root", keep="last")
api = HfApi() api = HfApi()
present = {p[: -len("/meta/info.json")] present = {p[: -len("/meta/info.json")]
for p in api.list_repo_files(args.dst_repo, repo_type="dataset") for p in api.list_repo_files(args.dst_repo, repo_type="dataset")
if p.endswith("/meta/info.json")} if p.endswith("/meta/info.json")}
if args.list_missing:
missing = df[~df["root"].isin(present)]
for _, row in missing.sort_values("action").iterrows():
print(f"{str(row.get('action') or '')[:90]:92s} {row['root']}")
print(f"\n{len(missing)} dataset(s) in manifest missing from {args.dst_repo} "
f"({df['root'].nunique()} unique manifest rows, {len(present)} datasets in repo).",
file=sys.stderr)
return
to_delete = select(df, present, args.errored, args.mislabeled_so) to_delete = select(df, present, args.errored, args.mislabeled_so)
for root, reason in sorted(to_delete.items()): for root, reason in sorted(to_delete.items()):
print(f"{reason:14s} {root}") print(f"{reason:14s} {root}")