admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-23 17:56:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

migration: add --report to list errored/mislabeled-so/missing together

Browse Source 
Non-destructive listing of all three categories in one pass, marking with '*'
which datasets are absent from the destination repo.
This commit is contained in:
CarolinePascal
2026-07-17 21:07:19 +02:00
parent ba6cf118cf
commit 34f9f07c6d
1 changed files with 18 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
community_v3_migration/prune_destination.py
+18 -9
View File
@@ -72,9 +72,12 @@ def main():
ap.add_argument("--list-missing", action="store_true", ap.add_argument("--list-missing", action="store_true",
help="Report datasets in the manifest that are NOT present in the destination " help="Report datasets in the manifest that are NOT present in the destination "
"repo (grouped by their migration action), then exit without deleting.") "repo (grouped by their migration action), then exit without deleting.")
ap.add_argument("--report", action="store_true",
help="List all three categories (errored, mislabeled-so, missing) without "
"deleting anything, then exit. '*' marks datasets absent from the repo.")
args = ap.parse_args() args = ap.parse_args()
if not (args.errored or args.mislabeled_so or args.list_missing): if not (args.errored or args.mislabeled_so or args.list_missing or args.report):
ap.error("enable at least one of: --errored, --mislabeled-so, --list-missing") ap.error("enable at least one of: --errored, --mislabeled-so, --list-missing, --report")
df = pd.concat([pd.read_csv(p) for p in args.manifest], ignore_index=True) df = pd.concat([pd.read_csv(p) for p in args.manifest], ignore_index=True)
if "root" not in df.columns: if "root" not in df.columns:
@@ -86,13 +89,19 @@ def main():
for p in api.list_repo_files(args.dst_repo, repo_type="dataset") for p in api.list_repo_files(args.dst_repo, repo_type="dataset")
if p.endswith("/meta/info.json")} if p.endswith("/meta/info.json")}
if args.list_missing: if args.report or args.list_missing:
missing = df[~df["root"].isin(present)] def _dump(title, sub):
for _, row in missing.sort_values("action").iterrows(): print(f"== {title} ({len(sub)}) ==")
print(f"{str(row.get('action') or '')[:90]:92s} {row['root']}") for root in sorted(sub):
print(f"\n{len(missing)} dataset(s) in manifest missing from {args.dst_repo} " print(f" {' ' if root in present else '*'} {root}")
f"({df['root'].nunique()} unique manifest rows, {len(present)} datasets in repo).", print()
file=sys.stderr) missing = set(df.loc[~df["root"].isin(present), "root"])
if args.report:
_dump("errored", set(df.loc[df.apply(_is_errored, axis=1), "root"]))
_dump("mislabeled-so", set(df.loc[df.apply(_is_mislabeled_so, axis=1), "root"]))
_dump("missing from repo", missing)
print(f"{df['root'].nunique()} unique manifest rows, {len(present)} datasets in "
f"{args.dst_repo}. '*' = absent from repo.", file=sys.stderr)
return return
to_delete = select(df, present, args.errored, args.mislabeled_so) to_delete = select(df, present, args.errored, args.mislabeled_so)