admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-23 17:56:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "migration: prefer success over errored row when de-duplicating manifest"

Browse Source 
This reverts commit d0f5849989.
This commit is contained in:
CarolinePascal
2026-07-17 21:16:02 +02:00
parent d0f5849989
commit ef81f9a62d
1 changed files with 1 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
community_v3_migration/prune_destination.py
+1 -7
View File
@@ -82,13 +82,7 @@ def main():
df = pd.concat([pd.read_csv(p) for p in args.manifest], ignore_index=True) df = pd.concat([pd.read_csv(p) for p in args.manifest], ignore_index=True)
if "root" not in df.columns: if "root" not in df.columns:
ap.error("manifest has no 'root' column; is this a run_migration.py manifest?") ap.error("manifest has no 'root' column; is this a run_migration.py manifest?")
# A dataset can appear multiple times across per-rank / resumed manifests (e.g. it timed out df = df.drop_duplicates(subset="root", keep="last")
# once then succeeded on retry). Collapse to one row per dataset, preferring a SUCCESS over an
# errored attempt so a later successful migration isn't shadowed by a stale ERROR row.
df = (df.assign(_err=df.apply(_is_errored, axis=1))
.sort_values("_err", kind="stable")
.drop_duplicates(subset="root", keep="first")
.drop(columns="_err"))
api = HfApi() api = HfApi()
present = {p[: -len("/meta/info.json")] present = {p[: -len("/meta/info.json")]