mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-23 17:56:07 +00:00
migration: prefer success over errored row when de-duplicating manifest
A dataset can appear multiple times across per-rank/resumed manifests (timed out once, then succeeded on retry). keep='last' could keep the stale ERROR row and wrongly flag an already-migrated dataset (e.g. VoicAndrei/so100_kitchen) for deletion. De-dup now prefers a successful attempt over an errored one.
This commit is contained in:
@@ -82,7 +82,13 @@ def main():
|
||||
df = pd.concat([pd.read_csv(p) for p in args.manifest], ignore_index=True)
|
||||
if "root" not in df.columns:
|
||||
ap.error("manifest has no 'root' column; is this a run_migration.py manifest?")
|
||||
df = df.drop_duplicates(subset="root", keep="last")
|
||||
# A dataset can appear multiple times across per-rank / resumed manifests (e.g. it timed out
|
||||
# once then succeeded on retry). Collapse to one row per dataset, preferring a SUCCESS over an
|
||||
# errored attempt so a later successful migration isn't shadowed by a stale ERROR row.
|
||||
df = (df.assign(_err=df.apply(_is_errored, axis=1))
|
||||
.sort_values("_err", kind="stable")
|
||||
.drop_duplicates(subset="root", keep="first")
|
||||
.drop(columns="_err"))
|
||||
|
||||
api = HfApi()
|
||||
present = {p[: -len("/meta/info.json")]
|
||||
|
||||
Reference in New Issue
Block a user