From d0f58499898b5ff3583a7247d14e5d990d0d1f19 Mon Sep 17 00:00:00 2001 From: CarolinePascal Date: Fri, 17 Jul 2026 21:13:33 +0200 Subject: [PATCH] migration: prefer success over errored row when de-duplicating manifest A dataset can appear multiple times across per-rank/resumed manifests (timed out once, then succeeded on retry). keep='last' could keep the stale ERROR row and wrongly flag an already-migrated dataset (e.g. VoicAndrei/so100_kitchen) for deletion. De-dup now prefers a successful attempt over an errored one. --- community_v3_migration/prune_destination.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/community_v3_migration/prune_destination.py b/community_v3_migration/prune_destination.py index 9f9884fa9..8c5599d88 100644 --- a/community_v3_migration/prune_destination.py +++ b/community_v3_migration/prune_destination.py @@ -82,7 +82,13 @@ def main(): df = pd.concat([pd.read_csv(p) for p in args.manifest], ignore_index=True) if "root" not in df.columns: ap.error("manifest has no 'root' column; is this a run_migration.py manifest?") - df = df.drop_duplicates(subset="root", keep="last") + # A dataset can appear multiple times across per-rank / resumed manifests (e.g. it timed out + # once then succeeded on retry). Collapse to one row per dataset, preferring a SUCCESS over an + # errored attempt so a later successful migration isn't shadowed by a stale ERROR row. + df = (df.assign(_err=df.apply(_is_errored, axis=1)) + .sort_values("_err", kind="stable") + .drop_duplicates(subset="root", keep="first") + .drop(columns="_err")) api = HfApi() present = {p[: -len("/meta/info.json")]