From 7fc9f5730382d6e9184f5ab0dfea752b19d09f34 Mon Sep 17 00:00:00 2001 From: CarolinePascal Date: Fri, 17 Jul 2026 21:21:00 +0200 Subject: [PATCH] migration: make prune deletions resilient to transient Hub timeouts Retry each delete_folder up to 3x with backoff and continue past failures instead of aborting the whole run on the first ReadTimeout. Re-running is safe (present set is recomputed, already-deleted datasets drop out). --- community_v3_migration/prune_destination.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/community_v3_migration/prune_destination.py b/community_v3_migration/prune_destination.py index 9f9884fa9..df2363966 100644 --- a/community_v3_migration/prune_destination.py +++ b/community_v3_migration/prune_destination.py @@ -17,6 +17,7 @@ to perform the deletions. """ import argparse import sys +import time import pandas as pd from huggingface_hub import HfApi @@ -113,10 +114,23 @@ def main(): if not args.yes: print("dry-run: nothing deleted. re-run with --yes to delete.", file=sys.stderr) return + failed = [] for root, reason in sorted(to_delete.items()): - api.delete_folder(path_in_repo=root, repo_id=args.dst_repo, repo_type="dataset", - commit_message=f"Prune {root} ({reason})") - print(f"deleted {root} ({reason})", file=sys.stderr) + for attempt in range(1, 4): # transient Hub ReadTimeouts are common; retry with backoff + try: + api.delete_folder(path_in_repo=root, repo_id=args.dst_repo, repo_type="dataset", + commit_message=f"Prune {root} ({reason})") + print(f"deleted {root} ({reason})", file=sys.stderr) + break + except Exception as e: + if attempt == 3: + failed.append(root) + print(f"FAILED {root}: {e}", file=sys.stderr) + else: + time.sleep(2 ** attempt) + if failed: + print(f"\n{len(failed)} deletion(s) failed (likely transient); safe to re-run to retry: " + f"{failed}", file=sys.stderr) if __name__ == "__main__":