fix the number of workers to prevent contention

examples/behavior_1k/upload.slurm

@@ -2,10 +2,10 @@
 #SBATCH -J b1k-upload
 #SBATCH -p hopper-cpu
 #SBATCH --qos=high
-#SBATCH -c 8
+#SBATCH -c 1
 #SBATCH -t 48:00:00
-#SBATCH --mem=8G
-#SBATCH --array=0-49%8
+#SBATCH --mem=4G
+#SBATCH --array=0-49%2
 #SBATCH -D /admin/home/francesco_capuano/lerobot
 #SBATCH -o /admin/home/francesco_capuano/lerobot/examples/behavior_1k/logs/%x-%A_%a.out
 #SBATCH -e /admin/home/francesco_capuano/lerobot/examples/behavior_1k/logs/%x-%A_%a.err
@@ -28,8 +28,14 @@ TASK_ID=${SLURM_ARRAY_TASK_ID}
 # Configuration
 ROOT_PATH="/fsx/francesco_capuano/behavior1k-v3"
 HF_USER="fracapuano"
+# Limit upload workers to reduce network contention (default in HF Hub is 4)
+# For I/O-bound uploads, 2-4 workers per task is optimal
+NUM_WORKERS=2
+
+echo "Task ${TASK_ID}: uploading with ${NUM_WORKERS} workers from ${ROOT_PATH}"
 
 python examples/behavior_1k/upload_folders.py \
   --task-id ${TASK_ID} \
   --root-path ${ROOT_PATH} \
-  --hf-user ${HF_USER}
+  --hf-user ${HF_USER} \
+  --num-workers ${NUM_WORKERS}