admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-05-21 11:39:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix the number of workers to prevent contention

Browse Source
This commit is contained in:
fracapuano
2025-11-21 09:28:36 +00:00
parent c912b1dd03
commit 2b2ff19366
1 changed files with 10 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/behavior_1k/upload.slurm
+10 -4
View File
@@ -2,10 +2,10 @@
#SBATCH -J b1k-upload #SBATCH -J b1k-upload
#SBATCH -p hopper-cpu #SBATCH -p hopper-cpu
#SBATCH --qos=high #SBATCH --qos=high
#SBATCH -c 8 #SBATCH -c 1
#SBATCH -t 48:00:00 #SBATCH -t 48:00:00
#SBATCH --mem=8G #SBATCH --mem=4G
#SBATCH --array=0-49%8 #SBATCH --array=0-49%2
#SBATCH -D /admin/home/francesco_capuano/lerobot #SBATCH -D /admin/home/francesco_capuano/lerobot
#SBATCH -o /admin/home/francesco_capuano/lerobot/examples/behavior_1k/logs/%x-%A_%a.out #SBATCH -o /admin/home/francesco_capuano/lerobot/examples/behavior_1k/logs/%x-%A_%a.out
#SBATCH -e /admin/home/francesco_capuano/lerobot/examples/behavior_1k/logs/%x-%A_%a.err #SBATCH -e /admin/home/francesco_capuano/lerobot/examples/behavior_1k/logs/%x-%A_%a.err
@@ -28,8 +28,14 @@ TASK_ID=${SLURM_ARRAY_TASK_ID}
# Configuration # Configuration
ROOT_PATH="/fsx/francesco_capuano/behavior1k-v3" ROOT_PATH="/fsx/francesco_capuano/behavior1k-v3"
HF_USER="fracapuano" HF_USER="fracapuano"
# Limit upload workers to reduce network contention (default in HF Hub is 4)
# For I/O-bound uploads, 2-4 workers per task is optimal
NUM_WORKERS=2
echo "Task ${TASK_ID}: uploading with ${NUM_WORKERS} workers from ${ROOT_PATH}"
python examples/behavior_1k/upload_folders.py \ python examples/behavior_1k/upload_folders.py \
--task-id ${TASK_ID} \ --task-id ${TASK_ID} \
--root-path ${ROOT_PATH} \ --root-path ${ROOT_PATH} \
--hf-user ${HF_USER} --hf-user ${HF_USER} \
--num-workers ${NUM_WORKERS}