mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-24 18:26:11 +00:00
Add dataloader_multiprocessing_context, default to spawn
Make the DataLoader multiprocessing start method configurable on TrainPipelineConfig and default it to 'spawn'. The previous default (fork on Linux) is unsafe with libraries that hold non-fork-safe state in the parent process — common ones in this codebase are PyAV, torchcodec, and the ffmpeg shared libs they wrap. Symptoms reported in #2488, #2209, and observed locally include: - multiprocessing.context.AuthenticationError: digest received was wrong - RuntimeError: Pin memory thread exited unexpectedly - RuntimeError: DataLoader worker exited unexpectedly - Random SIGSEGV inside worker processes during video decode Switching to spawn re-imports modules cleanly in each worker and eliminates these failure modes. Added the setting as a config field rather than hard-coding so users on platforms where fork is preferred can opt back in via --dataloader-multiprocessing-context=fork.
This commit is contained in:
@@ -101,6 +101,14 @@ class TrainPipelineConfig(HubMixin):
|
||||
batch_size: int = 8
|
||||
prefetch_factor: int = 4
|
||||
persistent_workers: bool = True
|
||||
# DataLoader multiprocessing start method. "spawn" is the safe default on
|
||||
# Linux because workers do not inherit fork-time state from the parent —
|
||||
# "fork" can crash with non-fork-safe libraries that the parent has loaded
|
||||
# (e.g. PyAV / torchcodec / ffmpeg) with errors like
|
||||
# `multiprocessing.context.AuthenticationError: digest received was wrong`,
|
||||
# `Pin memory thread exited unexpectedly`, or random worker segfaults.
|
||||
# See https://github.com/huggingface/lerobot/issues/2488.
|
||||
dataloader_multiprocessing_context: str = "spawn"
|
||||
steps: int = 100_000
|
||||
# Run policy in the simulation environment every N steps to measure reward/success (0 = disabled).
|
||||
env_eval_freq: int = 20_000
|
||||
|
||||
@@ -475,6 +475,7 @@ def train(cfg: TrainPipelineConfig, accelerator: "Accelerator | None" = None):
|
||||
collate_fn=collate_fn,
|
||||
prefetch_factor=cfg.prefetch_factor if cfg.num_workers > 0 else None,
|
||||
persistent_workers=cfg.persistent_workers and cfg.num_workers > 0,
|
||||
multiprocessing_context=cfg.dataloader_multiprocessing_context if cfg.num_workers > 0 else None,
|
||||
)
|
||||
|
||||
# Build eval dataloader if a held-out split exists
|
||||
|
||||
Reference in New Issue
Block a user