admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-28 12:15:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Harden production dataset streaming pipeline

Browse Source
This commit is contained in:
Pepijn
2026-07-27 16:07:48 +02:00
parent c0e9b0bbff
commit 79cfb52a71
14 changed files with 397 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench_streaming_dataset.py
+15
View File
@@ -50,6 +50,11 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--episode-pool-size", type=int, default=32)
parser.add_argument("--prefetch-episodes", type=int, default=8)
parser.add_argument("--byte-budget-gb", type=float, default=8.0)
parser.add_argument("--decode-threads", type=int, default=2)
parser.add_argument("--decoded-queue-size", type=int, default=8)
parser.add_argument("--max-open-decoders", type=int, default=64)
parser.add_argument("--native-http-connections", type=int, default=None)
parser.add_argument("--native-http-subranges", type=int, default=1)
parser.add_argument("--warmup-batches", type=int, default=8)
parser.add_argument("--measure-batches", type=int, default=128)
parser.add_argument("--summary-json", type=Path, default=None)
@@ -105,6 +110,11 @@ def main() -> None:
episode_pool_size=args.episode_pool_size,
prefetch_episodes=args.prefetch_episodes,
byte_budget_gb=args.byte_budget_gb,
decode_threads=args.decode_threads,
decoded_queue_size=args.decoded_queue_size,
max_open_decoders=args.max_open_decoders,
native_http_connections=args.native_http_connections,
native_http_subranges=args.native_http_subranges,
max_num_shards=max(1, args.fetch_workers),
return_uint8=True,
)
@@ -184,6 +194,11 @@ def main() -> None:
"episode_pool_size": args.episode_pool_size,
"prefetch_episodes": args.prefetch_episodes,
"byte_budget_gb": args.byte_budget_gb,
"decode_threads": args.decode_threads,
"decoded_queue_size": args.decoded_queue_size,
"max_open_decoders": args.max_open_decoders,
"native_http_connections": args.native_http_connections,
"native_http_subranges": args.native_http_subranges,
"warmup_batches": args.warmup_batches,
"measure_batches": args.measure_batches,
},