admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-28 12:15:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Harden production dataset streaming pipeline

Browse Source
This commit is contained in:
Pepijn
2026-07-27 16:07:48 +02:00
parent c0e9b0bbff
commit 79cfb52a71
14 changed files with 397 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench_episode_byte_cache.py
+6 -1
View File
@@ -70,7 +70,7 @@ def parse_args() -> argparse.Namespace:
type=int,
default=8,
help="Concurrent camera-fetch jobs. Total connections ~= workers x range-subranges; "
"the HF bucket path saturates around 64 connections per host, so keep the product near 64.",
"measure the host-specific saturation point before increasing this value.",
)
parser.add_argument(
"--range-subranges",
@@ -443,6 +443,7 @@ def run_exact_coverage_stream(
decode_pool.shutdown(wait=True)
elapsed = time.perf_counter() - start
replacements = max(0, pool.admitted_count - min(pool_size, len(order)))
result = {
"target_samples_s": target_samples_s,
"actual_samples_s": samples_done / elapsed if elapsed > 0 else float("inf"),
@@ -453,6 +454,10 @@ def run_exact_coverage_stream(
"shard_total_frames": float(total_frames),
"epoch_complete": 1.0 if epoch_complete else 0.0,
"prefetch_ahead": float(prefetch_ahead),
"prefetch_episodes": float(prefetch_ahead),
"replacements": float(replacements),
"replacement_episodes_s": replacements / elapsed if elapsed > 0 else float("inf"),
"samples_per_episode": samples_done / replacements if replacements else float(samples_done),
"batch_size": float(batch_size),
"decode_workers": float(decode_workers),
"kept_up": 1.0
scripts/bench_streaming_dataset.py
+15
View File
@@ -50,6 +50,11 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--episode-pool-size", type=int, default=32)
parser.add_argument("--prefetch-episodes", type=int, default=8)
parser.add_argument("--byte-budget-gb", type=float, default=8.0)
parser.add_argument("--decode-threads", type=int, default=2)
parser.add_argument("--decoded-queue-size", type=int, default=8)
parser.add_argument("--max-open-decoders", type=int, default=64)
parser.add_argument("--native-http-connections", type=int, default=None)
parser.add_argument("--native-http-subranges", type=int, default=1)
parser.add_argument("--warmup-batches", type=int, default=8)
parser.add_argument("--measure-batches", type=int, default=128)
parser.add_argument("--summary-json", type=Path, default=None)
@@ -105,6 +110,11 @@ def main() -> None:
episode_pool_size=args.episode_pool_size,
prefetch_episodes=args.prefetch_episodes,
byte_budget_gb=args.byte_budget_gb,
decode_threads=args.decode_threads,
decoded_queue_size=args.decoded_queue_size,
max_open_decoders=args.max_open_decoders,
native_http_connections=args.native_http_connections,
native_http_subranges=args.native_http_subranges,
max_num_shards=max(1, args.fetch_workers),
return_uint8=True,
)
@@ -184,6 +194,11 @@ def main() -> None:
"episode_pool_size": args.episode_pool_size,
"prefetch_episodes": args.prefetch_episodes,
"byte_budget_gb": args.byte_budget_gb,
"decode_threads": args.decode_threads,
"decoded_queue_size": args.decoded_queue_size,
"max_open_decoders": args.max_open_decoders,
"native_http_connections": args.native_http_connections,
"native_http_subranges": args.native_http_subranges,
"warmup_batches": args.warmup_batches,
"measure_batches": args.measure_batches,
},