admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-24 10:16:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow native HTTP sidecar range diagnostics

Browse Source
This commit is contained in:
Pepijn
2026-06-17 21:36:57 +02:00
parent 04ab43b8d2
commit 51ea18cb7a
1 changed files with 14 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench_episode_byte_cache.py
+14 -7
View File
@@ -53,6 +53,12 @@ def parse_args() -> argparse.Namespace:
default="both", default="both",
help=argparse.SUPPRESS, help=argparse.SUPPRESS,
) )
parser.add_argument(
"--range-backend",
choices=("fsspec", "native-http"),
default="fsspec",
help="Range reader used by indexed/full episode-pool fetch tracks.",
)
parser.add_argument("--num-episodes", type=int, default=512) parser.add_argument("--num-episodes", type=int, default=512)
parser.add_argument( parser.add_argument(
"--manifest-episodes", "--manifest-episodes",
@@ -730,6 +736,8 @@ def main() -> None:
args = parse_args() args = parse_args()
if args.strategy == "full": if args.strategy == "full":
args.strategy = "both" args.strategy = "both"
if args.strategy == "native-http":
args.range_backend = "native-http"
data_root = args.data_root data_root = args.data_root
if data_root.startswith("hf://") and not args.no_hub_branch_assert: if data_root.startswith("hf://") and not args.no_hub_branch_assert:
assert_hf_hub_range_cache_branch() assert_hf_hub_range_cache_branch()
@@ -753,8 +761,8 @@ def main() -> None:
data_root, data_root,
args, args,
parquet_reader, parquet_reader,
range_backend="fsspec", range_backend=args.range_backend,
label="indexed-sidecar", label=f"indexed-sidecar-{args.range_backend}",
sidecar_path=str(sidecar_path), sidecar_path=str(sidecar_path),
) )
return return
@@ -764,20 +772,19 @@ def main() -> None:
data_root, data_root,
args, args,
parquet_reader, parquet_reader,
range_backend="fsspec", range_backend=args.range_backend,
label="indexed-sidecar", label=f"indexed-sidecar-{args.range_backend}",
sidecar_path=str(sidecar_path), sidecar_path=str(sidecar_path),
) )
return return
if sidecar_path is not None and args.strategy == "native-http": if sidecar_path is not None and args.strategy == "native-http":
print("using_indexed_sidecar_for_native_http: sidecar mode uses HfFileSystem range reads")
run_indexed_strategy( run_indexed_strategy(
meta, meta,
data_root, data_root,
args, args,
parquet_reader, parquet_reader,
range_backend="fsspec", range_backend="native-http",
label="indexed-sidecar", label="indexed-sidecar-native-http",
sidecar_path=str(sidecar_path), sidecar_path=str(sidecar_path),
) )
return return