From 3a09d0c48a88a7a1c450595802470ac277e7b5b9 Mon Sep 17 00:00:00 2001 From: Pepijn Date: Mon, 22 Jun 2026 11:27:08 +0200 Subject: [PATCH] Track native HTTP failed attempt timing --- scripts/bench_episode_byte_cache.py | 11 +++++++++++ src/lerobot/datasets/episode_video_streaming.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/scripts/bench_episode_byte_cache.py b/scripts/bench_episode_byte_cache.py index 4b64432c9..86e7105f4 100644 --- a/scripts/bench_episode_byte_cache.py +++ b/scripts/bench_episode_byte_cache.py @@ -620,6 +620,7 @@ def _print_range_timing_summary(fetch_pool: dict[str, float]) -> None: ("range_body_s", "http body drain"), ("range_chunk_gap_s", "http chunk wait"), ("range_join_s", "join response chunks"), + ("range_failed_attempt_s", "http failed attempts"), ("range_retry_sleep_s", "http retry sleep"), ): value = fetch_pool.get(key) @@ -627,8 +628,18 @@ def _print_range_timing_summary(fetch_pool: dict[str, float]) -> None: print(f"| {label} | {value * 1000 / range_jobs:.3f} |") if "range_retry_attempts" in fetch_pool: print(f"| http retries | {fetch_pool['range_retry_attempts'] / range_jobs:.3f} |") + if "range_exception_attempts" in fetch_pool: + print(f"| http exceptions | {fetch_pool['range_exception_attempts'] / range_jobs:.3f} |") if fetch_pool.get("range_failed_requests"): print(f"| http failed requests | {fetch_pool['range_failed_requests']:.0f} |") + exception_counts = { + key.removeprefix("range_exception_"): value + for key, value in fetch_pool.items() + if key.startswith("range_exception_") + } + if exception_counts: + summary = ", ".join(f"{name}={count:.0f}" for name, count in sorted(exception_counts.items())) + print(f"| http exception counts | {summary} |") status_counts = { key.removeprefix("range_status_"): value for key, value in fetch_pool.items() diff --git a/src/lerobot/datasets/episode_video_streaming.py b/src/lerobot/datasets/episode_video_streaming.py index 22fea4116..b84b31e08 100644 --- a/src/lerobot/datasets/episode_video_streaming.py +++ b/src/lerobot/datasets/episode_video_streaming.py @@ -344,14 +344,25 @@ class NativeHTTPRangeFetcher: last_exc: Exception | None = None retry_attempts = 0.0 retry_sleep_s = 0.0 + failed_attempt_s = 0.0 + exception_attempts = 0.0 + exception_counts: dict[str, float] = {} for attempt in range(self.max_retries + 1): + attempt_start = time.perf_counter() try: payload, status_code, timings = self._read_range_response_once(url, headers) timings["range_retry_attempts"] = retry_attempts timings["range_retry_sleep_s"] = retry_sleep_s + timings["range_failed_attempt_s"] = failed_attempt_s + timings["range_exception_attempts"] = exception_attempts + timings.update(exception_counts) return payload, status_code, timings except self._RETRYABLE_EXCEPTIONS as exc: last_exc = exc + failed_attempt_s += time.perf_counter() - attempt_start + exception_attempts += 1.0 + exception_key = f"range_exception_{type(exc).__name__}" + exception_counts[exception_key] = exception_counts.get(exception_key, 0.0) + 1.0 if attempt >= self.max_retries: break retry_attempts += 1.0 @@ -362,6 +373,9 @@ class NativeHTTPRangeFetcher: range_failed_requests=1.0, range_retry_attempts=retry_attempts, range_retry_sleep_s=retry_sleep_s, + range_failed_attempt_s=failed_attempt_s, + range_exception_attempts=exception_attempts, + **exception_counts, ) if last_exc is None: raise RuntimeError("HTTP range request failed without an exception")