From bb3ef3537f296d94e804492e69dfd00dab63ff96 Mon Sep 17 00:00:00 2001 From: pranjalthebhatia Date: Thu, 4 Jun 2026 17:42:55 -0400 Subject: [PATCH] fix(datasets): support features with a zero-width dimension (shape=(0,)) Declaring a numeric feature with `shape=(0,)` crashed `save_episode()` in two distinct places, leaving `dtype: "string"` as the only (type-lossy) workaround: - `compute_episode_stats` -> `RunningQuantileStats.update` reshaped a size-0 array, raising "ValueError: cannot reshape array of size 0 into shape (0)". - `get_hf_features_from_features` mapped it to a fixed-size Arrow list of length 0, which pyarrow rejects ("list_size needs to be a strict positive integer"). The issue only reported the first error; the second surfaces once the first is fixed. This change handles both: - Skip zero-width features during episode stats, exactly as string/language features are already skipped. - Store 1-D zero-width features as a variable-length sequence (length=-1) so each per-frame value is simply an empty list. Adds a unit test (stats layer) and an integration test that records, saves, and reads back a zero-width feature, asserting it round-trips as an empty vector and is excluded from stats. Fixes #3654 Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lerobot/datasets/compute_stats.py | 7 +++++++ src/lerobot/datasets/feature_utils.py | 9 ++++++--- tests/datasets/test_compute_stats.py | 25 ++++++++++++++++++++++++ tests/datasets/test_dataset_writer.py | 28 +++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/src/lerobot/datasets/compute_stats.py b/src/lerobot/datasets/compute_stats.py index 438ac7fba..d86c684a7 100644 --- a/src/lerobot/datasets/compute_stats.py +++ b/src/lerobot/datasets/compute_stats.py @@ -515,6 +515,13 @@ def compute_episode_stats( if features[key]["dtype"] in {"string", "language"}: continue + # Features with a zero-width dimension (e.g. shape=(0,)) carry no + # statistics-bearing values. Skip them like strings instead of letting + # get_feature_stats -> RunningQuantileStats.update reshape a size-0 array, + # which raises "ValueError: cannot reshape array of size 0". + if any(dim == 0 for dim in features[key].get("shape", ())): + continue + if features[key]["dtype"] in ["image", "video"]: ep_ft_array = sample_images(data) axes_to_reduce = (0, 2, 3) diff --git a/src/lerobot/datasets/feature_utils.py b/src/lerobot/datasets/feature_utils.py index 56264408f..ec303c7c5 100644 --- a/src/lerobot/datasets/feature_utils.py +++ b/src/lerobot/datasets/feature_utils.py @@ -67,9 +67,12 @@ def get_hf_features_from_features(features: dict) -> datasets.Features: elif ft["shape"] == (1,): hf_features[key] = datasets.Value(dtype=ft["dtype"]) elif len(ft["shape"]) == 1: - hf_features[key] = datasets.Sequence( - length=ft["shape"][0], feature=datasets.Value(dtype=ft["dtype"]) - ) + # A zero-width feature (shape=(0,)) has no fixed-size Arrow representation: + # pyarrow rejects a fixed-size list of length 0 ("list_size needs to be a + # strict positive integer"). Store it as a variable-length sequence + # (length=-1) so each per-frame value is simply an empty list. + seq_length = ft["shape"][0] if ft["shape"][0] > 0 else -1 + hf_features[key] = datasets.Sequence(length=seq_length, feature=datasets.Value(dtype=ft["dtype"])) elif len(ft["shape"]) == 2: hf_features[key] = datasets.Array2D(shape=ft["shape"], dtype=ft["dtype"]) elif len(ft["shape"]) == 3: diff --git a/tests/datasets/test_compute_stats.py b/tests/datasets/test_compute_stats.py index 70ba42378..91ba821fa 100644 --- a/tests/datasets/test_compute_stats.py +++ b/tests/datasets/test_compute_stats.py @@ -643,6 +643,31 @@ def test_compute_episode_stats_string_features_skipped(): assert "q01" in stats["action"] +def test_compute_episode_stats_zero_width_feature_skipped(): + """Features with a zero-width dimension carry no values and must be skipped, not crash. + + Regression test for https://github.com/huggingface/lerobot/issues/3654: + a feature declared with shape=(0,) previously reached RunningQuantileStats.update, + where ``batch.reshape(-1, batch.shape[-1])`` raised + "ValueError: cannot reshape array of size 0 into shape (0)". + """ + episode_data = { + "action": np.random.normal(0, 1, (100, 5)).astype(np.float32), + "target": np.zeros((100, 0), dtype=np.float32), # zero-width feature + } + features = { + "action": {"dtype": "float32", "shape": (5,)}, + "target": {"dtype": "float32", "shape": (0,)}, + } + + stats = compute_episode_stats(episode_data, features) + + # Zero-width features are skipped, just like strings; non-empty features are unaffected. + assert "target" not in stats + assert "action" in stats + assert "q01" in stats["action"] + + def test_aggregate_feature_stats_with_quantiles(): """Test aggregating feature stats that include quantiles.""" stats_ft_list = [ diff --git a/tests/datasets/test_dataset_writer.py b/tests/datasets/test_dataset_writer.py index 8670aeebc..9addc8c4f 100644 --- a/tests/datasets/test_dataset_writer.py +++ b/tests/datasets/test_dataset_writer.py @@ -189,6 +189,34 @@ def test_save_multiple_episodes(tmp_path): assert dataset.meta.total_frames == total_frames +def test_save_episode_with_zero_width_feature(tmp_path): + """save_episode() succeeds when a feature has a zero-width dimension (shape=(0,)). + + Regression test for https://github.com/huggingface/lerobot/issues/3654: such a + feature previously crashed stats computation with + "ValueError: cannot reshape array of size 0 into shape (0)". + """ + features = { + **SIMPLE_FEATURES, + "target": {"dtype": "float32", "shape": (0,), "names": None}, + } + root = tmp_path / "ds" + dataset = LeRobotDataset.create(repo_id=DUMMY_REPO_ID, fps=DEFAULT_FPS, features=features, root=root) + for _ in range(4): + dataset.add_frame(_make_frame(features)) + dataset.save_episode() # previously raised ValueError on the zero-width 'target' feature + dataset.finalize() + + assert dataset.meta.total_episodes == 1 + assert dataset.meta.total_frames == 4 + + # The zero-width feature round-trips back as an empty vector and is excluded from stats. + reloaded = LeRobotDataset(repo_id=DUMMY_REPO_ID, root=root) + target = np.asarray(reloaded[0]["target"]) + assert target.shape == (0,) + assert "target" not in (reloaded.meta.stats or {}) + + # ── clear / lifecycle ────────────────────────────────────────────────