mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-28 04:06:00 +00:00
fix(datasets): support features with a zero-width dimension (shape=(0,))
Declaring a numeric feature with `shape=(0,)` crashed `save_episode()` in two
distinct places, leaving `dtype: "string"` as the only (type-lossy) workaround:
- `compute_episode_stats` -> `RunningQuantileStats.update` reshaped a size-0
array, raising "ValueError: cannot reshape array of size 0 into shape (0)".
- `get_hf_features_from_features` mapped it to a fixed-size Arrow list of
length 0, which pyarrow rejects ("list_size needs to be a strict positive
integer").
The issue only reported the first error; the second surfaces once the first is
fixed. This change handles both:
- Skip zero-width features during episode stats, exactly as string/language
features are already skipped.
- Store 1-D zero-width features as a variable-length sequence (length=-1) so
each per-frame value is simply an empty list.
Adds a unit test (stats layer) and an integration test that records, saves, and
reads back a zero-width feature, asserting it round-trips as an empty vector and
is excluded from stats.
Fixes #3654
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -515,6 +515,13 @@ def compute_episode_stats(
|
||||
if features[key]["dtype"] in {"string", "language"}:
|
||||
continue
|
||||
|
||||
# Features with a zero-width dimension (e.g. shape=(0,)) carry no
|
||||
# statistics-bearing values. Skip them like strings instead of letting
|
||||
# get_feature_stats -> RunningQuantileStats.update reshape a size-0 array,
|
||||
# which raises "ValueError: cannot reshape array of size 0".
|
||||
if any(dim == 0 for dim in features[key].get("shape", ())):
|
||||
continue
|
||||
|
||||
if features[key]["dtype"] in ["image", "video"]:
|
||||
ep_ft_array = sample_images(data)
|
||||
axes_to_reduce = (0, 2, 3)
|
||||
|
||||
@@ -67,9 +67,12 @@ def get_hf_features_from_features(features: dict) -> datasets.Features:
|
||||
elif ft["shape"] == (1,):
|
||||
hf_features[key] = datasets.Value(dtype=ft["dtype"])
|
||||
elif len(ft["shape"]) == 1:
|
||||
hf_features[key] = datasets.Sequence(
|
||||
length=ft["shape"][0], feature=datasets.Value(dtype=ft["dtype"])
|
||||
)
|
||||
# A zero-width feature (shape=(0,)) has no fixed-size Arrow representation:
|
||||
# pyarrow rejects a fixed-size list of length 0 ("list_size needs to be a
|
||||
# strict positive integer"). Store it as a variable-length sequence
|
||||
# (length=-1) so each per-frame value is simply an empty list.
|
||||
seq_length = ft["shape"][0] if ft["shape"][0] > 0 else -1
|
||||
hf_features[key] = datasets.Sequence(length=seq_length, feature=datasets.Value(dtype=ft["dtype"]))
|
||||
elif len(ft["shape"]) == 2:
|
||||
hf_features[key] = datasets.Array2D(shape=ft["shape"], dtype=ft["dtype"])
|
||||
elif len(ft["shape"]) == 3:
|
||||
|
||||
@@ -643,6 +643,31 @@ def test_compute_episode_stats_string_features_skipped():
|
||||
assert "q01" in stats["action"]
|
||||
|
||||
|
||||
def test_compute_episode_stats_zero_width_feature_skipped():
|
||||
"""Features with a zero-width dimension carry no values and must be skipped, not crash.
|
||||
|
||||
Regression test for https://github.com/huggingface/lerobot/issues/3654:
|
||||
a feature declared with shape=(0,) previously reached RunningQuantileStats.update,
|
||||
where ``batch.reshape(-1, batch.shape[-1])`` raised
|
||||
"ValueError: cannot reshape array of size 0 into shape (0)".
|
||||
"""
|
||||
episode_data = {
|
||||
"action": np.random.normal(0, 1, (100, 5)).astype(np.float32),
|
||||
"target": np.zeros((100, 0), dtype=np.float32), # zero-width feature
|
||||
}
|
||||
features = {
|
||||
"action": {"dtype": "float32", "shape": (5,)},
|
||||
"target": {"dtype": "float32", "shape": (0,)},
|
||||
}
|
||||
|
||||
stats = compute_episode_stats(episode_data, features)
|
||||
|
||||
# Zero-width features are skipped, just like strings; non-empty features are unaffected.
|
||||
assert "target" not in stats
|
||||
assert "action" in stats
|
||||
assert "q01" in stats["action"]
|
||||
|
||||
|
||||
def test_aggregate_feature_stats_with_quantiles():
|
||||
"""Test aggregating feature stats that include quantiles."""
|
||||
stats_ft_list = [
|
||||
|
||||
@@ -189,6 +189,34 @@ def test_save_multiple_episodes(tmp_path):
|
||||
assert dataset.meta.total_frames == total_frames
|
||||
|
||||
|
||||
def test_save_episode_with_zero_width_feature(tmp_path):
|
||||
"""save_episode() succeeds when a feature has a zero-width dimension (shape=(0,)).
|
||||
|
||||
Regression test for https://github.com/huggingface/lerobot/issues/3654: such a
|
||||
feature previously crashed stats computation with
|
||||
"ValueError: cannot reshape array of size 0 into shape (0)".
|
||||
"""
|
||||
features = {
|
||||
**SIMPLE_FEATURES,
|
||||
"target": {"dtype": "float32", "shape": (0,), "names": None},
|
||||
}
|
||||
root = tmp_path / "ds"
|
||||
dataset = LeRobotDataset.create(repo_id=DUMMY_REPO_ID, fps=DEFAULT_FPS, features=features, root=root)
|
||||
for _ in range(4):
|
||||
dataset.add_frame(_make_frame(features))
|
||||
dataset.save_episode() # previously raised ValueError on the zero-width 'target' feature
|
||||
dataset.finalize()
|
||||
|
||||
assert dataset.meta.total_episodes == 1
|
||||
assert dataset.meta.total_frames == 4
|
||||
|
||||
# The zero-width feature round-trips back as an empty vector and is excluded from stats.
|
||||
reloaded = LeRobotDataset(repo_id=DUMMY_REPO_ID, root=root)
|
||||
target = np.asarray(reloaded[0]["target"])
|
||||
assert target.shape == (0,)
|
||||
assert "target" not in (reloaded.meta.stats or {})
|
||||
|
||||
|
||||
# ── clear / lifecycle ────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user