From 42d4788e4abb13a106d7770885997679209912ee Mon Sep 17 00:00:00 2001 From: Pepijn Date: Tue, 9 Jun 2026 17:24:30 +0200 Subject: [PATCH] fix(streaming): drop undeclared parquet columns that break batch collation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The data_files_root/bucket path reads an unversioned source (e.g. `main`), which can carry extra annotation columns not in the dataset's feature contract — notably `language_events`, a variable-length list (length 0..N per frame). Passed through to the sample, these break default DataLoader collation ("each element in list of batch should be of equal size"), which is why bucket jobs failed while the hub path (pinned to the clean v3.0 revision) succeeded. Drop any hf_dataset column not in meta.features after load. No-op on a clean revision; removes language_events/language_persistent on main. Verified by reproducing the bucket code path locally via --data_files_root hf://datasets/ (parquet builder + main columns): now decodes and collates instead of raising. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lerobot/datasets/streaming_dataset.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lerobot/datasets/streaming_dataset.py b/src/lerobot/datasets/streaming_dataset.py index e7a7b6bb3..7cf61a6ed 100644 --- a/src/lerobot/datasets/streaming_dataset.py +++ b/src/lerobot/datasets/streaming_dataset.py @@ -372,6 +372,15 @@ class StreamingLeRobotDataset(torch.utils.data.IterableDataset): revision=self.revision, ) + # Drop any parquet columns not declared in the dataset's feature contract. Some revisions / sources + # (e.g. an unversioned bucket holding `main`) carry extra, possibly variable-length annotation + # columns such as `language_events`; left in, they leak into the sample and break default DataLoader + # collation across frames of differing length. On a clean revision this is a no-op. + known_columns = set(self.meta.features) + extra_columns = [c for c in (self.hf_dataset.column_names or []) if c not in known_columns] + if extra_columns: + self.hf_dataset = self.hf_dataset.remove_columns(extra_columns) + self.num_shards = min(self.hf_dataset.num_shards, max_num_shards) @property