Support streaming from HF Storage Buckets in StreamingLeRobotDataset

Add an opt-in repo_type="bucket" parameter to StreamingLeRobotDataset
and LeRobotDatasetMetadata so a dataset can be streamed directly from an
HF Storage Bucket (hf://buckets/...) with no local download.

When repo_type="bucket":
- skip git-version resolution (buckets have no refs/tags),
- pull the meta/ directory via HfFileSystem.get,
- point url_root at hf://buckets/{repo_id},
- read parquet shards via load_dataset("parquet",
  data_files="hf://buckets/{repo_id}/data/*/*.parquet", ...).

The default repo_type="dataset" preserves all existing behavior.
LeRobotDataset (non-streaming) and create() are unchanged.

Closes #3969

Signed-off-by: Sundar Raghavan <sdraghav@amazon.com>

tests/datasets/test_streaming.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from types import SimpleNamespace
-from unittest.mock import Mock
+from unittest.mock import MagicMock, Mock, patch
 
 import numpy as np
 import pytest
@@ -23,6 +23,7 @@ import torch
 pytest.importorskip("datasets", reason="datasets is required (install lerobot[dataset])")
 
 import lerobot.datasets.streaming_dataset as streaming_dataset_module
+from lerobot.datasets.dataset_metadata import LeRobotDatasetMetadata
 from lerobot.datasets.streaming_dataset import StreamingLeRobotDataset
 from lerobot.datasets.utils import safe_shard
 from lerobot.utils.constants import ACTION
@@ -100,6 +101,7 @@ def test_streaming_dataset_forwards_hub_token_only_for_remote_data(tmp_path, mon
         requested_root,
         streaming_dataset_module.CODEBASE_VERSION,
         force_cache_sync=False,
+        repo_type="dataset",
         token=token,
     )
     if from_local:
@@ -430,3 +432,79 @@ def test_frames_with_delta_consistency_with_shards(
         assert all(t[1] for t in key_checks), (
             f"Checking {list(filter(lambda t: not t[1], key_checks))[0][0]} left and right were found different (i: {i}, frame_idx: {frame_idx})"
         )
+
+
+class _StopConstructionError(Exception):
+    """Sentinel raised from a patched load_dataset to halt __init__ after the branch under test."""
+
+
+def _fake_meta(*args, **kwargs):
+    """Minimal LeRobotDatasetMetadata stand-in exposing only what __init__ reads."""
+    meta = type("_Meta", (), {})()
+    meta.root = kwargs.get("root") or "/tmp/_streaming_meta"
+    meta.revision = kwargs.get("revision") or "v0"
+    meta._version = "v3.0"
+    meta.depth_keys = []
+    meta.image_keys = []
+    meta.rescale_depth_stats = lambda *_a, **_k: None
+    meta.repo_type = kwargs.get("repo_type", "dataset")
+    return meta
+
+
+@pytest.mark.parametrize(
+    "repo_type, expected_source, expected_data_files",
+    [
+        ("bucket", "parquet", "hf://buckets/{repo_id}/data/*/*.parquet"),
+        ("dataset", "{repo_id}", "data/*/*.parquet"),
+    ],
+)
+def test_streaming_repo_type_routes_load_dataset(repo_type, expected_source, expected_data_files):
+    """repo_type='bucket' loads parquet from hf://buckets/...; 'dataset' keeps the Hub-repo path."""
+    captured = {}
+
+    def fake_load_dataset(source, **kwargs):
+        captured["source"] = source
+        captured["data_files"] = kwargs.get("data_files")
+        raise _StopConstructionError
+
+    with (
+        patch("lerobot.datasets.streaming_dataset.LeRobotDatasetMetadata", _fake_meta),
+        patch("lerobot.datasets.streaming_dataset.check_version_compatibility", lambda *a, **k: None),
+        patch("lerobot.datasets.streaming_dataset.load_dataset", fake_load_dataset),
+        pytest.raises(_StopConstructionError),
+    ):
+        StreamingLeRobotDataset(DUMMY_REPO_ID, repo_type=repo_type)
+
+    assert captured["source"] == expected_source.format(repo_id=DUMMY_REPO_ID)
+    assert captured["data_files"] == expected_data_files.format(repo_id=DUMMY_REPO_ID)
+
+
+def test_bucket_metadata_url_root(tmp_path):
+    """repo_type='bucket' produces url_root pointing at hf://buckets/..."""
+    mock_fs = MagicMock()
+    with (
+        patch("huggingface_hub.HfFileSystem", return_value=mock_fs),
+        patch.object(LeRobotDatasetMetadata, "_load_metadata"),
+    ):
+        meta = LeRobotDatasetMetadata(
+            DUMMY_REPO_ID,
+            root=tmp_path,
+            repo_type="bucket",
+        )
+    assert meta.url_root == f"hf://buckets/{DUMMY_REPO_ID}"
+
+
+def test_bucket_skips_get_safe_version(tmp_path):
+    """repo_type='bucket' must NOT call get_safe_version (buckets have no git refs)."""
+    mock_fs = MagicMock()
+    with (
+        patch("lerobot.datasets.dataset_metadata.get_safe_version") as mock_gsv,
+        patch("huggingface_hub.HfFileSystem", return_value=mock_fs),
+        patch.object(LeRobotDatasetMetadata, "_load_metadata"),
+    ):
+        LeRobotDatasetMetadata(
+            DUMMY_REPO_ID,
+            root=tmp_path,
+            repo_type="bucket",
+        )
+    mock_gsv.assert_not_called()