mirror of
https://github.com/huggingface/lerobot.git
synced 2026-08-08 17:39:44 +00:00
test(datasets): support the multi-file video layout in dataset fixtures
Add `episodes_per_video_file` to `episodes_factory`: episode `i` goes to `file_index = i // episodes_per_video_file` and `from_timestamp` restarts at 0 on each rollover. `create_videos` encodes one .mp4 per file, and `mock_snapshot_download` lists and creates every video file rather than only file-000. Without the option the fixtures emit the same single-file layout as before. Add streaming tests over that layout, on the plain and the delta path.
This commit is contained in:
committed by
CarolinePascal
parent
e0226b23c8
commit
e2804c9fbd
Vendored
+8
-2
@@ -29,6 +29,7 @@ from lerobot.datasets.utils import (
|
||||
STATS_PATH,
|
||||
)
|
||||
from tests.fixtures.constants import LEROBOT_TEST_DIR
|
||||
from tests.fixtures.dataset_factories import video_file_frames
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
@@ -99,7 +100,12 @@ def mock_snapshot_download_factory(
|
||||
|
||||
video_keys = [key for key, feats in info.features.items() if feats["dtype"] == "video"]
|
||||
for key in video_keys:
|
||||
all_files.append(DEFAULT_VIDEO_PATH.format(video_key=key, chunk_index=0, file_index=0))
|
||||
# A video key spans one file per `videos/<key>/file_index` in the episodes
|
||||
# metadata — several of them once v3.0 rolls a video over.
|
||||
for file_index in video_file_frames(episodes, key, info.total_frames):
|
||||
all_files.append(
|
||||
DEFAULT_VIDEO_PATH.format(video_key=key, chunk_index=0, file_index=file_index)
|
||||
)
|
||||
|
||||
allowed_files = filter_repo_objects(
|
||||
all_files, allow_patterns=allow_patterns, ignore_patterns=ignore_patterns
|
||||
@@ -138,7 +144,7 @@ def mock_snapshot_download_factory(
|
||||
if request_data:
|
||||
create_hf_dataset(local_dir, hf_dataset, data_files_size_in_mb, chunks_size)
|
||||
if request_videos:
|
||||
create_videos(root=local_dir, info=info)
|
||||
create_videos(root=local_dir, info=info, episodes=episodes)
|
||||
|
||||
return str(local_dir)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user