mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-16 17:20:05 +00:00
feat(processor): enhance type safety with generic DataProcessorPipeline for policy and robot pipelines (#1915)
* refactor(processor): enhance type annotations for processors in record, replay, teleoperate, and control utils - Updated type annotations for preprocessor and postprocessor parameters in record_loop and predict_action functions to specify the expected dictionary types. - Adjusted robot_action_processor type in ReplayConfig and TeleoperateConfig to improve clarity and maintainability. - Ensured consistency in type definitions across multiple files, enhancing overall code readability. * refactor(processor): enhance type annotations for RobotProcessorPipeline in various files - Updated type annotations for RobotProcessorPipeline instances in evaluate.py, record.py, replay.py, teleoperate.py, and other related files to specify input and output types more clearly. - Introduced new type conversions for PolicyAction and EnvTransition to improve type safety and maintainability across the processing pipelines. - Ensured consistency in type definitions, enhancing overall code readability and reducing potential runtime errors. * refactor(processor): update transition handling in processors to use transition_to_batch - Replaced direct transition handling with transition_to_batch in various processor tests and implementations to ensure consistent batching of input data. - Updated assertions in tests to reflect changes in data structure, enhancing clarity and maintainability. - Improved overall code readability by standardizing the way transitions are processed across different processor types. * refactor(tests): standardize transition key usage in processor tests - Updated assertions in processor test files to utilize the TransitionKey for action references, enhancing consistency across tests. - Replaced direct string references with TransitionKey constants for improved readability and maintainability. - Ensured that all relevant tests reflect these changes, contributing to a more uniform approach in handling transitions.
This commit is contained in:
Adil Zouitine
@@ -33,7 +33,7 @@ from lerobot.processor import (
|
||||
TransitionKey,
|
||||
UnnormalizerProcessorStep,
|
||||
)
|
||||
from lerobot.processor.converters import create_transition, identity_transition
|
||||
from lerobot.processor.converters import create_transition, transition_to_batch
|
||||
|
||||
|
||||
def create_default_config():
|
||||
@@ -72,8 +72,6 @@ def test_make_vqbet_processor_basic():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Check processor names
|
||||
@@ -101,8 +99,6 @@ def test_vqbet_processor_with_images():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Create test data with images and states
|
||||
@@ -113,13 +109,16 @@ def test_vqbet_processor_with_images():
|
||||
action = torch.randn(7)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
# Process through preprocessor
|
||||
processed = preprocessor(transition)
|
||||
|
||||
processed = preprocessor(batch)
|
||||
|
||||
# Check that data is batched
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_STATE].shape == (1, 8)
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_IMAGE].shape == (1, 3, 224, 224)
|
||||
assert processed[TransitionKey.ACTION].shape == (1, 7)
|
||||
assert processed[OBS_STATE].shape == (1, 8)
|
||||
assert processed[OBS_IMAGE].shape == (1, 3, 224, 224)
|
||||
assert processed[TransitionKey.ACTION.value].shape == (1, 7)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
@@ -132,8 +131,6 @@ def test_vqbet_processor_cuda():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Create CPU data
|
||||
@@ -144,20 +141,22 @@ def test_vqbet_processor_cuda():
|
||||
action = torch.randn(7)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
# Process through preprocessor
|
||||
processed = preprocessor(transition)
|
||||
|
||||
processed = preprocessor(batch)
|
||||
|
||||
# Check that data is on CUDA
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_STATE].device.type == "cuda"
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_IMAGE].device.type == "cuda"
|
||||
assert processed[TransitionKey.ACTION].device.type == "cuda"
|
||||
assert processed[OBS_STATE].device.type == "cuda"
|
||||
assert processed[OBS_IMAGE].device.type == "cuda"
|
||||
assert processed[TransitionKey.ACTION.value].device.type == "cuda"
|
||||
|
||||
# Process through postprocessor
|
||||
action_transition = create_transition(action=processed[TransitionKey.ACTION])
|
||||
postprocessed = postprocessor(action_transition)
|
||||
postprocessed = postprocessor(processed[TransitionKey.ACTION.value])
|
||||
|
||||
# Check that action is back on CPU
|
||||
assert postprocessed[TransitionKey.ACTION].device.type == "cpu"
|
||||
assert postprocessed.device.type == "cpu"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
@@ -170,8 +169,6 @@ def test_vqbet_processor_accelerate_scenario():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Simulate Accelerate: data already on GPU and batched
|
||||
@@ -183,13 +180,16 @@ def test_vqbet_processor_accelerate_scenario():
|
||||
action = torch.randn(1, 7).to(device)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
# Process through preprocessor
|
||||
processed = preprocessor(transition)
|
||||
|
||||
processed = preprocessor(batch)
|
||||
|
||||
# Check that data stays on same GPU
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_STATE].device == device
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_IMAGE].device == device
|
||||
assert processed[TransitionKey.ACTION].device == device
|
||||
assert processed[OBS_STATE].device == device
|
||||
assert processed[OBS_IMAGE].device == device
|
||||
assert processed[TransitionKey.ACTION.value].device == device
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="Requires at least 2 GPUs")
|
||||
@@ -202,8 +202,6 @@ def test_vqbet_processor_multi_gpu():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Simulate data on different GPU
|
||||
@@ -215,35 +213,23 @@ def test_vqbet_processor_multi_gpu():
|
||||
action = torch.randn(1, 7).to(device)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
# Process through preprocessor
|
||||
processed = preprocessor(transition)
|
||||
|
||||
processed = preprocessor(batch)
|
||||
|
||||
# Check that data stays on cuda:1
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_STATE].device == device
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_IMAGE].device == device
|
||||
assert processed[TransitionKey.ACTION].device == device
|
||||
assert processed[OBS_STATE].device == device
|
||||
assert processed[OBS_IMAGE].device == device
|
||||
assert processed[TransitionKey.ACTION.value].device == device
|
||||
|
||||
|
||||
def test_vqbet_processor_without_stats():
|
||||
"""Test VQBeT processor creation without dataset statistics."""
|
||||
config = create_default_config()
|
||||
|
||||
# Get the steps from the factory function
|
||||
factory_preprocessor, factory_postprocessor = make_vqbet_pre_post_processors(config, dataset_stats=None)
|
||||
|
||||
# Create new processors with EnvTransition input/output
|
||||
preprocessor = DataProcessorPipeline(
|
||||
factory_preprocessor.steps,
|
||||
name=factory_preprocessor.name,
|
||||
to_transition=identity_transition,
|
||||
to_output=identity_transition,
|
||||
)
|
||||
postprocessor = DataProcessorPipeline(
|
||||
factory_postprocessor.steps,
|
||||
name=factory_postprocessor.name,
|
||||
to_transition=identity_transition,
|
||||
to_output=identity_transition,
|
||||
)
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(config, dataset_stats=None)
|
||||
|
||||
# Should still create processors
|
||||
assert preprocessor is not None
|
||||
@@ -257,7 +243,9 @@ def test_vqbet_processor_without_stats():
|
||||
action = torch.randn(7)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
processed = preprocessor(transition)
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
processed = preprocessor(batch)
|
||||
assert processed is not None
|
||||
|
||||
|
||||
@@ -269,8 +257,6 @@ def test_vqbet_processor_save_and_load():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
@@ -278,9 +264,7 @@ def test_vqbet_processor_save_and_load():
|
||||
preprocessor.save_pretrained(tmpdir)
|
||||
|
||||
# Load preprocessor
|
||||
loaded_preprocessor = DataProcessorPipeline.from_pretrained(
|
||||
tmpdir, to_transition=identity_transition, to_output=identity_transition
|
||||
)
|
||||
loaded_preprocessor = DataProcessorPipeline.from_pretrained(tmpdir)
|
||||
|
||||
# Test that loaded processor works
|
||||
observation = {
|
||||
@@ -290,10 +274,11 @@ def test_vqbet_processor_save_and_load():
|
||||
action = torch.randn(7)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
processed = loaded_preprocessor(transition)
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_STATE].shape == (1, 8)
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_IMAGE].shape == (1, 3, 224, 224)
|
||||
assert processed[TransitionKey.ACTION].shape == (1, 7)
|
||||
batch = transition_to_batch(transition)
|
||||
processed = loaded_preprocessor(batch)
|
||||
assert processed[OBS_STATE].shape == (1, 8)
|
||||
assert processed[OBS_IMAGE].shape == (1, 3, 224, 224)
|
||||
assert processed[TransitionKey.ACTION.value].shape == (1, 7)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
@@ -307,8 +292,6 @@ def test_vqbet_processor_mixed_precision():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Replace DeviceProcessorStep with one that uses float16
|
||||
@@ -339,13 +322,16 @@ def test_vqbet_processor_mixed_precision():
|
||||
action = torch.randn(7, dtype=torch.float32)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
# Process through preprocessor
|
||||
processed = preprocessor(transition)
|
||||
|
||||
processed = preprocessor(batch)
|
||||
|
||||
# Check that data is converted to float16
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_STATE].dtype == torch.float16
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_IMAGE].dtype == torch.float16
|
||||
assert processed[TransitionKey.ACTION].dtype == torch.float16
|
||||
assert processed[OBS_STATE].dtype == torch.float16
|
||||
assert processed[OBS_IMAGE].dtype == torch.float16
|
||||
assert processed[TransitionKey.ACTION.value].dtype == torch.float16
|
||||
|
||||
|
||||
def test_vqbet_processor_large_batch():
|
||||
@@ -356,8 +342,6 @@ def test_vqbet_processor_large_batch():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Test with large batch
|
||||
@@ -369,13 +353,16 @@ def test_vqbet_processor_large_batch():
|
||||
action = torch.randn(batch_size, 7)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
# Process through preprocessor
|
||||
processed = preprocessor(transition)
|
||||
|
||||
processed = preprocessor(batch)
|
||||
|
||||
# Check that batch dimension is preserved
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_STATE].shape == (batch_size, 8)
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_IMAGE].shape == (batch_size, 3, 224, 224)
|
||||
assert processed[TransitionKey.ACTION].shape == (batch_size, 7)
|
||||
assert processed[OBS_STATE].shape == (batch_size, 8)
|
||||
assert processed[OBS_IMAGE].shape == (batch_size, 3, 224, 224)
|
||||
assert processed[TransitionKey.ACTION.value].shape == (batch_size, 7)
|
||||
|
||||
|
||||
def test_vqbet_processor_sequential_processing():
|
||||
@@ -386,8 +373,6 @@ def test_vqbet_processor_sequential_processing():
|
||||
preprocessor, postprocessor = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Process multiple samples sequentially
|
||||
@@ -400,14 +385,16 @@ def test_vqbet_processor_sequential_processing():
|
||||
action = torch.randn(7)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
processed = preprocessor(transition)
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
processed = preprocessor(batch)
|
||||
results.append(processed)
|
||||
|
||||
# Check that all results are consistent
|
||||
for result in results:
|
||||
assert result[TransitionKey.OBSERVATION][OBS_STATE].shape == (1, 8)
|
||||
assert result[TransitionKey.OBSERVATION][OBS_IMAGE].shape == (1, 3, 224, 224)
|
||||
assert result[TransitionKey.ACTION].shape == (1, 7)
|
||||
assert result[OBS_STATE].shape == (1, 8)
|
||||
assert result[OBS_IMAGE].shape == (1, 3, 224, 224)
|
||||
assert result[TransitionKey.ACTION.value].shape == (1, 7)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
@@ -420,8 +407,6 @@ def test_vqbet_processor_bfloat16_device_float32_normalizer():
|
||||
preprocessor, _ = make_vqbet_pre_post_processors(
|
||||
config,
|
||||
stats,
|
||||
preprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
postprocessor_kwargs={"to_transition": identity_transition, "to_output": identity_transition},
|
||||
)
|
||||
|
||||
# Modify the pipeline to use bfloat16 device processor with float32 normalizer
|
||||
@@ -457,15 +442,15 @@ def test_vqbet_processor_bfloat16_device_float32_normalizer():
|
||||
action = torch.randn(7, dtype=torch.float32)
|
||||
transition = create_transition(observation, action)
|
||||
|
||||
batch = transition_to_batch(transition)
|
||||
|
||||
# Process through full pipeline
|
||||
processed = preprocessor(transition)
|
||||
processed = preprocessor(batch)
|
||||
|
||||
# Verify: DeviceProcessor → bfloat16, NormalizerProcessor adapts → final output is bfloat16
|
||||
assert processed[TransitionKey.OBSERVATION][OBS_STATE].dtype == torch.bfloat16
|
||||
assert (
|
||||
processed[TransitionKey.OBSERVATION][OBS_IMAGE].dtype == torch.bfloat16
|
||||
) # IDENTITY normalization still gets dtype conversion
|
||||
assert processed[TransitionKey.ACTION].dtype == torch.bfloat16
|
||||
assert processed[OBS_STATE].dtype == torch.bfloat16
|
||||
assert processed[OBS_IMAGE].dtype == torch.bfloat16 # IDENTITY normalization still gets dtype conversion
|
||||
assert processed[TransitionKey.ACTION.value].dtype == torch.bfloat16
|
||||
|
||||
# Verify normalizer automatically adapted its internal state
|
||||
assert normalizer_step.dtype == torch.bfloat16
|
||||
|
||||
Reference in New Issue
Block a user