refactor(converters): gather converters and refactor the logic (#1833)

* refactor(converters): move batch transition functions to converters module

- Moved `_default_batch_to_transition` and `_default_transition_to_batch` functions from `pipeline.py` to `converters.py` for better organization and separation of concerns.
- Updated references in `RobotProcessor` to use the new location of these functions.
- Added tests to ensure correct functionality of the transition functions, including handling of index and task_index fields.
- Removed redundant tests from `pipeline.py` to streamline the test suite.

* refactor(processor): reorganize EnvTransition and TransitionKey definitions

- Moved `EnvTransition` and `TransitionKey` classes from `pipeline.py` to a new `core.py` module for better structure and maintainability.
- Updated import statements across relevant modules to reflect the new location of these definitions, ensuring consistent access throughout the codebase.

* refactor(converters): rename and update dataset frame conversion functions

- Replaced `to_dataset_frame` with `transition_to_dataset_frame` for clarity and consistency in naming.
- Updated references in `record.py`, `pipeline.py`, and tests to use the new function name.
- Introduced `merge_transitions` to streamline the merging of transitions, enhancing readability and maintainability.
- Adjusted related tests to ensure correct functionality with the new naming conventions.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix(processor): solve conflict artefacts

* refactor(converters): remove unused identity function and update type hints for merge_transitions

* refactor(processor): remove unused identity import and clean up gym_manipulator.py

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Steven Palma <steven.palma@huggingface.co>

tests/processor/test_pipeline.py

@@ -1659,109 +1659,6 @@ def test_state_file_naming_with_multiple_processors():
         assert loaded_post.steps[0].window_size == 10
 
 
-def test_default_batch_to_transition_with_index_fields():
-    """Test that _default_batch_to_transition handles index and task_index fields correctly."""
-    from lerobot.processor.pipeline import _default_batch_to_transition
-
-    # Create batch with index and task_index fields
-    batch = {
-        "observation.state": torch.randn(1, 7),
-        "action": torch.randn(1, 4),
-        "next.reward": 1.5,
-        "next.done": False,
-        "task": ["pick_cube"],
-        "index": torch.tensor([42], dtype=torch.int64),
-        "task_index": torch.tensor([3], dtype=torch.int64),
-    }
-
-    transition = _default_batch_to_transition(batch)
-
-    # Check basic transition structure
-    assert TransitionKey.OBSERVATION in transition
-    assert TransitionKey.ACTION in transition
-    assert TransitionKey.COMPLEMENTARY_DATA in transition
-
-    # Check that index and task_index are in complementary_data
-    comp_data = transition[TransitionKey.COMPLEMENTARY_DATA]
-    assert "index" in comp_data
-    assert "task_index" in comp_data
-    assert "task" in comp_data
-
-    # Verify values
-    assert torch.equal(comp_data["index"], batch["index"])
-    assert torch.equal(comp_data["task_index"], batch["task_index"])
-    assert comp_data["task"] == batch["task"]
-
-
-def test_default_transition_to_batch_with_index_fields():
-    """Test that _default_transition_to_batch handles index and task_index fields correctly."""
-    from lerobot.processor.pipeline import _default_transition_to_batch
-
-    # Create transition with index and task_index in complementary_data
-    transition = create_transition(
-        observation={"observation.state": torch.randn(1, 7)},
-        action=torch.randn(1, 4),
-        reward=1.5,
-        done=False,
-        complementary_data={
-            "task": ["navigate"],
-            "index": torch.tensor([100], dtype=torch.int64),
-            "task_index": torch.tensor([5], dtype=torch.int64),
-        },
-    )
-
-    batch = _default_transition_to_batch(transition)
-
-    # Check that index and task_index are in the batch
-    assert "index" in batch
-    assert "task_index" in batch
-    assert "task" in batch
-
-    # Verify values
-    assert torch.equal(batch["index"], transition[TransitionKey.COMPLEMENTARY_DATA]["index"])
-    assert torch.equal(batch["task_index"], transition[TransitionKey.COMPLEMENTARY_DATA]["task_index"])
-    assert batch["task"] == transition[TransitionKey.COMPLEMENTARY_DATA]["task"]
-
-
-def test_batch_to_transition_without_index_fields():
-    """Test that conversion works without index and task_index fields."""
-    from lerobot.processor.pipeline import _default_batch_to_transition
-
-    # Batch without index/task_index
-    batch = {
-        "observation.state": torch.randn(1, 7),
-        "action": torch.randn(1, 4),
-        "task": ["pick_cube"],
-    }
-
-    transition = _default_batch_to_transition(batch)
-    comp_data = transition[TransitionKey.COMPLEMENTARY_DATA]
-
-    # Should have task but not index/task_index
-    assert "task" in comp_data
-    assert "index" not in comp_data
-    assert "task_index" not in comp_data
-
-
-def test_transition_to_batch_without_index_fields():
-    """Test that conversion works without index and task_index fields."""
-    from lerobot.processor.pipeline import _default_transition_to_batch
-
-    # Transition without index/task_index
-    transition = create_transition(
-        observation={"observation.state": torch.randn(1, 7)},
-        action=torch.randn(1, 4),
-        complementary_data={"task": ["navigate"]},
-    )
-
-    batch = _default_transition_to_batch(transition)
-
-    # Should have task but not index/task_index
-    assert "task" in batch
-    assert "index" not in batch
-    assert "task_index" not in batch
-
-
 def test_override_with_device_strings():
     """Test overriding device parameters with string values."""