mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-20 16:31:55 +00:00
fix pi052 FAST training consistency
Align tokenizer fitting and loss reduction with the effective training dataset, and fail early when FAST supervision cannot be produced safely. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -94,6 +94,7 @@ def test_action_tokenizer_config_preserves_token_mapping():
|
||||
processor.max_action_tokens = 384
|
||||
processor.fast_skip_tokens = 64
|
||||
processor.paligemma_tokenizer_name = "custom/paligemma"
|
||||
processor.allow_truncation = False
|
||||
processor.action_tokenizer_name = "custom/fast"
|
||||
processor.action_tokenizer_input_object = None
|
||||
|
||||
@@ -102,10 +103,31 @@ def test_action_tokenizer_config_preserves_token_mapping():
|
||||
"max_action_tokens": 384,
|
||||
"fast_skip_tokens": 64,
|
||||
"paligemma_tokenizer_name": "custom/paligemma",
|
||||
"allow_truncation": False,
|
||||
"action_tokenizer_name": "custom/fast",
|
||||
}
|
||||
|
||||
|
||||
def test_action_tokenizer_can_reject_truncated_sequences():
|
||||
processor = object.__new__(ActionTokenizerProcessorStep)
|
||||
processor.max_action_tokens = 4
|
||||
processor.fast_skip_tokens = 128
|
||||
processor.allow_truncation = False
|
||||
processor.action_tokenizer = lambda _actions: [1, 2, 3]
|
||||
processor._paligemma_tokenizer = type(
|
||||
"Tokenizer",
|
||||
(),
|
||||
{
|
||||
"vocab_size": 1000,
|
||||
"bos_token_id": 2,
|
||||
"encode": lambda _self, text, **_kwargs: [10, 11] if text == "Action: " else [12, 1],
|
||||
},
|
||||
)()
|
||||
|
||||
with pytest.raises(ValueError, match="max_action_tokens=4"):
|
||||
processor._tokenize_action(torch.zeros(1, 2, 1))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
"""Provide a mock tokenizer for testing."""
|
||||
|
||||
Reference in New Issue
Block a user