From ede1fc2978445f6d826f982797bd70b0c21b4c48 Mon Sep 17 00:00:00 2001 From: Xingdong Zuo Date: Thu, 30 Jul 2026 20:17:54 +0900 Subject: [PATCH] fix(smolvla): freeze the intended VLM layers when train_expert_only=False (#4019) * fix(smolvla): freeze the intended VLM layers when train_expert_only=False The partial-freeze patterns in set_requires_grad() used a 'text_model.model.' prefix that does not exist in SmolVLM parameter names ('SmolVLMModel.text_model' is a bare LlamaModel, with no nested '.model'). As a result the last VLM layer and the final norm were silently left trainable, defeating the freeze that was added to avoid unused-parameter errors with DDP; only lm_head was frozen by substring luck. Use the real flat names, and raise if any freeze pattern stops matching so a future transformers renaming cannot silently reintroduce the bug. Add a CPU regression test covering both last_layers branches. Fixes #4018 * test(smolvla): drop regression test per review --------- Co-authored-by: Steven Palma --- .../policies/smolvla/smolvlm_with_expert.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/lerobot/policies/smolvla/smolvlm_with_expert.py b/src/lerobot/policies/smolvla/smolvlm_with_expert.py index e4cbd3d52..289dc43e2 100644 --- a/src/lerobot/policies/smolvla/smolvlm_with_expert.py +++ b/src/lerobot/policies/smolvla/smolvlm_with_expert.py @@ -168,14 +168,23 @@ class SmolVLMWithExpertModel(nn.Module): last_layers.append(self.num_vlm_layers - 2) frozen_layers = [ "lm_head", - "text_model.model.norm.weight", + "text_model.norm.weight", ] for layer in last_layers: - frozen_layers.append(f"text_model.model.layers.{layer}.") + frozen_layers.append(f"text_model.layers.{layer}.") + unmatched_patterns = set(frozen_layers) for name, params in self.vlm.named_parameters(): - if any(k in name for k in frozen_layers): + matched_patterns = [k for k in frozen_layers if k in name] + if matched_patterns: params.requires_grad = False + unmatched_patterns.difference_update(matched_patterns) + if unmatched_patterns: + raise RuntimeError( + "Some frozen layer patterns matched no VLM parameters, so the corresponding layers " + "would silently remain trainable (parameter naming may have changed in transformers): " + f"{sorted(unmatched_patterns)}" + ) # To avoid unused params issue with distributed training for name, params in self.lm_expert.named_parameters(): if "lm_head" in name: