admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-29 20:49:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add fine_tune_vision_encoder flag to SmolVLA

Browse Source 
Fixes #1774
This commit is contained in:
Functionhx
2026-07-09 21:51:59 +08:00
committed by Steven Palma
parent dd08d4eb53
commit 9e30807eeb
2 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lerobot/policies/smolvla/configuration_smolvla.py
+1
View File
@@ -67,6 +67,7 @@ class SmolVLAConfig(PreTrainedConfig):
# Finetuning settings # Finetuning settings
freeze_vision_encoder: bool = True freeze_vision_encoder: bool = True
fine_tune_vision_encoder: bool = False
train_expert_only: bool = True train_expert_only: bool = True
train_state_proj: bool = True train_state_proj: bool = True
src/lerobot/policies/smolvla/modeling_smolvla.py
+8
View File
@@ -162,6 +162,14 @@ class SmolVLAPolicy(PreTrainedPolicy):
self.config = config self.config = config
self.init_rtc_processor() self.init_rtc_processor()
self.model = VLAFlowMatching(config, rtc_processor=self.rtc_processor) self.model = VLAFlowMatching(config, rtc_processor=self.rtc_processor)
if self.config.fine_tune_vision_encoder:
self.model.vlm_with_expert.freeze_vision_encoder = False
for params in self.model.vlm_with_expert.get_vlm_model().vision_model.parameters():
params.requires_grad = True
for params in self.model.vlm_with_expert.get_vlm_model().connector.parameters():
params.requires_grad = True
self.reset() self.reset()
def reset(self): def reset(self):