This commit is contained in:
Pepijn
2025-08-28 08:52:48 +02:00
parent 34ca077d78
commit a4c88d6340
3 changed files with 16 additions and 12 deletions
@@ -59,7 +59,7 @@ class RLearNConfig(PreTrainedConfig):
use_tanh_head: bool = False # when True, bound outputs in [-1, 1] use_tanh_head: bool = False # when True, bound outputs in [-1, 1]
# Training # Training
learning_rate: float = 5e-5 # Reduced for stability learning_rate: float = 1e-4
weight_decay: float = 0.01 weight_decay: float = 0.01
loss_type: str = "composite" # Always use composite loss with spatial awareness loss_type: str = "composite" # Always use composite loss with spatial awareness
ranking_margin: float = 0.1 ranking_margin: float = 0.1
@@ -384,7 +384,10 @@ class RLearNPolicy(PreTrainedPolicy):
target = torch.cat([padding, target], dim=1) # Prepend padding target = torch.cat([padding, target], dim=1) # Prepend padding
import logging import logging
logging.debug(f"Padded targets from {target.shape[1] - padding_needed} to {self.config.max_seq_len}")
logging.debug(
f"Padded targets from {target.shape[1] - padding_needed} to {self.config.max_seq_len}"
)
# Now safely index with idx # Now safely index with idx
target = target[:, idx] target = target[:, idx]
@@ -682,6 +685,7 @@ def extract_visual_sequence(batch: dict[str, Tensor], target_seq_len: int = None
frames = torch.cat([padding, frames], dim=1) # Prepend padding frames = torch.cat([padding, frames], dim=1) # Prepend padding
import logging import logging
logging.debug(f"Padded sequence from {T} to {target_seq_len} frames by repeating first frame") logging.debug(f"Padded sequence from {T} to {target_seq_len} frames by repeating first frame")
return frames return frames
+1 -1
View File
@@ -132,7 +132,7 @@ Default weights: $\lambda_{\text{prog}}=1.0$, $\lambda_{\text{spatial-nce}}=0.5$
- Try DINO v3 as encoder Base 86 M: https://huggingface.co/facebook/dinov3-vitb16-pretrain-lvd1689m with HuggingFaceTB/SmolLM2-135M-Instruct ? [] - Try DINO v3 as encoder Base 86 M: https://huggingface.co/facebook/dinov3-vitb16-pretrain-lvd1689m with HuggingFaceTB/SmolLM2-135M-Instruct ? []
- Add more artificial text to dataset generated by vlm (google gemini) [] - Add more artificial text to dataset generated by vlm (google gemini) []
- See google gemini vlm caption [] https://gemini.google.com/app/7e332ffaf32580f2 - See google gemini vlm caption [] https://gemini.google.com/app/7e332ffaf32580f2
- Multiple captions per video, creat method to generate as much data as possible etc [] https://arxiv.org/abs/2508.13446 - Multiple captions per video, creat method to generate as much data as possible etc [] https://arxiv.org/abs/2508.13446, https://arxiv.org/pdf/2412.04453
- How can we improve spatial aware learning? co generating captions for each frame with language decoder? - How can we improve spatial aware learning? co generating captions for each frame with language decoder?
- Add droid [] - Add droid []
- Extend evaluation [] - Extend evaluation []