'resume': False, 'save_checkpoint': True, 'save_freq': 20000, 'scheduler': {'decay_lr': 2.5e-06, 'num_decay_steps': 30000, 'num_warmup_steps': 1000, 'peak_lr': 0.0001, 'type': 'cosine_decay_with_warmup'}, 'seed': 1000, 'steps': 100000, 'use_policy_training_preset': True, 'wandb': {'disable_artifact': False, 'enable': False, 'entity': None, 'mode': None, 'notes': None, 'project': 'lerobot', 'run_id': None}} INFO 2025-09-08 13:23:15 ils/utils.py:48 Cuda backend detected, using cuda. WARNING 2025-09-08 13:23:15 /policies.py:81 Device 'None' is not available. Switching to 'cuda'. INFO 2025-09-08 13:23:15 ccelerate.py:99 {'batch_size': 32, 'dataset': {'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'kwargs': {'brightness': [0.8, 1.2]}, 'type': 'ColorJitter', 'weight': 1.0}, 'contrast': {'kwargs': {'contrast': [0.8, 1.2]}, 'type': 'ColorJitter', 'weight': 1.0}, 'hue': {'kwargs': {'hue': [-0.05, 0.05]}, 'type': 'ColorJitter', 'weight': 1.0}, 'saturation': {'kwargs': {'saturation': [0.5, 1.5]}, 'type': 'ColorJitter', 'weight': 1.0}, 'sharpness': {'kwargs': {'sharpness': [0.5, 1.5]}, 'type': 'SharpnessJitter', 'weight': 1.0}}}, 'repo_id': 'HuggingFaceVLA/libero', 'revision': None, 'root': '/raid/jade/.cache/huggingface/datasets', 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': {'camera_name': 'agentview_image,robot0_eye_in_hand_image', 'episode_length': 520, 'features': {'action': {'shape': [7], 'type': }, 'agent_pos': {'shape': [8], 'type': }, 'pixels/agentview_image': {'shape': [360, 360, 3], 'type': }, 'pixels/robot0_eye_in_hand_image': {'shape': [360, 360, 3], 'type': }}, 'features_map': {'action': 'action', 'agent_pos': 'observation.state', 'pixels/agentview_image': 'observation.images.image', 'pixels/robot0_eye_in_hand_image': 'observation.images.image2'}, 'fps': 30, 'init_states': True, 'max_parallel_tasks': 5, 'multitask_eval': True, 'obs_type': 'pixels_agent_pos', 'render_mode': 'rgb_array', 'task': 'libero_spatial', 'type': 'libero'}, 'eval': {'batch_size': 1, 'n_episodes': 1, 'use_async_envs': False}, 'eval_freq': 0, 'job_name': 'libero_smolvla', 'log_freq': 200, 'num_workers': 4, 'optimizer': {'betas': [0.9, 0.95], 'eps': 1e-08, 'grad_clip_norm': 10, 'lr': 0.0001, 'type': 'adamw', 'weight_decay': 1e-10}, 'output_dir': '/raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla_lr1e-4bs32steps100000', 'policy': {'adapt_to_pi_aloha': False, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'chunk_size': 50, 'device': 'cuda', 'empty_cameras': 0, 'expert_width_multiplier': 0.75, 'freeze_vision_encoder': True, 'gradient_accumulation_steps': 1, 'input_features': {}, 'license': None, 'load_vlm_weights': False, 'max_action_dim': 32, 'max_period': 4.0, 'max_state_dim': 32, 'min_period': 0.004, 'n_action_steps': 1, 'n_obs_steps': 1, 'normalization_mapping': {'ACTION': , 'STATE': , 'VISUAL': }, 'num_expert_layers': -1, 'num_steps': 10, 'num_vlm_layers': 16, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_grad_clip_norm': 10, 'optimizer_lr': 0.0001, 'optimizer_weight_decay': 1e-10, 'output_features': {}, 'pad_language_to': 'longest', 'prefix_length': -1, 'private': None, 'push_to_hub': True, 'repo_id': 'None', 'resize_imgs_with_padding': [512, 512], 'scheduler_decay_lr': 2.5e-06, 'scheduler_decay_steps': 30000, 'scheduler_warmup_steps': 1000, 'self_attn_every_n_layers': 2, 'tags': None, 'tokenizer_max_length': 48, 'train_expert_only': True, 'train_state_proj': True, 'type': 'smolvla', 'use_amp': True, 'use_cache': True, 'use_delta_joint_actions_aloha': False, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Instruct'}, 'resume': False, 'save_checkpoint': True, 'save_freq': 20000, 'scheduler': {'decay_lr': 2.5e-06, 'num_decay_steps': 30000, 'num_warmup_steps': 1000, 'peak_lr': 0.0001, 'type': 'cosine_decay_with_warmup'}, 'seed': 1000, 'steps': 100000, 'use_policy_training_preset': True, 'wandb': {'disable_artifact': False, 'enable': False, 'entity': None, 'mode': None, 'notes': None, 'project': 'lerobot', 'run_id': None}} WARNING 2025-09-08 13:23:15 ls/other.py:512 Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher . WARNING 2025-09-08 13:23:15 ls/other.py:512 Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher . INFO 2025-09-08 13:23:15 celerate.py:149 Creating dataset Resolving data files: 100%|█████████████████████████████████| 1693/1693 [00:00<00:00, 35414.48it/s] Loading dataset shards: 100%|████████████████████████████████████| 69/69 [00:00<00:00, 5660.00it/s] Resolving data files: 100%|█████████████████████████████████| 1693/1693 [00:00<00:00, 43760.67it/s] Loading dataset shards: 100%|████████████████████████████████████| 69/69 [00:00<00:00, 5629.72it/s] c INFO 2025-09-08 13:23:22 celerate.py:160 Creating policy /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarnin g: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. warnings.warn( c /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u ser. warnings.warn( # warn only once [rank1]:[W908 13:23:22.785516795 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used b y this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can pecify device_id in init_process_group() to force use of a particular device. Reducing the number of VLM layers to 16 ... /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u ser. warnings.warn( # warn only once [rank0]:[W908 13:23:43.028071493 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used b y this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can pecify device_id in init_process_group() to force use of a particular device. INFO 2025-09-08 13:23:43 celerate.py:171 Creating optimizer and scheduler /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarnin g: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. warnings.warn( Reducing the number of VLM layers to 16 ... INFO 2025-09-08 13:24:04 celerate.py:211 Output dir: /raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla _lr1e-4bs32steps100000 INFO 2025-09-08 13:24:04 celerate.py:213 cfg.env.task='libero_spatial' INFO 2025-09-08 13:24:04 celerate.py:214 cfg.steps=100000 (100K) INFO 2025-09-08 13:24:04 celerate.py:215 dataset.num_frames=273465 (273K) INFO 2025-09-08 13:24:04 celerate.py:216 dataset.num_episodes=1693 INFO 2025-09-08 13:24:04 celerate.py:217 num_learnable_params=99880992 (100M) INFO 2025-09-08 13:24:04 celerate.py:218 num_total_params=450046220 (450M) INFO 2025-09-08 13:24:04 celerate.py:219 Number of processes: 2 INFO 2025-09-08 13:24:04 celerate.py:220 Device: cuda:0 INFO 2025-09-08 13:24:04 celerate.py:221 Mixed precision: bf16 INFO 2025-09-08 13:24:04 celerate.py:243 Start offline training on a fixed dataset bach: dict_keys(['observation.images.image', 'observation.images.image2', 'observation.state', 'action', 'timestamp ', 'frame_index', 'episode_index', 'index', 'task_index', 'observation.images.image_is_pad', 'observation.images.ima ge2_is_pad', 'observation.state_is_pad', 'action_is_pad', 'task']) > /home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py(263)train() -> train_tracker, output_dict = update_policy( (Pdb) bach: dict_keys(['observation.images.image', 'observation.images.image2', 'observation.state', 'action', 'timestamp ', 'frame_index', 'episode_index', 'index', 'task_index', 'observation.images.image_is_pad', 'observation.images.ima ge2_is_pad', 'observation.state_is_pad', 'action_is_pad', 'task']) > /home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py(263)train() -> train_tracker, output_dict = update_policy( (Pdb) batch.keys()[rank0]:[W908 13:24:43.868440913 reducer.cpp:1430] Warning: find_unused_parameters=True was specif ied in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra tr aversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never h as any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator()) policy.config.input_features *** SyntaxError: invalid syntax (Pdb) policy.config.input_features *** AttributeError: 'DistributedDataParallel' object has no attribute 'config' (Pdb) policy DistributedDataParallel( (module): SmolVLAPolicy( (normalize_inputs): Normalize( (buffer_observation_state): ParameterDict( (mean): Parameter containing: [torch.cuda.FloatTensor of size 8 (cuda:1)] (std): Parameter containing: [torch.cuda.FloatTensor of size 8 (cuda:1)] ) ) (normalize_targets): Normalize( (buffer_action): ParameterDict( (mean): Parameter containing: [torch.cuda.FloatTensor of size 7 (cuda:1)] (std): Parameter containing: [torch.cuda.FloatTensor of size 7 (cuda:1)] ) ) (unnormalize_outputs): Unnormalize( (buffer_action): ParameterDict( (mean): Parameter containing: [torch.cuda.FloatTensor of size 7 (cuda:1)] (std): Parameter containing: [torch.cuda.FloatTensor of size 7 (cuda:1)] ) ) (model): VLAFlowMatching( (vlm_with_expert): SmolVLMWithExpertModel( (vlm): SmolVLMForConditionalGeneration( (model): SmolVLMModel( (vision_model): SmolVLMVisionTransformer( (embeddings): SmolVLMVisionEmbeddings( (patch_embedding): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), padding=valid) (position_embedding): Embedding(1024, 768) ) (encoder): SmolVLMEncoder( (layers): ModuleList( (0-11): 12 x SmolVLMEncoderLayer( (self_attn): SmolVLMVisionAttention( (k_proj): Linear(in_features=768, out_features=768, bias=True) (v_proj): Linear(in_features=768, out_features=768, bias=True) (q_proj): Linear(in_features=768, out_features=768, bias=True) (out_proj): Linear(in_features=768, out_features=768, bias=True) ) (layer_norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True) (mlp): SmolVLMVisionMLP( (activation_fn): PytorchGELUTanh() (fc1): Linear(in_features=768, out_features=3072, bias=True) (fc2): Linear(in_features=3072, out_features=768, bias=True) ) (layer_norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True) ) ) ) (post_layernorm): LayerNorm((768,), eps=1e-06, elementwise_affine=True) ) (connector): SmolVLMConnector( (modality_projection): SmolVLMSimpleMLP( (proj): Linear(in_features=12288, out_features=960, bias=False) ) ) (text_model): LlamaModel( (embed_tokens): Embedding(49280, 960, padding_idx=2) (layers): ModuleList( (0-15): 16 x LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=960, out_features=960, bias=False) (k_proj): Linear(in_features=960, out_features=320, bias=False) (v_proj): Linear(in_features=960, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=960, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=960, out_features=2560, bias=False) (up_proj): Linear(in_features=960, out_features=2560, bias=False) (down_proj): Linear(in_features=2560, out_features=960, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((960,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((960,), eps=1e-05) ) ) (norm): LlamaRMSNorm((960,), eps=1e-05) (rotary_emb): LlamaRotaryEmbedding() ) ) (lm_head): Linear(in_features=960, out_features=49280, bias=False) ) (lm_expert): LlamaModel( (embed_tokens): None (layers): ModuleList( (0): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=720, out_features=320, bias=False) (v_proj): Linear(in_features=720, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (1): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=320, out_features=320, bias=False) (v_proj): Linear(in_features=320, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (2): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=720, out_features=320, bias=False) (v_proj): Linear(in_features=720, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (3): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=320, out_features=320, bias=False) (v_proj): Linear(in_features=320, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (4): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=720, out_features=320, bias=False) (v_proj): Linear(in_features=720, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (5): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=320, out_features=320, bias=False) (v_proj): Linear(in_features=320, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (6): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=720, out_features=320, bias=False) (v_proj): Linear(in_features=720, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (7): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=320, out_features=320, bias=False) (v_proj): Linear(in_features=320, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (8): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=720, out_features=320, bias=False) (v_proj): Linear(in_features=720, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (9): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=320, out_features=320, bias=False) (v_proj): Linear(in_features=320, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (10): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=720, out_features=320, bias=False) (v_proj): Linear(in_features=720, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (11): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=320, out_features=320, bias=False) (v_proj): Linear(in_features=320, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (12): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=720, out_features=320, bias=False) (v_proj): Linear(in_features=720, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (13): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=320, out_features=320, bias=False) (v_proj): Linear(in_features=320, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (14): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=720, out_features=320, bias=False) (v_proj): Linear(in_features=720, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) (15): LlamaDecoderLayer( (self_attn): LlamaAttention( (q_proj): Linear(in_features=720, out_features=960, bias=False) (k_proj): Linear(in_features=320, out_features=320, bias=False) (v_proj): Linear(in_features=320, out_features=320, bias=False) (o_proj): Linear(in_features=960, out_features=720, bias=False) ) (mlp): LlamaMLP( (gate_proj): Linear(in_features=720, out_features=2048, bias=False) (up_proj): Linear(in_features=720, out_features=2048, bias=False) (down_proj): Linear(in_features=2048, out_features=720, bias=False) (act_fn): SiLU() ) (input_layernorm): LlamaRMSNorm((720,), eps=1e-05) (post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05) ) ) (norm): LlamaRMSNorm((720,), eps=1e-05) (rotary_emb): LlamaRotaryEmbedding() ) ) (state_proj): Linear(in_features=32, out_features=960, bias=True) (action_in_proj): Linear(in_features=32, out_features=720, bias=True) (action_out_proj): Linear(in_features=720, out_features=32, bias=True) (action_time_mlp_in): Linear(in_features=1440, out_features=720, bias=True) (action_time_mlp_out): Linear(in_features=720, out_features=720, bias=True) ) ) ) (Pdb) policy.config *** AttributeError: 'DistributedDataParallel' object has no attribute 'config' (Pdb) policy.input_features *** AttributeError: 'DistributedDataParallel' object has no attribute 'input_features' (Pdb) quit() [rank1]: Traceback (most recent call last): [rank1]: File "/home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py", line 368, in [rank1]: train() [rank1]: File "/home/jade_choghari/lerobot/src/lerobot/configs/parser.py", line 225, in wrapper_inner [rank1]: response = fn(cfg, *args, **kwargs) [rank1]: File "/home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py", line 263, in train [rank1]: train_tracker, output_dict = update_policy( [rank1]: File "/home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py", line 263, in train [rank1]: train_tracker, output_dict = update_policy( [rank1]: File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/bdb.py", line 90, in trace_dispatch [rank1]: return self.dispatch_line(frame) [rank1]: File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/bdb.py", line 115, in dispatch_line [rank1]: if self.quitting: raise BdbQuit [rank1]: bdb.BdbQuit W0908 13:25:34.274000 776579 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 776 663 closing signal SIGTERM E0908 13:25:34.589000 776579 site-packages/torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1 ) local_rank: 1 (pid: 776664) of binary: /home/jade_choghari/miniconda3/envs/lerobot/bin/python Traceback (most recent call last): File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/runpy.py", line 196, in _run_module_as_main return _run_code(code, main_globals, None, File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/runpy.py", line 86, in _run_code exec(code, run_globals) File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/commands/launch.py", lin e 1245, in main() File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/commands/launch.py", lin e 1241, in main launch_command(args) File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/commands/launch.py", lin e 1226, in launch_command multi_gpu_launcher(args) File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/commands/launch.py", lin e 853, in multi_gpu_launcher distrib_run.run(args) File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/run.py", line 883 , in run elastic_launch( File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 139, in __call__ return launch_agent(self._config, self._entrypoint, list(args)) File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 270, in launch_agent raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ src/lerobot/scripts/train_accelerate.py FAILED ------------------------------------------------------------ Failures: ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2025-09-08_13:25:34 host : hf-dgx-01 rank : 1 (local_rank: 1) exitcode : 1 (pid: 776664) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ (lerobot) jade_choghari@hf-dgx-01:~/lerobot$ clear (lerobot) jade_choghari@hf-dgx-01:~/lerobot$ clear (lerobot) jade_choghari@hf-dgx-01:~/lerobot$ bash examples/7_train_acc.sh Training dir: /raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla_lr1e-4bs32steps100000 The following values were not passed to `accelerate launch` and had defaults used instead: More than one GPU was found, enabling multi-GPU training. If this was unintended please pass in `--num_processes=1`. `--dynamo_backend` was set to a value of `'no'` To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/utils/launch.py:238: UserWarning : Port `29522` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open por t automatically for this session. If this current attempt fails, or for more control in future runs, please specify a different port (e.g., `--main_process_port `) or use `--main_process_port 0` for automatic selec tion in your launch command or Accelerate config file. warnings.warn( INFO 2025-09-08 13:33:47 ils/utils.py:48 Cuda backend detected, using cuda. WARNING 2025-09-08 13:33:47 /policies.py:81 Device 'None' is not available. Switching to 'cuda'. INFO 2025-09-08 13:33:47 ccelerate.py:99 {'batch_size': 32, 'dataset': {'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'kwargs': {'brightness': [0.8, 1.2]}, 'type': 'ColorJitter', 'weight': 1.0}, 'contrast': {'kwargs': {'contrast': [0.8, 1.2]}, 'type': 'ColorJitter', 'weight': 1.0}, 'hue': {'kwargs': {'hue': [-0.05, 0.05]}, 'type': 'ColorJitter', 'weight': 1.0}, 'saturation': {'kwargs': {'saturation': [0.5, 1.5]}, 'type': 'ColorJitter', 'weight': 1.0}, 'sharpness': {'kwargs': {'sharpness': [0.5, 1.5]}, 'type': 'SharpnessJitter', 'weight': 1.0}}}, 'repo_id': 'HuggingFaceVLA/libero', 'revision': None, 'root': '/raid/jade/.cache/huggingface/datasets', 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': {'camera_name': 'agentview_image,robot0_eye_in_hand_image', 'episode_length': 520, 'features': {'action': {'shape': [7], 'type': }, 'agent_pos': {'shape': [8], 'type': }, 'pixels/agentview_image': {'shape': [360, 360, 3], 'type': }, 'pixels/robot0_eye_in_hand_image': {'shape': [360, 360, 3], 'type': }}, 'features_map': {'action': 'action', 'agent_pos': 'observation.state', 'pixels/agentview_image': 'observation.images.image', 'pixels/robot0_eye_in_hand_image': 'observation.images.image2'}, 'fps': 30, 'init_states': True, 'max_parallel_tasks': 5, 'multitask_eval': True, 'obs_type': 'pixels_agent_pos', 'render_mode': 'rgb_array', 'task': 'libero_spatial', 'type': 'libero'}, 'eval': {'batch_size': 1, 'n_episodes': 1, 'use_async_envs': False}, 'eval_freq': 0, 'job_name': 'libero_smolvla', 'log_freq': 200, 'num_workers': 4, 'optimizer': {'betas': [0.9, 0.95], 'eps': 1e-08, 'grad_clip_norm': 10, 'lr': 0.0001, 'type': 'adamw', 'weight_decay': 1e-10}, 'output_dir': '/raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla_lr1e-4bs32steps100000', 'policy': {'adapt_to_pi_aloha': False, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'chunk_size': 50, 'device': 'cuda', 'empty_cameras': 0, 'expert_width_multiplier': 0.75, 'freeze_vision_encoder': True, 'gradient_accumulation_steps': 1, 'input_features': {}, 'license': None, 'load_vlm_weights': False, 'max_action_dim': 32, 'max_period': 4.0, 'max_state_dim': 32, 'min_period': 0.004, 'n_action_steps': 1, 'n_obs_steps': 1, 'normalization_mapping': {'ACTION': , 'STATE': , 'VISUAL': }, 'num_expert_layers': -1, 'num_steps': 10, 'num_vlm_layers': 16, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_grad_clip_norm': 10, 'optimizer_lr': 0.0001, 'optimizer_weight_decay': 1e-10, 'output_features': {}, 'pad_language_to': 'longest', 'prefix_length': -1, 'private': None, 'push_to_hub': True, 'repo_id': 'None', 'resize_imgs_with_padding': [512, 512], 'scheduler_decay_lr': 2.5e-06, 'scheduler_decay_steps': 30000, 'scheduler_warmup_steps': 1000, 'self_attn_every_n_layers': 2, 'tags': None, 'tokenizer_max_length': 48, 'train_expert_only': True, 'train_state_proj': True, 'type': 'smolvla', 'use_amp': True, 'use_cache': True, 'use_delta_joint_actions_aloha': False, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Instruct'}, 'resume': False, 'save_checkpoint': True, 'save_freq': 20000, 'scheduler': {'decay_lr': 2.5e-06, 'num_decay_steps': 30000, 'num_warmup_steps': 1000, 'peak_lr': 0.0001, 'type': 'cosine_decay_with_warmup'}, 'seed': 1000, 'steps': 100000, 'use_policy_training_preset': True, 'wandb': {'disable_artifact': False, 'enable': False, 'entity': None, 'mode': None, 'notes': None, 'project': 'lerobot', 'run_id': None}} INFO 2025-09-08 13:33:47 ils/utils.py:48 Cuda backend detected, using cuda. WARNING 2025-09-08 13:33:47 /policies.py:81 Device 'None' is not available. Switching to 'cuda'. INFO 2025-09-08 13:33:47 ccelerate.py:99 {'batch_size': 32, 'dataset': {'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'kwargs': {'brightness': [0.8, 1.2]}, 'type': 'ColorJitter', 'weight': 1.0}, 'contrast': {'kwargs': {'contrast': [0.8, 1.2]}, 'type': 'ColorJitter', 'weight': 1.0}, 'hue': {'kwargs': {'hue': [-0.05, 0.05]}, 'type': 'ColorJitter', 'weight': 1.0}, 'saturation': {'kwargs': {'saturation': [0.5, 1.5]}, 'type': 'ColorJitter', 'weight': 1.0}, 'sharpness': {'kwargs': {'sharpness': [0.5, 1.5]}, 'type': 'SharpnessJitter', 'weight': 1.0}}}, 'repo_id': 'HuggingFaceVLA/libero', 'revision': None, 'root': '/raid/jade/.cache/huggingface/datasets', 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': {'camera_name': 'agentview_image,robot0_eye_in_hand_image', 'episode_length': 520, 'features': {'action': {'shape': [7], 'type': }, 'agent_pos': {'shape': [8], 'type': }, 'pixels/agentview_image': {'shape': [360, 360, 3], 'type': }, 'pixels/robot0_eye_in_hand_image': {'shape': [360, 360, 3], 'type': }}, 'features_map': {'action': 'action', 'agent_pos': 'observation.state', 'pixels/agentview_image': 'observation.images.image', 'pixels/robot0_eye_in_hand_image': 'observation.images.image2'}, 'fps': 30, 'init_states': True, 'max_parallel_tasks': 5, 'multitask_eval': True, 'obs_type': 'pixels_agent_pos', 'render_mode': 'rgb_array', 'task': 'libero_spatial', 'type': 'libero'}, 'eval': {'batch_size': 1, 'n_episodes': 1, 'use_async_envs': False}, 'eval_freq': 0, 'job_name': 'libero_smolvla', 'log_freq': 200, 'num_workers': 4, 'optimizer': {'betas': [0.9, 0.95], 'eps': 1e-08, 'grad_clip_norm': 10, 'lr': 0.0001, 'type': 'adamw', 'weight_decay': 1e-10}, 'output_dir': '/raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla_lr1e-4bs32steps100000', 'policy': {'adapt_to_pi_aloha': False, 'add_image_special_tokens': False, 'attention_mode': 'cross_attn', 'chunk_size': 50, 'device': 'cuda', 'empty_cameras': 0, 'expert_width_multiplier': 0.75, 'freeze_vision_encoder': True, 'gradient_accumulation_steps': 1, 'input_features': {}, 'license': None, 'load_vlm_weights': False, 'max_action_dim': 32, 'max_period': 4.0, 'max_state_dim': 32, 'min_period': 0.004, 'n_action_steps': 1, 'n_obs_steps': 1, 'normalization_mapping': {'ACTION': , 'STATE': , 'VISUAL': }, 'num_expert_layers': -1, 'num_steps': 10, 'num_vlm_layers': 16, 'optimizer_betas': [0.9, 0.95], 'optimizer_eps': 1e-08, 'optimizer_grad_clip_norm': 10, 'optimizer_lr': 0.0001, 'optimizer_weight_decay': 1e-10, 'output_features': {}, 'pad_language_to': 'longest', 'prefix_length': -1, 'private': None, 'push_to_hub': True, 'repo_id': 'None', 'resize_imgs_with_padding': [512, 512], 'scheduler_decay_lr': 2.5e-06, 'scheduler_decay_steps': 30000, 'scheduler_warmup_steps': 1000, 'self_attn_every_n_layers': 2, 'tags': None, 'tokenizer_max_length': 48, 'train_expert_only': True, 'train_state_proj': True, 'type': 'smolvla', 'use_amp': True, 'use_cache': True, 'use_delta_joint_actions_aloha': False, 'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Instruct'}, 'resume': False, 'save_checkpoint': True, 'save_freq': 20000, 'scheduler': {'decay_lr': 2.5e-06, 'num_decay_steps': 30000, 'num_warmup_steps': 1000, 'peak_lr': 0.0001, 'type': 'cosine_decay_with_warmup'}, 'seed': 1000, 'steps': 100000, 'use_policy_training_preset': True, 'wandb': {'disable_artifact': False, 'enable': False, 'entity': None, 'mode': None, 'notes': None, 'project': 'lerobot', 'run_id': None}} WARNING 2025-09-08 13:33:47 ls/other.py:512 Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher . WARNING 2025-09-08 13:33:47 ls/other.py:512 Detected kernel version 5.4.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher . INFO 2025-09-08 13:33:47 celerate.py:149 Creating dataset Resolving data files: 100%|████████████████████████████████| 1693/1693 [00:00<00:00, 103295.66it/s] Loading dataset shards: 100%|████████████████████████████████████| 69/69 [00:00<00:00, 5229.81it/s] Resolving data files: 100%|████████████████████████████████| 1693/1693 [00:00<00:00, 360601.09it/s] Loading dataset shards: 100%|████████████████████████████████████| 69/69 [00:00<00:00, 4881.54it/s] c INFO 2025-09-08 13:33:53 celerate.py:160 Creating policy /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarnin g: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. warnings.warn( c /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u ser. warnings.warn( # warn only once [rank1]:[W908 13:33:54.613597516 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used b y this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can pecify device_id in init_process_group() to force use of a particular device. Reducing the number of VLM layers to 16 ... /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u ser. warnings.warn( # warn only once [rank0]:[W908 13:34:15.806448425 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used b y this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can pecify device_id in init_process_group() to force use of a particular device. INFO 2025-09-08 13:34:15 celerate.py:171 Creating optimizer and scheduler /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarnin g: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead. warnings.warn( Reducing the number of VLM layers to 16 ... INFO 2025-09-08 13:34:36 celerate.py:211 Output dir: /raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla _lr1e-4bs32steps100000 INFO 2025-09-08 13:34:36 celerate.py:213 cfg.env.task='libero_spatial' INFO 2025-09-08 13:34:36 celerate.py:214 cfg.steps=100000 (100K) INFO 2025-09-08 13:34:36 celerate.py:215 dataset.num_frames=273465 (273K) INFO 2025-09-08 13:34:36 celerate.py:216 dataset.num_episodes=1693 INFO 2025-09-08 13:34:36 celerate.py:217 num_learnable_params=99880992 (100M) INFO 2025-09-08 13:34:36 celerate.py:218 num_total_params=450046220 (450M) INFO 2025-09-08 13:34:36 celerate.py:219 Number of processes: 2 INFO 2025-09-08 13:34:36 celerate.py:220 Device: cuda:0 INFO 2025-09-08 13:34:36 celerate.py:221 Mixed precision: bf16 INFO 2025-09-08 13:34:36 celerate.py:243 Start offline training on a fixed dataset [rank1]:[W908 13:34:39.454560620 reducer.cpp:1430] Warning: find_unused_parameters=True was specified in DDP constru ctor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the aut ograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused para meters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your m odel has flow control causing later iterations to have unused parameters. (function operator()) [rank0]:[W908 13:34:40.502702504 reducer.cpp:1430] Warning: find_unused_parameters=True was specified in DDP constru ctor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the aut ograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused para meters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your m odel has flow control causing later iterations to have unused parameters. (function operator()) INFO 2025-09-08 13:36:23 celerate.py:281 step:200 smpl:13K ep:79 epch:0.05 loss:0.963 grdn:2.699 lr:2.0e-05 updt_s:0 .506 data_s:0.027 INFO 2025-09-08 13:38:09 celerate.py:281 step:400 smpl:26K ep:158 epch:0.09 loss:0.389 grdn:3.127 lr:6.0e-05 updt_s: 0.525 data_s:0.003 INFO 2025-09-08 13:39:53 celerate.py:281 step:600 smpl:38K ep:238 epch:0.14 loss:0.261 grdn:2.618 lr:9.5e-05 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 13:41:37 celerate.py:281 step:800 smpl:51K ep:317 epch:0.19 loss:0.231 grdn:1.684 lr:9.9e-05 updt_s: 0.516 data_s:0.003 INFO 2025-09-08 13:43:21 celerate.py:281 step:1K smpl:64K ep:396 epch:0.23 loss:0.211 grdn:1.258 lr:9.9e-05 updt_s:0 .514 data_s:0.003 INFO 2025-09-08 13:45:05 celerate.py:281 step:1K smpl:77K ep:475 epch:0.28 loss:0.198 grdn:1.032 lr:9.9e-05 updt_s:0 .517 data_s:0.003 INFO 2025-09-08 13:46:49 celerate.py:281 step:1K smpl:90K ep:555 epch:0.33 loss:0.182 grdn:0.880 lr:9.8e-05 updt_s:0 .515 data_s:0.003 INFO 2025-09-08 13:48:33 celerate.py:281 step:2K smpl:102K ep:634 epch:0.37 loss:0.167 grdn:0.744 lr:9.8e-05 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 13:50:17 celerate.py:281 step:2K smpl:115K ep:713 epch:0.42 loss:0.157 grdn:0.680 lr:9.7e-05 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 13:52:01 celerate.py:281 step:2K smpl:128K ep:792 epch:0.47 loss:0.147 grdn:0.612 lr:9.6e-05 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 13:53:44 celerate.py:281 step:2K smpl:141K ep:872 epch:0.51 loss:0.142 grdn:0.576 lr:9.5e-05 updt_s: 0.510 data_s:0.003 INFO 2025-09-08 13:55:27 celerate.py:281 step:2K smpl:154K ep:951 epch:0.56 loss:0.136 grdn:0.523 lr:9.4e-05 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 13:57:11 celerate.py:281 step:3K smpl:166K ep:1K epch:0.61 loss:0.132 grdn:0.509 lr:9.3e-05 updt_s:0 .516 data_s:0.003 INFO 2025-09-08 13:58:57 celerate.py:281 step:3K smpl:179K ep:1K epch:0.66 loss:0.126 grdn:0.492 lr:9.2e-05 updt_s:0 .525 data_s:0.003 INFO 2025-09-08 14:00:43 celerate.py:281 step:3K smpl:192K ep:1K epch:0.70 loss:0.124 grdn:0.467 lr:9.1e-05 updt_s:0 .525 data_s:0.003 INFO 2025-09-08 14:02:26 celerate.py:281 step:3K smpl:205K ep:1K epch:0.75 loss:0.119 grdn:0.438 lr:9.0e-05 updt_s:0 .508 data_s:0.003 INFO 2025-09-08 14:04:27 celerate.py:281 step:3K smpl:218K ep:1K epch:0.80 loss:0.118 grdn:0.426 lr:8.9e-05 updt_s:0 .564 data_s:0.039 INFO 2025-09-08 14:06:10 celerate.py:281 step:4K smpl:230K ep:1K epch:0.84 loss:0.116 grdn:0.422 lr:8.7e-05 updt_s:0 .511 data_s:0.004 INFO 2025-09-08 14:07:55 celerate.py:281 step:4K smpl:243K ep:2K epch:0.89 loss:0.113 grdn:0.395 lr:8.6e-05 updt_s:0 .517 data_s:0.003 INFO 2025-09-08 14:09:38 celerate.py:281 step:4K smpl:256K ep:2K epch:0.94 loss:0.111 grdn:0.401 lr:8.5e-05 updt_s:0 .511 data_s:0.003 INFO 2025-09-08 14:11:21 celerate.py:281 step:4K smpl:269K ep:2K epch:0.98 loss:0.110 grdn:0.380 lr:8.3e-05 updt_s:0 .511 data_s:0.003 INFO 2025-09-08 14:13:08 celerate.py:281 step:4K smpl:282K ep:2K epch:1.03 loss:0.109 grdn:0.381 lr:8.2e-05 updt_s:0 .413 data_s:0.119 INFO 2025-09-08 14:14:52 celerate.py:281 step:5K smpl:294K ep:2K epch:1.08 loss:0.107 grdn:0.387 lr:8.0e-05 updt_s:0 .373 data_s:0.146 INFO 2025-09-08 14:16:36 celerate.py:281 step:5K smpl:307K ep:2K epch:1.12 loss:0.107 grdn:0.366 lr:7.8e-05 updt_s:0 .446 data_s:0.072 INFO 2025-09-08 14:18:19 celerate.py:281 step:5K smpl:320K ep:2K epch:1.17 loss:0.105 grdn:0.347 lr:7.6e-05 updt_s:0 .468 data_s:0.045 INFO 2025-09-08 14:20:01 celerate.py:281 step:5K smpl:333K ep:2K epch:1.22 loss:0.103 grdn:0.350 lr:7.5e-05 updt_s:0 .510 data_s:0.003 INFO 2025-09-08 14:21:46 celerate.py:281 step:5K smpl:346K ep:2K epch:1.26 loss:0.101 grdn:0.336 lr:7.3e-05 updt_s:0 .512 data_s:0.011 INFO 2025-09-08 14:23:30 celerate.py:281 step:6K smpl:358K ep:2K epch:1.31 loss:0.102 grdn:0.345 lr:7.1e-05 updt_s:0 .515 data_s:0.003 INFO 2025-09-08 14:25:15 celerate.py:281 step:6K smpl:371K ep:2K epch:1.36 loss:0.100 grdn:0.333 lr:6.9e-05 updt_s:0 .521 data_s:0.003 INFO 2025-09-08 14:26:59 celerate.py:281 step:6K smpl:384K ep:2K epch:1.40 loss:0.100 grdn:0.328 lr:6.7e-05 updt_s:0 .516 data_s:0.003 INFO 2025-09-08 14:28:43 celerate.py:281 step:6K smpl:397K ep:2K epch:1.45 loss:0.099 grdn:0.319 lr:6.5e-05 updt_s:0 .512 data_s:0.003 INFO 2025-09-08 14:30:26 celerate.py:281 step:6K smpl:410K ep:3K epch:1.50 loss:0.098 grdn:0.313 lr:6.3e-05 updt_s:0 .515 data_s:0.003 INFO 2025-09-08 14:32:11 celerate.py:281 step:7K smpl:422K ep:3K epch:1.54 loss:0.097 grdn:0.319 lr:6.1e-05 updt_s:0 .519 data_s:0.004 INFO 2025-09-08 14:33:55 celerate.py:281 step:7K smpl:435K ep:3K epch:1.59 loss:0.097 grdn:0.312 lr:5.9e-05 updt_s:0 .506 data_s:0.010 INFO 2025-09-08 14:35:39 celerate.py:281 step:7K smpl:448K ep:3K epch:1.64 loss:0.097 grdn:0.307 lr:5.7e-05 updt_s:0 .516 data_s:0.003 INFO 2025-09-08 14:37:23 celerate.py:281 step:7K smpl:461K ep:3K epch:1.69 loss:0.095 grdn:0.294 lr:5.5e-05 updt_s:0 .518 data_s:0.003 INFO 2025-09-08 14:39:07 celerate.py:281 step:7K smpl:474K ep:3K epch:1.73 loss:0.095 grdn:0.299 lr:5.3e-05 updt_s:0 .507 data_s:0.007 INFO 2025-09-08 14:40:52 celerate.py:281 step:8K smpl:486K ep:3K epch:1.78 loss:0.094 grdn:0.283 lr:5.1e-05 updt_s:0 .523 data_s:0.003 INFO 2025-09-08 14:42:36 celerate.py:281 step:8K smpl:499K ep:3K epch:1.83 loss:0.093 grdn:0.284 lr:4.9e-05 updt_s:0 .517 data_s:0.003 INFO 2025-09-08 14:44:22 celerate.py:281 step:8K smpl:512K ep:3K epch:1.87 loss:0.092 grdn:0.284 lr:4.7e-05 updt_s:0 .465 data_s:0.060 INFO 2025-09-08 14:46:06 celerate.py:281 step:8K smpl:525K ep:3K epch:1.92 loss:0.093 grdn:0.292 lr:4.5e-05 updt_s:0 .456 data_s:0.066 INFO 2025-09-08 14:47:49 celerate.py:281 step:8K smpl:538K ep:3K epch:1.97 loss:0.093 grdn:0.290 lr:4.3e-05 updt_s:0 .510 data_s:0.003 INFO 2025-09-08 14:49:37 celerate.py:281 step:9K smpl:550K ep:3K epch:2.01 loss:0.092 grdn:0.283 lr:4.1e-05 updt_s:0 .419 data_s:0.117 INFO 2025-09-08 14:51:20 celerate.py:281 step:9K smpl:563K ep:3K epch:2.06 loss:0.092 grdn:0.275 lr:3.9e-05 updt_s:0 .463 data_s:0.053 INFO 2025-09-08 14:53:05 celerate.py:281 step:9K smpl:576K ep:4K epch:2.11 loss:0.090 grdn:0.272 lr:3.7e-05 updt_s:0 .517 data_s:0.003 INFO 2025-09-08 14:54:49 celerate.py:281 step:9K smpl:589K ep:4K epch:2.15 loss:0.090 grdn:0.268 lr:3.5e-05 updt_s:0 .506 data_s:0.013 INFO 2025-09-08 14:56:32 celerate.py:281 step:9K smpl:602K ep:4K epch:2.20 loss:0.090 grdn:0.271 lr:3.3e-05 updt_s:0 .513 data_s:0.003 INFO 2025-09-08 14:58:17 celerate.py:281 step:10K smpl:614K ep:4K epch:2.25 loss:0.090 grdn:0.268 lr:3.1e-05 updt_s: 0.520 data_s:0.003 INFO 2025-09-08 15:00:02 celerate.py:281 step:10K smpl:627K ep:4K epch:2.29 loss:0.089 grdn:0.261 lr:3.0e-05 updt_s: 0.519 data_s:0.003 INFO 2025-09-08 15:01:48 celerate.py:281 step:10K smpl:640K ep:4K epch:2.34 loss:0.090 grdn:0.271 lr:2.8e-05 updt_s: 0.526 data_s:0.003 INFO 2025-09-08 15:03:33 celerate.py:281 step:10K smpl:653K ep:4K epch:2.39 loss:0.089 grdn:0.262 lr:2.6e-05 updt_s: 0.521 data_s:0.003 INFO 2025-09-08 15:05:18 celerate.py:281 step:10K smpl:666K ep:4K epch:2.43 loss:0.090 grdn:0.264 lr:2.4e-05 updt_s: 0.519 data_s:0.003 INFO 2025-09-08 15:07:32 celerate.py:281 step:11K smpl:678K ep:4K epch:2.48 loss:0.089 grdn:0.255 lr:2.3e-05 updt_s: 0.663 data_s:0.004 INFO 2025-09-08 15:09:21 celerate.py:281 step:11K smpl:691K ep:4K epch:2.53 loss:0.090 grdn:0.263 lr:2.1e-05 updt_s: 0.514 data_s:0.030 INFO 2025-09-08 15:11:06 celerate.py:281 step:11K smpl:704K ep:4K epch:2.57 loss:0.088 grdn:0.254 lr:1.9e-05 updt_s: 0.517 data_s:0.006 INFO 2025-09-08 15:12:51 celerate.py:281 step:11K smpl:717K ep:4K epch:2.62 loss:0.088 grdn:0.252 lr:1.8e-05 updt_s: 0.517 data_s:0.005 INFO 2025-09-08 15:14:38 celerate.py:281 step:11K smpl:730K ep:5K epch:2.67 loss:0.088 grdn:0.251 lr:1.6e-05 updt_s: 0.532 data_s:0.003 INFO 2025-09-08 15:16:23 celerate.py:281 step:12K smpl:742K ep:5K epch:2.71 loss:0.088 grdn:0.253 lr:1.5e-05 updt_s: 0.520 data_s:0.003 INFO 2025-09-08 15:18:08 celerate.py:281 step:12K smpl:755K ep:5K epch:2.76 loss:0.087 grdn:0.244 lr:1.4e-05 updt_s: 0.521 data_s:0.003 INFO 2025-09-08 15:19:54 celerate.py:281 step:12K smpl:768K ep:5K epch:2.81 loss:0.088 grdn:0.247 lr:1.2e-05 updt_s: 0.524 data_s:0.003 INFO 2025-09-08 15:21:39 celerate.py:281 step:12K smpl:781K ep:5K epch:2.86 loss:0.087 grdn:0.242 lr:1.1e-05 updt_s: 0.520 data_s:0.003 INFO 2025-09-08 15:23:32 celerate.py:281 step:12K smpl:794K ep:5K epch:2.90 loss:0.088 grdn:0.243 lr:1.0e-05 updt_s: 0.560 data_s:0.003 INFO 2025-09-08 15:25:48 celerate.py:281 step:13K smpl:806K ep:5K epch:2.95 loss:0.087 grdn:0.240 lr:9.0e-06 updt_s: 0.674 data_s:0.005 INFO 2025-09-08 15:28:02 celerate.py:281 step:13K smpl:819K ep:5K epch:3.00 loss:0.088 grdn:0.245 lr:8.0e-06 updt_s: 0.662 data_s:0.004 INFO 2025-09-08 15:31:06 celerate.py:281 step:13K smpl:832K ep:5K epch:3.04 loss:0.086 grdn:0.236 lr:7.1e-06 updt_s: 0.688 data_s:0.231 INFO 2025-09-08 15:32:52 celerate.py:281 step:13K smpl:845K ep:5K epch:3.09 loss:0.087 grdn:0.231 lr:6.3e-06 updt_s: 0.521 data_s:0.003 INFO 2025-09-08 15:35:46 celerate.py:281 step:13K smpl:858K ep:5K epch:3.14 loss:0.088 grdn:0.235 lr:5.6e-06 updt_s: 0.637 data_s:0.232 INFO 2025-09-08 15:37:34 celerate.py:281 step:14K smpl:870K ep:5K epch:3.18 loss:0.087 grdn:0.238 lr:4.9e-06 updt_s: 0.514 data_s:0.025 INFO 2025-09-08 15:39:18 celerate.py:281 step:14K smpl:883K ep:5K epch:3.23 loss:0.087 grdn:0.226 lr:4.3e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 15:41:02 celerate.py:281 step:14K smpl:896K ep:6K epch:3.28 loss:0.087 grdn:0.230 lr:3.8e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 15:42:45 celerate.py:281 step:14K smpl:909K ep:6K epch:3.32 loss:0.086 grdn:0.229 lr:3.4e-06 updt_s: 0.507 data_s:0.008 INFO 2025-09-08 15:44:29 celerate.py:281 step:14K smpl:922K ep:6K epch:3.37 loss:0.087 grdn:0.229 lr:3.0e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 15:46:12 celerate.py:281 step:15K smpl:934K ep:6K epch:3.42 loss:0.087 grdn:0.228 lr:2.8e-06 updt_s: 0.502 data_s:0.011 INFO 2025-09-08 15:47:56 celerate.py:281 step:15K smpl:947K ep:6K epch:3.46 loss:0.086 grdn:0.232 lr:2.6e-06 updt_s: 0.515 data_s:0.004 INFO 2025-09-08 15:49:39 celerate.py:281 step:15K smpl:960K ep:6K epch:3.51 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s: 0.365 data_s:0.147 INFO 2025-09-08 15:51:22 celerate.py:281 step:15K smpl:973K ep:6K epch:3.56 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s: 0.333 data_s:0.179 INFO 2025-09-08 15:53:07 celerate.py:281 step:15K smpl:986K ep:6K epch:3.60 loss:0.087 grdn:0.229 lr:2.5e-06 updt_s: 0.357 data_s:0.164 INFO 2025-09-08 15:54:50 celerate.py:281 step:16K smpl:998K ep:6K epch:3.65 loss:0.087 grdn:0.230 lr:2.5e-06 updt_s: 0.365 data_s:0.151 INFO 2025-09-08 15:56:35 celerate.py:281 step:16K smpl:1M ep:6K epch:3.70 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0. 450 data_s:0.071 INFO 2025-09-08 15:58:19 celerate.py:281 step:16K smpl:1M ep:6K epch:3.74 loss:0.087 grdn:0.232 lr:2.5e-06 updt_s:0. 495 data_s:0.023 INFO 2025-09-08 16:00:04 celerate.py:281 step:16K smpl:1M ep:6K epch:3.79 loss:0.087 grdn:0.227 lr:2.5e-06 updt_s:0. 393 data_s:0.131 INFO 2025-09-08 16:01:48 celerate.py:281 step:16K smpl:1M ep:6K epch:3.84 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:0. 515 data_s:0.003 INFO 2025-09-08 16:03:32 celerate.py:281 step:17K smpl:1M ep:7K epch:3.88 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0. 518 data_s:0.003 INFO 2025-09-08 16:05:17 celerate.py:281 step:17K smpl:1M ep:7K epch:3.93 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0. 397 data_s:0.125 INFO 2025-09-08 16:12:41 celerate.py:281 step:17K smpl:1M ep:7K epch:3.98 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:1. 496 data_s:0.719 INFO 2025-09-08 16:14:28 celerate.py:281 step:17K smpl:1M ep:7K epch:4.03 loss:0.087 grdn:0.228 lr:2.5e-06 updt_s:0. 413 data_s:0.124 INFO 2025-09-08 16:16:13 celerate.py:281 step:17K smpl:1M ep:7K epch:4.07 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0. 469 data_s:0.050 INFO 2025-09-08 16:17:57 celerate.py:281 step:18K smpl:1M ep:7K epch:4.12 loss:0.087 grdn:0.228 lr:2.5e-06 updt_s:0. 375 data_s:0.145 INFO 2025-09-08 16:19:42 celerate.py:281 step:18K smpl:1M ep:7K epch:4.17 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:0. 333 data_s:0.189 INFO 2025-09-08 16:21:26 celerate.py:281 step:18K smpl:1M ep:7K epch:4.21 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0. 334 data_s:0.184 INFO 2025-09-08 16:23:09 celerate.py:281 step:18K smpl:1M ep:7K epch:4.26 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0. 331 data_s:0.185 INFO 2025-09-08 16:24:53 celerate.py:281 step:18K smpl:1M ep:7K epch:4.31 loss:0.088 grdn:0.236 lr:2.5e-06 updt_s:0. 333 data_s:0.182 INFO 2025-09-08 16:26:38 celerate.py:281 step:19K smpl:1M ep:7K epch:4.35 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:0. 337 data_s:0.188 INFO 2025-09-08 16:28:22 celerate.py:281 step:19K smpl:1M ep:7K epch:4.40 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0. 420 data_s:0.099 INFO 2025-09-08 16:30:06 celerate.py:281 step:19K smpl:1M ep:8K epch:4.45 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0. 444 data_s:0.075 INFO 2025-09-08 16:31:49 celerate.py:281 step:19K smpl:1M ep:8K epch:4.49 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0. 475 data_s:0.036 INFO 2025-09-08 16:33:33 celerate.py:281 step:19K smpl:1M ep:8K epch:4.54 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0. 379 data_s:0.139 INFO 2025-09-08 16:35:17 celerate.py:281 step:20K smpl:1M ep:8K epch:4.59 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0. 348 data_s:0.171 INFO 2025-09-08 16:37:01 celerate.py:281 step:20K smpl:1M ep:8K epch:4.63 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0. 332 data_s:0.185 /home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u ser. warnings.warn( # warn only once INFO 2025-09-08 16:38:46 celerate.py:281 step:20K smpl:1M ep:8K epch:4.68 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0. 486 data_s:0.037 INFO 2025-09-08 16:38:46 celerate.py:295 Checkpoint policy after step 20000 INFO 2025-09-08 16:40:30 celerate.py:281 step:20K smpl:1M ep:8K epch:4.73 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0. 509 data_s:0.003 INFO 2025-09-08 16:42:16 celerate.py:281 step:20K smpl:1M ep:8K epch:4.77 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0. 527 data_s:0.003 INFO 2025-09-08 16:44:01 celerate.py:281 step:21K smpl:1M ep:8K epch:4.82 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:0. 519 data_s:0.003 INFO 2025-09-08 16:45:45 celerate.py:281 step:21K smpl:1M ep:8K epch:4.87 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0. 504 data_s:0.013 INFO 2025-09-08 16:47:29 celerate.py:281 step:21K smpl:1M ep:8K epch:4.91 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0. 509 data_s:0.011 INFO 2025-09-08 16:49:19 celerate.py:281 step:21K smpl:1M ep:8K epch:4.96 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0. 544 data_s:0.003 INFO 2025-09-08 16:51:04 celerate.py:281 step:21K smpl:1M ep:8K epch:5.01 loss:0.086 grdn:0.225 lr:2.5e-06 updt_s:0. 488 data_s:0.039 INFO 2025-09-08 16:52:51 celerate.py:281 step:22K smpl:1M ep:9K epch:5.06 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0. 430 data_s:0.099 INFO 2025-09-08 16:54:36 celerate.py:281 step:22K smpl:1M ep:9K epch:5.10 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0. 521 data_s:0.003 INFO 2025-09-08 16:56:23 celerate.py:281 step:22K smpl:1M ep:9K epch:5.15 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0. 521 data_s:0.014 INFO 2025-09-08 16:58:09 celerate.py:281 step:22K smpl:1M ep:9K epch:5.20 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0. 525 data_s:0.003 INFO 2025-09-08 17:00:04 celerate.py:281 step:22K smpl:1M ep:9K epch:5.24 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0. 568 data_s:0.003 INFO 2025-09-08 17:02:00 celerate.py:281 step:23K smpl:1M ep:9K epch:5.29 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:0. 575 data_s:0.003 INFO 2025-09-08 17:03:49 celerate.py:281 step:23K smpl:1M ep:9K epch:5.34 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0. 513 data_s:0.030 INFO 2025-09-08 17:05:39 celerate.py:281 step:23K smpl:1M ep:9K epch:5.38 loss:0.085 grdn:0.227 lr:2.5e-06 updt_s:0. 523 data_s:0.027 INFO 2025-09-08 17:07:26 celerate.py:281 step:23K smpl:1M ep:9K epch:5.43 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0. 529 data_s:0.003 INFO 2025-09-08 17:09:12 celerate.py:281 step:23K smpl:1M ep:9K epch:5.48 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0. 526 data_s:0.003 INFO 2025-09-08 17:10:55 celerate.py:281 step:24K smpl:2M ep:9K epch:5.52 loss:0.087 grdn:0.230 lr:2.5e-06 updt_s:0. 443 data_s:0.072 INFO 2025-09-08 17:12:40 celerate.py:281 step:24K smpl:2M ep:9K epch:5.57 loss:0.087 grdn:0.229 lr:2.5e-06 updt_s:0. 518 data_s:0.004 INFO 2025-09-08 17:14:25 celerate.py:281 step:24K smpl:2M ep:10K epch:5.62 loss:0.087 grdn:0.232 lr:2.5e-06 updt_s:0 .521 data_s:0.003 INFO 2025-09-08 17:16:11 celerate.py:281 step:24K smpl:2M ep:10K epch:5.66 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:0 .523 data_s:0.003 INFO 2025-09-08 17:17:55 celerate.py:281 step:24K smpl:2M ep:10K epch:5.71 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0 .515 data_s:0.005 INFO 2025-09-08 17:19:39 celerate.py:281 step:25K smpl:2M ep:10K epch:5.76 loss:0.087 grdn:0.229 lr:2.5e-06 updt_s:0 .415 data_s:0.106 INFO 2025-09-08 17:21:24 celerate.py:281 step:25K smpl:2M ep:10K epch:5.80 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0 .507 data_s:0.016 INFO 2025-09-08 17:23:08 celerate.py:281 step:25K smpl:2M ep:10K epch:5.85 loss:0.085 grdn:0.229 lr:2.5e-06 updt_s:0 .514 data_s:0.003 INFO 2025-09-08 17:24:54 celerate.py:281 step:25K smpl:2M ep:10K epch:5.90 loss:0.087 grdn:0.227 lr:2.5e-06 updt_s:0 .518 data_s:0.008 INFO 2025-09-08 17:26:41 celerate.py:281 step:25K smpl:2M ep:10K epch:5.94 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0 .529 data_s:0.003 INFO 2025-09-08 17:28:24 celerate.py:281 step:26K smpl:2M ep:10K epch:5.99 loss:0.087 grdn:0.232 lr:2.5e-06 updt_s:0 .513 data_s:0.003 INFO 2025-09-08 17:30:11 celerate.py:281 step:26K smpl:2M ep:10K epch:6.04 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0 .370 data_s:0.164 INFO 2025-09-08 17:31:55 celerate.py:281 step:26K smpl:2M ep:10K epch:6.08 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0 .385 data_s:0.132 INFO 2025-09-08 17:33:39 celerate.py:281 step:26K smpl:2M ep:10K epch:6.13 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0 .450 data_s:0.069 INFO 2025-09-08 17:35:24 celerate.py:281 step:26K smpl:2M ep:10K epch:6.18 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:0 .468 data_s:0.052 INFO 2025-09-08 17:37:07 celerate.py:281 step:27K smpl:2M ep:11K epch:6.23 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0 .514 data_s:0.004 INFO 2025-09-08 17:38:52 celerate.py:281 step:27K smpl:2M ep:11K epch:6.27 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0 .519 data_s:0.003 INFO 2025-09-08 17:40:40 celerate.py:281 step:27K smpl:2M ep:11K epch:6.32 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0 .534 data_s:0.003 INFO 2025-09-08 17:42:57 celerate.py:281 step:27K smpl:2M ep:11K epch:6.37 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0 .678 data_s:0.007 INFO 2025-09-08 17:46:13 celerate.py:281 step:27K smpl:2M ep:11K epch:6.41 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0 .968 data_s:0.009 INFO 2025-09-08 17:49:20 celerate.py:281 step:28K smpl:2M ep:11K epch:6.46 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0 .895 data_s:0.037 INFO 2025-09-08 17:51:22 celerate.py:281 step:28K smpl:2M ep:11K epch:6.51 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0 .604 data_s:0.003 INFO 2025-09-08 17:53:07 celerate.py:281 step:28K smpl:2M ep:11K epch:6.55 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:0 .521 data_s:0.003 INFO 2025-09-08 17:54:51 celerate.py:281 step:28K smpl:2M ep:11K epch:6.60 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0 .516 data_s:0.003 INFO 2025-09-08 17:56:36 celerate.py:281 step:28K smpl:2M ep:11K epch:6.65 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0 .519 data_s:0.003 INFO 2025-09-08 17:58:21 celerate.py:281 step:29K smpl:2M ep:11K epch:6.69 loss:0.085 grdn:0.228 lr:2.5e-06 updt_s:0 .521 data_s:0.003 INFO 2025-09-08 18:00:06 celerate.py:281 step:29K smpl:2M ep:11K epch:6.74 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0 .513 data_s:0.011 INFO 2025-09-08 18:01:50 celerate.py:281 step:29K smpl:2M ep:11K epch:6.79 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0 .476 data_s:0.041 INFO 2025-09-08 18:03:34 celerate.py:281 step:29K smpl:2M ep:12K epch:6.83 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:0 .506 data_s:0.012 INFO 2025-09-08 18:05:21 celerate.py:281 step:29K smpl:2M ep:12K epch:6.88 loss:0.086 grdn:0.229 lr:2.5e-06 updt_s:0 .455 data_s:0.075 INFO 2025-09-08 18:07:04 celerate.py:281 step:30K smpl:2M ep:12K epch:6.93 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0 .514 data_s:0.003 INFO 2025-09-08 18:08:47 celerate.py:281 step:30K smpl:2M ep:12K epch:6.97 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:0 .509 data_s:0.003 INFO 2025-09-08 18:10:33 celerate.py:281 step:30K smpl:2M ep:12K epch:7.02 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0 .422 data_s:0.105 INFO 2025-09-08 18:12:19 celerate.py:281 step:30K smpl:2M ep:12K epch:7.07 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0 .347 data_s:0.182 INFO 2025-09-08 18:14:05 celerate.py:281 step:30K smpl:2M ep:12K epch:7.11 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0 .473 data_s:0.053 INFO 2025-09-08 18:15:52 celerate.py:281 step:31K smpl:2M ep:12K epch:7.16 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:0 .531 data_s:0.005 INFO 2025-09-08 18:17:37 celerate.py:281 step:31K smpl:2M ep:12K epch:7.21 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0 .520 data_s:0.003 INFO 2025-09-08 18:19:22 celerate.py:281 step:31K smpl:2M ep:12K epch:7.26 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0 .500 data_s:0.020 INFO 2025-09-08 18:21:06 celerate.py:281 step:31K smpl:2M ep:12K epch:7.30 loss:0.087 grdn:0.243 lr:2.5e-06 updt_s:0 .511 data_s:0.009 INFO 2025-09-08 18:22:50 celerate.py:281 step:31K smpl:2M ep:12K epch:7.35 loss:0.087 grdn:0.227 lr:2.5e-06 updt_s:0 .518 data_s:0.003 INFO 2025-09-08 18:24:33 celerate.py:281 step:32K smpl:2M ep:13K epch:7.40 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0 .507 data_s:0.007 INFO 2025-09-08 18:26:16 celerate.py:281 step:32K smpl:2M ep:13K epch:7.44 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:0 .463 data_s:0.047 INFO 2025-09-08 18:27:59 celerate.py:281 step:32K smpl:2M ep:13K epch:7.49 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s:0 .509 data_s:0.007 INFO 2025-09-08 18:29:43 celerate.py:281 step:32K smpl:2M ep:13K epch:7.54 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0 .514 data_s:0.003 INFO 2025-09-08 18:31:26 celerate.py:281 step:32K smpl:2M ep:13K epch:7.58 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0 .511 data_s:0.003 INFO 2025-09-08 18:33:11 celerate.py:281 step:33K smpl:2M ep:13K epch:7.63 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0 .518 data_s:0.003 INFO 2025-09-08 18:34:57 celerate.py:281 step:33K smpl:2M ep:13K epch:7.68 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0 .512 data_s:0.016 INFO 2025-09-08 18:36:41 celerate.py:281 step:33K smpl:2M ep:13K epch:7.72 loss:0.086 grdn:0.229 lr:2.5e-06 updt_s:0 .517 data_s:0.003 INFO 2025-09-08 18:38:25 celerate.py:281 step:33K smpl:2M ep:13K epch:7.77 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0 .516 data_s:0.003 INFO 2025-09-08 18:40:07 celerate.py:281 step:33K smpl:2M ep:13K epch:7.82 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0 .506 data_s:0.003 INFO 2025-09-08 18:41:51 celerate.py:281 step:34K smpl:2M ep:13K epch:7.86 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0 .509 data_s:0.006 INFO 2025-09-08 18:43:36 celerate.py:281 step:34K smpl:2M ep:13K epch:7.91 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:0 .521 data_s:0.003 INFO 2025-09-08 18:45:19 celerate.py:281 step:34K smpl:2M ep:13K epch:7.96 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0 .511 data_s:0.003 INFO 2025-09-08 18:47:05 celerate.py:281 step:34K smpl:2M ep:14K epch:8.00 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0 .495 data_s:0.035 INFO 2025-09-08 18:48:51 celerate.py:281 step:34K smpl:2M ep:14K epch:8.05 loss:0.087 grdn:0.243 lr:2.5e-06 updt_s:0 .413 data_s:0.112 INFO 2025-09-08 18:50:34 celerate.py:281 step:35K smpl:2M ep:14K epch:8.10 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0 .515 data_s:0.003 INFO 2025-09-08 18:52:19 celerate.py:281 step:35K smpl:2M ep:14K epch:8.14 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0 .520 data_s:0.003 INFO 2025-09-08 18:54:03 celerate.py:281 step:35K smpl:2M ep:14K epch:8.19 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0 .515 data_s:0.003 INFO 2025-09-08 18:55:48 celerate.py:281 step:35K smpl:2M ep:14K epch:8.24 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0 .420 data_s:0.101 INFO 2025-09-08 18:57:33 celerate.py:281 step:35K smpl:2M ep:14K epch:8.28 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0 .506 data_s:0.022 INFO 2025-09-08 18:59:19 celerate.py:281 step:36K smpl:2M ep:14K epch:8.33 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0 .525 data_s:0.003 INFO 2025-09-08 19:01:03 celerate.py:281 step:36K smpl:2M ep:14K epch:8.38 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0 .516 data_s:0.003 INFO 2025-09-08 19:02:48 celerate.py:281 step:36K smpl:2M ep:14K epch:8.43 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0 .516 data_s:0.003 INFO 2025-09-08 19:04:32 celerate.py:281 step:36K smpl:2M ep:14K epch:8.47 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0 .505 data_s:0.013 INFO 2025-09-08 19:06:15 celerate.py:281 step:36K smpl:2M ep:14K epch:8.52 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0 .384 data_s:0.130 INFO 2025-09-08 19:07:58 celerate.py:281 step:37K smpl:2M ep:15K epch:8.57 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0 .430 data_s:0.084 INFO 2025-09-08 19:09:41 celerate.py:281 step:37K smpl:2M ep:15K epch:8.61 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0 .351 data_s:0.162 INFO 2025-09-08 19:11:25 celerate.py:281 step:37K smpl:2M ep:15K epch:8.66 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:0 .337 data_s:0.181 INFO 2025-09-08 19:13:08 celerate.py:281 step:37K smpl:2M ep:15K epch:8.71 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0 .336 data_s:0.177 INFO 2025-09-08 19:14:52 celerate.py:281 step:37K smpl:2M ep:15K epch:8.75 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0 .344 data_s:0.174 INFO 2025-09-08 19:16:35 celerate.py:281 step:38K smpl:2M ep:15K epch:8.80 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0 .332 data_s:0.182 INFO 2025-09-08 19:18:18 celerate.py:281 step:38K smpl:2M ep:15K epch:8.85 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0 .332 data_s:0.182 INFO 2025-09-08 19:20:01 celerate.py:281 step:38K smpl:2M ep:15K epch:8.89 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0 .337 data_s:0.177 INFO 2025-09-08 19:21:45 celerate.py:281 step:38K smpl:2M ep:15K epch:8.94 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0 .495 data_s:0.022 INFO 2025-09-08 19:23:29 celerate.py:281 step:38K smpl:2M ep:15K epch:8.99 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0 .513 data_s:0.003 INFO 2025-09-08 19:25:15 celerate.py:281 step:39K smpl:2M ep:15K epch:9.03 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0 .436 data_s:0.097 INFO 2025-09-08 19:26:59 celerate.py:281 step:39K smpl:2M ep:15K epch:9.08 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0 .378 data_s:0.138 INFO 2025-09-08 19:28:43 celerate.py:281 step:39K smpl:2M ep:15K epch:9.13 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s:0 .497 data_s:0.023 INFO 2025-09-08 19:30:27 celerate.py:281 step:39K smpl:3M ep:16K epch:9.17 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0 .515 data_s:0.003 INFO 2025-09-08 19:32:14 celerate.py:281 step:39K smpl:3M ep:16K epch:9.22 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0 .526 data_s:0.003 INFO 2025-09-08 19:33:57 celerate.py:281 step:40K smpl:3M ep:16K epch:9.27 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0 .506 data_s:0.011 INFO 2025-09-08 19:35:42 celerate.py:281 step:40K smpl:3M ep:16K epch:9.31 loss:0.087 grdn:0.229 lr:2.5e-06 updt_s:0 .455 data_s:0.066 INFO 2025-09-08 19:37:26 celerate.py:281 step:40K smpl:3M ep:16K epch:9.36 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:0 .493 data_s:0.026 INFO 2025-09-08 19:37:26 celerate.py:295 Checkpoint policy after step 40000 INFO 2025-09-08 19:39:10 celerate.py:281 step:40K smpl:3M ep:16K epch:9.41 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:0 .397 data_s:0.114 INFO 2025-09-08 19:40:53 celerate.py:281 step:40K smpl:3M ep:16K epch:9.45 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:0 .344 data_s:0.168 INFO 2025-09-08 19:42:37 celerate.py:281 step:41K smpl:3M ep:16K epch:9.50 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0 .480 data_s:0.036 INFO 2025-09-08 19:44:21 celerate.py:281 step:41K smpl:3M ep:16K epch:9.55 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0 .517 data_s:0.003 INFO 2025-09-08 19:46:05 celerate.py:281 step:41K smpl:3M ep:16K epch:9.60 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0 .517 data_s:0.003 INFO 2025-09-08 19:47:49 celerate.py:281 step:41K smpl:3M ep:16K epch:9.64 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0 .513 data_s:0.003 INFO 2025-09-08 19:49:33 celerate.py:281 step:41K smpl:3M ep:16K epch:9.69 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0 .515 data_s:0.003 INFO 2025-09-08 19:51:17 celerate.py:281 step:42K smpl:3M ep:16K epch:9.74 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0 .515 data_s:0.003 INFO 2025-09-08 19:53:00 celerate.py:281 step:42K smpl:3M ep:17K epch:9.78 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0 .513 data_s:0.003 INFO 2025-09-08 19:54:44 celerate.py:281 step:42K smpl:3M ep:17K epch:9.83 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0 .516 data_s:0.003 INFO 2025-09-08 19:56:28 celerate.py:281 step:42K smpl:3M ep:17K epch:9.88 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0 .512 data_s:0.003 INFO 2025-09-08 19:58:11 celerate.py:281 step:42K smpl:3M ep:17K epch:9.92 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0 .514 data_s:0.003 INFO 2025-09-08 19:59:55 celerate.py:281 step:43K smpl:3M ep:17K epch:9.97 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:0 .514 data_s:0.003 INFO 2025-09-08 20:01:42 celerate.py:281 step:43K smpl:3M ep:17K epch:10.02 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s: 0.476 data_s:0.057 INFO 2025-09-08 20:03:25 celerate.py:281 step:43K smpl:3M ep:17K epch:10.06 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s: 0.471 data_s:0.043 INFO 2025-09-08 20:05:09 celerate.py:281 step:43K smpl:3M ep:17K epch:10.11 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.515 data_s:0.004 INFO 2025-09-08 20:06:53 celerate.py:281 step:43K smpl:3M ep:17K epch:10.16 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s: 0.505 data_s:0.013 INFO 2025-09-08 20:08:36 celerate.py:281 step:44K smpl:3M ep:17K epch:10.20 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-08 20:10:20 celerate.py:281 step:44K smpl:3M ep:17K epch:10.25 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-08 20:12:04 celerate.py:281 step:44K smpl:3M ep:17K epch:10.30 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-08 20:13:47 celerate.py:281 step:44K smpl:3M ep:18K epch:10.34 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s: 0.503 data_s:0.011 INFO 2025-09-08 20:15:31 celerate.py:281 step:44K smpl:3M ep:18K epch:10.39 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.416 data_s:0.102 INFO 2025-09-08 20:17:15 celerate.py:281 step:45K smpl:3M ep:18K epch:10.44 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s: 0.502 data_s:0.017 INFO 2025-09-08 20:18:58 celerate.py:281 step:45K smpl:3M ep:18K epch:10.48 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-08 20:20:41 celerate.py:281 step:45K smpl:3M ep:18K epch:10.53 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.496 data_s:0.017 INFO 2025-09-08 20:22:24 celerate.py:281 step:45K smpl:3M ep:18K epch:10.58 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.493 data_s:0.022 INFO 2025-09-08 20:24:08 celerate.py:281 step:45K smpl:3M ep:18K epch:10.63 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.485 data_s:0.031 INFO 2025-09-08 20:25:52 celerate.py:281 step:46K smpl:3M ep:18K epch:10.67 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s: 0.518 data_s:0.003 INFO 2025-09-08 20:27:36 celerate.py:281 step:46K smpl:3M ep:18K epch:10.72 loss:0.085 grdn:0.228 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-08 20:29:19 celerate.py:281 step:46K smpl:3M ep:18K epch:10.77 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 20:31:04 celerate.py:281 step:46K smpl:3M ep:18K epch:10.81 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-08 20:32:47 celerate.py:281 step:46K smpl:3M ep:18K epch:10.86 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-08 20:34:32 celerate.py:281 step:47K smpl:3M ep:18K epch:10.91 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s: 0.518 data_s:0.003 INFO 2025-09-08 20:36:15 celerate.py:281 step:47K smpl:3M ep:19K epch:10.95 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 20:38:01 celerate.py:281 step:47K smpl:3M ep:19K epch:11.00 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.416 data_s:0.114 INFO 2025-09-08 20:39:45 celerate.py:281 step:47K smpl:3M ep:19K epch:11.05 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.429 data_s:0.086 INFO 2025-09-08 20:41:28 celerate.py:281 step:47K smpl:3M ep:19K epch:11.09 loss:0.086 grdn:0.229 lr:2.5e-06 updt_s: 0.409 data_s:0.106 INFO 2025-09-08 20:43:12 celerate.py:281 step:48K smpl:3M ep:19K epch:11.14 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.470 data_s:0.050 INFO 2025-09-08 20:44:56 celerate.py:281 step:48K smpl:3M ep:19K epch:11.19 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-08 20:46:41 celerate.py:281 step:48K smpl:3M ep:19K epch:11.23 loss:0.087 grdn:0.243 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 20:48:25 celerate.py:281 step:48K smpl:3M ep:19K epch:11.28 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.518 data_s:0.003 INFO 2025-09-08 20:50:08 celerate.py:281 step:48K smpl:3M ep:19K epch:11.33 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.505 data_s:0.009 INFO 2025-09-08 20:51:52 celerate.py:281 step:49K smpl:3M ep:19K epch:11.37 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s: 0.449 data_s:0.067 INFO 2025-09-08 20:53:35 celerate.py:281 step:49K smpl:3M ep:19K epch:11.42 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.420 data_s:0.094 INFO 2025-09-08 20:55:19 celerate.py:281 step:49K smpl:3M ep:19K epch:11.47 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 20:57:01 celerate.py:281 step:49K smpl:3M ep:19K epch:11.51 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s: 0.505 data_s:0.003 INFO 2025-09-08 20:58:44 celerate.py:281 step:49K smpl:3M ep:20K epch:11.56 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-08 21:00:28 celerate.py:281 step:50K smpl:3M ep:20K epch:11.61 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-08 21:02:11 celerate.py:281 step:50K smpl:3M ep:20K epch:11.65 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.402 data_s:0.110 INFO 2025-09-08 21:03:54 celerate.py:281 step:50K smpl:3M ep:20K epch:11.70 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.332 data_s:0.184 INFO 2025-09-08 21:05:37 celerate.py:281 step:50K smpl:3M ep:20K epch:11.75 loss:0.086 grdn:0.229 lr:2.5e-06 updt_s: 0.332 data_s:0.182 INFO 2025-09-08 21:07:21 celerate.py:281 step:50K smpl:3M ep:20K epch:11.80 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s: 0.466 data_s:0.049 INFO 2025-09-08 21:09:05 celerate.py:281 step:51K smpl:3M ep:20K epch:11.84 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 21:10:49 celerate.py:281 step:51K smpl:3M ep:20K epch:11.89 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s: 0.512 data_s:0.004 INFO 2025-09-08 21:12:32 celerate.py:281 step:51K smpl:3M ep:20K epch:11.94 loss:0.085 grdn:0.234 lr:2.5e-06 updt_s: 0.484 data_s:0.032 INFO 2025-09-08 21:14:17 celerate.py:281 step:51K smpl:3M ep:20K epch:11.98 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s: 0.517 data_s:0.004 INFO 2025-09-08 21:16:03 celerate.py:281 step:51K smpl:3M ep:20K epch:12.03 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.424 data_s:0.105 INFO 2025-09-08 21:17:46 celerate.py:281 step:52K smpl:3M ep:20K epch:12.08 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.442 data_s:0.073 INFO 2025-09-08 21:19:30 celerate.py:281 step:52K smpl:3M ep:21K epch:12.12 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s: 0.511 data_s:0.007 INFO 2025-09-08 21:21:15 celerate.py:281 step:52K smpl:3M ep:21K epch:12.17 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.520 data_s:0.003 INFO 2025-09-08 21:22:59 celerate.py:281 step:52K smpl:3M ep:21K epch:12.22 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 21:24:43 celerate.py:281 step:52K smpl:3M ep:21K epch:12.26 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.518 data_s:0.003 INFO 2025-09-08 21:26:27 celerate.py:281 step:53K smpl:3M ep:21K epch:12.31 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 21:28:11 celerate.py:281 step:53K smpl:3M ep:21K epch:12.36 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 21:29:55 celerate.py:281 step:53K smpl:3M ep:21K epch:12.40 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 21:31:39 celerate.py:281 step:53K smpl:3M ep:21K epch:12.45 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 21:33:23 celerate.py:281 step:53K smpl:3M ep:21K epch:12.50 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 21:35:08 celerate.py:281 step:54K smpl:3M ep:21K epch:12.54 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.519 data_s:0.003 INFO 2025-09-08 21:36:51 celerate.py:281 step:54K smpl:3M ep:21K epch:12.59 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-08 21:38:36 celerate.py:281 step:54K smpl:3M ep:21K epch:12.64 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 21:40:18 celerate.py:281 step:54K smpl:3M ep:21K epch:12.68 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.473 data_s:0.038 INFO 2025-09-08 21:42:02 celerate.py:281 step:54K smpl:3M ep:22K epch:12.73 loss:0.085 grdn:0.232 lr:2.5e-06 updt_s: 0.408 data_s:0.109 INFO 2025-09-08 21:43:45 celerate.py:281 step:55K smpl:3M ep:22K epch:12.78 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.377 data_s:0.136 INFO 2025-09-08 21:45:29 celerate.py:281 step:55K smpl:4M ep:22K epch:12.83 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s: 0.497 data_s:0.022 INFO 2025-09-08 21:47:12 celerate.py:281 step:55K smpl:4M ep:22K epch:12.87 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-08 21:48:56 celerate.py:281 step:55K smpl:4M ep:22K epch:12.92 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.429 data_s:0.086 INFO 2025-09-08 21:50:39 celerate.py:281 step:55K smpl:4M ep:22K epch:12.97 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s: 0.454 data_s:0.059 INFO 2025-09-08 21:52:25 celerate.py:281 step:56K smpl:4M ep:22K epch:13.01 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.459 data_s:0.072 INFO 2025-09-08 21:54:08 celerate.py:281 step:56K smpl:4M ep:22K epch:13.06 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s: 0.382 data_s:0.132 INFO 2025-09-08 21:55:51 celerate.py:281 step:56K smpl:4M ep:22K epch:13.11 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.500 data_s:0.016 INFO 2025-09-08 21:57:36 celerate.py:281 step:56K smpl:4M ep:22K epch:13.15 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 21:59:20 celerate.py:281 step:56K smpl:4M ep:22K epch:13.20 loss:0.085 grdn:0.235 lr:2.5e-06 updt_s: 0.518 data_s:0.003 INFO 2025-09-08 22:01:03 celerate.py:281 step:57K smpl:4M ep:22K epch:13.25 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-08 22:02:48 celerate.py:281 step:57K smpl:4M ep:23K epch:13.29 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 22:04:32 celerate.py:281 step:57K smpl:4M ep:23K epch:13.34 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 22:06:14 celerate.py:281 step:57K smpl:4M ep:23K epch:13.39 loss:0.087 grdn:0.244 lr:2.5e-06 updt_s: 0.505 data_s:0.005 bINFO 2025-09-08 22:07:59 celerate.py:281 step:57K smpl:4M ep:23K epch:13.43 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s :0.496 data_s:0.026 INFO 2025-09-08 22:09:43 celerate.py:281 step:58K smpl:4M ep:23K epch:13.48 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s: 0.438 data_s:0.080 INFO 2025-09-08 22:11:27 celerate.py:281 step:58K smpl:4M ep:23K epch:13.53 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s: 0.444 data_s:0.073 INFO 2025-09-08 22:13:11 celerate.py:281 step:58K smpl:4M ep:23K epch:13.57 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 22:14:55 celerate.py:281 step:58K smpl:4M ep:23K epch:13.62 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.518 data_s:0.003 INFO 2025-09-08 22:16:39 celerate.py:281 step:58K smpl:4M ep:23K epch:13.67 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-08 22:18:22 celerate.py:281 step:59K smpl:4M ep:23K epch:13.71 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-08 22:20:05 celerate.py:281 step:59K smpl:4M ep:23K epch:13.76 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s: 0.508 data_s:0.003 INFO 2025-09-08 22:21:48 celerate.py:281 step:59K smpl:4M ep:23K epch:13.81 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s: 0.505 data_s:0.008 INFO 2025-09-08 22:23:31 celerate.py:281 step:59K smpl:4M ep:23K epch:13.85 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.505 data_s:0.008 INFO 2025-09-08 22:25:15 celerate.py:281 step:59K smpl:4M ep:24K epch:13.90 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 22:26:58 celerate.py:281 step:60K smpl:4M ep:24K epch:13.95 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s: 0.493 data_s:0.022 INFO 2025-09-08 22:28:42 celerate.py:281 step:60K smpl:4M ep:24K epch:14.00 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 22:30:29 celerate.py:281 step:60K smpl:4M ep:24K epch:14.04 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.435 data_s:0.101 INFO 2025-09-08 22:30:29 celerate.py:295 Checkpoint policy after step 60000 INFO 2025-09-08 22:32:14 celerate.py:281 step:60K smpl:4M ep:24K epch:14.09 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.508 data_s:0.003 INFO 2025-09-08 22:33:58 celerate.py:281 step:60K smpl:4M ep:24K epch:14.14 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-08 22:35:42 celerate.py:281 step:61K smpl:4M ep:24K epch:14.18 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 22:37:25 celerate.py:281 step:61K smpl:4M ep:24K epch:14.23 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-08 22:39:09 celerate.py:281 step:61K smpl:4M ep:24K epch:14.28 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 22:40:52 celerate.py:281 step:61K smpl:4M ep:24K epch:14.32 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.509 data_s:0.003 INFO 2025-09-08 22:42:35 celerate.py:281 step:61K smpl:4M ep:24K epch:14.37 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-08 22:44:18 celerate.py:281 step:62K smpl:4M ep:24K epch:14.42 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-08 22:46:02 celerate.py:281 step:62K smpl:4M ep:24K epch:14.46 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 22:47:47 celerate.py:281 step:62K smpl:4M ep:25K epch:14.51 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 22:49:30 celerate.py:281 step:62K smpl:4M ep:25K epch:14.56 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-08 22:51:14 celerate.py:281 step:62K smpl:4M ep:25K epch:14.60 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 22:52:58 celerate.py:281 step:63K smpl:4M ep:25K epch:14.65 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s: 0.484 data_s:0.033 INFO 2025-09-08 22:54:41 celerate.py:281 step:63K smpl:4M ep:25K epch:14.70 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.501 data_s:0.016 INFO 2025-09-08 22:56:25 celerate.py:281 step:63K smpl:4M ep:25K epch:14.74 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.436 data_s:0.081 INFO 2025-09-08 22:58:08 celerate.py:281 step:63K smpl:4M ep:25K epch:14.79 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s: 0.436 data_s:0.080 INFO 2025-09-08 22:59:51 celerate.py:281 step:63K smpl:4M ep:25K epch:14.84 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s: 0.344 data_s:0.168 INFO 2025-09-08 23:01:34 celerate.py:281 step:64K smpl:4M ep:25K epch:14.88 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.479 data_s:0.035 INFO 2025-09-08 23:03:18 celerate.py:281 step:64K smpl:4M ep:25K epch:14.93 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-08 23:05:02 celerate.py:281 step:64K smpl:4M ep:25K epch:14.98 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-08 23:06:47 celerate.py:281 step:64K smpl:4M ep:25K epch:15.02 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s: 0.405 data_s:0.119 INFO 2025-09-08 23:08:30 celerate.py:281 step:64K smpl:4M ep:26K epch:15.07 loss:0.087 grdn:0.248 lr:2.5e-06 updt_s: 0.393 data_s:0.121 INFO 2025-09-08 23:10:13 celerate.py:281 step:65K smpl:4M ep:26K epch:15.12 loss:0.087 grdn:0.242 lr:2.5e-06 updt_s: 0.369 data_s:0.142 INFO 2025-09-08 23:11:56 celerate.py:281 step:65K smpl:4M ep:26K epch:15.17 loss:0.085 grdn:0.230 lr:2.5e-06 updt_s: 0.360 data_s:0.156 INFO 2025-09-08 23:13:40 celerate.py:281 step:65K smpl:4M ep:26K epch:15.21 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.333 data_s:0.182 INFO 2025-09-08 23:15:23 celerate.py:281 step:65K smpl:4M ep:26K epch:15.26 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s: 0.376 data_s:0.136 INFO 2025-09-08 23:17:05 celerate.py:281 step:65K smpl:4M ep:26K epch:15.31 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s: 0.439 data_s:0.069 INFO 2025-09-08 23:18:49 celerate.py:281 step:66K smpl:4M ep:26K epch:15.35 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.512 data_s:0.006 INFO 2025-09-08 23:20:32 celerate.py:281 step:66K smpl:4M ep:26K epch:15.40 loss:0.087 grdn:0.242 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-08 23:22:15 celerate.py:281 step:66K smpl:4M ep:26K epch:15.45 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.508 data_s:0.004 INFO 2025-09-08 23:23:59 celerate.py:281 step:66K smpl:4M ep:26K epch:15.49 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.474 data_s:0.046 INFO 2025-09-08 23:25:42 celerate.py:281 step:66K smpl:4M ep:26K epch:15.54 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-08 23:27:25 celerate.py:281 step:67K smpl:4M ep:26K epch:15.59 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.487 data_s:0.025 INFO 2025-09-08 23:29:08 celerate.py:281 step:67K smpl:4M ep:26K epch:15.63 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.510 data_s:0.005 INFO 2025-09-08 23:30:50 celerate.py:281 step:67K smpl:4M ep:27K epch:15.68 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.506 data_s:0.003 INFO 2025-09-08 23:32:33 celerate.py:281 step:67K smpl:4M ep:27K epch:15.73 loss:0.086 grdn:0.245 lr:2.5e-06 updt_s: 0.509 data_s:0.003 INFO 2025-09-08 23:34:16 celerate.py:281 step:67K smpl:4M ep:27K epch:15.77 loss:0.087 grdn:0.244 lr:2.5e-06 updt_s: 0.503 data_s:0.014 INFO 2025-09-08 23:35:59 celerate.py:281 step:68K smpl:4M ep:27K epch:15.82 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-08 23:37:42 celerate.py:281 step:68K smpl:4M ep:27K epch:15.87 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.509 data_s:0.003 INFO 2025-09-08 23:39:26 celerate.py:281 step:68K smpl:4M ep:27K epch:15.91 loss:0.085 grdn:0.235 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-08 23:41:10 celerate.py:281 step:68K smpl:4M ep:27K epch:15.96 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-08 23:42:56 celerate.py:281 step:68K smpl:4M ep:27K epch:16.01 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.469 data_s:0.061 INFO 2025-09-08 23:44:40 celerate.py:281 step:69K smpl:4M ep:27K epch:16.05 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.339 data_s:0.179 INFO 2025-09-08 23:46:22 celerate.py:281 step:69K smpl:4M ep:27K epch:16.10 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.386 data_s:0.124 INFO 2025-09-08 23:48:06 celerate.py:281 step:69K smpl:4M ep:27K epch:16.15 loss:0.085 grdn:0.239 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-08 23:49:50 celerate.py:281 step:69K smpl:4M ep:27K epch:16.20 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-08 23:51:33 celerate.py:281 step:69K smpl:4M ep:27K epch:16.24 loss:0.087 grdn:0.246 lr:2.5e-06 updt_s: 0.501 data_s:0.014 INFO 2025-09-08 23:53:17 celerate.py:281 step:70K smpl:4M ep:28K epch:16.29 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-08 23:54:59 celerate.py:281 step:70K smpl:4M ep:28K epch:16.34 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s: 0.507 data_s:0.005 INFO 2025-09-08 23:56:43 celerate.py:281 step:70K smpl:4M ep:28K epch:16.38 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-08 23:58:27 celerate.py:281 step:70K smpl:4M ep:28K epch:16.43 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-09 00:00:12 celerate.py:281 step:70K smpl:5M ep:28K epch:16.48 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.518 data_s:0.003 INFO 2025-09-09 00:01:55 celerate.py:281 step:71K smpl:5M ep:28K epch:16.52 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-09 00:03:37 celerate.py:281 step:71K smpl:5M ep:28K epch:16.57 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s: 0.508 data_s:0.003 INFO 2025-09-09 00:05:20 celerate.py:281 step:71K smpl:5M ep:28K epch:16.62 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.509 data_s:0.003 INFO 2025-09-09 00:07:04 celerate.py:281 step:71K smpl:5M ep:28K epch:16.66 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.519 data_s:0.003 INFO 2025-09-09 00:08:47 celerate.py:281 step:71K smpl:5M ep:28K epch:16.71 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s: 0.509 data_s:0.003 INFO 2025-09-09 00:10:30 celerate.py:281 step:72K smpl:5M ep:28K epch:16.76 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-09 00:12:13 celerate.py:281 step:72K smpl:5M ep:28K epch:16.80 loss:0.085 grdn:0.230 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-09 00:13:58 celerate.py:281 step:72K smpl:5M ep:29K epch:16.85 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.522 data_s:0.003 INFO 2025-09-09 00:15:40 celerate.py:281 step:72K smpl:5M ep:29K epch:16.90 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.506 data_s:0.003 INFO 2025-09-09 00:17:23 celerate.py:281 step:72K smpl:5M ep:29K epch:16.94 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.510 data_s:0.004 INFO 2025-09-09 00:19:05 celerate.py:281 step:73K smpl:5M ep:29K epch:16.99 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.505 data_s:0.003 INFO 2025-09-09 00:20:51 celerate.py:281 step:73K smpl:5M ep:29K epch:17.04 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.437 data_s:0.092 INFO 2025-09-09 00:22:34 celerate.py:281 step:73K smpl:5M ep:29K epch:17.08 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s: 0.489 data_s:0.025 INFO 2025-09-09 00:24:17 celerate.py:281 step:73K smpl:5M ep:29K epch:17.13 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-09 00:26:01 celerate.py:281 step:73K smpl:5M ep:29K epch:17.18 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-09 00:27:45 celerate.py:281 step:74K smpl:5M ep:29K epch:17.22 loss:0.087 grdn:0.246 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-09 00:29:28 celerate.py:281 step:74K smpl:5M ep:29K epch:17.27 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.509 data_s:0.003 INFO 2025-09-09 00:31:11 celerate.py:281 step:74K smpl:5M ep:29K epch:17.32 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-09 00:32:54 celerate.py:281 step:74K smpl:5M ep:29K epch:17.37 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-09 00:34:38 celerate.py:281 step:74K smpl:5M ep:29K epch:17.41 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-09 00:36:21 celerate.py:281 step:75K smpl:5M ep:30K epch:17.46 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-09 00:38:05 celerate.py:281 step:75K smpl:5M ep:30K epch:17.51 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-09 00:39:49 celerate.py:281 step:75K smpl:5M ep:30K epch:17.55 loss:0.086 grdn:0.251 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-09 00:41:32 celerate.py:281 step:75K smpl:5M ep:30K epch:17.60 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.506 data_s:0.005 INFO 2025-09-09 00:43:14 celerate.py:281 step:75K smpl:5M ep:30K epch:17.65 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.390 data_s:0.122 INFO 2025-09-09 00:44:58 celerate.py:281 step:76K smpl:5M ep:30K epch:17.69 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s: 0.410 data_s:0.107 INFO 2025-09-09 00:46:41 celerate.py:281 step:76K smpl:5M ep:30K epch:17.74 loss:0.087 grdn:0.247 lr:2.5e-06 updt_s: 0.427 data_s:0.085 INFO 2025-09-09 00:48:24 celerate.py:281 step:76K smpl:5M ep:30K epch:17.79 loss:0.085 grdn:0.241 lr:2.5e-06 updt_s: 0.492 data_s:0.025 INFO 2025-09-09 00:50:08 celerate.py:281 step:76K smpl:5M ep:30K epch:17.83 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-09 00:51:51 celerate.py:281 step:76K smpl:5M ep:30K epch:17.88 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-09 00:53:33 celerate.py:281 step:77K smpl:5M ep:30K epch:17.93 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-09 00:55:18 celerate.py:281 step:77K smpl:5M ep:30K epch:17.97 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.519 data_s:0.003 INFO 2025-09-09 00:57:05 celerate.py:281 step:77K smpl:5M ep:31K epch:18.02 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.486 data_s:0.046 INFO 2025-09-09 00:58:47 celerate.py:281 step:77K smpl:5M ep:31K epch:18.07 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.509 data_s:0.003 INFO 2025-09-09 01:00:31 celerate.py:281 step:77K smpl:5M ep:31K epch:18.11 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-09 01:02:15 celerate.py:281 step:78K smpl:5M ep:31K epch:18.16 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-09 01:03:56 celerate.py:281 step:78K smpl:5M ep:31K epch:18.21 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.498 data_s:0.008 INFO 2025-09-09 01:05:40 celerate.py:281 step:78K smpl:5M ep:31K epch:18.25 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.508 data_s:0.010 INFO 2025-09-09 01:07:24 celerate.py:281 step:78K smpl:5M ep:31K epch:18.30 loss:0.085 grdn:0.240 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-09 01:09:07 celerate.py:281 step:78K smpl:5M ep:31K epch:18.35 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-09 01:10:51 celerate.py:281 step:79K smpl:5M ep:31K epch:18.40 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-09 01:12:35 celerate.py:281 step:79K smpl:5M ep:31K epch:18.44 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-09 01:14:17 celerate.py:281 step:79K smpl:5M ep:31K epch:18.49 loss:0.086 grdn:0.247 lr:2.5e-06 updt_s: 0.508 data_s:0.003 INFO 2025-09-09 01:16:01 celerate.py:281 step:79K smpl:5M ep:31K epch:18.54 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-09 01:17:43 celerate.py:281 step:79K smpl:5M ep:31K epch:18.58 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s: 0.507 data_s:0.003 INFO 2025-09-09 01:19:26 celerate.py:281 step:80K smpl:5M ep:32K epch:18.63 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-09 01:21:09 celerate.py:281 step:80K smpl:5M ep:32K epch:18.68 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-09 01:22:52 celerate.py:281 step:80K smpl:5M ep:32K epch:18.72 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.501 data_s:0.011 INFO 2025-09-09 01:22:52 celerate.py:295 Checkpoint policy after step 80000 INFO 2025-09-09 01:24:37 celerate.py:281 step:80K smpl:5M ep:32K epch:18.77 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-09 01:26:20 celerate.py:281 step:80K smpl:5M ep:32K epch:18.82 loss:0.087 grdn:0.242 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-09 01:28:05 celerate.py:281 step:81K smpl:5M ep:32K epch:18.86 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-09 01:29:48 celerate.py:281 step:81K smpl:5M ep:32K epch:18.91 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-09 01:31:32 celerate.py:281 step:81K smpl:5M ep:32K epch:18.96 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.517 data_s:0.003 INFO 2025-09-09 01:33:18 celerate.py:281 step:81K smpl:5M ep:32K epch:19.00 loss:0.087 grdn:0.244 lr:2.5e-06 updt_s: 0.487 data_s:0.042 INFO 2025-09-09 01:35:01 celerate.py:281 step:81K smpl:5M ep:32K epch:19.05 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.461 data_s:0.054 INFO 2025-09-09 01:36:46 celerate.py:281 step:82K smpl:5M ep:32K epch:19.10 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.450 data_s:0.071 INFO 2025-09-09 01:38:29 celerate.py:281 step:82K smpl:5M ep:32K epch:19.14 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.451 data_s:0.064 INFO 2025-09-09 01:40:12 celerate.py:281 step:82K smpl:5M ep:32K epch:19.19 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-09 01:41:54 celerate.py:281 step:82K smpl:5M ep:33K epch:19.24 loss:0.085 grdn:0.237 lr:2.5e-06 updt_s: 0.480 data_s:0.030 INFO 2025-09-09 01:43:37 celerate.py:281 step:82K smpl:5M ep:33K epch:19.28 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.434 data_s:0.080 INFO 2025-09-09 01:45:20 celerate.py:281 step:83K smpl:5M ep:33K epch:19.33 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-09 01:47:04 celerate.py:281 step:83K smpl:5M ep:33K epch:19.38 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-09 01:48:49 celerate.py:281 step:83K smpl:5M ep:33K epch:19.42 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.520 data_s:0.003 INFO 2025-09-09 01:50:32 celerate.py:281 step:83K smpl:5M ep:33K epch:19.47 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-09 01:52:15 celerate.py:281 step:83K smpl:5M ep:33K epch:19.52 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-09 01:53:59 celerate.py:281 step:84K smpl:5M ep:33K epch:19.57 loss:0.085 grdn:0.236 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-09 01:55:41 celerate.py:281 step:84K smpl:5M ep:33K epch:19.61 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.508 data_s:0.003 INFO 2025-09-09 01:57:24 celerate.py:281 step:84K smpl:5M ep:33K epch:19.66 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s: 0.500 data_s:0.013 INFO 2025-09-09 01:59:07 celerate.py:281 step:84K smpl:5M ep:33K epch:19.71 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-09 02:00:50 celerate.py:281 step:84K smpl:5M ep:33K epch:19.75 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-09 02:02:34 celerate.py:281 step:85K smpl:5M ep:34K epch:19.80 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-09 02:04:17 celerate.py:281 step:85K smpl:5M ep:34K epch:19.85 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.511 data_s:0.003 INFO 2025-09-09 02:05:59 celerate.py:281 step:85K smpl:5M ep:34K epch:19.89 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.507 data_s:0.004 INFO 2025-09-09 02:07:43 celerate.py:281 step:85K smpl:5M ep:34K epch:19.94 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.505 data_s:0.011 INFO 2025-09-09 02:09:26 celerate.py:281 step:85K smpl:5M ep:34K epch:19.99 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.392 data_s:0.124 INFO 2025-09-09 02:11:14 celerate.py:281 step:86K smpl:5M ep:34K epch:20.03 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.435 data_s:0.100 INFO 2025-09-09 02:12:57 celerate.py:281 step:86K smpl:5M ep:34K epch:20.08 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-09 02:14:41 celerate.py:281 step:86K smpl:6M ep:34K epch:20.13 loss:0.085 grdn:0.234 lr:2.5e-06 updt_s: 0.516 data_s:0.003 INFO 2025-09-09 02:16:25 celerate.py:281 step:86K smpl:6M ep:34K epch:20.17 loss:0.085 grdn:0.234 lr:2.5e-06 updt_s: 0.514 data_s:0.003 INFO 2025-09-09 02:18:09 celerate.py:281 step:86K smpl:6M ep:34K epch:20.22 loss:0.085 grdn:0.238 lr:2.5e-06 updt_s: 0.506 data_s:0.010 INFO 2025-09-09 02:19:53 celerate.py:281 step:87K smpl:6M ep:34K epch:20.27 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.515 data_s:0.003 INFO 2025-09-09 02:21:35 celerate.py:281 step:87K smpl:6M ep:34K epch:20.31 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.501 data_s:0.007 INFO 2025-09-09 02:23:17 celerate.py:281 step:87K smpl:6M ep:34K epch:20.36 loss:0.085 grdn:0.242 lr:2.5e-06 updt_s: 0.432 data_s:0.080 INFO 2025-09-09 02:25:01 celerate.py:281 step:87K smpl:6M ep:35K epch:20.41 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.445 data_s:0.073 INFO 2025-09-09 02:26:43 celerate.py:281 step:87K smpl:6M ep:35K epch:20.45 loss:0.085 grdn:0.239 lr:2.5e-06 updt_s: 0.401 data_s:0.107 INFO 2025-09-09 02:28:26 celerate.py:281 step:88K smpl:6M ep:35K epch:20.50 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.502 data_s:0.009 INFO 2025-09-09 02:30:09 celerate.py:281 step:88K smpl:6M ep:35K epch:20.55 loss:0.087 grdn:0.248 lr:2.5e-06 updt_s: 0.491 data_s:0.021 INFO 2025-09-09 02:31:52 celerate.py:281 step:88K smpl:6M ep:35K epch:20.59 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.391 data_s:0.124 INFO 2025-09-09 02:33:35 celerate.py:281 step:88K smpl:6M ep:35K epch:20.64 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.332 data_s:0.180 INFO 2025-09-09 02:35:18 celerate.py:281 step:88K smpl:6M ep:35K epch:20.69 loss:0.087 grdn:0.243 lr:2.5e-06 updt_s: 0.333 data_s:0.181 INFO 2025-09-09 02:37:02 celerate.py:281 step:89K smpl:6M ep:35K epch:20.74 loss:0.086 grdn:0.245 lr:2.5e-06 updt_s: 0.332 data_s:0.185 INFO 2025-09-09 02:38:47 celerate.py:281 step:89K smpl:6M ep:35K epch:20.78 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.332 data_s:0.190 INFO 2025-09-09 02:40:30 celerate.py:281 step:89K smpl:6M ep:35K epch:20.83 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.330 data_s:0.187 INFO 2025-09-09 02:42:12 celerate.py:281 step:89K smpl:6M ep:35K epch:20.88 loss:0.085 grdn:0.243 lr:2.5e-06 updt_s: 0.331 data_s:0.177 INFO 2025-09-09 02:43:56 celerate.py:281 step:89K smpl:6M ep:35K epch:20.92 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.330 data_s:0.185 INFO 2025-09-09 02:45:39 celerate.py:281 step:90K smpl:6M ep:36K epch:20.97 loss:0.087 grdn:0.252 lr:2.5e-06 updt_s: 0.335 data_s:0.182 INFO 2025-09-09 02:47:25 celerate.py:281 step:90K smpl:6M ep:36K epch:21.02 loss:0.087 grdn:0.247 lr:2.5e-06 updt_s: 0.330 data_s:0.197 INFO 2025-09-09 02:49:08 celerate.py:281 step:90K smpl:6M ep:36K epch:21.06 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.330 data_s:0.182 INFO 2025-09-09 02:50:51 celerate.py:281 step:90K smpl:6M ep:36K epch:21.11 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.334 data_s:0.181 INFO 2025-09-09 02:52:34 celerate.py:281 step:90K smpl:6M ep:36K epch:21.16 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.330 data_s:0.183 INFO 2025-09-09 02:54:17 celerate.py:281 step:91K smpl:6M ep:36K epch:21.20 loss:0.085 grdn:0.235 lr:2.5e-06 updt_s: 0.342 data_s:0.173 INFO 2025-09-09 02:56:01 celerate.py:281 step:91K smpl:6M ep:36K epch:21.25 loss:0.086 grdn:0.246 lr:2.5e-06 updt_s: 0.331 data_s:0.189 INFO 2025-09-09 02:57:44 celerate.py:281 step:91K smpl:6M ep:36K epch:21.30 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.329 data_s:0.182 INFO 2025-09-09 02:59:27 celerate.py:281 step:91K smpl:6M ep:36K epch:21.34 loss:0.087 grdn:0.246 lr:2.5e-06 updt_s: 0.341 data_s:0.175 INFO 2025-09-09 03:01:11 celerate.py:281 step:91K smpl:6M ep:36K epch:21.39 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.330 data_s:0.188 INFO 2025-09-09 03:02:54 celerate.py:281 step:92K smpl:6M ep:36K epch:21.44 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.328 data_s:0.186 INFO 2025-09-09 03:04:39 celerate.py:281 step:92K smpl:6M ep:36K epch:21.48 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.329 data_s:0.195 INFO 2025-09-09 03:06:22 celerate.py:281 step:92K smpl:6M ep:36K epch:21.53 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.330 data_s:0.183 INFO 2025-09-09 03:08:04 celerate.py:281 step:92K smpl:6M ep:37K epch:21.58 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.462 data_s:0.051 INFO 2025-09-09 03:09:48 celerate.py:281 step:92K smpl:6M ep:37K epch:21.62 loss:0.086 grdn:0.248 lr:2.5e-06 updt_s: 0.407 data_s:0.108 INFO 2025-09-09 03:11:32 celerate.py:281 step:93K smpl:6M ep:37K epch:21.67 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s: 0.333 data_s:0.185 INFO 2025-09-09 03:13:15 celerate.py:281 step:93K smpl:6M ep:37K epch:21.72 loss:0.085 grdn:0.242 lr:2.5e-06 updt_s: 0.329 data_s:0.187 INFO 2025-09-09 03:14:58 celerate.py:281 step:93K smpl:6M ep:37K epch:21.77 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.357 data_s:0.156 INFO 2025-09-09 03:16:41 celerate.py:281 step:93K smpl:6M ep:37K epch:21.81 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.487 data_s:0.027 INFO 2025-09-09 03:18:25 celerate.py:281 step:93K smpl:6M ep:37K epch:21.86 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-09 03:20:08 celerate.py:281 step:94K smpl:6M ep:37K epch:21.91 loss:0.087 grdn:0.247 lr:2.5e-06 updt_s: 0.512 data_s:0.003 INFO 2025-09-09 03:21:51 celerate.py:281 step:94K smpl:6M ep:37K epch:21.95 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.508 data_s:0.004 INFO 2025-09-09 03:23:38 celerate.py:281 step:94K smpl:6M ep:37K epch:22.00 loss:0.085 grdn:0.239 lr:2.5e-06 updt_s: 0.429 data_s:0.104 INFO 2025-09-09 03:25:20 celerate.py:281 step:94K smpl:6M ep:37K epch:22.05 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.328 data_s:0.183 INFO 2025-09-09 03:27:04 celerate.py:281 step:94K smpl:6M ep:37K epch:22.09 loss:0.086 grdn:0.246 lr:2.5e-06 updt_s: 0.329 data_s:0.191 INFO 2025-09-09 03:28:47 celerate.py:281 step:95K smpl:6M ep:37K epch:22.14 loss:0.086 grdn:0.246 lr:2.5e-06 updt_s: 0.329 data_s:0.186 INFO 2025-09-09 03:30:30 celerate.py:281 step:95K smpl:6M ep:38K epch:22.19 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.346 data_s:0.166 INFO 2025-09-09 03:32:13 celerate.py:281 step:95K smpl:6M ep:38K epch:22.23 loss:0.086 grdn:0.247 lr:2.5e-06 updt_s: 0.334 data_s:0.181 INFO 2025-09-09 03:33:56 celerate.py:281 step:95K smpl:6M ep:38K epch:22.28 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.402 data_s:0.110 INFO 2025-09-09 03:35:39 celerate.py:281 step:95K smpl:6M ep:38K epch:22.33 loss:0.085 grdn:0.237 lr:2.5e-06 updt_s: 0.353 data_s:0.161 INFO 2025-09-09 03:37:22 celerate.py:281 step:96K smpl:6M ep:38K epch:22.37 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s: 0.356 data_s:0.158 INFO 2025-09-09 03:39:04 celerate.py:281 step:96K smpl:6M ep:38K epch:22.42 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.379 data_s:0.131 INFO 2025-09-09 03:40:49 celerate.py:281 step:96K smpl:6M ep:38K epch:22.47 loss:0.085 grdn:0.239 lr:2.5e-06 updt_s: 0.344 data_s:0.175 INFO 2025-09-09 03:42:32 celerate.py:281 step:96K smpl:6M ep:38K epch:22.51 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s: 0.331 data_s:0.185 INFO 2025-09-09 03:44:15 celerate.py:281 step:96K smpl:6M ep:38K epch:22.56 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s: 0.331 data_s:0.183 INFO 2025-09-09 03:45:58 celerate.py:281 step:97K smpl:6M ep:38K epch:22.61 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s: 0.330 data_s:0.184 INFO 2025-09-09 03:47:43 celerate.py:281 step:97K smpl:6M ep:38K epch:22.65 loss:0.086 grdn:0.248 lr:2.5e-06 updt_s: 0.331 data_s:0.188 INFO 2025-09-09 03:49:27 celerate.py:281 step:97K smpl:6M ep:38K epch:22.70 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.333 data_s:0.185 INFO 2025-09-09 03:51:10 celerate.py:281 step:97K smpl:6M ep:39K epch:22.75 loss:0.085 grdn:0.241 lr:2.5e-06 updt_s: 0.330 data_s:0.185 INFO 2025-09-09 03:52:54 celerate.py:281 step:97K smpl:6M ep:39K epch:22.79 loss:0.086 grdn:0.247 lr:2.5e-06 updt_s: 0.330 data_s:0.192 INFO 2025-09-09 03:54:37 celerate.py:281 step:98K smpl:6M ep:39K epch:22.84 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.329 data_s:0.185 INFO 2025-09-09 03:56:21 celerate.py:281 step:98K smpl:6M ep:39K epch:22.89 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s: 0.329 data_s:0.187 INFO 2025-09-09 03:58:04 celerate.py:281 step:98K smpl:6M ep:39K epch:22.94 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.329 data_s:0.185 INFO 2025-09-09 03:59:46 celerate.py:281 step:98K smpl:6M ep:39K epch:22.98 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s: 0.329 data_s:0.183 INFO 2025-09-09 04:01:32 celerate.py:281 step:98K smpl:6M ep:39K epch:23.03 loss:0.087 grdn:0.250 lr:2.5e-06 updt_s: 0.376 data_s:0.151 INFO 2025-09-09 04:03:16 celerate.py:281 step:99K smpl:6M ep:39K epch:23.08 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s: 0.329 data_s:0.187 INFO 2025-09-09 04:04:59 celerate.py:281 step:99K smpl:6M ep:39K epch:23.12 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s: 0.379 data_s:0.136 INFO 2025-09-09 04:06:42 celerate.py:281 step:99K smpl:6M ep:39K epch:23.17 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s: 0.513 data_s:0.003 INFO 2025-09-09 04:08:25 celerate.py:281 step:99K smpl:6M ep:39K epch:23.22 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s: 0.510 data_s:0.003 INFO 2025-09-09 04:10:07 celerate.py:281 step:99K smpl:6M ep:39K epch:23.26 loss:0.087 grdn:0.242 lr:2.5e-06 updt_s: 0.470 data_s:0.039 INFO 2025-09-09 04:11:50 celerate.py:281 step:100K smpl:6M ep:39K epch:23.31 loss:0.086 grdn:0.247 lr:2.5e-06 updt_s :0.347 data_s:0.169 INFO 2025-09-09 04:13:34 celerate.py:281 step:100K smpl:6M ep:40K epch:23.36 loss:0.085 grdn:0.237 lr:2.5e-06 updt_s :0.330 data_s:0.185 INFO 2025-09-09 04:15:17 celerate.py:281 step:100K smpl:6M ep:40K epch:23.40 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s :0.356 data_s:0.156 INFO 2025-09-09 04:15:17 celerate.py:295 Checkpoint policy after step 100000 INFO 2025-09-09 04:15:18 celerate.py:359 End of training (lerobot) jade_choghari@hf-dgx-01:~/lerobot$ clear (lerobot) jade_choghari@hf-dgx-01:~/lerobot$ tmux capture-pane -pS - > tmux_log.txt