mirror of
https://github.com/huggingface/lerobot.git
synced 2026-05-23 20:50:02 +00:00
Jade Choghari
2009 lines
121 KiB
Plaintext
2009 lines
121 KiB
Plaintext
'resume': False,
|
|
'save_checkpoint': True,
|
|
'save_freq': 20000,
|
|
'scheduler': {'decay_lr': 2.5e-06,
|
|
'num_decay_steps': 30000,
|
|
'num_warmup_steps': 1000,
|
|
'peak_lr': 0.0001,
|
|
'type': 'cosine_decay_with_warmup'},
|
|
'seed': 1000,
|
|
'steps': 100000,
|
|
'use_policy_training_preset': True,
|
|
'wandb': {'disable_artifact': False,
|
|
'enable': False,
|
|
'entity': None,
|
|
'mode': None,
|
|
'notes': None,
|
|
'project': 'lerobot',
|
|
'run_id': None}}
|
|
INFO 2025-09-08 13:23:15 ils/utils.py:48 Cuda backend detected, using cuda.
|
|
WARNING 2025-09-08 13:23:15 /policies.py:81 Device 'None' is not available. Switching to 'cuda'.
|
|
INFO 2025-09-08 13:23:15 ccelerate.py:99 {'batch_size': 32,
|
|
'dataset': {'episodes': None,
|
|
'image_transforms': {'enable': False,
|
|
'max_num_transforms': 3,
|
|
'random_order': False,
|
|
'tfs': {'brightness': {'kwargs': {'brightness': [0.8,
|
|
1.2]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'contrast': {'kwargs': {'contrast': [0.8,
|
|
1.2]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'hue': {'kwargs': {'hue': [-0.05,
|
|
0.05]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'saturation': {'kwargs': {'saturation': [0.5,
|
|
1.5]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'sharpness': {'kwargs': {'sharpness': [0.5,
|
|
1.5]},
|
|
'type': 'SharpnessJitter',
|
|
'weight': 1.0}}},
|
|
'repo_id': 'HuggingFaceVLA/libero',
|
|
'revision': None,
|
|
'root': '/raid/jade/.cache/huggingface/datasets',
|
|
'use_imagenet_stats': True,
|
|
'video_backend': 'torchcodec'},
|
|
'env': {'camera_name': 'agentview_image,robot0_eye_in_hand_image',
|
|
'episode_length': 520,
|
|
'features': {'action': {'shape': [7],
|
|
'type': <FeatureType.ACTION: 'ACTION'>},
|
|
'agent_pos': {'shape': [8],
|
|
'type': <FeatureType.STATE: 'STATE'>},
|
|
'pixels/agentview_image': {'shape': [360, 360, 3],
|
|
'type': <FeatureType.VISUAL: 'VISUAL'>},
|
|
'pixels/robot0_eye_in_hand_image': {'shape': [360,
|
|
360,
|
|
3],
|
|
'type': <FeatureType.VISUAL: 'VISUAL'>}},
|
|
'features_map': {'action': 'action',
|
|
'agent_pos': 'observation.state',
|
|
'pixels/agentview_image': 'observation.images.image',
|
|
'pixels/robot0_eye_in_hand_image': 'observation.images.image2'},
|
|
'fps': 30,
|
|
'init_states': True,
|
|
'max_parallel_tasks': 5,
|
|
'multitask_eval': True,
|
|
'obs_type': 'pixels_agent_pos',
|
|
'render_mode': 'rgb_array',
|
|
'task': 'libero_spatial',
|
|
'type': 'libero'},
|
|
'eval': {'batch_size': 1, 'n_episodes': 1, 'use_async_envs': False},
|
|
'eval_freq': 0,
|
|
'job_name': 'libero_smolvla',
|
|
'log_freq': 200,
|
|
'num_workers': 4,
|
|
'optimizer': {'betas': [0.9, 0.95],
|
|
'eps': 1e-08,
|
|
'grad_clip_norm': 10,
|
|
'lr': 0.0001,
|
|
'type': 'adamw',
|
|
'weight_decay': 1e-10},
|
|
'output_dir': '/raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla_lr1e-4bs32steps100000',
|
|
'policy': {'adapt_to_pi_aloha': False,
|
|
'add_image_special_tokens': False,
|
|
'attention_mode': 'cross_attn',
|
|
'chunk_size': 50,
|
|
'device': 'cuda',
|
|
'empty_cameras': 0,
|
|
'expert_width_multiplier': 0.75,
|
|
'freeze_vision_encoder': True,
|
|
'gradient_accumulation_steps': 1,
|
|
'input_features': {},
|
|
'license': None,
|
|
'load_vlm_weights': False,
|
|
'max_action_dim': 32,
|
|
'max_period': 4.0,
|
|
'max_state_dim': 32,
|
|
'min_period': 0.004,
|
|
'n_action_steps': 1,
|
|
'n_obs_steps': 1,
|
|
'normalization_mapping': {'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
|
|
'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
|
|
'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
|
|
'num_expert_layers': -1,
|
|
'num_steps': 10,
|
|
'num_vlm_layers': 16,
|
|
'optimizer_betas': [0.9, 0.95],
|
|
'optimizer_eps': 1e-08,
|
|
'optimizer_grad_clip_norm': 10,
|
|
'optimizer_lr': 0.0001,
|
|
'optimizer_weight_decay': 1e-10,
|
|
'output_features': {},
|
|
'pad_language_to': 'longest',
|
|
'prefix_length': -1,
|
|
'private': None,
|
|
'push_to_hub': True,
|
|
'repo_id': 'None',
|
|
'resize_imgs_with_padding': [512, 512],
|
|
'scheduler_decay_lr': 2.5e-06,
|
|
'scheduler_decay_steps': 30000,
|
|
'scheduler_warmup_steps': 1000,
|
|
'self_attn_every_n_layers': 2,
|
|
'tags': None,
|
|
'tokenizer_max_length': 48,
|
|
'train_expert_only': True,
|
|
'train_state_proj': True,
|
|
'type': 'smolvla',
|
|
'use_amp': True,
|
|
'use_cache': True,
|
|
'use_delta_joint_actions_aloha': False,
|
|
'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Instruct'},
|
|
'resume': False,
|
|
'save_checkpoint': True,
|
|
'save_freq': 20000,
|
|
'scheduler': {'decay_lr': 2.5e-06,
|
|
'num_decay_steps': 30000,
|
|
'num_warmup_steps': 1000,
|
|
'peak_lr': 0.0001,
|
|
'type': 'cosine_decay_with_warmup'},
|
|
'seed': 1000,
|
|
'steps': 100000,
|
|
'use_policy_training_preset': True,
|
|
'wandb': {'disable_artifact': False,
|
|
'enable': False,
|
|
'entity': None,
|
|
'mode': None,
|
|
'notes': None,
|
|
'project': 'lerobot',
|
|
'run_id': None}}
|
|
WARNING 2025-09-08 13:23:15 ls/other.py:512 Detected kernel version 5.4.0, which is below the recommended minimum of
|
|
5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher
|
|
.
|
|
WARNING 2025-09-08 13:23:15 ls/other.py:512 Detected kernel version 5.4.0, which is below the recommended minimum of
|
|
5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher
|
|
.
|
|
INFO 2025-09-08 13:23:15 celerate.py:149 Creating dataset
|
|
Resolving data files: 100%|█████████████████████████████████| 1693/1693 [00:00<00:00, 35414.48it/s]
|
|
Loading dataset shards: 100%|████████████████████████████████████| 69/69 [00:00<00:00, 5660.00it/s]
|
|
Resolving data files: 100%|█████████████████████████████████| 1693/1693 [00:00<00:00, 43760.67it/s]
|
|
Loading dataset shards: 100%|████████████████████████████████████| 69/69 [00:00<00:00, 5629.72it/s]
|
|
c
|
|
INFO 2025-09-08 13:23:22 celerate.py:160 Creating policy
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarnin
|
|
g: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.
|
|
warnings.warn(
|
|
c
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631:
|
|
UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u
|
|
ser.
|
|
warnings.warn( # warn only once
|
|
[rank1]:[W908 13:23:22.785516795 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used b
|
|
y this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You
|
|
can pecify device_id in init_process_group() to force use of a particular device.
|
|
Reducing the number of VLM layers to 16 ...
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631:
|
|
UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u
|
|
ser.
|
|
warnings.warn( # warn only once
|
|
[rank0]:[W908 13:23:43.028071493 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used b
|
|
y this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You
|
|
can pecify device_id in init_process_group() to force use of a particular device.
|
|
INFO 2025-09-08 13:23:43 celerate.py:171 Creating optimizer and scheduler
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarnin
|
|
g: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.
|
|
warnings.warn(
|
|
Reducing the number of VLM layers to 16 ...
|
|
INFO 2025-09-08 13:24:04 celerate.py:211 Output dir: /raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla
|
|
_lr1e-4bs32steps100000
|
|
INFO 2025-09-08 13:24:04 celerate.py:213 cfg.env.task='libero_spatial'
|
|
INFO 2025-09-08 13:24:04 celerate.py:214 cfg.steps=100000 (100K)
|
|
INFO 2025-09-08 13:24:04 celerate.py:215 dataset.num_frames=273465 (273K)
|
|
INFO 2025-09-08 13:24:04 celerate.py:216 dataset.num_episodes=1693
|
|
INFO 2025-09-08 13:24:04 celerate.py:217 num_learnable_params=99880992 (100M)
|
|
INFO 2025-09-08 13:24:04 celerate.py:218 num_total_params=450046220 (450M)
|
|
INFO 2025-09-08 13:24:04 celerate.py:219 Number of processes: 2
|
|
INFO 2025-09-08 13:24:04 celerate.py:220 Device: cuda:0
|
|
INFO 2025-09-08 13:24:04 celerate.py:221 Mixed precision: bf16
|
|
INFO 2025-09-08 13:24:04 celerate.py:243 Start offline training on a fixed dataset
|
|
|
|
bach: dict_keys(['observation.images.image', 'observation.images.image2', 'observation.state', 'action', 'timestamp
|
|
', 'frame_index', 'episode_index', 'index', 'task_index', 'observation.images.image_is_pad', 'observation.images.ima
|
|
ge2_is_pad', 'observation.state_is_pad', 'action_is_pad', 'task'])
|
|
> /home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py(263)train()
|
|
-> train_tracker, output_dict = update_policy(
|
|
(Pdb)
|
|
bach: dict_keys(['observation.images.image', 'observation.images.image2', 'observation.state', 'action', 'timestamp
|
|
', 'frame_index', 'episode_index', 'index', 'task_index', 'observation.images.image_is_pad', 'observation.images.ima
|
|
ge2_is_pad', 'observation.state_is_pad', 'action_is_pad', 'task'])
|
|
> /home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py(263)train()
|
|
-> train_tracker, output_dict = update_policy(
|
|
(Pdb) batch.keys()[rank0]:[W908 13:24:43.868440913 reducer.cpp:1430] Warning: find_unused_parameters=True was specif
|
|
ied in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra tr
|
|
aversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never h
|
|
as any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false
|
|
positive if your model has flow control causing later iterations to have unused parameters. (function operator())
|
|
policy.config.input_features
|
|
*** SyntaxError: invalid syntax
|
|
(Pdb) policy.config.input_features
|
|
*** AttributeError: 'DistributedDataParallel' object has no attribute 'config'
|
|
(Pdb) policy
|
|
DistributedDataParallel(
|
|
(module): SmolVLAPolicy(
|
|
(normalize_inputs): Normalize(
|
|
(buffer_observation_state): ParameterDict(
|
|
(mean): Parameter containing: [torch.cuda.FloatTensor of size 8 (cuda:1)]
|
|
(std): Parameter containing: [torch.cuda.FloatTensor of size 8 (cuda:1)]
|
|
)
|
|
)
|
|
(normalize_targets): Normalize(
|
|
(buffer_action): ParameterDict(
|
|
(mean): Parameter containing: [torch.cuda.FloatTensor of size 7 (cuda:1)]
|
|
(std): Parameter containing: [torch.cuda.FloatTensor of size 7 (cuda:1)]
|
|
)
|
|
)
|
|
(unnormalize_outputs): Unnormalize(
|
|
(buffer_action): ParameterDict(
|
|
(mean): Parameter containing: [torch.cuda.FloatTensor of size 7 (cuda:1)]
|
|
(std): Parameter containing: [torch.cuda.FloatTensor of size 7 (cuda:1)]
|
|
)
|
|
)
|
|
(model): VLAFlowMatching(
|
|
(vlm_with_expert): SmolVLMWithExpertModel(
|
|
(vlm): SmolVLMForConditionalGeneration(
|
|
(model): SmolVLMModel(
|
|
(vision_model): SmolVLMVisionTransformer(
|
|
(embeddings): SmolVLMVisionEmbeddings(
|
|
(patch_embedding): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), padding=valid)
|
|
(position_embedding): Embedding(1024, 768)
|
|
)
|
|
(encoder): SmolVLMEncoder(
|
|
(layers): ModuleList(
|
|
(0-11): 12 x SmolVLMEncoderLayer(
|
|
(self_attn): SmolVLMVisionAttention(
|
|
(k_proj): Linear(in_features=768, out_features=768, bias=True)
|
|
(v_proj): Linear(in_features=768, out_features=768, bias=True)
|
|
(q_proj): Linear(in_features=768, out_features=768, bias=True)
|
|
(out_proj): Linear(in_features=768, out_features=768, bias=True)
|
|
)
|
|
(layer_norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
|
|
(mlp): SmolVLMVisionMLP(
|
|
(activation_fn): PytorchGELUTanh()
|
|
(fc1): Linear(in_features=768, out_features=3072, bias=True)
|
|
(fc2): Linear(in_features=3072, out_features=768, bias=True)
|
|
)
|
|
(layer_norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
|
|
)
|
|
)
|
|
)
|
|
(post_layernorm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)
|
|
)
|
|
(connector): SmolVLMConnector(
|
|
(modality_projection): SmolVLMSimpleMLP(
|
|
(proj): Linear(in_features=12288, out_features=960, bias=False)
|
|
)
|
|
)
|
|
(text_model): LlamaModel(
|
|
(embed_tokens): Embedding(49280, 960, padding_idx=2)
|
|
(layers): ModuleList(
|
|
(0-15): 16 x LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=960, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=960, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=960, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=960, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=960, out_features=2560, bias=False)
|
|
(up_proj): Linear(in_features=960, out_features=2560, bias=False)
|
|
(down_proj): Linear(in_features=2560, out_features=960, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((960,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((960,), eps=1e-05)
|
|
)
|
|
)
|
|
(norm): LlamaRMSNorm((960,), eps=1e-05)
|
|
(rotary_emb): LlamaRotaryEmbedding()
|
|
)
|
|
)
|
|
(lm_head): Linear(in_features=960, out_features=49280, bias=False)
|
|
)
|
|
(lm_expert): LlamaModel(
|
|
(embed_tokens): None
|
|
(layers): ModuleList(
|
|
(0): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(1): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(2): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(3): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(4): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(5): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(6): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(7): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(8): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(9): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(10): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(11): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(12): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(13): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(14): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=720, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
(15): LlamaDecoderLayer(
|
|
(self_attn): LlamaAttention(
|
|
(q_proj): Linear(in_features=720, out_features=960, bias=False)
|
|
(k_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(v_proj): Linear(in_features=320, out_features=320, bias=False)
|
|
(o_proj): Linear(in_features=960, out_features=720, bias=False)
|
|
)
|
|
(mlp): LlamaMLP(
|
|
(gate_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(up_proj): Linear(in_features=720, out_features=2048, bias=False)
|
|
(down_proj): Linear(in_features=2048, out_features=720, bias=False)
|
|
(act_fn): SiLU()
|
|
)
|
|
(input_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(post_attention_layernorm): LlamaRMSNorm((720,), eps=1e-05)
|
|
)
|
|
)
|
|
(norm): LlamaRMSNorm((720,), eps=1e-05)
|
|
(rotary_emb): LlamaRotaryEmbedding()
|
|
)
|
|
)
|
|
(state_proj): Linear(in_features=32, out_features=960, bias=True)
|
|
(action_in_proj): Linear(in_features=32, out_features=720, bias=True)
|
|
(action_out_proj): Linear(in_features=720, out_features=32, bias=True)
|
|
(action_time_mlp_in): Linear(in_features=1440, out_features=720, bias=True)
|
|
(action_time_mlp_out): Linear(in_features=720, out_features=720, bias=True)
|
|
)
|
|
)
|
|
)
|
|
(Pdb) policy.config
|
|
*** AttributeError: 'DistributedDataParallel' object has no attribute 'config'
|
|
(Pdb) policy.input_features
|
|
*** AttributeError: 'DistributedDataParallel' object has no attribute 'input_features'
|
|
(Pdb) quit()
|
|
[rank1]: Traceback (most recent call last):
|
|
[rank1]: File "/home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py", line 368, in <module>
|
|
[rank1]: train()
|
|
[rank1]: File "/home/jade_choghari/lerobot/src/lerobot/configs/parser.py", line 225, in wrapper_inner
|
|
[rank1]: response = fn(cfg, *args, **kwargs)
|
|
[rank1]: File "/home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py", line 263, in train
|
|
[rank1]: train_tracker, output_dict = update_policy(
|
|
[rank1]: File "/home/jade_choghari/lerobot/src/lerobot/scripts/train_accelerate.py", line 263, in train
|
|
[rank1]: train_tracker, output_dict = update_policy(
|
|
[rank1]: File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/bdb.py", line 90, in trace_dispatch
|
|
[rank1]: return self.dispatch_line(frame)
|
|
[rank1]: File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/bdb.py", line 115, in dispatch_line
|
|
[rank1]: if self.quitting: raise BdbQuit
|
|
[rank1]: bdb.BdbQuit
|
|
W0908 13:25:34.274000 776579 site-packages/torch/distributed/elastic/multiprocessing/api.py:900] Sending process 776
|
|
663 closing signal SIGTERM
|
|
E0908 13:25:34.589000 776579 site-packages/torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1
|
|
) local_rank: 1 (pid: 776664) of binary: /home/jade_choghari/miniconda3/envs/lerobot/bin/python
|
|
Traceback (most recent call last):
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/runpy.py", line 196, in _run_module_as_main
|
|
return _run_code(code, main_globals, None,
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/runpy.py", line 86, in _run_code
|
|
exec(code, run_globals)
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/commands/launch.py", lin
|
|
e 1245, in <module>
|
|
main()
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/commands/launch.py", lin
|
|
e 1241, in main
|
|
launch_command(args)
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/commands/launch.py", lin
|
|
e 1226, in launch_command
|
|
multi_gpu_launcher(args)
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/commands/launch.py", lin
|
|
e 853, in multi_gpu_launcher
|
|
distrib_run.run(args)
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/run.py", line 883
|
|
, in run
|
|
elastic_launch(
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/launcher/api.py",
|
|
line 139, in __call__
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
File "/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/launcher/api.py",
|
|
line 270, in launch_agent
|
|
raise ChildFailedError(
|
|
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
|
|
============================================================
|
|
src/lerobot/scripts/train_accelerate.py FAILED
|
|
------------------------------------------------------------
|
|
Failures:
|
|
<NO_OTHER_FAILURES>
|
|
------------------------------------------------------------
|
|
Root Cause (first observed failure):
|
|
[0]:
|
|
time : 2025-09-08_13:25:34
|
|
host : hf-dgx-01
|
|
rank : 1 (local_rank: 1)
|
|
exitcode : 1 (pid: 776664)
|
|
error_file: <N/A>
|
|
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
|
|
============================================================
|
|
(lerobot) jade_choghari@hf-dgx-01:~/lerobot$ clear
|
|
(lerobot) jade_choghari@hf-dgx-01:~/lerobot$ clear
|
|
(lerobot) jade_choghari@hf-dgx-01:~/lerobot$ bash examples/7_train_acc.sh
|
|
Training dir: /raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla_lr1e-4bs32steps100000
|
|
The following values were not passed to `accelerate launch` and had defaults used instead:
|
|
More than one GPU was found, enabling multi-GPU training.
|
|
If this was unintended please pass in `--num_processes=1`.
|
|
`--dynamo_backend` was set to a value of `'no'`
|
|
To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/accelerate/utils/launch.py:238: UserWarning
|
|
: Port `29522` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open por
|
|
t automatically for this session. If this current attempt fails, or for more control in future runs, please specify
|
|
a different port (e.g., `--main_process_port <your_chosen_port>`) or use `--main_process_port 0` for automatic selec
|
|
tion in your launch command or Accelerate config file.
|
|
warnings.warn(
|
|
INFO 2025-09-08 13:33:47 ils/utils.py:48 Cuda backend detected, using cuda.
|
|
WARNING 2025-09-08 13:33:47 /policies.py:81 Device 'None' is not available. Switching to 'cuda'.
|
|
INFO 2025-09-08 13:33:47 ccelerate.py:99 {'batch_size': 32,
|
|
'dataset': {'episodes': None,
|
|
'image_transforms': {'enable': False,
|
|
'max_num_transforms': 3,
|
|
'random_order': False,
|
|
'tfs': {'brightness': {'kwargs': {'brightness': [0.8,
|
|
1.2]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'contrast': {'kwargs': {'contrast': [0.8,
|
|
1.2]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'hue': {'kwargs': {'hue': [-0.05,
|
|
0.05]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'saturation': {'kwargs': {'saturation': [0.5,
|
|
1.5]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'sharpness': {'kwargs': {'sharpness': [0.5,
|
|
1.5]},
|
|
'type': 'SharpnessJitter',
|
|
'weight': 1.0}}},
|
|
'repo_id': 'HuggingFaceVLA/libero',
|
|
'revision': None,
|
|
'root': '/raid/jade/.cache/huggingface/datasets',
|
|
'use_imagenet_stats': True,
|
|
'video_backend': 'torchcodec'},
|
|
'env': {'camera_name': 'agentview_image,robot0_eye_in_hand_image',
|
|
'episode_length': 520,
|
|
'features': {'action': {'shape': [7],
|
|
'type': <FeatureType.ACTION: 'ACTION'>},
|
|
'agent_pos': {'shape': [8],
|
|
'type': <FeatureType.STATE: 'STATE'>},
|
|
'pixels/agentview_image': {'shape': [360, 360, 3],
|
|
'type': <FeatureType.VISUAL: 'VISUAL'>},
|
|
'pixels/robot0_eye_in_hand_image': {'shape': [360,
|
|
360,
|
|
3],
|
|
'type': <FeatureType.VISUAL: 'VISUAL'>}},
|
|
'features_map': {'action': 'action',
|
|
'agent_pos': 'observation.state',
|
|
'pixels/agentview_image': 'observation.images.image',
|
|
'pixels/robot0_eye_in_hand_image': 'observation.images.image2'},
|
|
'fps': 30,
|
|
'init_states': True,
|
|
'max_parallel_tasks': 5,
|
|
'multitask_eval': True,
|
|
'obs_type': 'pixels_agent_pos',
|
|
'render_mode': 'rgb_array',
|
|
'task': 'libero_spatial',
|
|
'type': 'libero'},
|
|
'eval': {'batch_size': 1, 'n_episodes': 1, 'use_async_envs': False},
|
|
'eval_freq': 0,
|
|
'job_name': 'libero_smolvla',
|
|
'log_freq': 200,
|
|
'num_workers': 4,
|
|
'optimizer': {'betas': [0.9, 0.95],
|
|
'eps': 1e-08,
|
|
'grad_clip_norm': 10,
|
|
'lr': 0.0001,
|
|
'type': 'adamw',
|
|
'weight_decay': 1e-10},
|
|
'output_dir': '/raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla_lr1e-4bs32steps100000',
|
|
'policy': {'adapt_to_pi_aloha': False,
|
|
'add_image_special_tokens': False,
|
|
'attention_mode': 'cross_attn',
|
|
'chunk_size': 50,
|
|
'device': 'cuda',
|
|
'empty_cameras': 0,
|
|
'expert_width_multiplier': 0.75,
|
|
'freeze_vision_encoder': True,
|
|
'gradient_accumulation_steps': 1,
|
|
'input_features': {},
|
|
'license': None,
|
|
'load_vlm_weights': False,
|
|
'max_action_dim': 32,
|
|
'max_period': 4.0,
|
|
'max_state_dim': 32,
|
|
'min_period': 0.004,
|
|
'n_action_steps': 1,
|
|
'n_obs_steps': 1,
|
|
'normalization_mapping': {'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
|
|
'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
|
|
'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
|
|
'num_expert_layers': -1,
|
|
'num_steps': 10,
|
|
'num_vlm_layers': 16,
|
|
'optimizer_betas': [0.9, 0.95],
|
|
'optimizer_eps': 1e-08,
|
|
'optimizer_grad_clip_norm': 10,
|
|
'optimizer_lr': 0.0001,
|
|
'optimizer_weight_decay': 1e-10,
|
|
'output_features': {},
|
|
'pad_language_to': 'longest',
|
|
'prefix_length': -1,
|
|
'private': None,
|
|
'push_to_hub': True,
|
|
'repo_id': 'None',
|
|
'resize_imgs_with_padding': [512, 512],
|
|
'scheduler_decay_lr': 2.5e-06,
|
|
'scheduler_decay_steps': 30000,
|
|
'scheduler_warmup_steps': 1000,
|
|
'self_attn_every_n_layers': 2,
|
|
'tags': None,
|
|
'tokenizer_max_length': 48,
|
|
'train_expert_only': True,
|
|
'train_state_proj': True,
|
|
'type': 'smolvla',
|
|
'use_amp': True,
|
|
'use_cache': True,
|
|
'use_delta_joint_actions_aloha': False,
|
|
'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Instruct'},
|
|
'resume': False,
|
|
'save_checkpoint': True,
|
|
'save_freq': 20000,
|
|
'scheduler': {'decay_lr': 2.5e-06,
|
|
'num_decay_steps': 30000,
|
|
'num_warmup_steps': 1000,
|
|
'peak_lr': 0.0001,
|
|
'type': 'cosine_decay_with_warmup'},
|
|
'seed': 1000,
|
|
'steps': 100000,
|
|
'use_policy_training_preset': True,
|
|
'wandb': {'disable_artifact': False,
|
|
'enable': False,
|
|
'entity': None,
|
|
'mode': None,
|
|
'notes': None,
|
|
'project': 'lerobot',
|
|
'run_id': None}}
|
|
INFO 2025-09-08 13:33:47 ils/utils.py:48 Cuda backend detected, using cuda.
|
|
WARNING 2025-09-08 13:33:47 /policies.py:81 Device 'None' is not available. Switching to 'cuda'.
|
|
INFO 2025-09-08 13:33:47 ccelerate.py:99 {'batch_size': 32,
|
|
'dataset': {'episodes': None,
|
|
'image_transforms': {'enable': False,
|
|
'max_num_transforms': 3,
|
|
'random_order': False,
|
|
'tfs': {'brightness': {'kwargs': {'brightness': [0.8,
|
|
1.2]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'contrast': {'kwargs': {'contrast': [0.8,
|
|
1.2]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'hue': {'kwargs': {'hue': [-0.05,
|
|
0.05]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'saturation': {'kwargs': {'saturation': [0.5,
|
|
1.5]},
|
|
'type': 'ColorJitter',
|
|
'weight': 1.0},
|
|
'sharpness': {'kwargs': {'sharpness': [0.5,
|
|
1.5]},
|
|
'type': 'SharpnessJitter',
|
|
'weight': 1.0}}},
|
|
'repo_id': 'HuggingFaceVLA/libero',
|
|
'revision': None,
|
|
'root': '/raid/jade/.cache/huggingface/datasets',
|
|
'use_imagenet_stats': True,
|
|
'video_backend': 'torchcodec'},
|
|
'env': {'camera_name': 'agentview_image,robot0_eye_in_hand_image',
|
|
'episode_length': 520,
|
|
'features': {'action': {'shape': [7],
|
|
'type': <FeatureType.ACTION: 'ACTION'>},
|
|
'agent_pos': {'shape': [8],
|
|
'type': <FeatureType.STATE: 'STATE'>},
|
|
'pixels/agentview_image': {'shape': [360, 360, 3],
|
|
'type': <FeatureType.VISUAL: 'VISUAL'>},
|
|
'pixels/robot0_eye_in_hand_image': {'shape': [360,
|
|
360,
|
|
3],
|
|
'type': <FeatureType.VISUAL: 'VISUAL'>}},
|
|
'features_map': {'action': 'action',
|
|
'agent_pos': 'observation.state',
|
|
'pixels/agentview_image': 'observation.images.image',
|
|
'pixels/robot0_eye_in_hand_image': 'observation.images.image2'},
|
|
'fps': 30,
|
|
'init_states': True,
|
|
'max_parallel_tasks': 5,
|
|
'multitask_eval': True,
|
|
'obs_type': 'pixels_agent_pos',
|
|
'render_mode': 'rgb_array',
|
|
'task': 'libero_spatial',
|
|
'type': 'libero'},
|
|
'eval': {'batch_size': 1, 'n_episodes': 1, 'use_async_envs': False},
|
|
'eval_freq': 0,
|
|
'job_name': 'libero_smolvla',
|
|
'log_freq': 200,
|
|
'num_workers': 4,
|
|
'optimizer': {'betas': [0.9, 0.95],
|
|
'eps': 1e-08,
|
|
'grad_clip_norm': 10,
|
|
'lr': 0.0001,
|
|
'type': 'adamw',
|
|
'weight_decay': 1e-10},
|
|
'output_dir': '/raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla_lr1e-4bs32steps100000',
|
|
'policy': {'adapt_to_pi_aloha': False,
|
|
'add_image_special_tokens': False,
|
|
'attention_mode': 'cross_attn',
|
|
'chunk_size': 50,
|
|
'device': 'cuda',
|
|
'empty_cameras': 0,
|
|
'expert_width_multiplier': 0.75,
|
|
'freeze_vision_encoder': True,
|
|
'gradient_accumulation_steps': 1,
|
|
'input_features': {},
|
|
'license': None,
|
|
'load_vlm_weights': False,
|
|
'max_action_dim': 32,
|
|
'max_period': 4.0,
|
|
'max_state_dim': 32,
|
|
'min_period': 0.004,
|
|
'n_action_steps': 1,
|
|
'n_obs_steps': 1,
|
|
'normalization_mapping': {'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
|
|
'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>,
|
|
'VISUAL': <NormalizationMode.IDENTITY: 'IDENTITY'>},
|
|
'num_expert_layers': -1,
|
|
'num_steps': 10,
|
|
'num_vlm_layers': 16,
|
|
'optimizer_betas': [0.9, 0.95],
|
|
'optimizer_eps': 1e-08,
|
|
'optimizer_grad_clip_norm': 10,
|
|
'optimizer_lr': 0.0001,
|
|
'optimizer_weight_decay': 1e-10,
|
|
'output_features': {},
|
|
'pad_language_to': 'longest',
|
|
'prefix_length': -1,
|
|
'private': None,
|
|
'push_to_hub': True,
|
|
'repo_id': 'None',
|
|
'resize_imgs_with_padding': [512, 512],
|
|
'scheduler_decay_lr': 2.5e-06,
|
|
'scheduler_decay_steps': 30000,
|
|
'scheduler_warmup_steps': 1000,
|
|
'self_attn_every_n_layers': 2,
|
|
'tags': None,
|
|
'tokenizer_max_length': 48,
|
|
'train_expert_only': True,
|
|
'train_state_proj': True,
|
|
'type': 'smolvla',
|
|
'use_amp': True,
|
|
'use_cache': True,
|
|
'use_delta_joint_actions_aloha': False,
|
|
'vlm_model_name': 'HuggingFaceTB/SmolVLM2-500M-Instruct'},
|
|
'resume': False,
|
|
'save_checkpoint': True,
|
|
'save_freq': 20000,
|
|
'scheduler': {'decay_lr': 2.5e-06,
|
|
'num_decay_steps': 30000,
|
|
'num_warmup_steps': 1000,
|
|
'peak_lr': 0.0001,
|
|
'type': 'cosine_decay_with_warmup'},
|
|
'seed': 1000,
|
|
'steps': 100000,
|
|
'use_policy_training_preset': True,
|
|
'wandb': {'disable_artifact': False,
|
|
'enable': False,
|
|
'entity': None,
|
|
'mode': None,
|
|
'notes': None,
|
|
'project': 'lerobot',
|
|
'run_id': None}}
|
|
WARNING 2025-09-08 13:33:47 ls/other.py:512 Detected kernel version 5.4.0, which is below the recommended minimum of
|
|
5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher
|
|
.
|
|
WARNING 2025-09-08 13:33:47 ls/other.py:512 Detected kernel version 5.4.0, which is below the recommended minimum of
|
|
5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher
|
|
.
|
|
INFO 2025-09-08 13:33:47 celerate.py:149 Creating dataset
|
|
Resolving data files: 100%|████████████████████████████████| 1693/1693 [00:00<00:00, 103295.66it/s]
|
|
Loading dataset shards: 100%|████████████████████████████████████| 69/69 [00:00<00:00, 5229.81it/s]
|
|
Resolving data files: 100%|████████████████████████████████| 1693/1693 [00:00<00:00, 360601.09it/s]
|
|
Loading dataset shards: 100%|████████████████████████████████████| 69/69 [00:00<00:00, 4881.54it/s]
|
|
c
|
|
INFO 2025-09-08 13:33:53 celerate.py:160 Creating policy
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarnin
|
|
g: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.
|
|
warnings.warn(
|
|
c
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631:
|
|
UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u
|
|
ser.
|
|
warnings.warn( # warn only once
|
|
[rank1]:[W908 13:33:54.613597516 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used b
|
|
y this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You
|
|
can pecify device_id in init_process_group() to force use of a particular device.
|
|
Reducing the number of VLM layers to 16 ...
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631:
|
|
UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u
|
|
ser.
|
|
warnings.warn( # warn only once
|
|
[rank0]:[W908 13:34:15.806448425 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used b
|
|
y this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You
|
|
can pecify device_id in init_process_group() to force use of a particular device.
|
|
INFO 2025-09-08 13:34:15 celerate.py:171 Creating optimizer and scheduler
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/transformers/utils/hub.py:111: FutureWarnin
|
|
g: Using `TRANSFORMERS_CACHE` is deprecated and will be removed in v5 of Transformers. Use `HF_HOME` instead.
|
|
warnings.warn(
|
|
Reducing the number of VLM layers to 16 ...
|
|
INFO 2025-09-08 13:34:36 celerate.py:211 Output dir: /raid/jade/logs/lerobot/lerobot_2_HuggingFaceVLA_libero_smolvla
|
|
_lr1e-4bs32steps100000
|
|
INFO 2025-09-08 13:34:36 celerate.py:213 cfg.env.task='libero_spatial'
|
|
INFO 2025-09-08 13:34:36 celerate.py:214 cfg.steps=100000 (100K)
|
|
INFO 2025-09-08 13:34:36 celerate.py:215 dataset.num_frames=273465 (273K)
|
|
INFO 2025-09-08 13:34:36 celerate.py:216 dataset.num_episodes=1693
|
|
INFO 2025-09-08 13:34:36 celerate.py:217 num_learnable_params=99880992 (100M)
|
|
INFO 2025-09-08 13:34:36 celerate.py:218 num_total_params=450046220 (450M)
|
|
INFO 2025-09-08 13:34:36 celerate.py:219 Number of processes: 2
|
|
INFO 2025-09-08 13:34:36 celerate.py:220 Device: cuda:0
|
|
INFO 2025-09-08 13:34:36 celerate.py:221 Mixed precision: bf16
|
|
INFO 2025-09-08 13:34:36 celerate.py:243 Start offline training on a fixed dataset
|
|
[rank1]:[W908 13:34:39.454560620 reducer.cpp:1430] Warning: find_unused_parameters=True was specified in DDP constru
|
|
ctor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the aut
|
|
ograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused para
|
|
meters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your m
|
|
odel has flow control causing later iterations to have unused parameters. (function operator())
|
|
[rank0]:[W908 13:34:40.502702504 reducer.cpp:1430] Warning: find_unused_parameters=True was specified in DDP constru
|
|
ctor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the aut
|
|
ograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused para
|
|
meters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your m
|
|
odel has flow control causing later iterations to have unused parameters. (function operator())
|
|
INFO 2025-09-08 13:36:23 celerate.py:281 step:200 smpl:13K ep:79 epch:0.05 loss:0.963 grdn:2.699 lr:2.0e-05 updt_s:0
|
|
.506 data_s:0.027
|
|
INFO 2025-09-08 13:38:09 celerate.py:281 step:400 smpl:26K ep:158 epch:0.09 loss:0.389 grdn:3.127 lr:6.0e-05 updt_s:
|
|
0.525 data_s:0.003
|
|
INFO 2025-09-08 13:39:53 celerate.py:281 step:600 smpl:38K ep:238 epch:0.14 loss:0.261 grdn:2.618 lr:9.5e-05 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 13:41:37 celerate.py:281 step:800 smpl:51K ep:317 epch:0.19 loss:0.231 grdn:1.684 lr:9.9e-05 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-08 13:43:21 celerate.py:281 step:1K smpl:64K ep:396 epch:0.23 loss:0.211 grdn:1.258 lr:9.9e-05 updt_s:0
|
|
.514 data_s:0.003
|
|
INFO 2025-09-08 13:45:05 celerate.py:281 step:1K smpl:77K ep:475 epch:0.28 loss:0.198 grdn:1.032 lr:9.9e-05 updt_s:0
|
|
.517 data_s:0.003
|
|
INFO 2025-09-08 13:46:49 celerate.py:281 step:1K smpl:90K ep:555 epch:0.33 loss:0.182 grdn:0.880 lr:9.8e-05 updt_s:0
|
|
.515 data_s:0.003
|
|
INFO 2025-09-08 13:48:33 celerate.py:281 step:2K smpl:102K ep:634 epch:0.37 loss:0.167 grdn:0.744 lr:9.8e-05 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 13:50:17 celerate.py:281 step:2K smpl:115K ep:713 epch:0.42 loss:0.157 grdn:0.680 lr:9.7e-05 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 13:52:01 celerate.py:281 step:2K smpl:128K ep:792 epch:0.47 loss:0.147 grdn:0.612 lr:9.6e-05 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 13:53:44 celerate.py:281 step:2K smpl:141K ep:872 epch:0.51 loss:0.142 grdn:0.576 lr:9.5e-05 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-08 13:55:27 celerate.py:281 step:2K smpl:154K ep:951 epch:0.56 loss:0.136 grdn:0.523 lr:9.4e-05 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 13:57:11 celerate.py:281 step:3K smpl:166K ep:1K epch:0.61 loss:0.132 grdn:0.509 lr:9.3e-05 updt_s:0
|
|
.516 data_s:0.003
|
|
INFO 2025-09-08 13:58:57 celerate.py:281 step:3K smpl:179K ep:1K epch:0.66 loss:0.126 grdn:0.492 lr:9.2e-05 updt_s:0
|
|
.525 data_s:0.003
|
|
INFO 2025-09-08 14:00:43 celerate.py:281 step:3K smpl:192K ep:1K epch:0.70 loss:0.124 grdn:0.467 lr:9.1e-05 updt_s:0
|
|
.525 data_s:0.003
|
|
INFO 2025-09-08 14:02:26 celerate.py:281 step:3K smpl:205K ep:1K epch:0.75 loss:0.119 grdn:0.438 lr:9.0e-05 updt_s:0
|
|
.508 data_s:0.003
|
|
INFO 2025-09-08 14:04:27 celerate.py:281 step:3K smpl:218K ep:1K epch:0.80 loss:0.118 grdn:0.426 lr:8.9e-05 updt_s:0
|
|
.564 data_s:0.039
|
|
INFO 2025-09-08 14:06:10 celerate.py:281 step:4K smpl:230K ep:1K epch:0.84 loss:0.116 grdn:0.422 lr:8.7e-05 updt_s:0
|
|
.511 data_s:0.004
|
|
INFO 2025-09-08 14:07:55 celerate.py:281 step:4K smpl:243K ep:2K epch:0.89 loss:0.113 grdn:0.395 lr:8.6e-05 updt_s:0
|
|
.517 data_s:0.003
|
|
INFO 2025-09-08 14:09:38 celerate.py:281 step:4K smpl:256K ep:2K epch:0.94 loss:0.111 grdn:0.401 lr:8.5e-05 updt_s:0
|
|
.511 data_s:0.003
|
|
INFO 2025-09-08 14:11:21 celerate.py:281 step:4K smpl:269K ep:2K epch:0.98 loss:0.110 grdn:0.380 lr:8.3e-05 updt_s:0
|
|
.511 data_s:0.003
|
|
INFO 2025-09-08 14:13:08 celerate.py:281 step:4K smpl:282K ep:2K epch:1.03 loss:0.109 grdn:0.381 lr:8.2e-05 updt_s:0
|
|
.413 data_s:0.119
|
|
INFO 2025-09-08 14:14:52 celerate.py:281 step:5K smpl:294K ep:2K epch:1.08 loss:0.107 grdn:0.387 lr:8.0e-05 updt_s:0
|
|
.373 data_s:0.146
|
|
INFO 2025-09-08 14:16:36 celerate.py:281 step:5K smpl:307K ep:2K epch:1.12 loss:0.107 grdn:0.366 lr:7.8e-05 updt_s:0
|
|
.446 data_s:0.072
|
|
INFO 2025-09-08 14:18:19 celerate.py:281 step:5K smpl:320K ep:2K epch:1.17 loss:0.105 grdn:0.347 lr:7.6e-05 updt_s:0
|
|
.468 data_s:0.045
|
|
INFO 2025-09-08 14:20:01 celerate.py:281 step:5K smpl:333K ep:2K epch:1.22 loss:0.103 grdn:0.350 lr:7.5e-05 updt_s:0
|
|
.510 data_s:0.003
|
|
INFO 2025-09-08 14:21:46 celerate.py:281 step:5K smpl:346K ep:2K epch:1.26 loss:0.101 grdn:0.336 lr:7.3e-05 updt_s:0
|
|
.512 data_s:0.011
|
|
INFO 2025-09-08 14:23:30 celerate.py:281 step:6K smpl:358K ep:2K epch:1.31 loss:0.102 grdn:0.345 lr:7.1e-05 updt_s:0
|
|
.515 data_s:0.003
|
|
INFO 2025-09-08 14:25:15 celerate.py:281 step:6K smpl:371K ep:2K epch:1.36 loss:0.100 grdn:0.333 lr:6.9e-05 updt_s:0
|
|
.521 data_s:0.003
|
|
INFO 2025-09-08 14:26:59 celerate.py:281 step:6K smpl:384K ep:2K epch:1.40 loss:0.100 grdn:0.328 lr:6.7e-05 updt_s:0
|
|
.516 data_s:0.003
|
|
INFO 2025-09-08 14:28:43 celerate.py:281 step:6K smpl:397K ep:2K epch:1.45 loss:0.099 grdn:0.319 lr:6.5e-05 updt_s:0
|
|
.512 data_s:0.003
|
|
INFO 2025-09-08 14:30:26 celerate.py:281 step:6K smpl:410K ep:3K epch:1.50 loss:0.098 grdn:0.313 lr:6.3e-05 updt_s:0
|
|
.515 data_s:0.003
|
|
INFO 2025-09-08 14:32:11 celerate.py:281 step:7K smpl:422K ep:3K epch:1.54 loss:0.097 grdn:0.319 lr:6.1e-05 updt_s:0
|
|
.519 data_s:0.004
|
|
INFO 2025-09-08 14:33:55 celerate.py:281 step:7K smpl:435K ep:3K epch:1.59 loss:0.097 grdn:0.312 lr:5.9e-05 updt_s:0
|
|
.506 data_s:0.010
|
|
INFO 2025-09-08 14:35:39 celerate.py:281 step:7K smpl:448K ep:3K epch:1.64 loss:0.097 grdn:0.307 lr:5.7e-05 updt_s:0
|
|
.516 data_s:0.003
|
|
INFO 2025-09-08 14:37:23 celerate.py:281 step:7K smpl:461K ep:3K epch:1.69 loss:0.095 grdn:0.294 lr:5.5e-05 updt_s:0
|
|
.518 data_s:0.003
|
|
INFO 2025-09-08 14:39:07 celerate.py:281 step:7K smpl:474K ep:3K epch:1.73 loss:0.095 grdn:0.299 lr:5.3e-05 updt_s:0
|
|
.507 data_s:0.007
|
|
INFO 2025-09-08 14:40:52 celerate.py:281 step:8K smpl:486K ep:3K epch:1.78 loss:0.094 grdn:0.283 lr:5.1e-05 updt_s:0
|
|
.523 data_s:0.003
|
|
INFO 2025-09-08 14:42:36 celerate.py:281 step:8K smpl:499K ep:3K epch:1.83 loss:0.093 grdn:0.284 lr:4.9e-05 updt_s:0
|
|
.517 data_s:0.003
|
|
INFO 2025-09-08 14:44:22 celerate.py:281 step:8K smpl:512K ep:3K epch:1.87 loss:0.092 grdn:0.284 lr:4.7e-05 updt_s:0
|
|
.465 data_s:0.060
|
|
INFO 2025-09-08 14:46:06 celerate.py:281 step:8K smpl:525K ep:3K epch:1.92 loss:0.093 grdn:0.292 lr:4.5e-05 updt_s:0
|
|
.456 data_s:0.066
|
|
INFO 2025-09-08 14:47:49 celerate.py:281 step:8K smpl:538K ep:3K epch:1.97 loss:0.093 grdn:0.290 lr:4.3e-05 updt_s:0
|
|
.510 data_s:0.003
|
|
INFO 2025-09-08 14:49:37 celerate.py:281 step:9K smpl:550K ep:3K epch:2.01 loss:0.092 grdn:0.283 lr:4.1e-05 updt_s:0
|
|
.419 data_s:0.117
|
|
INFO 2025-09-08 14:51:20 celerate.py:281 step:9K smpl:563K ep:3K epch:2.06 loss:0.092 grdn:0.275 lr:3.9e-05 updt_s:0
|
|
.463 data_s:0.053
|
|
INFO 2025-09-08 14:53:05 celerate.py:281 step:9K smpl:576K ep:4K epch:2.11 loss:0.090 grdn:0.272 lr:3.7e-05 updt_s:0
|
|
.517 data_s:0.003
|
|
INFO 2025-09-08 14:54:49 celerate.py:281 step:9K smpl:589K ep:4K epch:2.15 loss:0.090 grdn:0.268 lr:3.5e-05 updt_s:0
|
|
.506 data_s:0.013
|
|
INFO 2025-09-08 14:56:32 celerate.py:281 step:9K smpl:602K ep:4K epch:2.20 loss:0.090 grdn:0.271 lr:3.3e-05 updt_s:0
|
|
.513 data_s:0.003
|
|
INFO 2025-09-08 14:58:17 celerate.py:281 step:10K smpl:614K ep:4K epch:2.25 loss:0.090 grdn:0.268 lr:3.1e-05 updt_s:
|
|
0.520 data_s:0.003
|
|
INFO 2025-09-08 15:00:02 celerate.py:281 step:10K smpl:627K ep:4K epch:2.29 loss:0.089 grdn:0.261 lr:3.0e-05 updt_s:
|
|
0.519 data_s:0.003
|
|
INFO 2025-09-08 15:01:48 celerate.py:281 step:10K smpl:640K ep:4K epch:2.34 loss:0.090 grdn:0.271 lr:2.8e-05 updt_s:
|
|
0.526 data_s:0.003
|
|
INFO 2025-09-08 15:03:33 celerate.py:281 step:10K smpl:653K ep:4K epch:2.39 loss:0.089 grdn:0.262 lr:2.6e-05 updt_s:
|
|
0.521 data_s:0.003
|
|
INFO 2025-09-08 15:05:18 celerate.py:281 step:10K smpl:666K ep:4K epch:2.43 loss:0.090 grdn:0.264 lr:2.4e-05 updt_s:
|
|
0.519 data_s:0.003
|
|
INFO 2025-09-08 15:07:32 celerate.py:281 step:11K smpl:678K ep:4K epch:2.48 loss:0.089 grdn:0.255 lr:2.3e-05 updt_s:
|
|
0.663 data_s:0.004
|
|
INFO 2025-09-08 15:09:21 celerate.py:281 step:11K smpl:691K ep:4K epch:2.53 loss:0.090 grdn:0.263 lr:2.1e-05 updt_s:
|
|
0.514 data_s:0.030
|
|
INFO 2025-09-08 15:11:06 celerate.py:281 step:11K smpl:704K ep:4K epch:2.57 loss:0.088 grdn:0.254 lr:1.9e-05 updt_s:
|
|
0.517 data_s:0.006
|
|
INFO 2025-09-08 15:12:51 celerate.py:281 step:11K smpl:717K ep:4K epch:2.62 loss:0.088 grdn:0.252 lr:1.8e-05 updt_s:
|
|
0.517 data_s:0.005
|
|
INFO 2025-09-08 15:14:38 celerate.py:281 step:11K smpl:730K ep:5K epch:2.67 loss:0.088 grdn:0.251 lr:1.6e-05 updt_s:
|
|
0.532 data_s:0.003
|
|
INFO 2025-09-08 15:16:23 celerate.py:281 step:12K smpl:742K ep:5K epch:2.71 loss:0.088 grdn:0.253 lr:1.5e-05 updt_s:
|
|
0.520 data_s:0.003
|
|
INFO 2025-09-08 15:18:08 celerate.py:281 step:12K smpl:755K ep:5K epch:2.76 loss:0.087 grdn:0.244 lr:1.4e-05 updt_s:
|
|
0.521 data_s:0.003
|
|
INFO 2025-09-08 15:19:54 celerate.py:281 step:12K smpl:768K ep:5K epch:2.81 loss:0.088 grdn:0.247 lr:1.2e-05 updt_s:
|
|
0.524 data_s:0.003
|
|
INFO 2025-09-08 15:21:39 celerate.py:281 step:12K smpl:781K ep:5K epch:2.86 loss:0.087 grdn:0.242 lr:1.1e-05 updt_s:
|
|
0.520 data_s:0.003
|
|
INFO 2025-09-08 15:23:32 celerate.py:281 step:12K smpl:794K ep:5K epch:2.90 loss:0.088 grdn:0.243 lr:1.0e-05 updt_s:
|
|
0.560 data_s:0.003
|
|
INFO 2025-09-08 15:25:48 celerate.py:281 step:13K smpl:806K ep:5K epch:2.95 loss:0.087 grdn:0.240 lr:9.0e-06 updt_s:
|
|
0.674 data_s:0.005
|
|
INFO 2025-09-08 15:28:02 celerate.py:281 step:13K smpl:819K ep:5K epch:3.00 loss:0.088 grdn:0.245 lr:8.0e-06 updt_s:
|
|
0.662 data_s:0.004
|
|
INFO 2025-09-08 15:31:06 celerate.py:281 step:13K smpl:832K ep:5K epch:3.04 loss:0.086 grdn:0.236 lr:7.1e-06 updt_s:
|
|
0.688 data_s:0.231
|
|
INFO 2025-09-08 15:32:52 celerate.py:281 step:13K smpl:845K ep:5K epch:3.09 loss:0.087 grdn:0.231 lr:6.3e-06 updt_s:
|
|
0.521 data_s:0.003
|
|
INFO 2025-09-08 15:35:46 celerate.py:281 step:13K smpl:858K ep:5K epch:3.14 loss:0.088 grdn:0.235 lr:5.6e-06 updt_s:
|
|
0.637 data_s:0.232
|
|
INFO 2025-09-08 15:37:34 celerate.py:281 step:14K smpl:870K ep:5K epch:3.18 loss:0.087 grdn:0.238 lr:4.9e-06 updt_s:
|
|
0.514 data_s:0.025
|
|
INFO 2025-09-08 15:39:18 celerate.py:281 step:14K smpl:883K ep:5K epch:3.23 loss:0.087 grdn:0.226 lr:4.3e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 15:41:02 celerate.py:281 step:14K smpl:896K ep:6K epch:3.28 loss:0.087 grdn:0.230 lr:3.8e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 15:42:45 celerate.py:281 step:14K smpl:909K ep:6K epch:3.32 loss:0.086 grdn:0.229 lr:3.4e-06 updt_s:
|
|
0.507 data_s:0.008
|
|
INFO 2025-09-08 15:44:29 celerate.py:281 step:14K smpl:922K ep:6K epch:3.37 loss:0.087 grdn:0.229 lr:3.0e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 15:46:12 celerate.py:281 step:15K smpl:934K ep:6K epch:3.42 loss:0.087 grdn:0.228 lr:2.8e-06 updt_s:
|
|
0.502 data_s:0.011
|
|
INFO 2025-09-08 15:47:56 celerate.py:281 step:15K smpl:947K ep:6K epch:3.46 loss:0.086 grdn:0.232 lr:2.6e-06 updt_s:
|
|
0.515 data_s:0.004
|
|
INFO 2025-09-08 15:49:39 celerate.py:281 step:15K smpl:960K ep:6K epch:3.51 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.365 data_s:0.147
|
|
INFO 2025-09-08 15:51:22 celerate.py:281 step:15K smpl:973K ep:6K epch:3.56 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:
|
|
0.333 data_s:0.179
|
|
INFO 2025-09-08 15:53:07 celerate.py:281 step:15K smpl:986K ep:6K epch:3.60 loss:0.087 grdn:0.229 lr:2.5e-06 updt_s:
|
|
0.357 data_s:0.164
|
|
INFO 2025-09-08 15:54:50 celerate.py:281 step:16K smpl:998K ep:6K epch:3.65 loss:0.087 grdn:0.230 lr:2.5e-06 updt_s:
|
|
0.365 data_s:0.151
|
|
INFO 2025-09-08 15:56:35 celerate.py:281 step:16K smpl:1M ep:6K epch:3.70 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0.
|
|
450 data_s:0.071
|
|
INFO 2025-09-08 15:58:19 celerate.py:281 step:16K smpl:1M ep:6K epch:3.74 loss:0.087 grdn:0.232 lr:2.5e-06 updt_s:0.
|
|
495 data_s:0.023
|
|
INFO 2025-09-08 16:00:04 celerate.py:281 step:16K smpl:1M ep:6K epch:3.79 loss:0.087 grdn:0.227 lr:2.5e-06 updt_s:0.
|
|
393 data_s:0.131
|
|
INFO 2025-09-08 16:01:48 celerate.py:281 step:16K smpl:1M ep:6K epch:3.84 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:0.
|
|
515 data_s:0.003
|
|
INFO 2025-09-08 16:03:32 celerate.py:281 step:17K smpl:1M ep:7K epch:3.88 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0.
|
|
518 data_s:0.003
|
|
INFO 2025-09-08 16:05:17 celerate.py:281 step:17K smpl:1M ep:7K epch:3.93 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0.
|
|
397 data_s:0.125
|
|
INFO 2025-09-08 16:12:41 celerate.py:281 step:17K smpl:1M ep:7K epch:3.98 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:1.
|
|
496 data_s:0.719
|
|
INFO 2025-09-08 16:14:28 celerate.py:281 step:17K smpl:1M ep:7K epch:4.03 loss:0.087 grdn:0.228 lr:2.5e-06 updt_s:0.
|
|
413 data_s:0.124
|
|
INFO 2025-09-08 16:16:13 celerate.py:281 step:17K smpl:1M ep:7K epch:4.07 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0.
|
|
469 data_s:0.050
|
|
INFO 2025-09-08 16:17:57 celerate.py:281 step:18K smpl:1M ep:7K epch:4.12 loss:0.087 grdn:0.228 lr:2.5e-06 updt_s:0.
|
|
375 data_s:0.145
|
|
INFO 2025-09-08 16:19:42 celerate.py:281 step:18K smpl:1M ep:7K epch:4.17 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:0.
|
|
333 data_s:0.189
|
|
INFO 2025-09-08 16:21:26 celerate.py:281 step:18K smpl:1M ep:7K epch:4.21 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0.
|
|
334 data_s:0.184
|
|
INFO 2025-09-08 16:23:09 celerate.py:281 step:18K smpl:1M ep:7K epch:4.26 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0.
|
|
331 data_s:0.185
|
|
INFO 2025-09-08 16:24:53 celerate.py:281 step:18K smpl:1M ep:7K epch:4.31 loss:0.088 grdn:0.236 lr:2.5e-06 updt_s:0.
|
|
333 data_s:0.182
|
|
INFO 2025-09-08 16:26:38 celerate.py:281 step:19K smpl:1M ep:7K epch:4.35 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:0.
|
|
337 data_s:0.188
|
|
INFO 2025-09-08 16:28:22 celerate.py:281 step:19K smpl:1M ep:7K epch:4.40 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0.
|
|
420 data_s:0.099
|
|
INFO 2025-09-08 16:30:06 celerate.py:281 step:19K smpl:1M ep:8K epch:4.45 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0.
|
|
444 data_s:0.075
|
|
INFO 2025-09-08 16:31:49 celerate.py:281 step:19K smpl:1M ep:8K epch:4.49 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0.
|
|
475 data_s:0.036
|
|
INFO 2025-09-08 16:33:33 celerate.py:281 step:19K smpl:1M ep:8K epch:4.54 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0.
|
|
379 data_s:0.139
|
|
INFO 2025-09-08 16:35:17 celerate.py:281 step:20K smpl:1M ep:8K epch:4.59 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0.
|
|
348 data_s:0.171
|
|
INFO 2025-09-08 16:37:01 celerate.py:281 step:20K smpl:1M ep:8K epch:4.63 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0.
|
|
332 data_s:0.185
|
|
/home/jade_choghari/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631:
|
|
UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the u
|
|
ser.
|
|
warnings.warn( # warn only once
|
|
INFO 2025-09-08 16:38:46 celerate.py:281 step:20K smpl:1M ep:8K epch:4.68 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0.
|
|
486 data_s:0.037
|
|
INFO 2025-09-08 16:38:46 celerate.py:295 Checkpoint policy after step 20000
|
|
INFO 2025-09-08 16:40:30 celerate.py:281 step:20K smpl:1M ep:8K epch:4.73 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0.
|
|
509 data_s:0.003
|
|
INFO 2025-09-08 16:42:16 celerate.py:281 step:20K smpl:1M ep:8K epch:4.77 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0.
|
|
527 data_s:0.003
|
|
INFO 2025-09-08 16:44:01 celerate.py:281 step:21K smpl:1M ep:8K epch:4.82 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:0.
|
|
519 data_s:0.003
|
|
INFO 2025-09-08 16:45:45 celerate.py:281 step:21K smpl:1M ep:8K epch:4.87 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0.
|
|
504 data_s:0.013
|
|
INFO 2025-09-08 16:47:29 celerate.py:281 step:21K smpl:1M ep:8K epch:4.91 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0.
|
|
509 data_s:0.011
|
|
INFO 2025-09-08 16:49:19 celerate.py:281 step:21K smpl:1M ep:8K epch:4.96 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0.
|
|
544 data_s:0.003
|
|
INFO 2025-09-08 16:51:04 celerate.py:281 step:21K smpl:1M ep:8K epch:5.01 loss:0.086 grdn:0.225 lr:2.5e-06 updt_s:0.
|
|
488 data_s:0.039
|
|
INFO 2025-09-08 16:52:51 celerate.py:281 step:22K smpl:1M ep:9K epch:5.06 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0.
|
|
430 data_s:0.099
|
|
INFO 2025-09-08 16:54:36 celerate.py:281 step:22K smpl:1M ep:9K epch:5.10 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0.
|
|
521 data_s:0.003
|
|
INFO 2025-09-08 16:56:23 celerate.py:281 step:22K smpl:1M ep:9K epch:5.15 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0.
|
|
521 data_s:0.014
|
|
INFO 2025-09-08 16:58:09 celerate.py:281 step:22K smpl:1M ep:9K epch:5.20 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0.
|
|
525 data_s:0.003
|
|
INFO 2025-09-08 17:00:04 celerate.py:281 step:22K smpl:1M ep:9K epch:5.24 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0.
|
|
568 data_s:0.003
|
|
INFO 2025-09-08 17:02:00 celerate.py:281 step:23K smpl:1M ep:9K epch:5.29 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:0.
|
|
575 data_s:0.003
|
|
INFO 2025-09-08 17:03:49 celerate.py:281 step:23K smpl:1M ep:9K epch:5.34 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0.
|
|
513 data_s:0.030
|
|
INFO 2025-09-08 17:05:39 celerate.py:281 step:23K smpl:1M ep:9K epch:5.38 loss:0.085 grdn:0.227 lr:2.5e-06 updt_s:0.
|
|
523 data_s:0.027
|
|
INFO 2025-09-08 17:07:26 celerate.py:281 step:23K smpl:1M ep:9K epch:5.43 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0.
|
|
529 data_s:0.003
|
|
INFO 2025-09-08 17:09:12 celerate.py:281 step:23K smpl:1M ep:9K epch:5.48 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0.
|
|
526 data_s:0.003
|
|
INFO 2025-09-08 17:10:55 celerate.py:281 step:24K smpl:2M ep:9K epch:5.52 loss:0.087 grdn:0.230 lr:2.5e-06 updt_s:0.
|
|
443 data_s:0.072
|
|
INFO 2025-09-08 17:12:40 celerate.py:281 step:24K smpl:2M ep:9K epch:5.57 loss:0.087 grdn:0.229 lr:2.5e-06 updt_s:0.
|
|
518 data_s:0.004
|
|
INFO 2025-09-08 17:14:25 celerate.py:281 step:24K smpl:2M ep:10K epch:5.62 loss:0.087 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.521 data_s:0.003
|
|
INFO 2025-09-08 17:16:11 celerate.py:281 step:24K smpl:2M ep:10K epch:5.66 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:0
|
|
.523 data_s:0.003
|
|
INFO 2025-09-08 17:17:55 celerate.py:281 step:24K smpl:2M ep:10K epch:5.71 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0
|
|
.515 data_s:0.005
|
|
INFO 2025-09-08 17:19:39 celerate.py:281 step:25K smpl:2M ep:10K epch:5.76 loss:0.087 grdn:0.229 lr:2.5e-06 updt_s:0
|
|
.415 data_s:0.106
|
|
INFO 2025-09-08 17:21:24 celerate.py:281 step:25K smpl:2M ep:10K epch:5.80 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.507 data_s:0.016
|
|
INFO 2025-09-08 17:23:08 celerate.py:281 step:25K smpl:2M ep:10K epch:5.85 loss:0.085 grdn:0.229 lr:2.5e-06 updt_s:0
|
|
.514 data_s:0.003
|
|
INFO 2025-09-08 17:24:54 celerate.py:281 step:25K smpl:2M ep:10K epch:5.90 loss:0.087 grdn:0.227 lr:2.5e-06 updt_s:0
|
|
.518 data_s:0.008
|
|
INFO 2025-09-08 17:26:41 celerate.py:281 step:25K smpl:2M ep:10K epch:5.94 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.529 data_s:0.003
|
|
INFO 2025-09-08 17:28:24 celerate.py:281 step:26K smpl:2M ep:10K epch:5.99 loss:0.087 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.513 data_s:0.003
|
|
INFO 2025-09-08 17:30:11 celerate.py:281 step:26K smpl:2M ep:10K epch:6.04 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.370 data_s:0.164
|
|
INFO 2025-09-08 17:31:55 celerate.py:281 step:26K smpl:2M ep:10K epch:6.08 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0
|
|
.385 data_s:0.132
|
|
INFO 2025-09-08 17:33:39 celerate.py:281 step:26K smpl:2M ep:10K epch:6.13 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.450 data_s:0.069
|
|
INFO 2025-09-08 17:35:24 celerate.py:281 step:26K smpl:2M ep:10K epch:6.18 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:0
|
|
.468 data_s:0.052
|
|
INFO 2025-09-08 17:37:07 celerate.py:281 step:27K smpl:2M ep:11K epch:6.23 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.514 data_s:0.004
|
|
INFO 2025-09-08 17:38:52 celerate.py:281 step:27K smpl:2M ep:11K epch:6.27 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.519 data_s:0.003
|
|
INFO 2025-09-08 17:40:40 celerate.py:281 step:27K smpl:2M ep:11K epch:6.32 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.534 data_s:0.003
|
|
INFO 2025-09-08 17:42:57 celerate.py:281 step:27K smpl:2M ep:11K epch:6.37 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0
|
|
.678 data_s:0.007
|
|
INFO 2025-09-08 17:46:13 celerate.py:281 step:27K smpl:2M ep:11K epch:6.41 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.968 data_s:0.009
|
|
INFO 2025-09-08 17:49:20 celerate.py:281 step:28K smpl:2M ep:11K epch:6.46 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0
|
|
.895 data_s:0.037
|
|
INFO 2025-09-08 17:51:22 celerate.py:281 step:28K smpl:2M ep:11K epch:6.51 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.604 data_s:0.003
|
|
INFO 2025-09-08 17:53:07 celerate.py:281 step:28K smpl:2M ep:11K epch:6.55 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.521 data_s:0.003
|
|
INFO 2025-09-08 17:54:51 celerate.py:281 step:28K smpl:2M ep:11K epch:6.60 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.516 data_s:0.003
|
|
INFO 2025-09-08 17:56:36 celerate.py:281 step:28K smpl:2M ep:11K epch:6.65 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.519 data_s:0.003
|
|
INFO 2025-09-08 17:58:21 celerate.py:281 step:29K smpl:2M ep:11K epch:6.69 loss:0.085 grdn:0.228 lr:2.5e-06 updt_s:0
|
|
.521 data_s:0.003
|
|
INFO 2025-09-08 18:00:06 celerate.py:281 step:29K smpl:2M ep:11K epch:6.74 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.513 data_s:0.011
|
|
INFO 2025-09-08 18:01:50 celerate.py:281 step:29K smpl:2M ep:11K epch:6.79 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.476 data_s:0.041
|
|
INFO 2025-09-08 18:03:34 celerate.py:281 step:29K smpl:2M ep:12K epch:6.83 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.506 data_s:0.012
|
|
INFO 2025-09-08 18:05:21 celerate.py:281 step:29K smpl:2M ep:12K epch:6.88 loss:0.086 grdn:0.229 lr:2.5e-06 updt_s:0
|
|
.455 data_s:0.075
|
|
INFO 2025-09-08 18:07:04 celerate.py:281 step:30K smpl:2M ep:12K epch:6.93 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.514 data_s:0.003
|
|
INFO 2025-09-08 18:08:47 celerate.py:281 step:30K smpl:2M ep:12K epch:6.97 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.509 data_s:0.003
|
|
INFO 2025-09-08 18:10:33 celerate.py:281 step:30K smpl:2M ep:12K epch:7.02 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.422 data_s:0.105
|
|
INFO 2025-09-08 18:12:19 celerate.py:281 step:30K smpl:2M ep:12K epch:7.07 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.347 data_s:0.182
|
|
INFO 2025-09-08 18:14:05 celerate.py:281 step:30K smpl:2M ep:12K epch:7.11 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0
|
|
.473 data_s:0.053
|
|
INFO 2025-09-08 18:15:52 celerate.py:281 step:31K smpl:2M ep:12K epch:7.16 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:0
|
|
.531 data_s:0.005
|
|
INFO 2025-09-08 18:17:37 celerate.py:281 step:31K smpl:2M ep:12K epch:7.21 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.520 data_s:0.003
|
|
INFO 2025-09-08 18:19:22 celerate.py:281 step:31K smpl:2M ep:12K epch:7.26 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0
|
|
.500 data_s:0.020
|
|
INFO 2025-09-08 18:21:06 celerate.py:281 step:31K smpl:2M ep:12K epch:7.30 loss:0.087 grdn:0.243 lr:2.5e-06 updt_s:0
|
|
.511 data_s:0.009
|
|
INFO 2025-09-08 18:22:50 celerate.py:281 step:31K smpl:2M ep:12K epch:7.35 loss:0.087 grdn:0.227 lr:2.5e-06 updt_s:0
|
|
.518 data_s:0.003
|
|
INFO 2025-09-08 18:24:33 celerate.py:281 step:32K smpl:2M ep:13K epch:7.40 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.507 data_s:0.007
|
|
INFO 2025-09-08 18:26:16 celerate.py:281 step:32K smpl:2M ep:13K epch:7.44 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:0
|
|
.463 data_s:0.047
|
|
INFO 2025-09-08 18:27:59 celerate.py:281 step:32K smpl:2M ep:13K epch:7.49 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s:0
|
|
.509 data_s:0.007
|
|
INFO 2025-09-08 18:29:43 celerate.py:281 step:32K smpl:2M ep:13K epch:7.54 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.514 data_s:0.003
|
|
INFO 2025-09-08 18:31:26 celerate.py:281 step:32K smpl:2M ep:13K epch:7.58 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.511 data_s:0.003
|
|
INFO 2025-09-08 18:33:11 celerate.py:281 step:33K smpl:2M ep:13K epch:7.63 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.518 data_s:0.003
|
|
INFO 2025-09-08 18:34:57 celerate.py:281 step:33K smpl:2M ep:13K epch:7.68 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.512 data_s:0.016
|
|
INFO 2025-09-08 18:36:41 celerate.py:281 step:33K smpl:2M ep:13K epch:7.72 loss:0.086 grdn:0.229 lr:2.5e-06 updt_s:0
|
|
.517 data_s:0.003
|
|
INFO 2025-09-08 18:38:25 celerate.py:281 step:33K smpl:2M ep:13K epch:7.77 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.516 data_s:0.003
|
|
INFO 2025-09-08 18:40:07 celerate.py:281 step:33K smpl:2M ep:13K epch:7.82 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.506 data_s:0.003
|
|
INFO 2025-09-08 18:41:51 celerate.py:281 step:34K smpl:2M ep:13K epch:7.86 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.509 data_s:0.006
|
|
INFO 2025-09-08 18:43:36 celerate.py:281 step:34K smpl:2M ep:13K epch:7.91 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:0
|
|
.521 data_s:0.003
|
|
INFO 2025-09-08 18:45:19 celerate.py:281 step:34K smpl:2M ep:13K epch:7.96 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.511 data_s:0.003
|
|
INFO 2025-09-08 18:47:05 celerate.py:281 step:34K smpl:2M ep:14K epch:8.00 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.495 data_s:0.035
|
|
INFO 2025-09-08 18:48:51 celerate.py:281 step:34K smpl:2M ep:14K epch:8.05 loss:0.087 grdn:0.243 lr:2.5e-06 updt_s:0
|
|
.413 data_s:0.112
|
|
INFO 2025-09-08 18:50:34 celerate.py:281 step:35K smpl:2M ep:14K epch:8.10 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.515 data_s:0.003
|
|
INFO 2025-09-08 18:52:19 celerate.py:281 step:35K smpl:2M ep:14K epch:8.14 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.520 data_s:0.003
|
|
INFO 2025-09-08 18:54:03 celerate.py:281 step:35K smpl:2M ep:14K epch:8.19 loss:0.087 grdn:0.231 lr:2.5e-06 updt_s:0
|
|
.515 data_s:0.003
|
|
INFO 2025-09-08 18:55:48 celerate.py:281 step:35K smpl:2M ep:14K epch:8.24 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0
|
|
.420 data_s:0.101
|
|
INFO 2025-09-08 18:57:33 celerate.py:281 step:35K smpl:2M ep:14K epch:8.28 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.506 data_s:0.022
|
|
INFO 2025-09-08 18:59:19 celerate.py:281 step:36K smpl:2M ep:14K epch:8.33 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.525 data_s:0.003
|
|
INFO 2025-09-08 19:01:03 celerate.py:281 step:36K smpl:2M ep:14K epch:8.38 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0
|
|
.516 data_s:0.003
|
|
INFO 2025-09-08 19:02:48 celerate.py:281 step:36K smpl:2M ep:14K epch:8.43 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.516 data_s:0.003
|
|
INFO 2025-09-08 19:04:32 celerate.py:281 step:36K smpl:2M ep:14K epch:8.47 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.505 data_s:0.013
|
|
INFO 2025-09-08 19:06:15 celerate.py:281 step:36K smpl:2M ep:14K epch:8.52 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.384 data_s:0.130
|
|
INFO 2025-09-08 19:07:58 celerate.py:281 step:37K smpl:2M ep:15K epch:8.57 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.430 data_s:0.084
|
|
INFO 2025-09-08 19:09:41 celerate.py:281 step:37K smpl:2M ep:15K epch:8.61 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.351 data_s:0.162
|
|
INFO 2025-09-08 19:11:25 celerate.py:281 step:37K smpl:2M ep:15K epch:8.66 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:0
|
|
.337 data_s:0.181
|
|
INFO 2025-09-08 19:13:08 celerate.py:281 step:37K smpl:2M ep:15K epch:8.71 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.336 data_s:0.177
|
|
INFO 2025-09-08 19:14:52 celerate.py:281 step:37K smpl:2M ep:15K epch:8.75 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.344 data_s:0.174
|
|
INFO 2025-09-08 19:16:35 celerate.py:281 step:38K smpl:2M ep:15K epch:8.80 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:0
|
|
.332 data_s:0.182
|
|
INFO 2025-09-08 19:18:18 celerate.py:281 step:38K smpl:2M ep:15K epch:8.85 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.332 data_s:0.182
|
|
INFO 2025-09-08 19:20:01 celerate.py:281 step:38K smpl:2M ep:15K epch:8.89 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.337 data_s:0.177
|
|
INFO 2025-09-08 19:21:45 celerate.py:281 step:38K smpl:2M ep:15K epch:8.94 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.495 data_s:0.022
|
|
INFO 2025-09-08 19:23:29 celerate.py:281 step:38K smpl:2M ep:15K epch:8.99 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.513 data_s:0.003
|
|
INFO 2025-09-08 19:25:15 celerate.py:281 step:39K smpl:2M ep:15K epch:9.03 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.436 data_s:0.097
|
|
INFO 2025-09-08 19:26:59 celerate.py:281 step:39K smpl:2M ep:15K epch:9.08 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.378 data_s:0.138
|
|
INFO 2025-09-08 19:28:43 celerate.py:281 step:39K smpl:2M ep:15K epch:9.13 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s:0
|
|
.497 data_s:0.023
|
|
INFO 2025-09-08 19:30:27 celerate.py:281 step:39K smpl:3M ep:16K epch:9.17 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:0
|
|
.515 data_s:0.003
|
|
INFO 2025-09-08 19:32:14 celerate.py:281 step:39K smpl:3M ep:16K epch:9.22 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.526 data_s:0.003
|
|
INFO 2025-09-08 19:33:57 celerate.py:281 step:40K smpl:3M ep:16K epch:9.27 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.506 data_s:0.011
|
|
INFO 2025-09-08 19:35:42 celerate.py:281 step:40K smpl:3M ep:16K epch:9.31 loss:0.087 grdn:0.229 lr:2.5e-06 updt_s:0
|
|
.455 data_s:0.066
|
|
INFO 2025-09-08 19:37:26 celerate.py:281 step:40K smpl:3M ep:16K epch:9.36 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:0
|
|
.493 data_s:0.026
|
|
INFO 2025-09-08 19:37:26 celerate.py:295 Checkpoint policy after step 40000
|
|
INFO 2025-09-08 19:39:10 celerate.py:281 step:40K smpl:3M ep:16K epch:9.41 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.397 data_s:0.114
|
|
INFO 2025-09-08 19:40:53 celerate.py:281 step:40K smpl:3M ep:16K epch:9.45 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:0
|
|
.344 data_s:0.168
|
|
INFO 2025-09-08 19:42:37 celerate.py:281 step:41K smpl:3M ep:16K epch:9.50 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.480 data_s:0.036
|
|
INFO 2025-09-08 19:44:21 celerate.py:281 step:41K smpl:3M ep:16K epch:9.55 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.517 data_s:0.003
|
|
INFO 2025-09-08 19:46:05 celerate.py:281 step:41K smpl:3M ep:16K epch:9.60 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.517 data_s:0.003
|
|
INFO 2025-09-08 19:47:49 celerate.py:281 step:41K smpl:3M ep:16K epch:9.64 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:0
|
|
.513 data_s:0.003
|
|
INFO 2025-09-08 19:49:33 celerate.py:281 step:41K smpl:3M ep:16K epch:9.69 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:0
|
|
.515 data_s:0.003
|
|
INFO 2025-09-08 19:51:17 celerate.py:281 step:42K smpl:3M ep:16K epch:9.74 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:0
|
|
.515 data_s:0.003
|
|
INFO 2025-09-08 19:53:00 celerate.py:281 step:42K smpl:3M ep:17K epch:9.78 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:0
|
|
.513 data_s:0.003
|
|
INFO 2025-09-08 19:54:44 celerate.py:281 step:42K smpl:3M ep:17K epch:9.83 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.516 data_s:0.003
|
|
INFO 2025-09-08 19:56:28 celerate.py:281 step:42K smpl:3M ep:17K epch:9.88 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:0
|
|
.512 data_s:0.003
|
|
INFO 2025-09-08 19:58:11 celerate.py:281 step:42K smpl:3M ep:17K epch:9.92 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:0
|
|
.514 data_s:0.003
|
|
INFO 2025-09-08 19:59:55 celerate.py:281 step:43K smpl:3M ep:17K epch:9.97 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:0
|
|
.514 data_s:0.003
|
|
INFO 2025-09-08 20:01:42 celerate.py:281 step:43K smpl:3M ep:17K epch:10.02 loss:0.087 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.476 data_s:0.057
|
|
INFO 2025-09-08 20:03:25 celerate.py:281 step:43K smpl:3M ep:17K epch:10.06 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.471 data_s:0.043
|
|
INFO 2025-09-08 20:05:09 celerate.py:281 step:43K smpl:3M ep:17K epch:10.11 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.004
|
|
INFO 2025-09-08 20:06:53 celerate.py:281 step:43K smpl:3M ep:17K epch:10.16 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.505 data_s:0.013
|
|
INFO 2025-09-08 20:08:36 celerate.py:281 step:44K smpl:3M ep:17K epch:10.20 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-08 20:10:20 celerate.py:281 step:44K smpl:3M ep:17K epch:10.25 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-08 20:12:04 celerate.py:281 step:44K smpl:3M ep:17K epch:10.30 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-08 20:13:47 celerate.py:281 step:44K smpl:3M ep:18K epch:10.34 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.503 data_s:0.011
|
|
INFO 2025-09-08 20:15:31 celerate.py:281 step:44K smpl:3M ep:18K epch:10.39 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.416 data_s:0.102
|
|
INFO 2025-09-08 20:17:15 celerate.py:281 step:45K smpl:3M ep:18K epch:10.44 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.502 data_s:0.017
|
|
INFO 2025-09-08 20:18:58 celerate.py:281 step:45K smpl:3M ep:18K epch:10.48 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-08 20:20:41 celerate.py:281 step:45K smpl:3M ep:18K epch:10.53 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.496 data_s:0.017
|
|
INFO 2025-09-08 20:22:24 celerate.py:281 step:45K smpl:3M ep:18K epch:10.58 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.493 data_s:0.022
|
|
INFO 2025-09-08 20:24:08 celerate.py:281 step:45K smpl:3M ep:18K epch:10.63 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.485 data_s:0.031
|
|
INFO 2025-09-08 20:25:52 celerate.py:281 step:46K smpl:3M ep:18K epch:10.67 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:
|
|
0.518 data_s:0.003
|
|
INFO 2025-09-08 20:27:36 celerate.py:281 step:46K smpl:3M ep:18K epch:10.72 loss:0.085 grdn:0.228 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-08 20:29:19 celerate.py:281 step:46K smpl:3M ep:18K epch:10.77 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 20:31:04 celerate.py:281 step:46K smpl:3M ep:18K epch:10.81 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-08 20:32:47 celerate.py:281 step:46K smpl:3M ep:18K epch:10.86 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-08 20:34:32 celerate.py:281 step:47K smpl:3M ep:18K epch:10.91 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.518 data_s:0.003
|
|
INFO 2025-09-08 20:36:15 celerate.py:281 step:47K smpl:3M ep:19K epch:10.95 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 20:38:01 celerate.py:281 step:47K smpl:3M ep:19K epch:11.00 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.416 data_s:0.114
|
|
INFO 2025-09-08 20:39:45 celerate.py:281 step:47K smpl:3M ep:19K epch:11.05 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.429 data_s:0.086
|
|
INFO 2025-09-08 20:41:28 celerate.py:281 step:47K smpl:3M ep:19K epch:11.09 loss:0.086 grdn:0.229 lr:2.5e-06 updt_s:
|
|
0.409 data_s:0.106
|
|
INFO 2025-09-08 20:43:12 celerate.py:281 step:48K smpl:3M ep:19K epch:11.14 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.470 data_s:0.050
|
|
INFO 2025-09-08 20:44:56 celerate.py:281 step:48K smpl:3M ep:19K epch:11.19 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-08 20:46:41 celerate.py:281 step:48K smpl:3M ep:19K epch:11.23 loss:0.087 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 20:48:25 celerate.py:281 step:48K smpl:3M ep:19K epch:11.28 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.518 data_s:0.003
|
|
INFO 2025-09-08 20:50:08 celerate.py:281 step:48K smpl:3M ep:19K epch:11.33 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.505 data_s:0.009
|
|
INFO 2025-09-08 20:51:52 celerate.py:281 step:49K smpl:3M ep:19K epch:11.37 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s:
|
|
0.449 data_s:0.067
|
|
INFO 2025-09-08 20:53:35 celerate.py:281 step:49K smpl:3M ep:19K epch:11.42 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.420 data_s:0.094
|
|
INFO 2025-09-08 20:55:19 celerate.py:281 step:49K smpl:3M ep:19K epch:11.47 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 20:57:01 celerate.py:281 step:49K smpl:3M ep:19K epch:11.51 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.505 data_s:0.003
|
|
INFO 2025-09-08 20:58:44 celerate.py:281 step:49K smpl:3M ep:20K epch:11.56 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-08 21:00:28 celerate.py:281 step:50K smpl:3M ep:20K epch:11.61 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-08 21:02:11 celerate.py:281 step:50K smpl:3M ep:20K epch:11.65 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.402 data_s:0.110
|
|
INFO 2025-09-08 21:03:54 celerate.py:281 step:50K smpl:3M ep:20K epch:11.70 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.332 data_s:0.184
|
|
INFO 2025-09-08 21:05:37 celerate.py:281 step:50K smpl:3M ep:20K epch:11.75 loss:0.086 grdn:0.229 lr:2.5e-06 updt_s:
|
|
0.332 data_s:0.182
|
|
INFO 2025-09-08 21:07:21 celerate.py:281 step:50K smpl:3M ep:20K epch:11.80 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.466 data_s:0.049
|
|
INFO 2025-09-08 21:09:05 celerate.py:281 step:51K smpl:3M ep:20K epch:11.84 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 21:10:49 celerate.py:281 step:51K smpl:3M ep:20K epch:11.89 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.004
|
|
INFO 2025-09-08 21:12:32 celerate.py:281 step:51K smpl:3M ep:20K epch:11.94 loss:0.085 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.484 data_s:0.032
|
|
INFO 2025-09-08 21:14:17 celerate.py:281 step:51K smpl:3M ep:20K epch:11.98 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.004
|
|
INFO 2025-09-08 21:16:03 celerate.py:281 step:51K smpl:3M ep:20K epch:12.03 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.424 data_s:0.105
|
|
INFO 2025-09-08 21:17:46 celerate.py:281 step:52K smpl:3M ep:20K epch:12.08 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.442 data_s:0.073
|
|
INFO 2025-09-08 21:19:30 celerate.py:281 step:52K smpl:3M ep:21K epch:12.12 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.007
|
|
INFO 2025-09-08 21:21:15 celerate.py:281 step:52K smpl:3M ep:21K epch:12.17 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.520 data_s:0.003
|
|
INFO 2025-09-08 21:22:59 celerate.py:281 step:52K smpl:3M ep:21K epch:12.22 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 21:24:43 celerate.py:281 step:52K smpl:3M ep:21K epch:12.26 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.518 data_s:0.003
|
|
INFO 2025-09-08 21:26:27 celerate.py:281 step:53K smpl:3M ep:21K epch:12.31 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 21:28:11 celerate.py:281 step:53K smpl:3M ep:21K epch:12.36 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 21:29:55 celerate.py:281 step:53K smpl:3M ep:21K epch:12.40 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 21:31:39 celerate.py:281 step:53K smpl:3M ep:21K epch:12.45 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 21:33:23 celerate.py:281 step:53K smpl:3M ep:21K epch:12.50 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 21:35:08 celerate.py:281 step:54K smpl:3M ep:21K epch:12.54 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.519 data_s:0.003
|
|
INFO 2025-09-08 21:36:51 celerate.py:281 step:54K smpl:3M ep:21K epch:12.59 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-08 21:38:36 celerate.py:281 step:54K smpl:3M ep:21K epch:12.64 loss:0.087 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 21:40:18 celerate.py:281 step:54K smpl:3M ep:21K epch:12.68 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.473 data_s:0.038
|
|
INFO 2025-09-08 21:42:02 celerate.py:281 step:54K smpl:3M ep:22K epch:12.73 loss:0.085 grdn:0.232 lr:2.5e-06 updt_s:
|
|
0.408 data_s:0.109
|
|
INFO 2025-09-08 21:43:45 celerate.py:281 step:55K smpl:3M ep:22K epch:12.78 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.377 data_s:0.136
|
|
INFO 2025-09-08 21:45:29 celerate.py:281 step:55K smpl:4M ep:22K epch:12.83 loss:0.087 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.497 data_s:0.022
|
|
INFO 2025-09-08 21:47:12 celerate.py:281 step:55K smpl:4M ep:22K epch:12.87 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-08 21:48:56 celerate.py:281 step:55K smpl:4M ep:22K epch:12.92 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.429 data_s:0.086
|
|
INFO 2025-09-08 21:50:39 celerate.py:281 step:55K smpl:4M ep:22K epch:12.97 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.454 data_s:0.059
|
|
INFO 2025-09-08 21:52:25 celerate.py:281 step:56K smpl:4M ep:22K epch:13.01 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.459 data_s:0.072
|
|
INFO 2025-09-08 21:54:08 celerate.py:281 step:56K smpl:4M ep:22K epch:13.06 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:
|
|
0.382 data_s:0.132
|
|
INFO 2025-09-08 21:55:51 celerate.py:281 step:56K smpl:4M ep:22K epch:13.11 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.500 data_s:0.016
|
|
INFO 2025-09-08 21:57:36 celerate.py:281 step:56K smpl:4M ep:22K epch:13.15 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 21:59:20 celerate.py:281 step:56K smpl:4M ep:22K epch:13.20 loss:0.085 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.518 data_s:0.003
|
|
INFO 2025-09-08 22:01:03 celerate.py:281 step:57K smpl:4M ep:22K epch:13.25 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-08 22:02:48 celerate.py:281 step:57K smpl:4M ep:23K epch:13.29 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 22:04:32 celerate.py:281 step:57K smpl:4M ep:23K epch:13.34 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 22:06:14 celerate.py:281 step:57K smpl:4M ep:23K epch:13.39 loss:0.087 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.505 data_s:0.005
|
|
bINFO 2025-09-08 22:07:59 celerate.py:281 step:57K smpl:4M ep:23K epch:13.43 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s
|
|
:0.496 data_s:0.026
|
|
INFO 2025-09-08 22:09:43 celerate.py:281 step:58K smpl:4M ep:23K epch:13.48 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.438 data_s:0.080
|
|
INFO 2025-09-08 22:11:27 celerate.py:281 step:58K smpl:4M ep:23K epch:13.53 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.444 data_s:0.073
|
|
INFO 2025-09-08 22:13:11 celerate.py:281 step:58K smpl:4M ep:23K epch:13.57 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 22:14:55 celerate.py:281 step:58K smpl:4M ep:23K epch:13.62 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.518 data_s:0.003
|
|
INFO 2025-09-08 22:16:39 celerate.py:281 step:58K smpl:4M ep:23K epch:13.67 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-08 22:18:22 celerate.py:281 step:59K smpl:4M ep:23K epch:13.71 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-08 22:20:05 celerate.py:281 step:59K smpl:4M ep:23K epch:13.76 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.508 data_s:0.003
|
|
INFO 2025-09-08 22:21:48 celerate.py:281 step:59K smpl:4M ep:23K epch:13.81 loss:0.086 grdn:0.228 lr:2.5e-06 updt_s:
|
|
0.505 data_s:0.008
|
|
INFO 2025-09-08 22:23:31 celerate.py:281 step:59K smpl:4M ep:23K epch:13.85 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.505 data_s:0.008
|
|
INFO 2025-09-08 22:25:15 celerate.py:281 step:59K smpl:4M ep:24K epch:13.90 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 22:26:58 celerate.py:281 step:60K smpl:4M ep:24K epch:13.95 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.493 data_s:0.022
|
|
INFO 2025-09-08 22:28:42 celerate.py:281 step:60K smpl:4M ep:24K epch:14.00 loss:0.086 grdn:0.231 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 22:30:29 celerate.py:281 step:60K smpl:4M ep:24K epch:14.04 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.435 data_s:0.101
|
|
INFO 2025-09-08 22:30:29 celerate.py:295 Checkpoint policy after step 60000
|
|
INFO 2025-09-08 22:32:14 celerate.py:281 step:60K smpl:4M ep:24K epch:14.09 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.508 data_s:0.003
|
|
INFO 2025-09-08 22:33:58 celerate.py:281 step:60K smpl:4M ep:24K epch:14.14 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-08 22:35:42 celerate.py:281 step:61K smpl:4M ep:24K epch:14.18 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 22:37:25 celerate.py:281 step:61K smpl:4M ep:24K epch:14.23 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-08 22:39:09 celerate.py:281 step:61K smpl:4M ep:24K epch:14.28 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 22:40:52 celerate.py:281 step:61K smpl:4M ep:24K epch:14.32 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.509 data_s:0.003
|
|
INFO 2025-09-08 22:42:35 celerate.py:281 step:61K smpl:4M ep:24K epch:14.37 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-08 22:44:18 celerate.py:281 step:62K smpl:4M ep:24K epch:14.42 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-08 22:46:02 celerate.py:281 step:62K smpl:4M ep:24K epch:14.46 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 22:47:47 celerate.py:281 step:62K smpl:4M ep:25K epch:14.51 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 22:49:30 celerate.py:281 step:62K smpl:4M ep:25K epch:14.56 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-08 22:51:14 celerate.py:281 step:62K smpl:4M ep:25K epch:14.60 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 22:52:58 celerate.py:281 step:63K smpl:4M ep:25K epch:14.65 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s:
|
|
0.484 data_s:0.033
|
|
INFO 2025-09-08 22:54:41 celerate.py:281 step:63K smpl:4M ep:25K epch:14.70 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.501 data_s:0.016
|
|
INFO 2025-09-08 22:56:25 celerate.py:281 step:63K smpl:4M ep:25K epch:14.74 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.436 data_s:0.081
|
|
INFO 2025-09-08 22:58:08 celerate.py:281 step:63K smpl:4M ep:25K epch:14.79 loss:0.087 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.436 data_s:0.080
|
|
INFO 2025-09-08 22:59:51 celerate.py:281 step:63K smpl:4M ep:25K epch:14.84 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.344 data_s:0.168
|
|
INFO 2025-09-08 23:01:34 celerate.py:281 step:64K smpl:4M ep:25K epch:14.88 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.479 data_s:0.035
|
|
INFO 2025-09-08 23:03:18 celerate.py:281 step:64K smpl:4M ep:25K epch:14.93 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-08 23:05:02 celerate.py:281 step:64K smpl:4M ep:25K epch:14.98 loss:0.086 grdn:0.230 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-08 23:06:47 celerate.py:281 step:64K smpl:4M ep:25K epch:15.02 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.405 data_s:0.119
|
|
INFO 2025-09-08 23:08:30 celerate.py:281 step:64K smpl:4M ep:26K epch:15.07 loss:0.087 grdn:0.248 lr:2.5e-06 updt_s:
|
|
0.393 data_s:0.121
|
|
INFO 2025-09-08 23:10:13 celerate.py:281 step:65K smpl:4M ep:26K epch:15.12 loss:0.087 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.369 data_s:0.142
|
|
INFO 2025-09-08 23:11:56 celerate.py:281 step:65K smpl:4M ep:26K epch:15.17 loss:0.085 grdn:0.230 lr:2.5e-06 updt_s:
|
|
0.360 data_s:0.156
|
|
INFO 2025-09-08 23:13:40 celerate.py:281 step:65K smpl:4M ep:26K epch:15.21 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.333 data_s:0.182
|
|
INFO 2025-09-08 23:15:23 celerate.py:281 step:65K smpl:4M ep:26K epch:15.26 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.376 data_s:0.136
|
|
INFO 2025-09-08 23:17:05 celerate.py:281 step:65K smpl:4M ep:26K epch:15.31 loss:0.087 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.439 data_s:0.069
|
|
INFO 2025-09-08 23:18:49 celerate.py:281 step:66K smpl:4M ep:26K epch:15.35 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.006
|
|
INFO 2025-09-08 23:20:32 celerate.py:281 step:66K smpl:4M ep:26K epch:15.40 loss:0.087 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-08 23:22:15 celerate.py:281 step:66K smpl:4M ep:26K epch:15.45 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.508 data_s:0.004
|
|
INFO 2025-09-08 23:23:59 celerate.py:281 step:66K smpl:4M ep:26K epch:15.49 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.474 data_s:0.046
|
|
INFO 2025-09-08 23:25:42 celerate.py:281 step:66K smpl:4M ep:26K epch:15.54 loss:0.087 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-08 23:27:25 celerate.py:281 step:67K smpl:4M ep:26K epch:15.59 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.487 data_s:0.025
|
|
INFO 2025-09-08 23:29:08 celerate.py:281 step:67K smpl:4M ep:26K epch:15.63 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.005
|
|
INFO 2025-09-08 23:30:50 celerate.py:281 step:67K smpl:4M ep:27K epch:15.68 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.506 data_s:0.003
|
|
INFO 2025-09-08 23:32:33 celerate.py:281 step:67K smpl:4M ep:27K epch:15.73 loss:0.086 grdn:0.245 lr:2.5e-06 updt_s:
|
|
0.509 data_s:0.003
|
|
INFO 2025-09-08 23:34:16 celerate.py:281 step:67K smpl:4M ep:27K epch:15.77 loss:0.087 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.503 data_s:0.014
|
|
INFO 2025-09-08 23:35:59 celerate.py:281 step:68K smpl:4M ep:27K epch:15.82 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-08 23:37:42 celerate.py:281 step:68K smpl:4M ep:27K epch:15.87 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.509 data_s:0.003
|
|
INFO 2025-09-08 23:39:26 celerate.py:281 step:68K smpl:4M ep:27K epch:15.91 loss:0.085 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-08 23:41:10 celerate.py:281 step:68K smpl:4M ep:27K epch:15.96 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-08 23:42:56 celerate.py:281 step:68K smpl:4M ep:27K epch:16.01 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.469 data_s:0.061
|
|
INFO 2025-09-08 23:44:40 celerate.py:281 step:69K smpl:4M ep:27K epch:16.05 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.339 data_s:0.179
|
|
INFO 2025-09-08 23:46:22 celerate.py:281 step:69K smpl:4M ep:27K epch:16.10 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.386 data_s:0.124
|
|
INFO 2025-09-08 23:48:06 celerate.py:281 step:69K smpl:4M ep:27K epch:16.15 loss:0.085 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-08 23:49:50 celerate.py:281 step:69K smpl:4M ep:27K epch:16.20 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-08 23:51:33 celerate.py:281 step:69K smpl:4M ep:27K epch:16.24 loss:0.087 grdn:0.246 lr:2.5e-06 updt_s:
|
|
0.501 data_s:0.014
|
|
INFO 2025-09-08 23:53:17 celerate.py:281 step:70K smpl:4M ep:28K epch:16.29 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-08 23:54:59 celerate.py:281 step:70K smpl:4M ep:28K epch:16.34 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s:
|
|
0.507 data_s:0.005
|
|
INFO 2025-09-08 23:56:43 celerate.py:281 step:70K smpl:4M ep:28K epch:16.38 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-08 23:58:27 celerate.py:281 step:70K smpl:4M ep:28K epch:16.43 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-09 00:00:12 celerate.py:281 step:70K smpl:5M ep:28K epch:16.48 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.518 data_s:0.003
|
|
INFO 2025-09-09 00:01:55 celerate.py:281 step:71K smpl:5M ep:28K epch:16.52 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-09 00:03:37 celerate.py:281 step:71K smpl:5M ep:28K epch:16.57 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.508 data_s:0.003
|
|
INFO 2025-09-09 00:05:20 celerate.py:281 step:71K smpl:5M ep:28K epch:16.62 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.509 data_s:0.003
|
|
INFO 2025-09-09 00:07:04 celerate.py:281 step:71K smpl:5M ep:28K epch:16.66 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.519 data_s:0.003
|
|
INFO 2025-09-09 00:08:47 celerate.py:281 step:71K smpl:5M ep:28K epch:16.71 loss:0.087 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.509 data_s:0.003
|
|
INFO 2025-09-09 00:10:30 celerate.py:281 step:72K smpl:5M ep:28K epch:16.76 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-09 00:12:13 celerate.py:281 step:72K smpl:5M ep:28K epch:16.80 loss:0.085 grdn:0.230 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-09 00:13:58 celerate.py:281 step:72K smpl:5M ep:29K epch:16.85 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.522 data_s:0.003
|
|
INFO 2025-09-09 00:15:40 celerate.py:281 step:72K smpl:5M ep:29K epch:16.90 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.506 data_s:0.003
|
|
INFO 2025-09-09 00:17:23 celerate.py:281 step:72K smpl:5M ep:29K epch:16.94 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.004
|
|
INFO 2025-09-09 00:19:05 celerate.py:281 step:73K smpl:5M ep:29K epch:16.99 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.505 data_s:0.003
|
|
INFO 2025-09-09 00:20:51 celerate.py:281 step:73K smpl:5M ep:29K epch:17.04 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.437 data_s:0.092
|
|
INFO 2025-09-09 00:22:34 celerate.py:281 step:73K smpl:5M ep:29K epch:17.08 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.489 data_s:0.025
|
|
INFO 2025-09-09 00:24:17 celerate.py:281 step:73K smpl:5M ep:29K epch:17.13 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-09 00:26:01 celerate.py:281 step:73K smpl:5M ep:29K epch:17.18 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-09 00:27:45 celerate.py:281 step:74K smpl:5M ep:29K epch:17.22 loss:0.087 grdn:0.246 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-09 00:29:28 celerate.py:281 step:74K smpl:5M ep:29K epch:17.27 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.509 data_s:0.003
|
|
INFO 2025-09-09 00:31:11 celerate.py:281 step:74K smpl:5M ep:29K epch:17.32 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-09 00:32:54 celerate.py:281 step:74K smpl:5M ep:29K epch:17.37 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-09 00:34:38 celerate.py:281 step:74K smpl:5M ep:29K epch:17.41 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-09 00:36:21 celerate.py:281 step:75K smpl:5M ep:30K epch:17.46 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-09 00:38:05 celerate.py:281 step:75K smpl:5M ep:30K epch:17.51 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-09 00:39:49 celerate.py:281 step:75K smpl:5M ep:30K epch:17.55 loss:0.086 grdn:0.251 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-09 00:41:32 celerate.py:281 step:75K smpl:5M ep:30K epch:17.60 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.506 data_s:0.005
|
|
INFO 2025-09-09 00:43:14 celerate.py:281 step:75K smpl:5M ep:30K epch:17.65 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.390 data_s:0.122
|
|
INFO 2025-09-09 00:44:58 celerate.py:281 step:76K smpl:5M ep:30K epch:17.69 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:
|
|
0.410 data_s:0.107
|
|
INFO 2025-09-09 00:46:41 celerate.py:281 step:76K smpl:5M ep:30K epch:17.74 loss:0.087 grdn:0.247 lr:2.5e-06 updt_s:
|
|
0.427 data_s:0.085
|
|
INFO 2025-09-09 00:48:24 celerate.py:281 step:76K smpl:5M ep:30K epch:17.79 loss:0.085 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.492 data_s:0.025
|
|
INFO 2025-09-09 00:50:08 celerate.py:281 step:76K smpl:5M ep:30K epch:17.83 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-09 00:51:51 celerate.py:281 step:76K smpl:5M ep:30K epch:17.88 loss:0.087 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-09 00:53:33 celerate.py:281 step:77K smpl:5M ep:30K epch:17.93 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-09 00:55:18 celerate.py:281 step:77K smpl:5M ep:30K epch:17.97 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.519 data_s:0.003
|
|
INFO 2025-09-09 00:57:05 celerate.py:281 step:77K smpl:5M ep:31K epch:18.02 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.486 data_s:0.046
|
|
INFO 2025-09-09 00:58:47 celerate.py:281 step:77K smpl:5M ep:31K epch:18.07 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.509 data_s:0.003
|
|
INFO 2025-09-09 01:00:31 celerate.py:281 step:77K smpl:5M ep:31K epch:18.11 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-09 01:02:15 celerate.py:281 step:78K smpl:5M ep:31K epch:18.16 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-09 01:03:56 celerate.py:281 step:78K smpl:5M ep:31K epch:18.21 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.498 data_s:0.008
|
|
INFO 2025-09-09 01:05:40 celerate.py:281 step:78K smpl:5M ep:31K epch:18.25 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.508 data_s:0.010
|
|
INFO 2025-09-09 01:07:24 celerate.py:281 step:78K smpl:5M ep:31K epch:18.30 loss:0.085 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-09 01:09:07 celerate.py:281 step:78K smpl:5M ep:31K epch:18.35 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-09 01:10:51 celerate.py:281 step:79K smpl:5M ep:31K epch:18.40 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-09 01:12:35 celerate.py:281 step:79K smpl:5M ep:31K epch:18.44 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-09 01:14:17 celerate.py:281 step:79K smpl:5M ep:31K epch:18.49 loss:0.086 grdn:0.247 lr:2.5e-06 updt_s:
|
|
0.508 data_s:0.003
|
|
INFO 2025-09-09 01:16:01 celerate.py:281 step:79K smpl:5M ep:31K epch:18.54 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-09 01:17:43 celerate.py:281 step:79K smpl:5M ep:31K epch:18.58 loss:0.086 grdn:0.233 lr:2.5e-06 updt_s:
|
|
0.507 data_s:0.003
|
|
INFO 2025-09-09 01:19:26 celerate.py:281 step:80K smpl:5M ep:32K epch:18.63 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-09 01:21:09 celerate.py:281 step:80K smpl:5M ep:32K epch:18.68 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-09 01:22:52 celerate.py:281 step:80K smpl:5M ep:32K epch:18.72 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.501 data_s:0.011
|
|
INFO 2025-09-09 01:22:52 celerate.py:295 Checkpoint policy after step 80000
|
|
INFO 2025-09-09 01:24:37 celerate.py:281 step:80K smpl:5M ep:32K epch:18.77 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-09 01:26:20 celerate.py:281 step:80K smpl:5M ep:32K epch:18.82 loss:0.087 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-09 01:28:05 celerate.py:281 step:81K smpl:5M ep:32K epch:18.86 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-09 01:29:48 celerate.py:281 step:81K smpl:5M ep:32K epch:18.91 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-09 01:31:32 celerate.py:281 step:81K smpl:5M ep:32K epch:18.96 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.517 data_s:0.003
|
|
INFO 2025-09-09 01:33:18 celerate.py:281 step:81K smpl:5M ep:32K epch:19.00 loss:0.087 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.487 data_s:0.042
|
|
INFO 2025-09-09 01:35:01 celerate.py:281 step:81K smpl:5M ep:32K epch:19.05 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.461 data_s:0.054
|
|
INFO 2025-09-09 01:36:46 celerate.py:281 step:82K smpl:5M ep:32K epch:19.10 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.450 data_s:0.071
|
|
INFO 2025-09-09 01:38:29 celerate.py:281 step:82K smpl:5M ep:32K epch:19.14 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.451 data_s:0.064
|
|
INFO 2025-09-09 01:40:12 celerate.py:281 step:82K smpl:5M ep:32K epch:19.19 loss:0.086 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-09 01:41:54 celerate.py:281 step:82K smpl:5M ep:33K epch:19.24 loss:0.085 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.480 data_s:0.030
|
|
INFO 2025-09-09 01:43:37 celerate.py:281 step:82K smpl:5M ep:33K epch:19.28 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.434 data_s:0.080
|
|
INFO 2025-09-09 01:45:20 celerate.py:281 step:83K smpl:5M ep:33K epch:19.33 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-09 01:47:04 celerate.py:281 step:83K smpl:5M ep:33K epch:19.38 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-09 01:48:49 celerate.py:281 step:83K smpl:5M ep:33K epch:19.42 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.520 data_s:0.003
|
|
INFO 2025-09-09 01:50:32 celerate.py:281 step:83K smpl:5M ep:33K epch:19.47 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-09 01:52:15 celerate.py:281 step:83K smpl:5M ep:33K epch:19.52 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-09 01:53:59 celerate.py:281 step:84K smpl:5M ep:33K epch:19.57 loss:0.085 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-09 01:55:41 celerate.py:281 step:84K smpl:5M ep:33K epch:19.61 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.508 data_s:0.003
|
|
INFO 2025-09-09 01:57:24 celerate.py:281 step:84K smpl:5M ep:33K epch:19.66 loss:0.087 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.500 data_s:0.013
|
|
INFO 2025-09-09 01:59:07 celerate.py:281 step:84K smpl:5M ep:33K epch:19.71 loss:0.087 grdn:0.245 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-09 02:00:50 celerate.py:281 step:84K smpl:5M ep:33K epch:19.75 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-09 02:02:34 celerate.py:281 step:85K smpl:5M ep:34K epch:19.80 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-09 02:04:17 celerate.py:281 step:85K smpl:5M ep:34K epch:19.85 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.511 data_s:0.003
|
|
INFO 2025-09-09 02:05:59 celerate.py:281 step:85K smpl:5M ep:34K epch:19.89 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.507 data_s:0.004
|
|
INFO 2025-09-09 02:07:43 celerate.py:281 step:85K smpl:5M ep:34K epch:19.94 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.505 data_s:0.011
|
|
INFO 2025-09-09 02:09:26 celerate.py:281 step:85K smpl:5M ep:34K epch:19.99 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.392 data_s:0.124
|
|
INFO 2025-09-09 02:11:14 celerate.py:281 step:86K smpl:5M ep:34K epch:20.03 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.435 data_s:0.100
|
|
INFO 2025-09-09 02:12:57 celerate.py:281 step:86K smpl:5M ep:34K epch:20.08 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-09 02:14:41 celerate.py:281 step:86K smpl:6M ep:34K epch:20.13 loss:0.085 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.516 data_s:0.003
|
|
INFO 2025-09-09 02:16:25 celerate.py:281 step:86K smpl:6M ep:34K epch:20.17 loss:0.085 grdn:0.234 lr:2.5e-06 updt_s:
|
|
0.514 data_s:0.003
|
|
INFO 2025-09-09 02:18:09 celerate.py:281 step:86K smpl:6M ep:34K epch:20.22 loss:0.085 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.506 data_s:0.010
|
|
INFO 2025-09-09 02:19:53 celerate.py:281 step:87K smpl:6M ep:34K epch:20.27 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.515 data_s:0.003
|
|
INFO 2025-09-09 02:21:35 celerate.py:281 step:87K smpl:6M ep:34K epch:20.31 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.501 data_s:0.007
|
|
INFO 2025-09-09 02:23:17 celerate.py:281 step:87K smpl:6M ep:34K epch:20.36 loss:0.085 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.432 data_s:0.080
|
|
INFO 2025-09-09 02:25:01 celerate.py:281 step:87K smpl:6M ep:35K epch:20.41 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.445 data_s:0.073
|
|
INFO 2025-09-09 02:26:43 celerate.py:281 step:87K smpl:6M ep:35K epch:20.45 loss:0.085 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.401 data_s:0.107
|
|
INFO 2025-09-09 02:28:26 celerate.py:281 step:88K smpl:6M ep:35K epch:20.50 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.502 data_s:0.009
|
|
INFO 2025-09-09 02:30:09 celerate.py:281 step:88K smpl:6M ep:35K epch:20.55 loss:0.087 grdn:0.248 lr:2.5e-06 updt_s:
|
|
0.491 data_s:0.021
|
|
INFO 2025-09-09 02:31:52 celerate.py:281 step:88K smpl:6M ep:35K epch:20.59 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.391 data_s:0.124
|
|
INFO 2025-09-09 02:33:35 celerate.py:281 step:88K smpl:6M ep:35K epch:20.64 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.332 data_s:0.180
|
|
INFO 2025-09-09 02:35:18 celerate.py:281 step:88K smpl:6M ep:35K epch:20.69 loss:0.087 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.333 data_s:0.181
|
|
INFO 2025-09-09 02:37:02 celerate.py:281 step:89K smpl:6M ep:35K epch:20.74 loss:0.086 grdn:0.245 lr:2.5e-06 updt_s:
|
|
0.332 data_s:0.185
|
|
INFO 2025-09-09 02:38:47 celerate.py:281 step:89K smpl:6M ep:35K epch:20.78 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.332 data_s:0.190
|
|
INFO 2025-09-09 02:40:30 celerate.py:281 step:89K smpl:6M ep:35K epch:20.83 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.187
|
|
INFO 2025-09-09 02:42:12 celerate.py:281 step:89K smpl:6M ep:35K epch:20.88 loss:0.085 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.331 data_s:0.177
|
|
INFO 2025-09-09 02:43:56 celerate.py:281 step:89K smpl:6M ep:35K epch:20.92 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.185
|
|
INFO 2025-09-09 02:45:39 celerate.py:281 step:90K smpl:6M ep:36K epch:20.97 loss:0.087 grdn:0.252 lr:2.5e-06 updt_s:
|
|
0.335 data_s:0.182
|
|
INFO 2025-09-09 02:47:25 celerate.py:281 step:90K smpl:6M ep:36K epch:21.02 loss:0.087 grdn:0.247 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.197
|
|
INFO 2025-09-09 02:49:08 celerate.py:281 step:90K smpl:6M ep:36K epch:21.06 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.182
|
|
INFO 2025-09-09 02:50:51 celerate.py:281 step:90K smpl:6M ep:36K epch:21.11 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.334 data_s:0.181
|
|
INFO 2025-09-09 02:52:34 celerate.py:281 step:90K smpl:6M ep:36K epch:21.16 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.183
|
|
INFO 2025-09-09 02:54:17 celerate.py:281 step:91K smpl:6M ep:36K epch:21.20 loss:0.085 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.342 data_s:0.173
|
|
INFO 2025-09-09 02:56:01 celerate.py:281 step:91K smpl:6M ep:36K epch:21.25 loss:0.086 grdn:0.246 lr:2.5e-06 updt_s:
|
|
0.331 data_s:0.189
|
|
INFO 2025-09-09 02:57:44 celerate.py:281 step:91K smpl:6M ep:36K epch:21.30 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.182
|
|
INFO 2025-09-09 02:59:27 celerate.py:281 step:91K smpl:6M ep:36K epch:21.34 loss:0.087 grdn:0.246 lr:2.5e-06 updt_s:
|
|
0.341 data_s:0.175
|
|
INFO 2025-09-09 03:01:11 celerate.py:281 step:91K smpl:6M ep:36K epch:21.39 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.188
|
|
INFO 2025-09-09 03:02:54 celerate.py:281 step:92K smpl:6M ep:36K epch:21.44 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.328 data_s:0.186
|
|
INFO 2025-09-09 03:04:39 celerate.py:281 step:92K smpl:6M ep:36K epch:21.48 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.195
|
|
INFO 2025-09-09 03:06:22 celerate.py:281 step:92K smpl:6M ep:36K epch:21.53 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.183
|
|
INFO 2025-09-09 03:08:04 celerate.py:281 step:92K smpl:6M ep:37K epch:21.58 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.462 data_s:0.051
|
|
INFO 2025-09-09 03:09:48 celerate.py:281 step:92K smpl:6M ep:37K epch:21.62 loss:0.086 grdn:0.248 lr:2.5e-06 updt_s:
|
|
0.407 data_s:0.108
|
|
INFO 2025-09-09 03:11:32 celerate.py:281 step:93K smpl:6M ep:37K epch:21.67 loss:0.086 grdn:0.232 lr:2.5e-06 updt_s:
|
|
0.333 data_s:0.185
|
|
INFO 2025-09-09 03:13:15 celerate.py:281 step:93K smpl:6M ep:37K epch:21.72 loss:0.085 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.187
|
|
INFO 2025-09-09 03:14:58 celerate.py:281 step:93K smpl:6M ep:37K epch:21.77 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.357 data_s:0.156
|
|
INFO 2025-09-09 03:16:41 celerate.py:281 step:93K smpl:6M ep:37K epch:21.81 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.487 data_s:0.027
|
|
INFO 2025-09-09 03:18:25 celerate.py:281 step:93K smpl:6M ep:37K epch:21.86 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-09 03:20:08 celerate.py:281 step:94K smpl:6M ep:37K epch:21.91 loss:0.087 grdn:0.247 lr:2.5e-06 updt_s:
|
|
0.512 data_s:0.003
|
|
INFO 2025-09-09 03:21:51 celerate.py:281 step:94K smpl:6M ep:37K epch:21.95 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.508 data_s:0.004
|
|
INFO 2025-09-09 03:23:38 celerate.py:281 step:94K smpl:6M ep:37K epch:22.00 loss:0.085 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.429 data_s:0.104
|
|
INFO 2025-09-09 03:25:20 celerate.py:281 step:94K smpl:6M ep:37K epch:22.05 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.328 data_s:0.183
|
|
INFO 2025-09-09 03:27:04 celerate.py:281 step:94K smpl:6M ep:37K epch:22.09 loss:0.086 grdn:0.246 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.191
|
|
INFO 2025-09-09 03:28:47 celerate.py:281 step:95K smpl:6M ep:37K epch:22.14 loss:0.086 grdn:0.246 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.186
|
|
INFO 2025-09-09 03:30:30 celerate.py:281 step:95K smpl:6M ep:38K epch:22.19 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.346 data_s:0.166
|
|
INFO 2025-09-09 03:32:13 celerate.py:281 step:95K smpl:6M ep:38K epch:22.23 loss:0.086 grdn:0.247 lr:2.5e-06 updt_s:
|
|
0.334 data_s:0.181
|
|
INFO 2025-09-09 03:33:56 celerate.py:281 step:95K smpl:6M ep:38K epch:22.28 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.402 data_s:0.110
|
|
INFO 2025-09-09 03:35:39 celerate.py:281 step:95K smpl:6M ep:38K epch:22.33 loss:0.085 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.353 data_s:0.161
|
|
INFO 2025-09-09 03:37:22 celerate.py:281 step:96K smpl:6M ep:38K epch:22.37 loss:0.086 grdn:0.235 lr:2.5e-06 updt_s:
|
|
0.356 data_s:0.158
|
|
INFO 2025-09-09 03:39:04 celerate.py:281 step:96K smpl:6M ep:38K epch:22.42 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.379 data_s:0.131
|
|
INFO 2025-09-09 03:40:49 celerate.py:281 step:96K smpl:6M ep:38K epch:22.47 loss:0.085 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.344 data_s:0.175
|
|
INFO 2025-09-09 03:42:32 celerate.py:281 step:96K smpl:6M ep:38K epch:22.51 loss:0.086 grdn:0.236 lr:2.5e-06 updt_s:
|
|
0.331 data_s:0.185
|
|
INFO 2025-09-09 03:44:15 celerate.py:281 step:96K smpl:6M ep:38K epch:22.56 loss:0.086 grdn:0.244 lr:2.5e-06 updt_s:
|
|
0.331 data_s:0.183
|
|
INFO 2025-09-09 03:45:58 celerate.py:281 step:97K smpl:6M ep:38K epch:22.61 loss:0.086 grdn:0.238 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.184
|
|
INFO 2025-09-09 03:47:43 celerate.py:281 step:97K smpl:6M ep:38K epch:22.65 loss:0.086 grdn:0.248 lr:2.5e-06 updt_s:
|
|
0.331 data_s:0.188
|
|
INFO 2025-09-09 03:49:27 celerate.py:281 step:97K smpl:6M ep:38K epch:22.70 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.333 data_s:0.185
|
|
INFO 2025-09-09 03:51:10 celerate.py:281 step:97K smpl:6M ep:39K epch:22.75 loss:0.085 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.185
|
|
INFO 2025-09-09 03:52:54 celerate.py:281 step:97K smpl:6M ep:39K epch:22.79 loss:0.086 grdn:0.247 lr:2.5e-06 updt_s:
|
|
0.330 data_s:0.192
|
|
INFO 2025-09-09 03:54:37 celerate.py:281 step:98K smpl:6M ep:39K epch:22.84 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.185
|
|
INFO 2025-09-09 03:56:21 celerate.py:281 step:98K smpl:6M ep:39K epch:22.89 loss:0.086 grdn:0.237 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.187
|
|
INFO 2025-09-09 03:58:04 celerate.py:281 step:98K smpl:6M ep:39K epch:22.94 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.185
|
|
INFO 2025-09-09 03:59:46 celerate.py:281 step:98K smpl:6M ep:39K epch:22.98 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.183
|
|
INFO 2025-09-09 04:01:32 celerate.py:281 step:98K smpl:6M ep:39K epch:23.03 loss:0.087 grdn:0.250 lr:2.5e-06 updt_s:
|
|
0.376 data_s:0.151
|
|
INFO 2025-09-09 04:03:16 celerate.py:281 step:99K smpl:6M ep:39K epch:23.08 loss:0.086 grdn:0.241 lr:2.5e-06 updt_s:
|
|
0.329 data_s:0.187
|
|
INFO 2025-09-09 04:04:59 celerate.py:281 step:99K smpl:6M ep:39K epch:23.12 loss:0.086 grdn:0.243 lr:2.5e-06 updt_s:
|
|
0.379 data_s:0.136
|
|
INFO 2025-09-09 04:06:42 celerate.py:281 step:99K smpl:6M ep:39K epch:23.17 loss:0.086 grdn:0.240 lr:2.5e-06 updt_s:
|
|
0.513 data_s:0.003
|
|
INFO 2025-09-09 04:08:25 celerate.py:281 step:99K smpl:6M ep:39K epch:23.22 loss:0.086 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.510 data_s:0.003
|
|
INFO 2025-09-09 04:10:07 celerate.py:281 step:99K smpl:6M ep:39K epch:23.26 loss:0.087 grdn:0.242 lr:2.5e-06 updt_s:
|
|
0.470 data_s:0.039
|
|
INFO 2025-09-09 04:11:50 celerate.py:281 step:100K smpl:6M ep:39K epch:23.31 loss:0.086 grdn:0.247 lr:2.5e-06 updt_s
|
|
:0.347 data_s:0.169
|
|
INFO 2025-09-09 04:13:34 celerate.py:281 step:100K smpl:6M ep:40K epch:23.36 loss:0.085 grdn:0.237 lr:2.5e-06 updt_s
|
|
:0.330 data_s:0.185
|
|
INFO 2025-09-09 04:15:17 celerate.py:281 step:100K smpl:6M ep:40K epch:23.40 loss:0.086 grdn:0.239 lr:2.5e-06 updt_s
|
|
:0.356 data_s:0.156
|
|
INFO 2025-09-09 04:15:17 celerate.py:295 Checkpoint policy after step 100000
|
|
INFO 2025-09-09 04:15:18 celerate.py:359 End of training
|
|
(lerobot) jade_choghari@hf-dgx-01:~/lerobot$ clear
|
|
(lerobot) jade_choghari@hf-dgx-01:~/lerobot$ tmux capture-pane -pS - > tmux_log.txt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|