mirror of
https://github.com/huggingface/lerobot.git
synced 2026-07-15 14:02:14 +00:00
refactor(profiling): remove cProfile, keep torch profiler only
Remove cProfile wrapping from the training loop and profiling utilities. The torch profiler already captures fine-grained timing and operator breakdowns; cProfile added redundant overhead without actionable insight for GPU-bound models. - Remove render_cprofile_summary, run_with_cprofile from profiling_utils - Replace cProfile-wrapped calls in lerobot_train with direct calls - Remove cprofile_summaries from artifact index in run_model_profiling - Update tests to match Made-with: Cursor
This commit is contained in:
@@ -54,7 +54,6 @@ from lerobot.utils.profiling_utils import (
|
||||
StepTimingCollector,
|
||||
ensure_dir,
|
||||
make_torch_profiler,
|
||||
run_with_cprofile,
|
||||
write_deterministic_forward_artifacts,
|
||||
write_torch_profiler_outputs,
|
||||
)
|
||||
@@ -231,10 +230,8 @@ def train(cfg: TrainPipelineConfig, accelerator: "Accelerator | None" = None):
|
||||
|
||||
profiling_enabled = cfg.profile_mode != "off"
|
||||
profile_output_dir = None
|
||||
cprofile_dir = None
|
||||
if profiling_enabled and is_main_process and cfg.profile_output_dir is not None:
|
||||
profile_output_dir = ensure_dir(Path(cfg.profile_output_dir))
|
||||
cprofile_dir = ensure_dir(profile_output_dir / "cprofile")
|
||||
logging.info("Profiling enabled. Artifacts will be written to %s", profile_output_dir)
|
||||
|
||||
# Initialize wandb only on main process
|
||||
@@ -260,10 +257,7 @@ def train(cfg: TrainPipelineConfig, accelerator: "Accelerator | None" = None):
|
||||
# Dataset loading synchronization: main process downloads first to avoid race conditions
|
||||
if is_main_process:
|
||||
logging.info("Creating dataset")
|
||||
if cprofile_dir is not None:
|
||||
dataset = run_with_cprofile("dataset_setup", cprofile_dir, make_dataset, cfg)
|
||||
else:
|
||||
dataset = make_dataset(cfg)
|
||||
dataset = make_dataset(cfg)
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
|
||||
@@ -281,21 +275,11 @@ def train(cfg: TrainPipelineConfig, accelerator: "Accelerator | None" = None):
|
||||
|
||||
if is_main_process:
|
||||
logging.info("Creating policy")
|
||||
if is_main_process and cprofile_dir is not None:
|
||||
policy = run_with_cprofile(
|
||||
"policy_setup",
|
||||
cprofile_dir,
|
||||
make_policy,
|
||||
cfg=cfg.policy,
|
||||
ds_meta=dataset.meta,
|
||||
rename_map=cfg.rename_map,
|
||||
)
|
||||
else:
|
||||
policy = make_policy(
|
||||
cfg=cfg.policy,
|
||||
ds_meta=dataset.meta,
|
||||
rename_map=cfg.rename_map,
|
||||
)
|
||||
policy = make_policy(
|
||||
cfg=cfg.policy,
|
||||
ds_meta=dataset.meta,
|
||||
rename_map=cfg.rename_map,
|
||||
)
|
||||
|
||||
if cfg.peft is not None:
|
||||
logging.info("Using PEFT! Wrapping model.")
|
||||
@@ -349,36 +333,16 @@ def train(cfg: TrainPipelineConfig, accelerator: "Accelerator | None" = None):
|
||||
},
|
||||
}
|
||||
|
||||
if is_main_process and cprofile_dir is not None:
|
||||
preprocessor, postprocessor = run_with_cprofile(
|
||||
"processor_setup",
|
||||
cprofile_dir,
|
||||
make_pre_post_processors,
|
||||
policy_cfg=cfg.policy,
|
||||
pretrained_path=processor_pretrained_path,
|
||||
**processor_kwargs,
|
||||
**postprocessor_kwargs,
|
||||
)
|
||||
else:
|
||||
preprocessor, postprocessor = make_pre_post_processors(
|
||||
policy_cfg=cfg.policy,
|
||||
pretrained_path=processor_pretrained_path,
|
||||
**processor_kwargs,
|
||||
**postprocessor_kwargs,
|
||||
)
|
||||
preprocessor, postprocessor = make_pre_post_processors(
|
||||
policy_cfg=cfg.policy,
|
||||
pretrained_path=processor_pretrained_path,
|
||||
**processor_kwargs,
|
||||
**postprocessor_kwargs,
|
||||
)
|
||||
|
||||
if is_main_process:
|
||||
logging.info("Creating optimizer and scheduler")
|
||||
if is_main_process and cprofile_dir is not None:
|
||||
optimizer, lr_scheduler = run_with_cprofile(
|
||||
"optimizer_setup",
|
||||
cprofile_dir,
|
||||
make_optimizer_and_scheduler,
|
||||
cfg,
|
||||
policy,
|
||||
)
|
||||
else:
|
||||
optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, policy)
|
||||
optimizer, lr_scheduler = make_optimizer_and_scheduler(cfg, policy)
|
||||
|
||||
if profiling_enabled and is_main_process and profile_output_dir is not None:
|
||||
logging.info("Recording deterministic forward-pass artifacts")
|
||||
|
||||
@@ -16,13 +16,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import cProfile
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import pstats
|
||||
import statistics
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from numbers import Real
|
||||
from pathlib import Path
|
||||
@@ -37,15 +33,6 @@ def ensure_dir(path: Path) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def render_cprofile_summary(
|
||||
profile: cProfile.Profile, *, sort_by: str = "cumulative", limit: int = 40
|
||||
) -> str:
|
||||
output = io.StringIO()
|
||||
stats = pstats.Stats(profile, stream=output).strip_dirs().sort_stats(sort_by)
|
||||
stats.print_stats(limit)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def write_profiler_table(
|
||||
profiler: Any,
|
||||
output_path: Path,
|
||||
@@ -103,26 +90,6 @@ def write_torch_profiler_outputs(
|
||||
write_profiler_table(profiler, tables_dir / "flops.txt", sort_by="flops")
|
||||
|
||||
|
||||
def run_with_cprofile[T](
|
||||
label: str,
|
||||
output_dir: Path,
|
||||
fn: Callable[..., T],
|
||||
*args: Any,
|
||||
sort_by: str = "cumulative",
|
||||
limit: int = 40,
|
||||
**kwargs: Any,
|
||||
) -> T:
|
||||
ensure_dir(output_dir)
|
||||
profile = cProfile.Profile()
|
||||
profile.enable()
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
profile.disable()
|
||||
summary = render_cprofile_summary(profile, sort_by=sort_by, limit=limit)
|
||||
(output_dir / f"{label}.txt").write_text(summary)
|
||||
|
||||
|
||||
def _stable_float(value: float | int | None) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user