admin/lerobot
1
0
Fork 0
mirror of https://github.com/huggingface/lerobot.git synced 2026-07-25 10:46:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

further memory optimizations needed due to calling pd.concat

Browse Source
This commit is contained in:
Michel Aractingi
2025-09-03 18:49:30 +02:00
parent 1db3401159
commit 992fb177c3
1 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lerobot/datasets/lerobot_dataset.py
+8
View File
@@ -307,6 +307,10 @@ class LeRobotDatasetMetadata:
latest_df = pd.read_parquet(latest_path) latest_df = pd.read_parquet(latest_path)
df = pd.concat([latest_df, df], ignore_index=True) df = pd.concat([latest_df, df], ignore_index=True)
# Memort optimization
del latest_df
gc.collect()
# Write the resulting dataframe from RAM to disk # Write the resulting dataframe from RAM to disk
path = self.root / DEFAULT_EPISODES_PATH.format(chunk_index=chunk_idx, file_index=file_idx) path = self.root / DEFAULT_EPISODES_PATH.format(chunk_index=chunk_idx, file_index=file_idx)
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
@@ -1048,6 +1052,10 @@ class LeRobotDataset(torch.utils.data.Dataset):
latest_df = pd.read_parquet(latest_path) latest_df = pd.read_parquet(latest_path)
df = pd.concat([latest_df, df], ignore_index=True) df = pd.concat([latest_df, df], ignore_index=True)
# Memort optimization
del latest_df
gc.collect()
# Write the resulting dataframe from RAM to disk # Write the resulting dataframe from RAM to disk
path = self.root / self.meta.data_path.format(chunk_index=chunk_idx, file_index=file_idx) path = self.root / self.meta.data_path.format(chunk_index=chunk_idx, file_index=file_idx)
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)