💥 Add generic converter pipeline (#104)

* Add generic converter pipeline Co-authored-by: Codex <codex@openai.com> * Update generic converter README Co-authored-by: Codex <codex@openai.com> * Simplify generic converter README Co-authored-by: Codex <codex@openai.com> * Simplify adapter task loading API Co-authored-by: Codex <codex@openai.com> * Require adapter output path Co-authored-by: Codex <codex@openai.com> * Use adapter temp output path for LIBERO Co-authored-by: Codex <codex@openai.com> * Remove LIBERO changes from generic converter PR Co-authored-by: Codex <codex@openai.com> * update readme --------- Co-authored-by: Codex <codex@openai.com>
2026-07-15 16:31:43 +00:00 · 2026-06-11 22:16:44 -07:00
parent 2ef2370d66
commit f40e09f481
6 changed files with 408 additions and 1 deletions
@@ -0,0 +1,39 @@
+import shutil
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+TaskMetadata = Mapping[str, Any]
+FeatureSpec = Mapping[str, dict]
+
+
+@dataclass(frozen=True)
+class ConversionTask:
+    """One independently convertible raw input file and adapter metadata."""
+
+    input_path: Path
+    output_path: Path
+    local_repo_id: str | None = None
+    metadata: TaskMetadata = field(default_factory=dict)
+
+
+def setup_logger():
+    import sys
+
+    from datatrove.utils.logging import logger
+
+    logger.remove()
+    logger.add(sys.stdout, level="INFO", colorize=True)
+    return logger
+
+
+def unique_strings(values: Sequence[str]) -> list[str]:
+    result = []
+    seen = set()
+    for value in values:
+        if value in seen:
+            continue
+        result.append(value)
+        seen.add(value)
+    return result