Compare commits

...

16 Commits

Author SHA1 Message Date
Michel Aractingi 2efc74012d Fix default version to v2.1 for vizualizer 2025-07-22 10:19:38 +02:00
Mishig Davaadorj 5e86d6228c rm unneeded comments 2025-06-10 17:01:55 +02:00
Mishig Davaadorj d922371f37 Merge remote-tracking branch 'refs/remotes/origin/update_html_visualizer' into update_html_visualizer 2025-06-10 14:26:58 +02:00
Mishig Davaadorj dcc4456ea7 fix 2025-06-10 14:26:38 +02:00
Mishig 8a01e45290 Merge branch 'main' into update_html_visualizer 2025-06-10 14:16:05 +02:00
pre-commit-ci[bot] 885d0ca618 [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
2025-06-10 10:31:08 +00:00
Mishig Davaadorj 9d9326975f fix 2025-06-10 12:28:50 +02:00
Mishig Davaadorj b39c2468c4 fix race condition 2025-06-10 12:23:34 +02:00
pre-commit-ci[bot] d70b67a330 [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
2025-06-10 09:40:17 +00:00
Mishig Davaadorj 8cc7ec87c9 wip 2025-06-10 11:39:53 +02:00
Mishig Davaadorj b2b88e57df rm html template files 2025-06-09 18:48:34 +02:00
Mishig Davaadorj 3fd24a5802 buildid 2025-06-09 18:47:30 +02:00
Mishig Davaadorj 3e587b42d0 fix typo 2025-06-09 14:54:53 +02:00
Mishig 02c76e3ec5 Merge branch 'main' into update_html_visualizer 2025-06-09 14:52:42 +02:00
pre-commit-ci[bot] 37b06de872 [pre-commit.ci] auto fixes from pre-commit.com hooks
for more information, see https://pre-commit.ci
2025-06-09 12:52:14 +00:00
Mishig Davaadorj ecdf066fae Update HTML visualizer 2025-06-09 14:51:45 +02:00
33 changed files with 8778 additions and 1001 deletions
-57
View File
@@ -17,11 +17,9 @@ import contextlib
import importlib.resources
import json
import logging
from collections.abc import Iterator
from itertools import accumulate
from pathlib import Path
from pprint import pformat
from types import SimpleNamespace
from typing import Any
import datasets
@@ -696,61 +694,6 @@ def create_lerobot_dataset_card(
)
class IterableNamespace(SimpleNamespace):
"""
A namespace object that supports both dictionary-like iteration and dot notation access.
Automatically converts nested dictionaries into IterableNamespaces.
This class extends SimpleNamespace to provide:
- Dictionary-style iteration over keys
- Access to items via both dot notation (obj.key) and brackets (obj["key"])
- Dictionary-like methods: items(), keys(), values()
- Recursive conversion of nested dictionaries
Args:
dictionary: Optional dictionary to initialize the namespace
**kwargs: Additional keyword arguments passed to SimpleNamespace
Examples:
>>> data = {"name": "Alice", "details": {"age": 25}}
>>> ns = IterableNamespace(data)
>>> ns.name
'Alice'
>>> ns.details.age
25
>>> list(ns.keys())
['name', 'details']
>>> for key, value in ns.items():
... print(f"{key}: {value}")
name: Alice
details: IterableNamespace(age=25)
"""
def __init__(self, dictionary: dict[str, Any] = None, **kwargs):
super().__init__(**kwargs)
if dictionary is not None:
for key, value in dictionary.items():
if isinstance(value, dict):
setattr(self, key, IterableNamespace(value))
else:
setattr(self, key, value)
def __iter__(self) -> Iterator[str]:
return iter(vars(self))
def __getitem__(self, key: str) -> Any:
return vars(self)[key]
def items(self):
return vars(self).items()
def values(self):
return vars(self).values()
def keys(self):
return vars(self).keys()
def validate_frame(frame: dict, features: dict):
expected_features = set(features) - set(DEFAULT_FEATURES)
actual_features = set(frame)
@@ -0,0 +1,42 @@
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.*
.yarn/*
!.yarn/patches
!.yarn/plugins
!.yarn/releases
!.yarn/versions
# testing
/coverage
# next.js
/.next/
/out/
/public
# production
/build
# misc
.DS_Store
*.pem
# debug
npm-debug.log*
yarn-debug.log*
yarn-error.log*
.pnpm-debug.log*
# env files (can opt-in for committing if needed)
.env*
# vercel
.vercel
# typescript
*.tsbuildinfo
next-env.d.ts
+57
View File
@@ -0,0 +1,57 @@
# LeRobot Dataset Visualizer
LeRobot Dataset Visualizer is a web application for interactive exploration and visualization of robotics datasets, particularly those in the LeRobot format. It enables users to browse, view, and analyze episodes from large-scale robotics datasets, combining synchronized video playback with rich, interactive data graphs.
## Project Overview
This tool is designed to help robotics researchers and practitioners quickly inspect and understand large, complex datasets. It fetches dataset metadata and episode data (including video and sensor/telemetry data), and provides a unified interface for:
- Navigating between organizations, datasets, and episodes
- Watching episode videos
- Exploring synchronized time-series data with interactive charts
- Paginating through large datasets efficiently
## Key Features
- **Dataset & Episode Navigation:** Quickly jump between organizations, datasets, and episodes using a sidebar and navigation controls.
- **Synchronized Video & Data:** Video playback is synchronized with interactive data graphs for detailed inspection of sensor and control signals.
- **Efficient Data Loading:** Uses parquet and JSON loading for large dataset support, with pagination and chunking.
- **Responsive UI:** Built with React, Next.js, and Tailwind CSS for a fast, modern user experience.
## Technologies Used
- **Next.js** (App Router)
- **React**
- **Recharts** (for data visualization)
- **hyparquet** (for reading Parquet files)
- **Tailwind CSS** (styling)
## Getting Started
First, run the development server:
```bash
npm run dev
# or
yarn dev
# or
pnpm dev
# or
bun dev
```
Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
You can start editing the page by modifying `src/app/page.tsx` or other files in the `src/` directory. The app supports hot-reloading for rapid development.
### Environment Variables
- `DATASET_URL`: (optional) Base URL for dataset hosting (defaults to HuggingFace Datasets).
## Contributing
Contributions, bug reports, and feature requests are welcome! Please open an issue or submit a pull request.
## License
This project is licensed under the MIT License.
@@ -0,0 +1,16 @@
import { dirname } from "path";
import { fileURLToPath } from "url";
import { FlatCompat } from "@eslint/eslintrc";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const compat = new FlatCompat({
baseDirectory: __dirname,
});
const eslintConfig = [
...compat.extends("next/core-web-vitals", "next/typescript"),
];
export default eslintConfig;
@@ -0,0 +1,15 @@
import type { NextConfig } from "next";
import packageJson from './package.json';
const nextConfig: NextConfig = {
typescript: {
ignoreBuildErrors: true,
},
eslint: {
ignoreDuringBuilds: true,
},
generateBuildId: () => packageJson.version,
};
export default nextConfig;
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,32 @@
{
"name": "lerobot-viewer",
"version": "0.1.0",
"private": true,
"scripts": {
"dev": "next dev",
"build": "next build",
"start": "next start",
"lint": "next lint",
"format": "prettier --write ."
},
"dependencies": {
"hyparquet": "^1.12.1",
"next": "15.3.1",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"react-icons": "^5.5.0",
"recharts": "^2.15.3"
},
"devDependencies": {
"@eslint/eslintrc": "^3",
"@tailwindcss/postcss": "^4",
"@types/node": "^20",
"@types/react": "^19",
"@types/react-dom": "^19",
"eslint": "^9",
"eslint-config-next": "15.3.1",
"prettier": "^3.5.3",
"tailwindcss": "^4",
"typescript": "^5"
}
}
@@ -0,0 +1,5 @@
const config = {
plugins: ["@tailwindcss/postcss"],
};
export default config;
@@ -0,0 +1,231 @@
"use client";
import { useState, useEffect, useRef } from "react";
import { useRouter, useSearchParams } from "next/navigation";
import { postParentMessageWithParams } from "@/utils/postParentMessage";
import VideosPlayer from "@/components/videos-player";
import DataRecharts from "@/components/data-recharts";
import PlaybackBar from "@/components/playback-bar";
import { TimeProvider, useTime } from "@/context/time-context";
import Sidebar from "@/components/side-nav";
import Loading from "@/components/loading-component";
export default function EpisodeViewer({
data,
error,
}: {
data?: any;
error?: string;
}) {
if (error) {
return (
<div className="flex h-screen items-center justify-center bg-slate-950 text-red-400">
<div className="max-w-xl p-8 rounded bg-slate-900 border border-red-500 shadow-lg">
<h2 className="text-2xl font-bold mb-4">Something went wrong</h2>
<p className="text-lg font-mono whitespace-pre-wrap mb-4">{error}</p>
</div>
</div>
);
}
return (
<TimeProvider duration={data.duration}>
<EpisodeViewerInner data={data} />
</TimeProvider>
);
}
function EpisodeViewerInner({ data }: { data: any }) {
const {
datasetInfo,
episodeId,
videosInfo,
chartDataGroups,
episodes,
ignoredColumns,
} = data;
const [videosReady, setVideosReady] = useState(!videosInfo.length);
const [chartsReady, setChartsReady] = useState(false);
const isLoading = !videosReady || !chartsReady;
const router = useRouter();
const searchParams = useSearchParams();
// State
// Use context for time sync
const { currentTime, setCurrentTime, setIsPlaying, isPlaying } = useTime();
// Pagination state
const pageSize = 100;
const [currentPage, setCurrentPage] = useState(1);
const totalPages = Math.ceil(episodes.length / pageSize);
const paginatedEpisodes = episodes.slice(
(currentPage - 1) * pageSize,
currentPage * pageSize,
);
// Initialize based on URL time parameter
useEffect(() => {
const timeParam = searchParams.get("t");
if (timeParam) {
const timeValue = parseFloat(timeParam);
if (!isNaN(timeValue)) {
setCurrentTime(timeValue);
}
}
}, []);
// sync with parent window hf.co/spaces
useEffect(() => {
postParentMessageWithParams((params: URLSearchParams) => {
params.set("path", window.location.pathname + window.location.search);
});
}, []);
// Initialize based on URL time parameter
useEffect(() => {
// Initialize page based on current episode
const episodeIndex = episodes.indexOf(episodeId);
if (episodeIndex !== -1) {
setCurrentPage(Math.floor(episodeIndex / pageSize) + 1);
}
// Add keyboard event listener
window.addEventListener("keydown", handleKeyDown);
return () => {
window.removeEventListener("keydown", handleKeyDown);
};
}, [episodes, episodeId, pageSize, searchParams]);
// Only update URL ?t= param when the integer second changes
const lastUrlSecondRef = useRef<number>(-1);
useEffect(() => {
if (isPlaying) return;
const currentSec = Math.floor(currentTime);
if (currentTime > 0 && lastUrlSecondRef.current !== currentSec) {
lastUrlSecondRef.current = currentSec;
const newParams = new URLSearchParams(searchParams.toString());
newParams.set("t", currentSec.toString());
// Replace state instead of pushing to avoid navigation stack bloat
window.history.replaceState(
{},
"",
`${window.location.pathname}?${newParams.toString()}`,
);
postParentMessageWithParams((params: URLSearchParams) => {
params.set("path", window.location.pathname + window.location.search);
});
}
}, [isPlaying, currentTime, searchParams]);
// Handle keyboard shortcuts
const handleKeyDown = (e: KeyboardEvent) => {
const { key } = e;
if (key === " ") {
e.preventDefault();
setIsPlaying((prev: boolean) => !prev);
} else if (key === "ArrowDown" || key === "ArrowUp") {
e.preventDefault();
const nextEpisodeId = key === "ArrowDown" ? episodeId + 1 : episodeId - 1;
const lowestEpisodeId = episodes[0];
const highestEpisodeId = episodes[episodes.length - 1];
if (
nextEpisodeId >= lowestEpisodeId &&
nextEpisodeId <= highestEpisodeId
) {
router.push(`./episode_${nextEpisodeId}`);
}
}
};
// Pagination functions
const nextPage = () => {
if (currentPage < totalPages) {
setCurrentPage((prev) => prev + 1);
}
};
const prevPage = () => {
if (currentPage > 1) {
setCurrentPage((prev) => prev - 1);
}
};
return (
<div className="flex h-screen max-h-screen bg-slate-950 text-gray-200">
{/* Sidebar */}
<Sidebar
datasetInfo={datasetInfo}
paginatedEpisodes={paginatedEpisodes}
episodeId={episodeId}
totalPages={totalPages}
currentPage={currentPage}
prevPage={prevPage}
nextPage={nextPage}
/>
{/* Content */}
<div
className={`flex max-h-screen flex-col gap-4 p-4 md:flex-1 relative ${isLoading ? "overflow-hidden" : "overflow-y-auto"}`}
>
{isLoading && <Loading />}
<div className="flex items-center justify-start my-4">
<a
href="https://github.com/huggingface/lerobot"
target="_blank"
className="block"
>
<img
src="https://github.com/huggingface/lerobot/raw/main/media/lerobot-logo-thumbnail.png"
alt="LeRobot Logo"
className="w-32"
/>
</a>
<div>
<a
href={`https://huggingface.co/datasets/${datasetInfo.repoId}`}
target="_blank"
>
<p className="text-lg font-semibold">{datasetInfo.repoId}</p>
</a>
<p className="font-mono text-lg font-semibold">
episode {episodeId}
</p>
</div>
</div>
{/* Videos */}
{videosInfo.length && (
<VideosPlayer
videosInfo={videosInfo}
onVideosReady={() => setVideosReady(true)}
/>
)}
{/* Graph */}
<div className="mb-4">
<DataRecharts
data={chartDataGroups}
onChartsReady={() => setChartsReady(true)}
/>
{ignoredColumns.length > 0 && (
<p className="mt-2 text-orange-700">
Columns{" "}
<span className="font-mono">{ignoredColumns.join(", ")}</span> are
NOT shown since the visualizer currently does not support 2D or 3D
data.
</p>
)}
</div>
<PlaybackBar />
</div>
</div>
);
}
@@ -0,0 +1,28 @@
"use client";
import React from "react";
export default function Error({
error,
reset,
}: {
error: Error & { digest?: string };
reset: () => void;
}) {
return (
<div className="flex h-screen items-center justify-center bg-slate-950 text-red-400">
<div className="max-w-xl p-8 rounded bg-slate-900 border border-red-500 shadow-lg">
<h2 className="text-2xl font-bold mb-4">Something went wrong</h2>
<p className="text-lg font-mono whitespace-pre-wrap mb-4">
{error.message}
</p>
<button
className="mt-4 px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600"
onClick={() => reset()}
>
Try Again
</button>
</div>
</div>
);
}
@@ -0,0 +1,280 @@
import {
DatasetMetadata,
fetchJson,
fetchParquetFile,
formatStringWithVars,
readParquetColumn,
} from "@/utils/parquetUtils";
import { pick } from "@/utils/pick";
const DATASET_URL =
process.env.DATASET_URL || "https://huggingface.co/datasets";
const DEFAULT_REVISION = "v2.1";
const SERIES_NAME_DELIMITER = " | ";
export async function getEpisodeData(
org: string,
dataset: string,
episodeId: number,
) {
const repoId = `${org}/${dataset}`;
try {
const episode_chunk = Math.floor(0 / 1000);
const jsonUrl = `${DATASET_URL}/${repoId}/resolve/${DEFAULT_REVISION}/meta/info.json`;
const info = await fetchJson<DatasetMetadata>(jsonUrl);
// Dataset information
const datasetInfo = {
repoId,
total_frames: info.total_frames,
total_episodes: info.total_episodes,
fps: info.fps,
};
// Generate list of episodes
const episodes =
process.env.EPISODES === undefined
? Array.from(
{ length: datasetInfo.total_episodes },
// episode id starts from 0
(_, i) => i,
)
: process.env.EPISODES
.split(/\s+/)
.map((x) => parseInt(x.trim(), 10))
.filter((x) => !isNaN(x));
// Videos information
const videosInfo = Object.entries(info.features)
.filter(([key, value]) => value.dtype === "video")
.map(([key, _]) => {
const videoPath = formatStringWithVars(info.video_path, {
video_key: key,
episode_chunk: episode_chunk.toString().padStart(3, "0"),
episode_index: episodeId.toString().padStart(6, "0"),
});
return {
filename: key,
url: `${DATASET_URL}/${repoId}/resolve/${DEFAULT_REVISION}/` + videoPath,
};
});
// Column data
const columnNames = Object.entries(info.features)
.filter(
([key, value]) =>
["float32", "int32"].includes(value.dtype) &&
value.shape.length === 1,
)
.map(([key, { shape }]) => ({ key, length: shape[0] }));
// Exclude specific columns
const excludedColumns = [
"timestamp",
"frame_index",
"episode_index",
"index",
"task_index",
];
const filteredColumns = columnNames.filter(
(column) => !excludedColumns.includes(column.key),
);
const filteredColumnNames = [
"timestamp",
...filteredColumns.map((column) => column.key),
];
const columns = filteredColumns.map(({ key }) => {
let column_names = info.features[key].names;
while (typeof column_names === "object") {
if (Array.isArray(column_names)) break;
column_names = Object.values(column_names ?? {})[0];
}
return {
key,
value: Array.isArray(column_names)
? column_names.map((name) => `${key}${SERIES_NAME_DELIMITER}${name}`)
: Array.from(
{ length: columnNames.find((c) => c.key === key)?.length ?? 1 },
(_, i) => `${key}${SERIES_NAME_DELIMITER}${i}`,
),
};
});
const parquetUrl =
`${DATASET_URL}/${repoId}/resolve/${DEFAULT_REVISION}/` +
formatStringWithVars(info.data_path, {
episode_chunk: episode_chunk.toString().padStart(3, "0"),
episode_index: episodeId.toString().padStart(6, "0"),
});
const arrayBuffer = await fetchParquetFile(parquetUrl);
const data = await readParquetColumn(arrayBuffer, filteredColumnNames);
// Flatten and map to array of objects for chartData
const seriesNames = [
"timestamp",
...columns.map(({ value }) => value).flat(),
];
const chartData = data.map((row) => {
const flatRow = row.flat();
const obj: Record<string, number> = {};
seriesNames.forEach((key, idx) => {
obj[key] = flatRow[idx];
});
return obj;
});
// List of columns that are ignored (e.g., 2D or 3D data)
const ignoredColumns = Object.entries(info.features)
.filter(
([key, value]) =>
["float32", "int32"].includes(value.dtype) && value.shape.length > 1,
)
.map(([key]) => key);
// 1. Group all numeric keys by suffix (excluding 'timestamp')
const numericKeys = seriesNames.filter((k) => k !== "timestamp");
const suffixGroupsMap: Record<string, string[]> = {};
for (const key of numericKeys) {
const parts = key.split(SERIES_NAME_DELIMITER);
const suffix = parts[1] || parts[0]; // fallback to key if no delimiter
if (!suffixGroupsMap[suffix]) suffixGroupsMap[suffix] = [];
suffixGroupsMap[suffix].push(key);
}
const suffixGroups = Object.values(suffixGroupsMap);
// 2. Compute min/max for each suffix group as a whole
const groupStats: Record<string, { min: number; max: number }> = {};
suffixGroups.forEach((group) => {
let min = Infinity,
max = -Infinity;
for (const row of chartData) {
for (const key of group) {
const v = row[key];
if (typeof v === "number" && !isNaN(v)) {
if (v < min) min = v;
if (v > max) max = v;
}
}
}
// Use the first key in the group as the group id
groupStats[group[0]] = { min, max };
});
// 3. Group suffix groups by similar scale (treat each suffix group as a unit)
const scaleGroups: Record<string, string[][]> = {};
const used = new Set<string>();
const SCALE_THRESHOLD = 2;
for (const group of suffixGroups) {
const groupId = group[0];
if (used.has(groupId)) continue;
const { min, max } = groupStats[groupId];
if (!isFinite(min) || !isFinite(max)) continue;
const logMin = Math.log10(Math.abs(min) + 1e-9);
const logMax = Math.log10(Math.abs(max) + 1e-9);
const unit: string[][] = [group];
used.add(groupId);
for (const other of suffixGroups) {
const otherId = other[0];
if (used.has(otherId) || otherId === groupId) continue;
const { min: omin, max: omax } = groupStats[otherId];
if (!isFinite(omin) || !isFinite(omax) || omin === omax) continue;
const ologMin = Math.log10(Math.abs(omin) + 1e-9);
const ologMax = Math.log10(Math.abs(omax) + 1e-9);
if (
Math.abs(logMin - ologMin) <= SCALE_THRESHOLD &&
Math.abs(logMax - ologMax) <= SCALE_THRESHOLD
) {
unit.push(other);
used.add(otherId);
}
}
scaleGroups[groupId] = unit;
}
// 4. Flatten scaleGroups into chartGroups (array of arrays of keys)
const chartGroups: string[][] = Object.values(scaleGroups)
.sort((a, b) => b.length - a.length)
.flatMap((suffixGroupArr) => {
// suffixGroupArr is array of suffix groups (each is array of keys)
const merged = suffixGroupArr.flat();
if (merged.length > 6) {
const subgroups = [];
for (let i = 0; i < merged.length; i += 6) {
subgroups.push(merged.slice(i, i + 6));
}
return subgroups;
}
return [merged];
});
const duration = chartData[chartData.length - 1].timestamp;
// Utility: group row keys by suffix
function groupRowBySuffix(row: Record<string, number>): Record<string, any> {
const result: Record<string, any> = {};
const suffixGroups: Record<string, Record<string, number>> = {};
for (const [key, value] of Object.entries(row)) {
if (key === "timestamp") {
result["timestamp"] = value;
continue;
}
const parts = key.split(SERIES_NAME_DELIMITER);
if (parts.length === 2) {
const [prefix, suffix] = parts;
if (!suffixGroups[suffix]) suffixGroups[suffix] = {};
suffixGroups[suffix][prefix] = value;
} else {
result[key] = value;
}
}
for (const [suffix, group] of Object.entries(suffixGroups)) {
const keys = Object.keys(group);
if (keys.length === 1) {
// Use the full original name as the key
const fullName = `${keys[0]}${SERIES_NAME_DELIMITER}${suffix}`;
result[fullName] = group[keys[0]];
} else {
result[suffix] = group;
}
}
return result;
}
const chartDataGroups = chartGroups.map((group) =>
chartData.map((row) => groupRowBySuffix(pick(row, [...group, "timestamp"])))
);
return {
datasetInfo,
episodeId,
videosInfo,
chartDataGroups,
episodes,
ignoredColumns,
duration,
};
} catch (err) {
console.error("Error loading episode data:", err);
throw err;
}
}
// Safe wrapper for UI error display
export async function getEpisodeDataSafe(
org: string,
dataset: string,
episodeId: number,
): Promise<{ data?: any; error?: string }> {
try {
const data = await getEpisodeData(org, dataset, episodeId);
return { data };
} catch (err: any) {
// Only expose the error message, not stack or sensitive info
return { error: err?.message || String(err) || "Unknown error" };
}
}
@@ -0,0 +1,28 @@
import EpisodeViewer from "./episode-viewer";
import { getEpisodeDataSafe } from "./fetch-data";
export const dynamic = "force-dynamic";
export async function generateMetadata({
params,
}: {
params: { org: string; dataset: string; episode: string };
}) {
const { org, dataset, episode } = params;
return {
title: `${org}/${dataset} | episode ${episode}`,
};
}
export default async function EpisodePage({
params,
}: {
params: { org: string; dataset: string; episode: string };
}) {
// episode is like 'episode_1'
const { org, dataset, episode } = params;
// fetchData should be updated if needed to support this path pattern
const episodeNumber = Number(episode.replace(/^episode_/, ""));
const { data, error } = await getEpisodeDataSafe(org, dataset, episodeNumber);
return <EpisodeViewer data={data} error={error} />;
}
@@ -0,0 +1,14 @@
import { redirect } from "next/navigation";
export default function DatasetRootPage({
params,
}: {
params: { org: string; dataset: string };
}) {
const episodeN = process.env.EPISODES
?.split(/\s+/)
.map((x) => parseInt(x.trim(), 10))
.filter((x) => !isNaN(x))[0] ?? 0;
redirect(`/${params.org}/${params.dataset}/episode_${episodeN}`);
}
@@ -0,0 +1,104 @@
"use client";
import React, { useEffect, useRef } from "react";
import Link from "next/link";
import { useRouter, useSearchParams } from "next/navigation";
import { postParentMessageWithParams } from "@/utils/postParentMessage";
type ExploreGridProps = {
datasets: Array<{ id: string; videoUrl: string | null }>;
currentPage: number;
totalPages: number;
};
export default function ExploreGrid({
datasets,
currentPage,
totalPages,
}: ExploreGridProps) {
// sync with parent window hf.co/spaces
useEffect(() => {
postParentMessageWithParams((params: URLSearchParams) => {
params.set("path", window.location.pathname + window.location.search);
});
}, []);
// Create an array of refs for each video
const videoRefs = useRef<(HTMLVideoElement | null)[]>([]);
return (
<main className="p-8">
<h1 className="text-2xl font-bold mb-6">Explore LeRobot Datasets</h1>
<div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 gap-6">
{datasets.map((ds, idx) => (
<Link
key={ds.id}
href={`/${ds.id}`}
className="relative border rounded-lg p-4 bg-white shadow hover:shadow-lg transition overflow-hidden h-48 flex items-end group"
onMouseEnter={() => {
const vid = videoRefs.current[idx];
if (vid) vid.play();
}}
onMouseLeave={() => {
const vid = videoRefs.current[idx];
if (vid) {
vid.pause();
vid.currentTime = 0;
}
}}
>
<video
ref={(el) => {
videoRefs.current[idx] = el;
}}
src={ds.videoUrl || undefined}
className="absolute top-0 left-0 w-full h-full object-cover object-center z-0"
loop
muted
playsInline
preload="metadata"
onTimeUpdate={(e) => {
const vid = e.currentTarget;
if (vid.currentTime >= 15) {
vid.pause();
vid.currentTime = 0;
}
}}
/>
<div className="absolute top-0 left-0 w-full h-full bg-black/40 z-10 pointer-events-none" />
<div className="relative z-20 font-mono text-blue-100 break-all text-sm bg-black/60 backdrop-blur px-2 py-1 rounded shadow">
{ds.id}
</div>
</Link>
))}
</div>
<div className="flex justify-center mt-8 gap-4">
{currentPage > 1 && (
<button
className="px-6 py-2 bg-gray-600 text-white rounded shadow hover:bg-gray-700 transition"
onClick={() => {
const params = new URLSearchParams(window.location.search);
params.set("p", (currentPage - 1).toString());
window.location.search = params.toString();
}}
>
Previous
</button>
)}
{currentPage < totalPages && (
<button
className="px-6 py-2 bg-blue-600 text-white rounded shadow hover:bg-blue-700 transition"
onClick={() => {
const params = new URLSearchParams(window.location.search);
params.set("p", (currentPage + 1).toString());
window.location.search = params.toString();
}}
>
Next
</button>
)}
</div>
</main>
);
}
@@ -0,0 +1,96 @@
import React from "react";
import ExploreGrid from "./explore-grid";
import {
DatasetMetadata,
fetchJson,
formatStringWithVars,
} from "@/utils/parquetUtils";
export default async function ExplorePage({
searchParams,
}: {
searchParams: { p?: string };
}) {
let datasets: any[] = [];
let currentPage = 1;
let totalPages = 1;
try {
const res = await fetch(
"https://huggingface.co/api/datasets?sort=lastModified&filter=LeRobot",
{
cache: "no-store",
},
);
if (!res.ok) throw new Error("Failed to fetch datasets");
const data = await res.json();
const allDatasets = data.datasets || data;
// Use searchParams from props
const page = parseInt(searchParams?.p || "1", 10);
const perPage = 30;
currentPage = page;
totalPages = Math.ceil(allDatasets.length / perPage);
const startIdx = (currentPage - 1) * perPage;
const endIdx = startIdx + perPage;
datasets = allDatasets.slice(startIdx, endIdx);
} catch (e) {
return <div className="p-8 text-red-600">Failed to load datasets.</div>;
}
// Default to v2.1 revision for dataset loading
const DEFAULT_REVISION = "v2.1";
// Fetch episode 0 data for each dataset
const datasetWithVideos = (
await Promise.all(
datasets.map(async (ds: any) => {
try {
const [org, dataset] = ds.id.split("/");
const repoId = `${org}/${dataset}`;
const jsonUrl = `https://huggingface.co/datasets/${repoId}/resolve/${DEFAULT_REVISION}/meta/info.json`;
const info = await fetchJson<DatasetMetadata>(jsonUrl);
const videoEntry = Object.entries(info.features).find(
([key, value]) => value.dtype === "video",
);
let videoUrl: string | null = null;
if (videoEntry) {
const [key] = videoEntry;
const videoPath = formatStringWithVars(info.video_path, {
video_key: key,
episode_chunk: "0".padStart(3, "0"),
episode_index: "0".padStart(6, "0"),
});
const url =
`https://huggingface.co/datasets/${repoId}/resolve/${DEFAULT_REVISION}/` +
videoPath;
// Check if videoUrl exists (status 200)
try {
const headRes = await fetch(url, { method: "HEAD" });
if (headRes.ok) {
videoUrl = url;
}
} catch (e) {
// If fetch fails, videoUrl remains null
}
}
return videoUrl ? { id: repoId, videoUrl } : null;
} catch (err) {
console.error(
`Failed to fetch or parse dataset info for ${ds.id}:`,
err,
);
return null;
}
}),
)
).filter(Boolean) as { id: string; videoUrl: string | null }[];
return (
<ExploreGrid
datasets={datasetWithVideos}
currentPage={currentPage}
totalPages={totalPages}
/>
);
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

@@ -0,0 +1,46 @@
@import "tailwindcss";
:root {
--background: #ffffff;
--foreground: #171717;
}
@theme inline {
--color-background: var(--background);
--color-foreground: var(--foreground);
--font-sans: var(--font-geist-sans);
--font-mono: var(--font-geist-mono);
}
@media (prefers-color-scheme: dark) {
:root {
--background: #0a0a0a;
--foreground: #ededed;
}
}
body {
background: var(--background);
color: var(--foreground);
font-family: Arial, Helvetica, sans-serif;
}
.video-background {
@apply fixed top-0 right-0 bottom-0 left-0 -z-10 overflow-hidden w-screen h-screen;
}
.video-background iframe {
@apply absolute top-1/2 left-1/2 border-0 pointer-events-none bg-black;
width: 100vw;
height: 100vh;
transform: translate(-50%, -50%);
}
@media (min-aspect-ratio: 16/9) {
.video-background iframe {
height: 56.25vw;
}
}
@media (max-aspect-ratio: 16/9) {
.video-background iframe {
width: 177.78vh;
}
}
@@ -0,0 +1,22 @@
import type { Metadata } from "next";
import { Inter } from "next/font/google";
import "./globals.css";
const inter = Inter({ subsets: ["latin"] });
export const metadata: Metadata = {
title: "LeRobot Dataset Visualizer",
description: "Visualization of LeRobot Datasets",
};
export default function RootLayout({
children,
}: {
children: React.ReactNode;
}) {
return (
<html lang="en">
<body className={inter.className}>{children}</body>
</html>
);
}
@@ -0,0 +1,177 @@
"use client";
import { useEffect, useRef } from "react";
import Link from "next/link";
import { useRouter } from "next/navigation";
import { redirect } from "next/navigation";
export default function Home({
searchParams,
}: {
searchParams: { [key: string]: string | undefined };
}) {
// Redirect to the first episode of the dataset if REPO_ID is defined
if (process.env.REPO_ID) {
const episodeN = process.env.EPISODES
?.split(/\s+/)
.map((x) => parseInt(x.trim(), 10))
.filter((x) => !isNaN(x))[0] ?? 0;
redirect(`/${process.env.REPO_ID}/episode_${episodeN}`);
}
// sync with hf.co/spaces URL params
if (searchParams.path) {
redirect(searchParams.path);
}
// legacy sync with hf.co/spaces URL params
let redirectUrl: string | null = null;
if (searchParams?.dataset && searchParams?.episode) {
redirectUrl = `/${searchParams.dataset}/episode_${searchParams.episode}`;
} else if (searchParams?.dataset) {
redirectUrl = `/${searchParams.dataset}`;
}
if (redirectUrl && searchParams?.t) {
redirectUrl += `?t=${searchParams.t}`;
}
if (redirectUrl) {
redirect(redirectUrl);
}
const playerRef = useRef<any>(null);
useEffect(() => {
// Load YouTube IFrame API if not already present
if (!(window as any).YT) {
const tag = document.createElement("script");
tag.src = "https://www.youtube.com/iframe_api";
document.body.appendChild(tag);
}
let interval: NodeJS.Timeout;
(window as any).onYouTubeIframeAPIReady = () => {
playerRef.current = new (window as any).YT.Player("yt-bg-player", {
videoId: "Er8SPJsIYr0",
playerVars: {
autoplay: 1,
mute: 1,
controls: 0,
showinfo: 0,
modestbranding: 1,
rel: 0,
loop: 1,
fs: 0,
playlist: "Er8SPJsIYr0",
start: 0,
},
events: {
onReady: (event: any) => {
event.target.playVideo();
event.target.mute();
interval = setInterval(() => {
const t = event.target.getCurrentTime();
if (t >= 60) {
event.target.seekTo(0);
}
}, 500);
},
},
});
};
return () => {
if (interval) clearInterval(interval);
if (playerRef.current && playerRef.current.destroy)
playerRef.current.destroy();
};
}, []);
const inputRef = useRef<HTMLInputElement>(null);
const router = useRouter();
const handleGo = (e: React.FormEvent) => {
e.preventDefault();
const value = inputRef.current?.value.trim();
if (value) {
router.push(value);
}
};
return (
<div className="relative h-screen w-screen overflow-hidden">
{/* YouTube Video Background */}
<div className="video-background">
<div id="yt-bg-player" />
</div>
{/* Overlay */}
<div className="fixed top-0 right-0 bottom-0 left-0 bg-black/60 -z-0" />
{/* Centered Content */}
<div className="relative z-10 h-screen flex flex-col items-center justify-center text-white text-center">
<h1 className="text-4xl md:text-5xl font-bold mb-6 drop-shadow-lg">
LeRobot Dataset Visualizer
</h1>
<a
href="https://x.com/RemiCadene/status/1825455895561859185"
target="_blank"
rel="noopener noreferrer"
className="text-sky-400 font-medium text-lg underline mb-8 inline-block hover:text-sky-300 transition-colors"
>
create & train your own robots
</a>
<form onSubmit={handleGo} className="flex gap-2 justify-center mt-6">
<input
ref={inputRef}
type="text"
placeholder="Enter dataset id (e.g. lerobot/visualize_dataset)"
className="px-4 py-2 rounded-md text-base text-white border-white border-1 focus:outline-none min-w-[220px] shadow-md"
onKeyDown={(e) => {
if (e.key === "Enter") {
// Prevent double submission if form onSubmit also fires
e.preventDefault();
handleGo(e as any);
}
}}
/>
<button
type="submit"
className="px-5 py-2 rounded-md bg-sky-400 text-black font-semibold text-base hover:bg-sky-300 transition-colors shadow-md"
>
Go
</button>
</form>
{/* Example Datasets */}
<div className="mt-8">
<div className="font-semibold mb-2 text-lg">Example Datasets:</div>
<div className="flex flex-col gap-2 items-center">
{[
"lerobot/aloha_static_cups_open",
"lerobot/columbia_cairlab_pusht_real",
"lerobot/taco_play",
].map((ds) => (
<button
key={ds}
type="button"
className="px-4 py-2 rounded bg-slate-700 text-sky-200 hover:bg-sky-700 hover:text-white transition-colors shadow"
onClick={() => {
if (inputRef.current) {
inputRef.current.value = ds;
inputRef.current.focus();
}
router.push(ds);
}}
>
{ds}
</button>
))}
</div>
</div>
<Link
href="/explore"
className="inline-block px-6 py-3 mt-8 rounded-md bg-sky-500 text-white font-semibold text-lg shadow-lg hover:bg-sky-400 transition-colors"
>
Explore Open Datasets
</Link>
</div>
</div>
);
}
@@ -0,0 +1,343 @@
"use client";
import { useEffect, useState } from "react";
import { useTime } from "../context/time-context";
import {
LineChart,
Line,
XAxis,
YAxis,
CartesianGrid,
ResponsiveContainer,
Tooltip,
} from "recharts";
type DataGraphProps = {
data: Array<Array<Record<string, number>>>;
onChartsReady?: () => void;
};
import React, { useMemo } from "react";
export const DataRecharts = React.memo(
({ data, onChartsReady }: DataGraphProps) => {
// Shared hoveredTime for all graphs
const [hoveredTime, setHoveredTime] = useState<number | null>(null);
if (!Array.isArray(data) || data.length === 0) return null;
useEffect(() => {
if (typeof onChartsReady === "function") {
onChartsReady();
}
}, [onChartsReady]);
return (
<div className="grid md:grid-cols-2 grid-cols-1 gap-4">
{data.map((group, idx) => (
<SingleDataGraph
key={idx}
data={group}
hoveredTime={hoveredTime}
setHoveredTime={setHoveredTime}
/>
))}
</div>
);
},
);
const NESTED_KEY_DELIMITER = ",";
const SingleDataGraph = React.memo(
({
data,
hoveredTime,
setHoveredTime,
}: {
data: Array<Record<string, number>>;
hoveredTime: number | null;
setHoveredTime: (t: number | null) => void;
}) => {
const { currentTime, setCurrentTime } = useTime();
function flattenRow(row: Record<string, any>, prefix = ""): Record<string, number> {
const result: Record<string, number> = {};
for (const [key, value] of Object.entries(row)) {
// Special case: if this is a group value that is a primitive, assign to prefix.key
if (typeof value === "number") {
if (prefix) {
result[`${prefix}${NESTED_KEY_DELIMITER}${key}`] = value;
} else {
result[key] = value;
}
} else if (value !== null && typeof value === "object" && !Array.isArray(value)) {
// If it's an object, recurse
Object.assign(result, flattenRow(value, prefix ? `${prefix}${NESTED_KEY_DELIMITER}${key}` : key));
}
}
// Always keep timestamp at top level if present
if ("timestamp" in row) {
result["timestamp"] = row["timestamp"];
}
return result;
}
// Flatten all rows for recharts
const chartData = useMemo(() => data.map(row => flattenRow(row)), [data]);
const [dataKeys, setDataKeys] = useState<string[]>([]);
const [visibleKeys, setVisibleKeys] = useState<string[]>([]);
useEffect(() => {
if (!chartData || chartData.length === 0) return;
// Get all keys except timestamp from the first row
const keys = Object.keys(chartData[0]).filter((k) => k !== "timestamp");
setDataKeys(keys);
setVisibleKeys(keys);
}, [chartData]);
// Parse dataKeys into groups (dot notation)
const groups: Record<string, string[]> = {};
const singles: string[] = [];
dataKeys.forEach((key) => {
const parts = key.split(NESTED_KEY_DELIMITER);
if (parts.length > 1) {
const group = parts[0];
if (!groups[group]) groups[group] = [];
groups[group].push(key);
} else {
singles.push(key);
}
});
// Assign a color per group (and for singles)
const allGroups = [...Object.keys(groups), ...singles];
const groupColorMap: Record<string, string> = {};
allGroups.forEach((group, idx) => {
groupColorMap[group] = `hsl(${idx * (360 / allGroups.length)}, 100%, 50%)`;
});
// Find the closest data point to the current time for highlighting
const findClosestDataIndex = (time: number) => {
if (!chartData.length) return 0;
// Find the index of the first data point whose timestamp is >= time (ceiling)
const idx = chartData.findIndex((point) => point.timestamp >= time);
if (idx !== -1) return idx;
// If all timestamps are less than time, return the last index
return chartData.length - 1;
};
const handleMouseLeave = () => {
setHoveredTime(null);
};
const handleClick = (data: any) => {
if (data && data.activePayload && data.activePayload.length) {
const timeValue = data.activePayload[0].payload.timestamp;
setCurrentTime(timeValue);
}
};
// Custom legend to show current value next to each series
const CustomLegend = () => {
const closestIndex = findClosestDataIndex(
hoveredTime != null ? hoveredTime : currentTime,
);
const currentData = chartData[closestIndex] || {};
// Parse dataKeys into groups (dot notation)
const groups: Record<string, string[]> = {};
const singles: string[] = [];
dataKeys.forEach((key) => {
const parts = key.split(NESTED_KEY_DELIMITER);
if (parts.length > 1) {
const group = parts[0];
if (!groups[group]) groups[group] = [];
groups[group].push(key);
} else {
singles.push(key);
}
});
// Assign a color per group (and for singles)
const allGroups = [...Object.keys(groups), ...singles];
const groupColorMap: Record<string, string> = {};
allGroups.forEach((group, idx) => {
groupColorMap[group] = `hsl(${idx * (360 / allGroups.length)}, 100%, 50%)`;
});
const isGroupChecked = (group: string) => groups[group].every(k => visibleKeys.includes(k));
const isGroupIndeterminate = (group: string) => groups[group].some(k => visibleKeys.includes(k)) && !isGroupChecked(group);
const handleGroupCheckboxChange = (group: string) => {
if (isGroupChecked(group)) {
// Uncheck all children
setVisibleKeys((prev) => prev.filter(k => !groups[group].includes(k)));
} else {
// Check all children
setVisibleKeys((prev) => Array.from(new Set([...prev, ...groups[group]])));
}
};
const handleCheckboxChange = (key: string) => {
setVisibleKeys((prev) =>
prev.includes(key) ? prev.filter((k) => k !== key) : [...prev, key]
);
};
return (
<div className="grid grid-cols-[repeat(auto-fit,250px)] gap-4 mx-8">
{/* Grouped keys */}
{Object.entries(groups).map(([group, children]) => {
const color = groupColorMap[group];
return (
<div key={group} className="mb-2">
<label className="flex gap-2 cursor-pointer select-none font-semibold">
<input
type="checkbox"
checked={isGroupChecked(group)}
ref={el => { if (el) el.indeterminate = isGroupIndeterminate(group); }}
onChange={() => handleGroupCheckboxChange(group)}
className="size-3.5 mt-1"
style={{ accentColor: color }}
/>
<span className="text-sm w-40 text-white">{group}</span>
</label>
<div className="pl-7 flex flex-col gap-1 mt-1">
{children.map((key) => (
<label key={key} className="flex gap-2 cursor-pointer select-none">
<input
type="checkbox"
checked={visibleKeys.includes(key)}
onChange={() => handleCheckboxChange(key)}
className="size-3.5 mt-1"
style={{ accentColor: color }}
/>
<span className={`text-xs break-all w-36 ${visibleKeys.includes(key) ? "text-white" : "text-gray-400"}`}>{key.slice(group.length + 1)}</span>
<span className={`text-xs font-mono ml-auto ${visibleKeys.includes(key) ? "text-orange-300" : "text-gray-500"}`}>
{typeof currentData[key] === "number" ? currentData[key].toFixed(2) : "--"}
</span>
</label>
))}
</div>
</div>
);
})}
{/* Singles (non-grouped) */}
{singles.map((key) => {
const color = groupColorMap[key];
return (
<label key={key} className="flex gap-2 cursor-pointer select-none">
<input
type="checkbox"
checked={visibleKeys.includes(key)}
onChange={() => handleCheckboxChange(key)}
className="size-3.5 mt-1"
style={{ accentColor: color }}
/>
<span className={`text-sm break-all w-40 ${visibleKeys.includes(key) ? "text-white" : "text-gray-400"}`}>{key}</span>
<span className={`text-sm font-mono ml-auto ${visibleKeys.includes(key) ? "text-orange-300" : "text-gray-500"}`}>
{typeof currentData[key] === "number" ? currentData[key].toFixed(2) : "--"}
</span>
</label>
);
})}
</div>
);
};
return (
<div className="w-full">
<div className="w-full h-80" onMouseLeave={handleMouseLeave}>
<ResponsiveContainer width="100%" height="100%">
<LineChart
data={chartData}
syncId="episode-sync"
margin={{ top: 24, right: 16, left: 0, bottom: 16 }}
onClick={handleClick}
onMouseMove={(state: any) => {
setHoveredTime(
state?.activePayload?.[0]?.payload?.timestamp ??
state?.activeLabel ??
null,
);
}}
onMouseLeave={handleMouseLeave}
>
<CartesianGrid strokeDasharray="3 3" stroke="#444" />
<XAxis
dataKey="timestamp"
label={{
value: "time",
position: "insideBottomLeft",
fill: "#cbd5e1",
}}
domain={[
chartData.at(0)?.timestamp ?? 0,
chartData.at(-1)?.timestamp ?? 0,
]}
ticks={useMemo(
() =>
Array.from(
new Set(chartData.map((d) => Math.ceil(d.timestamp))),
),
[chartData],
)}
stroke="#cbd5e1"
minTickGap={20} // Increased for fewer ticks
allowDataOverflow={true}
/>
<YAxis
domain={["auto", "auto"]}
stroke="#cbd5e1"
interval={0}
allowDataOverflow={true}
/>
<Tooltip
content={() => null}
active={true}
isAnimationActive={false}
defaultIndex={
!hoveredTime ? findClosestDataIndex(currentTime) : undefined
}
/>
{/* Render lines for visible dataKeys only */}
{dataKeys.map((key) => {
// Use group color for all keys in a group
const group = key.includes(NESTED_KEY_DELIMITER) ? key.split(NESTED_KEY_DELIMITER)[0] : key;
const color = groupColorMap[group];
let strokeDasharray: string | undefined = undefined;
if (groups[group] && groups[group].length > 1) {
const idxInGroup = groups[group].indexOf(key);
if (idxInGroup > 0) strokeDasharray = "5 5";
}
return (
visibleKeys.includes(key) && (
<Line
key={key}
type="monotone"
dataKey={key}
name={key}
stroke={color}
strokeDasharray={strokeDasharray}
dot={false}
activeDot={false}
strokeWidth={1.5}
isAnimationActive={false}
/>
)
);
})}
</LineChart>
</ResponsiveContainer>
</div>
<CustomLegend />
</div>
);
},
); // End React.memo
SingleDataGraph.displayName = "SingleDataGraph";
DataRecharts.displayName = "DataGraph";
export default DataRecharts;
@@ -0,0 +1,37 @@
"use client";
export default function Loading() {
return (
<div
className="absolute inset-0 flex flex-col items-center justify-center bg-slate-950/70 z-10 text-slate-100 animate-fade-in"
tabIndex={-1}
aria-modal="true"
role="dialog"
>
<svg
className="animate-spin mb-8"
width="64"
height="64"
viewBox="0 0 24 24"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<circle
className="opacity-25"
cx="12"
cy="12"
r="10"
stroke="currentColor"
strokeWidth="4"
/>
<path
className="opacity-75"
fill="currentColor"
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4z"
/>
</svg>
<h1 className="text-2xl font-bold mb-2">Loading...</h1>
<p className="text-slate-400">preparing data & videos</p>
</div>
);
}
@@ -0,0 +1,132 @@
import React from "react";
import { useTime } from "../context/time-context";
import {
FaPlay,
FaPause,
FaBackward,
FaForward,
FaUndoAlt,
FaArrowDown,
FaArrowUp,
} from "react-icons/fa";
import { debounce } from "@/utils/debounce";
const PlaybackBar: React.FC = () => {
const { duration, isPlaying, setIsPlaying, currentTime, setCurrentTime } =
useTime();
const sliderActiveRef = React.useRef(false);
const wasPlayingRef = React.useRef(false);
const [sliderValue, setSliderValue] = React.useState(currentTime);
// Only update sliderValue from context if not dragging
React.useEffect(() => {
if (!sliderActiveRef.current) {
setSliderValue(currentTime);
}
}, [currentTime]);
const updateTime = debounce((t: number) => {
setCurrentTime(t);
}, 200);
const handleSliderChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const t = Number(e.target.value);
setSliderValue(t);
updateTime(t);
};
const handleSliderMouseDown = () => {
sliderActiveRef.current = true;
wasPlayingRef.current = isPlaying;
setIsPlaying(false);
};
const handleSliderMouseUp = () => {
sliderActiveRef.current = false;
setCurrentTime(sliderValue); // Snap to final value
if (wasPlayingRef.current) {
setIsPlaying(true);
}
// If it was paused before, keep it paused
};
return (
<div className="flex items-center gap-4 w-full max-w-4xl mx-auto sticky bottom-0 bg-slate-900/95 px-4 py-3 rounded-3xl mt-auto">
<button
title="Jump backward 5 seconds"
onClick={() => setCurrentTime(Math.max(0, currentTime - 5))}
className="text-2xl hidden md:block"
>
<FaBackward size={24} />
</button>
<button
className={`text-3xl transition-transform ${isPlaying ? "scale-90 opacity-60" : "scale-110"}`}
title="Play. Toggle with Space"
onClick={() => setIsPlaying(true)}
style={{ display: isPlaying ? "none" : "inline-block" }}
>
<FaPlay size={24} />
</button>
<button
className={`text-3xl transition-transform ${!isPlaying ? "scale-90 opacity-60" : "scale-110"}`}
title="Pause. Toggle with Space"
onClick={() => setIsPlaying(false)}
style={{ display: !isPlaying ? "none" : "inline-block" }}
>
<FaPause size={24} />
</button>
<button
title="Jump forward 5 seconds"
onClick={() => setCurrentTime(Math.min(duration, currentTime + 5))}
className="text-2xl hidden md:block"
>
<FaForward size={24} />
</button>
<button
title="Rewind from start"
onClick={() => setCurrentTime(0)}
className="text-2xl hidden md:block"
>
<FaUndoAlt size={24} />
</button>
<input
type="range"
min={0}
max={duration}
step={0.01}
value={sliderValue}
onChange={handleSliderChange}
onMouseDown={handleSliderMouseDown}
onMouseUp={handleSliderMouseUp}
onTouchStart={handleSliderMouseDown}
onTouchEnd={handleSliderMouseUp}
className="flex-1 mx-2 accent-orange-500 focus:outline-none focus:ring-0"
aria-label="Seek video"
/>
<span className="w-16 text-right tabular-nums text-xs text-slate-200 shrink-0">
{Math.floor(sliderValue)} / {Math.floor(duration)}
</span>
<div className="text-xs text-slate-300 select-none ml-8 flex-col gap-y-0.5 hidden md:flex">
<p>
<span className="inline-flex items-center gap-1 font-mono align-middle">
<span className="px-2 py-0.5 rounded border border-slate-400 bg-slate-800 text-slate-200 text-xs shadow-inner">
Space
</span>
</span>{" "}
to pause/unpause
</p>
<p>
<span className="inline-flex items-center gap-1 font-mono align-middle">
<FaArrowUp size={14} />/<FaArrowDown size={14} />
</span>{" "}
to previous/next episode
</p>
</div>
</div>
);
};
export default PlaybackBar;
@@ -0,0 +1,119 @@
"use client";
import Link from "next/link";
import React from "react";
interface SidebarProps {
datasetInfo: any;
paginatedEpisodes: any[];
episodeId: any;
totalPages: number;
currentPage: number;
prevPage: () => void;
nextPage: () => void;
}
const Sidebar: React.FC<SidebarProps> = ({
datasetInfo,
paginatedEpisodes,
episodeId,
totalPages,
currentPage,
prevPage,
nextPage,
}) => {
const [sidebarVisible, setSidebarVisible] = React.useState(true);
const toggleSidebar = () => setSidebarVisible((prev) => !prev);
const sidebarRef = React.useRef<HTMLDivElement>(null);
React.useEffect(() => {
if (!sidebarVisible) return;
function handleClickOutside(event: MouseEvent) {
// If click is outside the sidebar nav
if (
sidebarRef.current &&
!sidebarRef.current.contains(event.target as Node)
) {
setTimeout(() => setSidebarVisible(false), 500);
}
}
document.addEventListener("mousedown", handleClickOutside);
return () => {
document.removeEventListener("mousedown", handleClickOutside);
};
}, [sidebarVisible]);
return (
<div className="flex z-10 min-h-screen absolute md:static" ref={sidebarRef}>
<nav
className={`shrink-0 overflow-y-auto bg-slate-900 p-5 break-words md:max-h-screen w-60 md:shrink ${
!sidebarVisible ? "hidden" : ""
}`}
aria-label="Sidebar navigation"
>
<ul>
<li>Number of samples/frames: {datasetInfo.total_frames}</li>
<li>Number of episodes: {datasetInfo.total_episodes}</li>
<li>Frames per second: {datasetInfo.fps}</li>
</ul>
<p>Episodes:</p>
{/* episodes menu for medium & large screens */}
<div className="ml-2 block">
<ul>
{paginatedEpisodes.map((episode) => (
<li key={episode} className="mt-0.5 font-mono text-sm">
<Link
href={`./episode_${episode}`}
className={`underline ${episode === episodeId ? "-ml-1 font-bold" : ""}`}
>
Episode {episode}
</Link>
</li>
))}
</ul>
{totalPages > 1 && (
<div className="mt-3 flex items-center text-xs">
<button
onClick={prevPage}
className={`mr-2 rounded bg-slate-800 px-2 py-1 ${
currentPage === 1 ? "cursor-not-allowed opacity-50" : ""
}`}
disabled={currentPage === 1}
>
« Prev
</button>
<span className="mr-2 font-mono">
{currentPage} / {totalPages}
</span>
<button
onClick={nextPage}
className={`rounded bg-slate-800 px-2 py-1 ${
currentPage === totalPages
? "cursor-not-allowed opacity-50"
: ""
}`}
disabled={currentPage === totalPages}
>
Next »
</button>
</div>
)}
</div>
</nav>
{/* Toggle sidebar button */}
<button
className="mx-1 flex items-center opacity-50 hover:opacity-100 focus:outline-none focus:ring-0"
onClick={toggleSidebar}
title="Toggle sidebar"
>
<div className="h-10 w-2 rounded-full bg-slate-500"></div>
</button>
</div>
);
};
export default Sidebar;
@@ -0,0 +1,343 @@
"use client";
import { useEffect, useRef, useState } from "react";
import { useTime } from "../context/time-context";
import { FaExpand, FaCompress, FaTimes, FaEye } from "react-icons/fa";
type VideoInfo = {
filename: string;
url: string;
};
type VideoPlayerProps = {
videosInfo: VideoInfo[];
onVideosReady?: () => void;
};
export const VideosPlayer = ({
videosInfo,
onVideosReady,
}: VideoPlayerProps) => {
const { currentTime, setCurrentTime, isPlaying, setIsPlaying } = useTime();
const videoRefs = useRef<HTMLVideoElement[]>([]);
// Hidden/enlarged state and hidden menu
const [hiddenVideos, setHiddenVideos] = useState<string[]>([]);
// Find the index of the first visible (not hidden) video
const firstVisibleIdx = videosInfo.findIndex(
(video) => !hiddenVideos.includes(video.filename),
);
// Count of visible videos
const visibleCount = videosInfo.filter(
(video) => !hiddenVideos.includes(video.filename),
).length;
const [enlargedVideo, setEnlargedVideo] = useState<string | null>(null);
// Track previous hiddenVideos for comparison
const prevHiddenVideosRef = useRef<string[]>([]);
const videoContainerRefs = useRef<Record<string, HTMLDivElement | null>>({});
const [showHiddenMenu, setShowHiddenMenu] = useState(false);
const hiddenMenuRef = useRef<HTMLDivElement | null>(null);
const showHiddenBtnRef = useRef<HTMLButtonElement | null>(null);
const [videoCodecError, setVideoCodecError] = useState(false);
// Initialize video refs
useEffect(() => {
videoRefs.current = videoRefs.current.slice(0, videosInfo.length);
}, [videosInfo]);
// When videos get unhidden, start playing them if it was playing
useEffect(() => {
// Find which videos were just unhidden
const prevHidden = prevHiddenVideosRef.current;
const newlyUnhidden = prevHidden.filter(
(filename) => !hiddenVideos.includes(filename),
);
if (newlyUnhidden.length > 0) {
videosInfo.forEach((video, idx) => {
if (newlyUnhidden.includes(video.filename)) {
const ref = videoRefs.current[idx];
if (ref) {
ref.currentTime = currentTime;
if (isPlaying) {
ref.play().catch(() => {});
}
}
}
});
}
prevHiddenVideosRef.current = hiddenVideos;
}, [hiddenVideos, isPlaying, videosInfo, currentTime]);
// Check video codec support
useEffect(() => {
const checkCodecSupport = () => {
const dummyVideo = document.createElement("video");
const canPlayVideos = dummyVideo.canPlayType(
'video/mp4; codecs="av01.0.05M.08"',
);
setVideoCodecError(!canPlayVideos);
};
checkCodecSupport();
}, []);
// Handle play/pause
useEffect(() => {
videoRefs.current.forEach((video) => {
if (video) {
if (isPlaying) {
video.play().catch((e) => console.error("Error playing video:", e));
} else {
video.pause();
}
}
});
}, [isPlaying]);
// Minimize enlarged video on Escape key
useEffect(() => {
if (!enlargedVideo) return;
const handleKeyDown = (e: KeyboardEvent) => {
if (e.key === "Escape") {
setEnlargedVideo(null);
}
};
window.addEventListener("keydown", handleKeyDown);
// Scroll enlarged video into view
const ref = videoContainerRefs.current[enlargedVideo];
if (ref) {
ref.scrollIntoView();
}
return () => {
window.removeEventListener("keydown", handleKeyDown);
};
}, [enlargedVideo]);
// Close hidden videos dropdown on outside click
useEffect(() => {
if (!showHiddenMenu) return;
function handleClick(e: MouseEvent) {
const menu = hiddenMenuRef.current;
const btn = showHiddenBtnRef.current;
if (
menu &&
!menu.contains(e.target as Node) &&
btn &&
!btn.contains(e.target as Node)
) {
setShowHiddenMenu(false);
}
}
document.addEventListener("mousedown", handleClick);
return () => document.removeEventListener("mousedown", handleClick);
}, [showHiddenMenu]);
// Close dropdown if no hidden videos
useEffect(() => {
if (hiddenVideos.length === 0 && showHiddenMenu) {
setShowHiddenMenu(false);
}
// Minimize if enlarged video is hidden
if (enlargedVideo && hiddenVideos.includes(enlargedVideo)) {
setEnlargedVideo(null);
}
}, [hiddenVideos, showHiddenMenu, enlargedVideo]);
// Sync video times
useEffect(() => {
videoRefs.current.forEach((video) => {
if (video && Math.abs(video.currentTime - currentTime) > 0.2) {
video.currentTime = currentTime;
}
});
}, [currentTime]);
// Handle time update
const handleTimeUpdate = (e: React.SyntheticEvent<HTMLVideoElement>) => {
const video = e.target as HTMLVideoElement;
if (video && video.duration) {
setCurrentTime(video.currentTime);
}
};
// Handle video ready
useEffect(() => {
let videosReadyCount = 0;
const onCanPlayThrough = () => {
videosReadyCount += 1;
if (videosReadyCount === videosInfo.length) {
if (typeof onVideosReady === "function") {
onVideosReady();
setIsPlaying(true);
}
}
};
videoRefs.current.forEach((video) => {
if (video) {
// If already ready, call the handler immediately
if (video.readyState >= 4) {
onCanPlayThrough();
} else {
video.addEventListener("canplaythrough", onCanPlayThrough);
}
}
});
return () => {
videoRefs.current.forEach((video) => {
if (video) {
video.removeEventListener("canplaythrough", onCanPlayThrough);
}
});
};
}, []);
return (
<>
{/* Error message */}
{videoCodecError && (
<div className="font-medium text-orange-700">
<p>
Videos could NOT play because{" "}
<a
href="https://en.wikipedia.org/wiki/AV1"
target="_blank"
className="underline"
>
AV1
</a>{" "}
decoding is not available on your browser.
</p>
<ul className="list-inside list-decimal">
<li>
If iPhone:{" "}
<span className="italic">
It is supported with A17 chip or higher.
</span>
</li>
<li>
If Mac with Safari:{" "}
<span className="italic">
It is supported on most browsers except Safari with M1 chip or
higher and on Safari with M3 chip or higher.
</span>
</li>
<li>
Other:{" "}
<span className="italic">
Contact the maintainers on LeRobot discord channel:
</span>
<a
href="https://discord.com/invite/s3KuuzsPFb"
target="_blank"
className="underline"
>
https://discord.com/invite/s3KuuzsPFb
</a>
</li>
</ul>
</div>
)}
{/* Show Hidden Videos Button */}
{hiddenVideos.length > 0 && (
<div className="relative">
<button
ref={showHiddenBtnRef}
className="flex items-center gap-2 rounded bg-slate-800 px-3 py-2 text-sm text-slate-100 hover:bg-slate-700 border border-slate-500"
onClick={() => setShowHiddenMenu((prev) => !prev)}
>
<FaEye /> Show Hidden Videos ({hiddenVideos.length})
</button>
{showHiddenMenu && (
<div
ref={hiddenMenuRef}
className="absolute left-0 mt-2 w-max rounded border border-slate-500 bg-slate-900 shadow-lg p-2 z-50"
>
<div className="mb-2 text-xs text-slate-300">
Restore hidden videos:
</div>
{hiddenVideos.map((filename) => (
<button
key={filename}
className="block w-full text-left px-2 py-1 rounded hover:bg-slate-700 text-slate-100"
onClick={() =>
setHiddenVideos((prev: string[]) =>
prev.filter((v: string) => v !== filename),
)
}
>
{filename}
</button>
))}
</div>
)}
</div>
)}
{/* Videos */}
<div className="flex flex-wrap gap-x-2 gap-y-6">
{videosInfo.map((video, idx) => {
if (hiddenVideos.includes(video.filename) || videoCodecError)
return null;
const isEnlarged = enlargedVideo === video.filename;
return (
<div
key={video.filename}
ref={(el) => {
videoContainerRefs.current[video.filename] = el;
}}
className={`${isEnlarged ? "z-40 fixed inset-0 bg-black bg-opacity-90 flex flex-col items-center justify-center" : "max-w-96"}`}
style={isEnlarged ? { height: "100vh", width: "100vw" } : {}}
>
<p className="truncate w-full rounded-t-xl bg-gray-800 px-2 text-sm text-gray-300 flex items-center justify-between">
<span>{video.filename}</span>
<span className="flex gap-1">
<button
title={isEnlarged ? "Minimize" : "Enlarge"}
className="ml-2 p-1 hover:bg-slate-700 rounded focus:outline-none focus:ring-0"
onClick={() =>
setEnlargedVideo(isEnlarged ? null : video.filename)
}
>
{isEnlarged ? <FaCompress /> : <FaExpand />}
</button>
<button
title="Hide Video"
className="ml-1 p-1 hover:bg-slate-700 rounded focus:outline-none focus:ring-0"
onClick={() =>
setHiddenVideos((prev: string[]) => [
...prev,
video.filename,
])
}
disabled={visibleCount === 1}
>
<FaTimes />
</button>
</span>
</p>
<video
ref={(el) => {
if (el) videoRefs.current[idx] = el;
}}
muted
loop
className={`w-full object-contain ${isEnlarged ? "max-h-[90vh] max-w-[90vw]" : ""}`}
onTimeUpdate={
idx === firstVisibleIdx ? handleTimeUpdate : undefined
}
style={isEnlarged ? { zIndex: 41 } : {}}
>
<source src={video.url} type="video/mp4" />
Your browser does not support the video tag.
</video>
</div>
);
})}
</div>
</>
);
};
export default VideosPlayer;
@@ -0,0 +1,64 @@
import React, {
createContext,
useContext,
useRef,
useState,
useCallback,
} from "react";
// The shape of our context
type TimeContextType = {
currentTime: number;
setCurrentTime: (t: number) => void;
subscribe: (cb: (t: number) => void) => () => void;
isPlaying: boolean;
setIsPlaying: React.Dispatch<React.SetStateAction<boolean>>;
duration: number;
setDuration: React.Dispatch<React.SetStateAction<number>>;
};
const TimeContext = createContext<TimeContextType | undefined>(undefined);
export const useTime = () => {
const ctx = useContext(TimeContext);
if (!ctx) throw new Error("useTime must be used within a TimeProvider");
return ctx;
};
export const TimeProvider: React.FC<{
children: React.ReactNode;
duration: number;
}> = ({ children, duration: initialDuration }) => {
const [currentTime, setCurrentTime] = useState(0);
const [isPlaying, setIsPlaying] = useState(false);
const [duration, setDuration] = useState(initialDuration);
const listeners = useRef<Set<(t: number) => void>>(new Set());
// Call this to update time and notify all listeners
const updateTime = useCallback((t: number) => {
setCurrentTime(t);
listeners.current.forEach((fn) => fn(t));
}, []);
// Components can subscribe to time changes (for imperative updates)
const subscribe = useCallback((cb: (t: number) => void) => {
listeners.current.add(cb);
return () => listeners.current.delete(cb);
}, []);
return (
<TimeContext.Provider
value={{
currentTime,
setCurrentTime: updateTime,
subscribe,
isPlaying,
setIsPlaying,
duration,
setDuration,
}}
>
{children}
</TimeContext.Provider>
);
};
@@ -0,0 +1,10 @@
export function debounce<F extends (...args: any[]) => any>(
func: F,
waitFor: number,
): (...args: Parameters<F>) => void {
let timeoutId: number;
return (...args: Parameters<F>) => {
clearTimeout(timeoutId);
timeoutId = window.setTimeout(() => func(...args), waitFor);
};
}
@@ -0,0 +1,97 @@
import { parquetRead } from "hyparquet";
export interface DatasetMetadata {
codebase_version: string;
robot_type: string;
total_episodes: number;
total_frames: number;
total_tasks: number;
total_videos: number;
total_chunks: number;
chunks_size: number;
fps: number;
splits: Record<string, string>;
data_path: string;
video_path: string;
features: Record<
string,
{
dtype: string;
shape: any[];
names: any[] | Record<string, any> | null;
info?: Record<string, any>;
}
>;
}
export async function fetchJson<T>(url: string): Promise<T> {
const res = await fetch(url);
if (!res.ok) {
throw new Error(
`Failed to fetch JSON ${url}: ${res.status} ${res.statusText}`,
);
}
return res.json() as Promise<T>;
}
export function formatStringWithVars(
format: string,
vars: Record<string, any>,
): string {
return format.replace(/{(\w+)(?::\d+d)?}/g, (_, key) => vars[key]);
}
// Fetch and parse the Parquet file
export async function fetchParquetFile(url: string): Promise<ArrayBuffer> {
const res = await fetch(url);
return res.arrayBuffer();
}
// Read specific columns from the Parquet file
export async function readParquetColumn(
fileBuffer: ArrayBuffer,
columns: string[],
): Promise<any[]> {
return new Promise((resolve) => {
parquetRead({
file: fileBuffer,
columns,
onComplete: (data: any[]) => resolve(data),
});
});
}
// Convert a 2D array to a CSV string
export function arrayToCSV(data: (number | string)[][]): string {
return data.map((row) => row.join(",")).join("\n");
}
// Get rows from the current frame data
export function getRows(currentFrameData: any[], columns: any[]) {
if (!currentFrameData || currentFrameData.length === 0) {
return [];
}
const rows = [];
const nRows = Math.max(...columns.map((column) => column.value.length));
let rowIndex = 0;
while (rowIndex < nRows) {
const row = [];
// number of states may NOT match number of actions. In this case, we null-pad the 2D array
const nullCell = { isNull: true };
// row consists of [state value, action value]
let idx = rowIndex;
for (const column of columns) {
const nColumn = column.value.length;
row.push(rowIndex < nColumn ? currentFrameData[idx] : nullCell);
idx += nColumn; // because currentFrameData = [state0, state1, ..., stateN, action0, action1, ..., actionN]
}
rowIndex += 1;
rows.push(row);
}
return rows;
}
@@ -0,0 +1,19 @@
/**
* Return copy of object, only keeping whitelisted properties.
*
* This doesn't add {p: undefined} anymore, for props not in the o object.
*/
export function pick<T, K extends keyof T>(
o: T,
props: K[] | ReadonlyArray<K>,
): Pick<T, K> {
// inspired by stackoverflow.com/questions/25553910/one-liner-to-take-some-properties-from-object-in-es-6
return Object.assign(
{},
...props.map((prop) => {
if (o[prop] !== undefined) {
return { [prop]: o[prop] };
}
}),
);
}
@@ -0,0 +1,12 @@
// Utility to post a message to the parent window with custom URLSearchParams
export function postParentMessageWithParams(
setParams: (params: URLSearchParams) => void,
) {
const parentOrigin = "https://huggingface.co";
const searchParams = new URLSearchParams();
setParams(searchParams);
window.parent.postMessage(
{ queryString: searchParams.toString() },
parentOrigin,
);
}
@@ -0,0 +1,27 @@
{
"compilerOptions": {
"target": "ES2017",
"lib": ["dom", "dom.iterable", "esnext"],
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"noEmit": true,
"esModuleInterop": true,
"module": "esnext",
"moduleResolution": "bundler",
"resolveJsonModule": true,
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"plugins": [
{
"name": "next"
}
],
"paths": {
"@/*": ["./src/*"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"exclude": ["node_modules"]
}
+159 -330
View File
@@ -53,80 +53,35 @@ python lerobot/scripts/visualize_dataset_html.py \
"""
import argparse
import csv
import atexit
import contextlib
import json
import logging
import re
import os
import shutil
import tempfile
from io import StringIO
import signal
import subprocess
import sys
from pathlib import Path
import numpy as np
import pandas as pd
import requests
from flask import Flask, redirect, render_template, request, url_for
from flask import Flask, jsonify, redirect, send_file, url_for
from lerobot import available_datasets
from lerobot.common.datasets.lerobot_dataset import LeRobotDataset
from lerobot.common.datasets.utils import IterableNamespace
from lerobot.common.datasets.utils import DEFAULT_PARQUET_PATH, DEFAULT_VIDEO_PATH, INFO_PATH
from lerobot.common.utils.utils import init_logging
def run_server(
dataset: LeRobotDataset | IterableNamespace | None,
episodes: list[int] | None,
def run_data_server(
dataset: LeRobotDataset | None,
host: str,
port: str,
static_folder: Path,
template_folder: Path,
):
app = Flask(__name__, static_folder=static_folder.resolve(), template_folder=template_folder.resolve())
app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 0 # specifying not to cache
port: int,
) -> Path | None:
init_logging()
@app.route("/")
def hommepage(dataset=dataset):
if dataset:
dataset_namespace, dataset_name = dataset.repo_id.split("/")
return redirect(
url_for(
"show_episode",
dataset_namespace=dataset_namespace,
dataset_name=dataset_name,
episode_id=0,
)
)
data_server = Flask(__name__)
data_server.config["SEND_FILE_MAX_AGE_DEFAULT"] = 0 # specifying not to cache
dataset_param, episode_param = None, None
all_params = request.args
if "dataset" in all_params:
dataset_param = all_params["dataset"]
if "episode" in all_params:
episode_param = int(all_params["episode"])
if dataset_param:
dataset_namespace, dataset_name = dataset_param.split("/")
return redirect(
url_for(
"show_episode",
dataset_namespace=dataset_namespace,
dataset_name=dataset_name,
episode_id=episode_param if episode_param is not None else 0,
)
)
featured_datasets = [
"lerobot/aloha_static_cups_open",
"lerobot/columbia_cairlab_pusht_real",
"lerobot/taco_play",
]
return render_template(
"visualize_dataset_homepage.html",
featured_datasets=featured_datasets,
lerobot_datasets=available_datasets,
)
@app.route("/<string:dataset_namespace>/<string:dataset_name>")
@data_server.route("/<string:dataset_namespace>/<string:dataset_name>")
def show_first_episode(dataset_namespace, dataset_name):
first_episode_id = 0
return redirect(
@@ -138,256 +93,144 @@ def run_server(
)
)
@app.route("/<string:dataset_namespace>/<string:dataset_name>/episode_<int:episode_id>")
def show_episode(dataset_namespace, dataset_name, episode_id, dataset=dataset, episodes=episodes):
repo_id = f"{dataset_namespace}/{dataset_name}"
@data_server.route("/<string:dataset_namespace>/<string:dataset_name>/resolve/main/meta/info.json")
def serve_info_json(dataset_namespace, dataset_name):
try:
if dataset is None:
dataset = get_dataset_info(repo_id)
return send_file(dataset.root / INFO_PATH, mimetype="application/json")
except FileNotFoundError:
return (
"Make sure to convert your LeRobotDataset to v2 & above. See how to convert your dataset at https://github.com/huggingface/lerobot/pull/461",
400,
)
dataset_version = (
str(dataset.meta._version) if isinstance(dataset, LeRobotDataset) else dataset.codebase_version
)
match = re.search(r"v(\d+)\.", dataset_version)
if match:
major_version = int(match.group(1))
if major_version < 2:
return "Make sure to convert your LeRobotDataset to v2 & above."
return jsonify({"error": "File not found"}), 404
except Exception as e:
return jsonify({"error": f"Server error: {str(e)}"}), 500
episode_data_csv_str, columns, ignored_columns = get_episode_data(dataset, episode_id)
dataset_info = {
"repo_id": f"{dataset_namespace}/{dataset_name}",
"num_samples": dataset.num_frames
if isinstance(dataset, LeRobotDataset)
else dataset.total_frames,
"num_episodes": dataset.num_episodes
if isinstance(dataset, LeRobotDataset)
else dataset.total_episodes,
"fps": dataset.fps,
}
if isinstance(dataset, LeRobotDataset):
video_paths = [
dataset.meta.get_video_file_path(episode_id, key) for key in dataset.meta.video_keys
]
videos_info = [
{
"url": url_for("static", filename=str(video_path).replace("\\", "/")),
"filename": video_path.parent.name,
}
for video_path in video_paths
]
tasks = dataset.meta.episodes[episode_id]["tasks"]
else:
video_keys = [key for key, ft in dataset.features.items() if ft["dtype"] == "video"]
videos_info = [
{
"url": f"https://huggingface.co/datasets/{repo_id}/resolve/main/"
+ dataset.video_path.format(
episode_chunk=int(episode_id) // dataset.chunks_size,
video_key=video_key,
episode_index=episode_id,
),
"filename": video_key,
}
for video_key in video_keys
]
response = requests.get(
f"https://huggingface.co/datasets/{repo_id}/resolve/main/meta/episodes.jsonl", timeout=5
)
response.raise_for_status()
# Split into lines and parse each line as JSON
tasks_jsonl = [json.loads(line) for line in response.text.splitlines() if line.strip()]
filtered_tasks_jsonl = [row for row in tasks_jsonl if row["episode_index"] == episode_id]
tasks = filtered_tasks_jsonl[0]["tasks"]
videos_info[0]["language_instruction"] = tasks
if episodes is None:
episodes = list(
range(dataset.num_episodes if isinstance(dataset, LeRobotDataset) else dataset.total_episodes)
)
return render_template(
"visualize_dataset_template.html",
episode_id=episode_id,
episodes=episodes,
dataset_info=dataset_info,
videos_info=videos_info,
episode_data_csv_str=episode_data_csv_str,
columns=columns,
ignored_columns=ignored_columns,
)
app.run(host=host, port=port)
def get_ep_csv_fname(episode_id: int):
ep_csv_fname = f"episode_{episode_id}.csv"
return ep_csv_fname
def get_episode_data(dataset: LeRobotDataset | IterableNamespace, episode_index):
"""Get a csv str containing timeseries data of an episode (e.g. state and action).
This file will be loaded by Dygraph javascript to plot data in real time."""
columns = []
selected_columns = [col for col, ft in dataset.features.items() if ft["dtype"] in ["float32", "int32"]]
selected_columns.remove("timestamp")
ignored_columns = []
for column_name in selected_columns:
shape = dataset.features[column_name]["shape"]
shape_dim = len(shape)
if shape_dim > 1:
selected_columns.remove(column_name)
ignored_columns.append(column_name)
# init header of csv with state and action names
header = ["timestamp"]
for column_name in selected_columns:
dim_state = (
dataset.meta.shapes[column_name][0]
if isinstance(dataset, LeRobotDataset)
else dataset.features[column_name].shape[0]
)
if "names" in dataset.features[column_name] and dataset.features[column_name]["names"]:
column_names = dataset.features[column_name]["names"]
while not isinstance(column_names, list):
column_names = list(column_names.values())[0]
else:
column_names = [f"{column_name}_{i}" for i in range(dim_state)]
columns.append({"key": column_name, "value": column_names})
header += column_names
selected_columns.insert(0, "timestamp")
if isinstance(dataset, LeRobotDataset):
from_idx = dataset.episode_data_index["from"][episode_index]
to_idx = dataset.episode_data_index["to"][episode_index]
data = (
dataset.hf_dataset.select(range(from_idx, to_idx))
.select_columns(selected_columns)
.with_format("pandas")
)
else:
repo_id = dataset.repo_id
url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/" + dataset.data_path.format(
episode_chunk=int(episode_index) // dataset.chunks_size, episode_index=episode_index
)
df = pd.read_parquet(url)
data = df[selected_columns] # Select specific columns
rows = np.hstack(
(
np.expand_dims(data["timestamp"], axis=1),
*[np.vstack(data[col]) for col in selected_columns[1:]],
)
).tolist()
# Convert data to CSV string
csv_buffer = StringIO()
csv_writer = csv.writer(csv_buffer)
# Write header
csv_writer.writerow(header)
# Write data rows
csv_writer.writerows(rows)
csv_string = csv_buffer.getvalue()
return csv_string, columns, ignored_columns
def get_episode_video_paths(dataset: LeRobotDataset, ep_index: int) -> list[str]:
# get first frame of episode (hack to get video_path of the episode)
first_frame_idx = dataset.episode_data_index["from"][ep_index].item()
return [
dataset.hf_dataset.select_columns(key)[first_frame_idx][key]["path"]
for key in dataset.meta.video_keys
]
def get_episode_language_instruction(dataset: LeRobotDataset, ep_index: int) -> list[str]:
# check if the dataset has language instructions
if "language_instruction" not in dataset.features:
return None
# get first frame index
first_frame_idx = dataset.episode_data_index["from"][ep_index].item()
language_instruction = dataset.hf_dataset[first_frame_idx]["language_instruction"]
# TODO (michel-aractingi) hack to get the sentence, some strings in openx are badly stored
# with the tf.tensor appearing in the string
return language_instruction.removeprefix("tf.Tensor(b'").removesuffix("', shape=(), dtype=string)")
def get_dataset_info(repo_id: str) -> IterableNamespace:
response = requests.get(
f"https://huggingface.co/datasets/{repo_id}/resolve/main/meta/info.json", timeout=5
@data_server.route(
"/<string:dataset_namespace>/<string:dataset_name>/resolve/main/data/chunk-<int:episode_chunk>/episode_<int:episode_index>.parquet"
)
response.raise_for_status() # Raises an HTTPError for bad responses
dataset_info = response.json()
dataset_info["repo_id"] = repo_id
return IterableNamespace(dataset_info)
def serve_parquet_file(dataset_namespace, dataset_name, episode_chunk, episode_index):
try:
# Format the path with the captured parameters
file_path = DEFAULT_PARQUET_PATH.format(episode_chunk=episode_chunk, episode_index=episode_index)
full_path = dataset.root / file_path
def visualize_dataset_html(
dataset: LeRobotDataset | None,
episodes: list[int] | None = None,
output_dir: Path | None = None,
serve: bool = True,
host: str = "127.0.0.1",
port: int = 9090,
force_override: bool = False,
) -> Path | None:
init_logging()
return send_file(full_path, mimetype="application/octet-stream")
except FileNotFoundError:
return jsonify({"error": "File not found"}), 404
except Exception as e:
return jsonify({"error": f"Server error: {str(e)}"}), 500
template_dir = Path(__file__).resolve().parent.parent / "templates"
if output_dir is None:
# Create a temporary directory that will be automatically cleaned up
output_dir = tempfile.mkdtemp(prefix="lerobot_visualize_dataset_")
output_dir = Path(output_dir)
if output_dir.exists():
if force_override:
shutil.rmtree(output_dir)
else:
logging.info(f"Output directory already exists. Loading from it: '{output_dir}'")
output_dir.mkdir(parents=True, exist_ok=True)
static_dir = output_dir / "static"
static_dir.mkdir(parents=True, exist_ok=True)
if dataset is None:
if serve:
run_server(
dataset=None,
episodes=None,
host=host,
port=port,
static_folder=static_dir,
template_folder=template_dir,
@data_server.route(
"/<string:dataset_namespace>/<string:dataset_name>/resolve/main/videos/chunk-<int:episode_chunk>/<string:video_key>/episode_<int:episode_index>.mp4"
)
def serve_video_file(dataset_namespace, dataset_name, episode_chunk, video_key, episode_index):
try:
# Format the path with the captured parameters
file_path = DEFAULT_VIDEO_PATH.format(
episode_chunk=episode_chunk, video_key=video_key, episode_index=episode_index
)
else:
# Create a simlink from the dataset video folder containing mp4 files to the output directory
# so that the http server can get access to the mp4 files.
if isinstance(dataset, LeRobotDataset):
ln_videos_dir = static_dir / "videos"
if not ln_videos_dir.exists():
ln_videos_dir.symlink_to((dataset.root / "videos").resolve().as_posix())
if serve:
run_server(dataset, episodes, host, port, static_dir, template_dir)
# Assuming 'dataset' object has a 'root' attribute
full_path = dataset.root / file_path
return send_file(full_path, mimetype="video/mp4")
except FileNotFoundError:
return jsonify({"error": "Video file not found"}), 404
except Exception as e:
return jsonify({"error": f"Server error: {str(e)}"}), 500
log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)
data_server.run(host=host, port=get_local_data_server_port(port))
def is_npm_available():
npm_path = shutil.which("npm")
if npm_path is None:
return False
try:
subprocess.run([npm_path, "--version"], capture_output=True, text=True, check=True)
return True
except (subprocess.CalledProcessError, FileNotFoundError):
return False
def build_react_app(script_dir: Path):
next_dir = script_dir.parent / "html_dataset_visualizer"
next_build_dir = next_dir / ".next"
npm_path = shutil.which("npm")
if npm_path is None:
raise RuntimeError(
"'npm' executable not found in PATH. Please ensure Node.js is installed and npm is available."
)
if not next_build_dir.exists() or not next_build_dir.is_dir():
print("Building React.js app ...")
subprocess.run([npm_path, "ci"], cwd=next_dir)
subprocess.run([npm_path, "run", "build"], cwd=next_dir)
package_json_path = next_dir / "package.json"
build_id_path = next_build_dir / "BUILD_ID"
with open(package_json_path) as f:
package_data = json.load(f)
package_version = package_data.get("version", "")
with open(build_id_path) as f:
build_id = f.read().strip()
if package_version != build_id:
print("Building React.js app ...")
subprocess.run([npm_path, "ci"], cwd=next_dir)
subprocess.run([npm_path, "run", "build"], cwd=next_dir)
def run_react_app(
repo_id: str,
script_dir: Path,
load_from_hf_hub: bool,
host: str,
port: int,
episodes: list[int] | None = None,
):
next_dir = script_dir.parent / "html_dataset_visualizer"
env = os.environ.copy()
env["REPO_ID"] = repo_id
if not load_from_hf_hub:
env["DATASET_URL"] = f"http://{host}:{get_local_data_server_port(port)}"
if episodes:
env["EPISODES"] = " ".join(map(str, episodes))
npm_path = shutil.which("npm")
if npm_path is None:
raise RuntimeError(
"'npm' executable not found in PATH. Please ensure Node.js is installed and npm is available."
)
process = subprocess.Popen(
[npm_path, "run", "start", "--", f"--port={port}"], cwd=next_dir, env=env, preexec_fn=os.setsid
)
def cleanup():
if process.poll() is None: # Process still running
print("Cleaning up React server...")
try:
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
process.wait(timeout=5)
except (ProcessLookupError, subprocess.TimeoutExpired):
# Force kill if graceful termination fails
with contextlib.suppress(ProcessLookupError):
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
def signal_handler(sig, frame):
cleanup()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
atexit.register(cleanup) # Also cleanup on normal exit
return process
def get_local_data_server_port(port: str):
"""Returns the port used by the local data server."""
return str(int(port) + 1)
def main():
@@ -396,8 +239,8 @@ def main():
parser.add_argument(
"--repo-id",
type=str,
default=None,
help="Name of hugging face repositery containing a LeRobotDataset dataset (e.g. `lerobot/pusht` for https://huggingface.co/datasets/lerobot/pusht).",
required=True,
)
parser.add_argument(
"--root",
@@ -418,18 +261,6 @@ def main():
default=None,
help="Episode indices to visualize (e.g. `0 1 5 6` to load episodes of index 0, 1, 5 and 6). By default loads all episodes.",
)
parser.add_argument(
"--output-dir",
type=Path,
default=None,
help="Directory path to write html files and kickoff a web server. By default write them to 'outputs/visualize_dataset/REPO_ID'.",
)
parser.add_argument(
"--serve",
type=int,
default=1,
help="Launch web server.",
)
parser.add_argument(
"--host",
type=str,
@@ -442,13 +273,6 @@ def main():
default=9090,
help="Web port used by the http server.",
)
parser.add_argument(
"--force-override",
type=int,
default=0,
help="Delete the output directory if it exists already.",
)
parser.add_argument(
"--tolerance-s",
type=float,
@@ -466,16 +290,21 @@ def main():
load_from_hf_hub = kwargs.pop("load_from_hf_hub")
root = kwargs.pop("root")
tolerance_s = kwargs.pop("tolerance_s")
host = kwargs.pop("host")
port = kwargs.pop("port")
episodes = kwargs.pop("episodes")
dataset = None
if repo_id:
dataset = (
LeRobotDataset(repo_id, root=root, tolerance_s=tolerance_s)
if not load_from_hf_hub
else get_dataset_info(repo_id)
)
if not is_npm_available():
raise RuntimeError("npm is not available. Please install it to use this script.")
visualize_dataset_html(dataset, **vars(args))
script_dir = Path(__file__).parent.absolute()
build_react_app(script_dir)
run_react_app(repo_id, script_dir, load_from_hf_hub, host, port, episodes)
if not load_from_hf_hub:
dataset = LeRobotDataset(repo_id, root=root, tolerance_s=tolerance_s)
run_data_server(dataset, host, port)
if __name__ == "__main__":
@@ -1,68 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Interactive Video Background Page</title>
<script src="https://cdn.tailwindcss.com"></script>
<script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
</head>
<body class="h-screen overflow-hidden font-mono text-white" x-data="{
inputValue: '',
navigateToDataset() {
const trimmedValue = this.inputValue.trim();
if (trimmedValue) {
window.location.href = `/${trimmedValue}`;
}
}
}">
<div class="fixed inset-0 w-full h-full overflow-hidden">
<video class="absolute min-w-full min-h-full w-auto h-auto top-1/2 left-1/2 transform -translate-x-1/2 -translate-y-1/2" autoplay muted loop>
<source src="https://huggingface.co/datasets/cadene/koch_bimanual_folding/resolve/v1.6/videos/observation.images.phone_episode_000037.mp4" type="video/mp4">
Your browser does not support HTML5 video.
</video>
</div>
<div class="fixed inset-0 bg-black bg-opacity-80"></div>
<div class="relative z-10 flex flex-col items-center justify-center h-screen">
<div class="text-center mb-8">
<h1 class="text-4xl font-bold mb-4">LeRobot Dataset Visualizer</h1>
<a href="https://x.com/RemiCadene/status/1825455895561859185" target="_blank" rel="noopener noreferrer" class="underline">create & train your own robots</a>
<p class="text-xl mb-4"></p>
<div class="text-left inline-block">
<h3 class="font-semibold mb-2 mt-4">Example Datasets:</h3>
<ul class="list-disc list-inside">
{% for dataset in featured_datasets %}
<li><a href="/{{ dataset }}" class="text-blue-300 hover:text-blue-100 hover:underline">{{ dataset }}</a></li>
{% endfor %}
</ul>
</div>
</div>
<div class="flex w-full max-w-lg px-4 mb-4">
<input
type="text"
x-model="inputValue"
@keyup.enter="navigateToDataset"
placeholder="enter dataset id (ex: lerobot/droid_100)"
class="flex-grow px-4 py-2 rounded-l bg-white bg-opacity-20 text-white placeholder-gray-300 focus:outline-none focus:ring-2 focus:ring-blue-300"
>
<button
@click="navigateToDataset"
class="px-4 py-2 bg-blue-500 text-white rounded-r hover:bg-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-300"
>
Go
</button>
</div>
<details class="mt-4 max-w-full px-4">
<summary>More example datasets</summary>
<ul class="list-disc list-inside max-h-28 overflow-y-auto break-all">
{% for dataset in lerobot_datasets %}
<li><a href="/{{ dataset }}" class="text-blue-300 hover:text-blue-100 hover:underline">{{ dataset }}</a></li>
{% endfor %}
</ul>
</details>
</div>
</body>
</html>
@@ -1,546 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<!-- # TODO(rcadene, mishig25): store the js files locally -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/alpinejs/3.13.5/cdn.min.js" defer></script>
<script src="https://cdn.jsdelivr.net/npm/dygraphs@2.2.1/dist/dygraph.min.js" type="text/javascript"></script>
<script src="https://cdn.tailwindcss.com"></script>
<title>{{ dataset_info.repo_id }} episode {{ episode_id }}</title>
</head>
<!-- Use [Alpin.js](https://alpinejs.dev), a lightweight and easy to learn JS framework -->
<!-- Use [tailwindcss](https://tailwindcss.com/), CSS classes for styling html -->
<!-- Use [dygraphs](https://dygraphs.com/), a lightweight JS charting library -->
<body class="flex flex-col md:flex-row h-screen max-h-screen bg-slate-950 text-gray-200" x-data="createAlpineData()">
<!-- Sidebar -->
<div x-ref="sidebar" class="bg-slate-900 p-5 break-words overflow-y-auto shrink-0 md:shrink md:w-60 md:max-h-screen">
<a href="https://github.com/huggingface/lerobot" target="_blank" class="hidden md:block">
<img src="https://github.com/huggingface/lerobot/raw/main/media/lerobot-logo-thumbnail.png">
</a>
<a href="https://huggingface.co/datasets/{{ dataset_info.repo_id }}" target="_blank">
<h1 class="mb-4 text-xl font-semibold">{{ dataset_info.repo_id }}</h1>
</a>
<ul>
<li>
Number of samples/frames: {{ dataset_info.num_samples }}
</li>
<li>
Number of episodes: {{ dataset_info.num_episodes }}
</li>
<li>
Frames per second: {{ dataset_info.fps }}
</li>
</ul>
<p>Episodes:</p>
<!-- episodes menu for medium & large screens -->
<div class="ml-2 hidden md:block" x-data="episodePagination">
<ul>
<template x-for="episode in paginatedEpisodes" :key="episode">
<li class="font-mono text-sm mt-0.5">
<a :href="'episode_' + episode"
:class="{'underline': true, 'font-bold -ml-1': episode == {{ episode_id }}}"
x-text="'Episode ' + episode"></a>
</li>
</template>
</ul>
<div class="flex items-center mt-3 text-xs" x-show="totalPages > 1">
<button @click="prevPage()"
class="px-2 py-1 bg-slate-800 rounded mr-2"
:class="{'opacity-50 cursor-not-allowed': page === 1}"
:disabled="page === 1">
&laquo; Prev
</button>
<span class="font-mono mr-2" x-text="` ${page} / ${totalPages}`"></span>
<button @click="nextPage()"
class="px-2 py-1 bg-slate-800 rounded"
:class="{'opacity-50 cursor-not-allowed': page === totalPages}"
:disabled="page === totalPages">
Next &raquo;
</button>
</div>
</div>
<!-- episodes menu for small screens -->
<div class="flex overflow-x-auto md:hidden" x-data="episodePagination">
<button @click="prevPage()"
class="px-2 bg-slate-800 rounded mr-2"
:class="{'opacity-50 cursor-not-allowed': page === 1}"
:disabled="page === 1">&laquo;</button>
<div class="flex">
<template x-for="(episode, index) in paginatedEpisodes" :key="episode">
<p class="font-mono text-sm mt-0.5 px-2"
:class="{
'font-bold': episode == {{ episode_id }},
'border-r': index !== paginatedEpisodes.length - 1
}">
<a :href="'episode_' + episode" x-text="episode"></a>
</p>
</template>
</div>
<button @click="nextPage()"
class="px-2 bg-slate-800 rounded ml-2"
:class="{'opacity-50 cursor-not-allowed': page === totalPages}"
:disabled="page === totalPages">&raquo; </button>
</div>
</div>
<!-- Toggle sidebar button -->
<button class="flex items-center opacity-50 hover:opacity-100 mx-1 hidden md:block"
@click="() => ($refs.sidebar.classList.toggle('hidden'))" title="Toggle sidebar">
<div class="bg-slate-500 w-2 h-10 rounded-full"></div>
</button>
<!-- Content -->
<div class="max-h-screen flex flex-col gap-4 overflow-y-auto md:flex-1">
<h1 class="text-xl font-bold mt-4 font-mono">
Episode {{ episode_id }}
</h1>
<!-- Error message -->
<div class="font-medium text-orange-700 hidden" :class="{ 'hidden': !videoCodecError }">
<p>Videos could NOT play because <a href="https://en.wikipedia.org/wiki/AV1" target="_blank" class="underline">AV1</a> decoding is not available on your browser.</p>
<ul class="list-decimal list-inside">
<li>If iPhone: <span class="italic">It is supported with A17 chip or higher.</span></li>
<li>If Mac with Safari: <span class="italic">It is supported on most browsers except Safari with M1 chip or higher and on Safari with M3 chip or higher.</span></li>
<li>Other: <span class="italic">Contact the maintainers on LeRobot discord channel:</span> <a href="https://discord.com/invite/s3KuuzsPFb" target="_blank" class="underline">https://discord.com/invite/s3KuuzsPFb</a></li>
</ul>
</div>
<!-- Videos -->
<div class="max-w-32 relative text-sm mb-4 select-none"
@click.outside="isVideosDropdownOpen = false">
<div
@click="isVideosDropdownOpen = !isVideosDropdownOpen"
class="p-2 border border-slate-500 rounded flex justify-between items-center cursor-pointer"
>
<span class="truncate">filter videos</span>
<div class="transition-transform" :class="{ 'rotate-180': isVideosDropdownOpen }">🔽</div>
</div>
<div x-show="isVideosDropdownOpen"
class="absolute mt-1 border border-slate-500 rounded shadow-lg z-10">
<div>
<template x-for="option in videosKeys" :key="option">
<div
@click="videosKeysSelected = videosKeysSelected.includes(option) ? videosKeysSelected.filter(v => v !== option) : [...videosKeysSelected, option]"
class="p-2 cursor-pointer bg-slate-900"
:class="{ 'bg-slate-700': videosKeysSelected.includes(option) }"
x-text="option"
></div>
</template>
</div>
</div>
</div>
<div class="flex flex-wrap gap-x-2 gap-y-6">
{% for video_info in videos_info %}
<div x-show="!videoCodecError && videosKeysSelected.includes('{{ video_info.filename }}')" class="max-w-96 relative">
<p class="absolute inset-x-0 -top-4 text-sm text-gray-300 bg-gray-800 px-2 rounded-t-xl truncate">{{ video_info.filename }}</p>
<video muted loop type="video/mp4" class="object-contain w-full h-full" @canplaythrough="videoCanPlay" @timeupdate="() => {
if (video.duration) {
const time = video.currentTime;
const pc = (100 / video.duration) * time;
$refs.slider.value = pc;
dygraphTime = time;
dygraphIndex = Math.floor(pc * dygraph.numRows() / 100);
dygraph.setSelection(dygraphIndex, undefined, true, true);
$refs.timer.textContent = formatTime(time) + ' / ' + formatTime(video.duration);
updateTimeQuery(time.toFixed(2));
}
}" @ended="() => {
$refs.btnPlay.classList.remove('hidden');
$refs.btnPause.classList.add('hidden');
}"
@loadedmetadata="() => ($refs.timer.textContent = formatTime(0) + ' / ' + formatTime(video.duration))">
<source src="{{ video_info.url }}">
Your browser does not support the video tag.
</video>
</div>
{% endfor %}
</div>
<!-- Language instruction -->
{% if videos_info[0].language_instruction %}
<p class="font-medium mt-2">
Language Instruction: <span class="italic">{{ videos_info[0].language_instruction }}</span>
</p>
{% endif %}
<!-- Shortcuts info -->
<div class="text-sm hidden md:block">
Hotkeys: <span class="font-mono">Space</span> to pause/unpause, <span class="font-mono">Arrow Down</span> to go to next episode, <span class="font-mono">Arrow Up</span> to go to previous episode.
</div>
<!-- Controllers -->
<div class="flex gap-1 text-3xl items-center">
<button x-ref="btnPlay" class="-rotate-90" class="-rotate-90" title="Play. Toggle with Space" @click="() => {
videos.forEach(video => video.play());
$refs.btnPlay.classList.toggle('hidden');
$refs.btnPause.classList.toggle('hidden');
}">🔽</button>
<button x-ref="btnPause" class="hidden" title="Pause. Toggle with Space" @click="() => {
videos.forEach(video => video.pause());
$refs.btnPlay.classList.toggle('hidden');
$refs.btnPause.classList.toggle('hidden');
}">⏸️</button>
<button title="Jump backward 5 seconds"
@click="() => (videos.forEach(video => (video.currentTime -= 5)))">⏪</button>
<button title="Jump forward 5 seconds"
@click="() => (videos.forEach(video => (video.currentTime += 5)))">⏩</button>
<button title="Rewind from start"
@click="() => (videos.forEach(video => (video.currentTime = 0.0)))">↩️</button>
<input x-ref="slider" max="100" min="0" step="1" type="range" value="0" class="w-80 mx-2" @input="() => {
const sliderValue = $refs.slider.value;
videos.forEach(video => {
const time = (video.duration * sliderValue) / 100;
video.currentTime = time;
});
}" />
<div x-ref="timer" class="font-mono text-sm border border-slate-500 rounded-lg px-1 py-0.5 shrink-0">0:00 /
0:00
</div>
</div>
<!-- Graph -->
<div class="flex gap-2 mb-4 flex-wrap">
<div>
<div id="graph" @mouseleave="() => {
dygraph.setSelection(dygraphIndex, undefined, true, true);
dygraphTime = video.currentTime;
}">
</div>
<p x-ref="graphTimer" class="font-mono ml-14 mt-4"
x-init="$watch('dygraphTime', value => ($refs.graphTimer.innerText = `Time: ${dygraphTime.toFixed(2)}s`))">
Time: 0.00s
</p>
</div>
<div>
<table class="text-sm border-collapse border border-slate-700" x-show="currentFrameData">
<thead>
<tr>
<th></th>
<template x-for="(_, colIndex) in Array.from({length: columns.length}, (_, index) => index)">
<th class="border border-slate-700">
<div class="flex gap-x-2 justify-between px-2">
<input type="checkbox" :checked="isColumnChecked(colIndex)"
@change="toggleColumn(colIndex)">
<p x-text="`${columns[colIndex].key}`"></p>
</div>
</th>
</template>
</tr>
</thead>
<tbody>
<template x-for="(row, rowIndex) in rows">
<tr class="odd:bg-gray-800 even:bg-gray-900">
<td class="border border-slate-700">
<div class="flex gap-x-2 max-w-64 font-semibold px-1 break-all">
<input type="checkbox" :checked="isRowChecked(rowIndex)"
@change="toggleRow(rowIndex)">
</div>
</td>
<template x-for="(cell, colIndex) in row">
<td x-show="cell" class="border border-slate-700">
<div class="flex gap-x-2 justify-between px-2" :class="{ 'hidden': cell.isNull }">
<div class="flex gap-x-2">
<input type="checkbox" x-model="cell.checked" @change="updateTableValues()">
<span x-text="`${!cell.isNull ? cell.label : null}`"></span>
</div>
<span class="w-14 text-right" x-text="`${!cell.isNull ? (typeof cell.value === 'number' ? cell.value.toFixed(2) : cell.value) : null}`"
:style="`color: ${cell.color}`"></span>
</div>
</td>
</template>
</tr>
</template>
</tbody>
</table>
<div id="labels" class="hidden">
</div>
{% if ignored_columns|length > 0 %}
<div class="m-2 text-orange-700 max-w-96">
Columns {{ ignored_columns }} are NOT shown since the visualizer currently does not support 2D or 3D data.
</div>
{% endif %}
</div>
</div>
</div>
<script>
const parentOrigin = "https://huggingface.co";
const searchParams = new URLSearchParams();
searchParams.set("dataset", "{{ dataset_info.repo_id }}");
searchParams.set("episode", "{{ episode_id }}");
window.parent.postMessage({ queryString: searchParams.toString() }, parentOrigin);
</script>
<script>
function createAlpineData() {
return {
// state
dygraph: null,
currentFrameData: null,
checked: [],
dygraphTime: 0.0,
dygraphIndex: 0,
videos: null,
video: null,
colors: null,
nVideos: {{ videos_info | length }},
nVideoReadyToPlay: 0,
videoCodecError: false,
isVideosDropdownOpen: false,
videosKeys: {{ videos_info | map(attribute='filename') | list | tojson }},
videosKeysSelected: [],
columns: {{ columns | tojson }},
// alpine initialization
init() {
// check if videos can play
const dummyVideo = document.createElement('video');
const canPlayVideos = dummyVideo.canPlayType('video/mp4; codecs="av01.0.05M.08"'); // codec source: https://huggingface.co/blog/video-encoding#results
if(!canPlayVideos){
this.videoCodecError = true;
}
this.videosKeysSelected = this.videosKeys.map(opt => opt)
// process CSV data
const csvDataStr = {{ episode_data_csv_str|tojson|safe }};
// Create a Blob with the CSV data
const blob = new Blob([csvDataStr], { type: 'text/csv;charset=utf-8;' });
// Create a URL for the Blob
const csvUrl = URL.createObjectURL(blob);
// process CSV data
this.videos = document.querySelectorAll('video');
this.video = this.videos[0];
this.dygraph = new Dygraph(document.getElementById("graph"), csvUrl, {
pixelsPerPoint: 0.01,
legend: 'always',
labelsDiv: document.getElementById('labels'),
labelsKMB: true,
strokeWidth: 1.5,
pointClickCallback: (event, point) => {
this.dygraphTime = point.xval;
this.updateTableValues(this.dygraphTime);
},
highlightCallback: (event, x, points, row, seriesName) => {
this.dygraphTime = x;
this.updateTableValues(this.dygraphTime);
},
drawCallback: (dygraph, is_initial) => {
if (is_initial) {
// dygraph initialization
this.dygraph.setSelection(this.dygraphIndex, undefined, true, true);
this.colors = this.dygraph.getColors();
this.checked = Array(this.colors.length).fill(true);
const colors = [];
let lightness = 30; // const LIGHTNESS = [30, 65, 85]; // state_lightness, action_lightness, pred_action_lightness
for(const column of this.columns){
const nValues = column.value.length;
for (let hue = 0; hue < 360; hue += parseInt(360/nValues)) {
const color = `hsl(${hue}, 100%, ${lightness}%)`;
colors.push(color);
}
lightness += 35;
}
this.dygraph.updateOptions({ colors });
this.colors = colors;
this.updateTableValues();
let url = new URL(window.location.href);
let params = new URLSearchParams(url.search);
let time = params.get("t");
if(time){
time = parseFloat(time);
this.videos.forEach(video => (video.currentTime = time));
}
}
},
});
},
//#region Table Data
// turn dygraph's 1D data (at a given time t) to 2D data that whose columns names are defined in this.columnNames.
// 2d data view is used to create html table element.
get rows() {
if (!this.currentFrameData) {
return [];
}
const rows = [];
const nRows = Math.max(...this.columns.map(column => column.value.length));
let rowIndex = 0;
while(rowIndex < nRows){
const row = [];
// number of states may NOT match number of actions. In this case, we null-pad the 2D array to make a fully rectangular 2d array
const nullCell = { isNull: true };
// row consists of [state value, action value]
let idx = rowIndex;
for(const column of this.columns){
const nColumn = column.value.length;
row.push(rowIndex < nColumn ? this.currentFrameData[idx] : nullCell);
idx += nColumn; // because this.currentFrameData = [state0, state1, ..., stateN, action0, action1, ..., actionN]
}
rowIndex += 1;
rows.push(row);
}
return rows;
},
isRowChecked(rowIndex) {
return this.rows[rowIndex].every(cell => cell && (cell.isNull || cell.checked));
},
isColumnChecked(colIndex) {
return this.rows.every(row => row[colIndex] && (row[colIndex].isNull || row[colIndex].checked));
},
toggleRow(rowIndex) {
const newState = !this.isRowChecked(rowIndex);
this.rows[rowIndex].forEach(cell => {
if (cell && !cell.isNull) cell.checked = newState;
});
this.updateTableValues();
},
toggleColumn(colIndex) {
const newState = !this.isColumnChecked(colIndex);
this.rows.forEach(row => {
if (row[colIndex] && !row[colIndex].isNull) row[colIndex].checked = newState;
});
this.updateTableValues();
},
// given time t, update the values in the html table with "data[t]"
updateTableValues(time) {
if (!this.colors) {
return;
}
let pc = (100 / this.video.duration) * (time === undefined ? this.video.currentTime : time);
if (isNaN(pc)) pc = 0;
const index = Math.floor(pc * this.dygraph.numRows() / 100);
// slice(1) to remove the timestamp point that we do not need
const labels = this.dygraph.getLabels().slice(1);
const values = this.dygraph.rawData_[index].slice(1);
const checkedNew = this.currentFrameData ? this.currentFrameData.map(cell => cell.checked) : Array(
this.colors.length).fill(true);
this.currentFrameData = labels.map((label, idx) => ({
label,
value: values[idx],
color: this.colors[idx],
checked: checkedNew[idx],
}));
const shouldUpdateVisibility = !this.checked.every((value, index) => value === checkedNew[index]);
if (shouldUpdateVisibility) {
this.checked = checkedNew;
this.dygraph.setVisibility(this.checked);
}
},
//#endregion
updateTimeQuery(time) {
let url = new URL(window.location.href);
let params = new URLSearchParams(url.search);
params.set("t", time);
url.search = params.toString();
window.history.replaceState({}, '', url.toString());
},
formatTime(time) {
var hours = Math.floor(time / 3600);
var minutes = Math.floor((time % 3600) / 60);
var seconds = Math.floor(time % 60);
return (hours > 0 ? hours + ':' : '') + (minutes < 10 ? '0' + minutes : minutes) + ':' + (seconds <
10 ?
'0' + seconds : seconds);
},
videoCanPlay() {
this.nVideoReadyToPlay += 1;
if(this.nVideoReadyToPlay == this.nVideos) {
// start autoplay all videos in sync
this.$refs.btnPlay.click();
}
}
};
}
document.addEventListener('alpine:init', () => {
// Episode pagination component
Alpine.data('episodePagination', () => ({
episodes: {{ episodes }},
pageSize: 100,
page: 1,
init() {
// Find which page contains the current episode_id
const currentEpisodeId = {{ episode_id }};
const episodeIndex = this.episodes.indexOf(currentEpisodeId);
if (episodeIndex !== -1) {
this.page = Math.floor(episodeIndex / this.pageSize) + 1;
}
},
get totalPages() {
return Math.ceil(this.episodes.length / this.pageSize);
},
get paginatedEpisodes() {
const start = (this.page - 1) * this.pageSize;
const end = start + this.pageSize;
return this.episodes.slice(start, end);
},
nextPage() {
if (this.page < this.totalPages) {
this.page++;
}
},
prevPage() {
if (this.page > 1) {
this.page--;
}
}
}));
});
</script>
<script>
window.addEventListener('keydown', (e) => {
// Use the space bar to play and pause, instead of default action (e.g. scrolling)
const { keyCode, key } = e;
if (keyCode === 32 || key === ' ') {
e.preventDefault();
const btnPause = document.querySelector('[x-ref="btnPause"]');
const btnPlay = document.querySelector('[x-ref="btnPlay"]');
btnPause.classList.contains('hidden') ? btnPlay.click() : btnPause.click();
} else if (key === 'ArrowDown' || key === 'ArrowUp') {
const episodes = {{ episodes }}; // Access episodes directly from the Jinja template
const nextEpisodeId = key === 'ArrowDown' ? {{ episode_id }} + 1 : {{ episode_id }} - 1;
const lowestEpisodeId = episodes.at(0);
const highestEpisodeId = episodes.at(-1);
if (nextEpisodeId >= lowestEpisodeId && nextEpisodeId <= highestEpisodeId) {
window.location.href = `./episode_${nextEpisodeId}`;
}
}
});
</script>
</body>
</html>