[software] 添加16DOF早期训练仿真与Sim2Real闭环

2026-07-21 16:15:14 +08:00
parent 9bd22225f9
commit e9e2c946b3
681 changed files with 137221 additions and 8 deletions
@@ -0,0 +1,76 @@
+# mjlab Nightly Benchmarks
+
+This directory contains scripts for automated nightly benchmarking of mjlab.
+
+## Overview
+
+The nightly benchmark system:
+1. Trains a tracking policy on the latest commit
+2. Evaluates the policy across 1024 trials
+3. Measures simulation throughput
+4. Generates an HTML report with historical trends
+5. Publishes results to GitHub Pages
+
+## Usage
+
+### Run the full nightly benchmark
+
+```bash
+./scripts/benchmarks/nightly_train.sh
+```
+
+### Skip training (regenerate report only)
+
+```bash
+SKIP_TRAINING=1 ./scripts/benchmarks/nightly_train.sh
+```
+
+### Skip training and throughput
+
+```bash
+SKIP_TRAINING=1 SKIP_THROUGHPUT=1 ./scripts/benchmarks/nightly_train.sh
+```
+
+### Regenerate report directly (no git operations)
+
+```bash
+uv run python scripts/benchmarks/generate_report.py \
+  --entity gcbc_researchers \
+  --tag nightly \
+  --output-dir benchmark_results
+```
+
+### Measure throughput only
+
+```bash
+uv run python scripts/benchmarks/measure_throughput.py \
+  --num-envs 4096 \
+  --output-dir benchmark_results
+```
+
+## Configuration
+
+Environment variables for `nightly_train.sh`:
+
+- `CUDA_DEVICE` - GPU device to use (default: 0)
+- `WANDB_TAGS` - Comma-separated tags for the run (default: nightly)
+- `SKIP_TRAINING` - Set to "1" to skip training
+- `SKIP_THROUGHPUT` - Set to "1" to skip throughput benchmarking
+
+## Automated Setup
+
+See [systemd/README.md](systemd/README.md) for instructions on setting up automated nightly runs using systemd timers.
+
+## Report Options
+
+The `generate_report.py` script supports:
+
+- `--eval-limit N` - Maximum number of NEW runs to evaluate per invocation (default: 10)
+  - Set to 0 for no limit
+  - Historical cached results are always preserved
+- `--tag TAG` - Filter runs by WandB tag (default: "nightly")
+- `--num-envs N` - Number of parallel environments for evaluation (default: 1024)
+
+## Viewing Results
+
+Reports are published to: https://mujocolab.github.io/mjlab/nightly/
@@ -0,0 +1,796 @@
+"""Generate benchmark report from evaluation metrics.
+
+This script runs policy evaluation on nightly runs and generates a static HTML
+dashboard for tracking policy performance over time.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from datetime import datetime
+from pathlib import Path
+
+import tyro
+import wandb
+
+import mjlab
+from mjlab.tasks.tracking.scripts.evaluate import EvaluateConfig, run_evaluate
+
+# Metrics to display: (key, label, unit, scale, higher_is_better)
+METRICS = [
+  ("success_rate", "Success Rate", "%", 100, True),
+  ("mpkpe", "MPKPE", "m", 1, False),
+  ("r_mpkpe", "R-MPKPE", "m", 1, False),
+  ("ee_pos_error", "EE Position Error", "m", 1, False),
+  ("ee_ori_error", "EE Orientation Error", "rad", 1, False),
+  ("joint_vel_error", "Joint Velocity Error", "rad/s", 1, False),
+]
+
+
+def evaluate_run(run_path: str, num_envs: int = 1024) -> dict:
+  """Evaluate a single run and return metrics with metadata."""
+  api = wandb.Api()
+  run = api.run(run_path)
+
+  print(f"Evaluating run: {run.name} ({run.id})")
+
+  cfg = EvaluateConfig(wandb_run_path=run_path, num_envs=num_envs)
+  metrics = run_evaluate("Mjlab-Tracking-Flat-Unitree-G1", cfg)
+
+  # Get commit SHA from run metadata.
+  commit = run.commit or run.config.get("commit", "unknown")
+
+  return {
+    "id": run.id,
+    "name": run.name,
+    "url": run.url,
+    "created_at": run.created_at,
+    "commit": commit[:7] if len(commit) > 7 else commit,
+    "metrics": metrics,
+  }
+
+
+def load_throughput_data(output_dir: Path) -> list[dict]:
+  """Load throughput benchmark data if available."""
+  data_file = output_dir / "throughput_data.json"
+  if not data_file.exists():
+    return []
+  with open(data_file) as f:
+    return json.load(f)
+
+
+def generate_html_report(runs: list[dict], output_dir: Path) -> None:
+  """Generate static HTML dashboard from evaluation data."""
+  output_dir.mkdir(parents=True, exist_ok=True)
+
+  # Save raw data.
+  with open(output_dir / "data.json", "w") as f:
+    json.dump(runs, f, indent=2, default=str)
+
+  # Copy task images for the throughput dashboard.
+  images_src = Path(__file__).parent / "nightly_images"
+  if images_src.is_dir():
+    images_dst = output_dir / "images"
+    if images_dst.exists():
+      shutil.rmtree(images_dst)
+    shutil.copytree(images_src, images_dst)
+
+  # Load throughput data if available.
+  throughput_data = load_throughput_data(output_dir)
+
+  html = generate_dashboard_html(runs, throughput_data)
+  with open(output_dir / "index.html", "w") as f:
+    f.write(html)
+
+  print(f"Report generated at {output_dir / 'index.html'}")
+
+
+def generate_dashboard_html(runs: list[dict], throughput_data: list[dict]) -> str:
+  """Generate the HTML dashboard content."""
+  runs_json = json.dumps(runs, default=str)
+  metrics_json = json.dumps(METRICS)
+  throughput_json = json.dumps(throughput_data, default=str)
+  timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
+  github_repo = "https://github.com/mujocolab/mjlab"
+
+  return f"""<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>mjlab Nightly Benchmark</title>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns"></script>
+    <style>
+        :root {{
+            --bg: #ffffff;
+            --bg-card: #f6f8fa;
+            --text: #1f2328;
+            --text-dim: #656d76;
+            --border: #d0d7de;
+            --accent: #0969da;
+            --green: #1a7f37;
+            --red: #cf222e;
+        }}
+        @media (prefers-color-scheme: dark) {{
+            :root:not([data-theme="light"]) {{
+                --bg: #0d1117;
+                --bg-card: #161b22;
+                --text: #c9d1d9;
+                --text-dim: #8b949e;
+                --border: #30363d;
+                --accent: #58a6ff;
+                --green: #3fb950;
+                --red: #f85149;
+            }}
+        }}
+        :root[data-theme="dark"] {{
+            --bg: #0d1117;
+            --bg-card: #161b22;
+            --text: #c9d1d9;
+            --text-dim: #8b949e;
+            --border: #30363d;
+            --accent: #58a6ff;
+            --green: #3fb950;
+            --red: #f85149;
+        }}
+        * {{ margin: 0; padding: 0; box-sizing: border-box; }}
+        body {{
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
+            background: var(--bg);
+            color: var(--text);
+            padding: 2rem;
+            max-width: 1400px;
+            margin: 0 auto;
+        }}
+        header {{
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            margin-bottom: 2rem;
+            padding-bottom: 1rem;
+            border-bottom: 1px solid var(--border);
+        }}
+        h1 {{ font-size: 1.5rem; }}
+        .subtitle {{
+            font-size: 0.875rem;
+            color: var(--text-dim);
+            margin-top: 0.25rem;
+        }}
+        .timestamp {{ color: var(--text-dim); font-size: 0.875rem; }}
+        .charts {{
+            display: grid;
+            grid-template-columns: 1fr;
+            gap: 1.5rem;
+            margin-bottom: 2rem;
+        }}
+        .chart-card {{
+            background: var(--bg-card);
+            border: 1px solid var(--border);
+            border-radius: 8px;
+            padding: 1rem;
+        }}
+        .chart-title {{
+            font-size: 0.875rem;
+            font-weight: 500;
+            margin-bottom: 0.5rem;
+            display: flex;
+            justify-content: space-between;
+        }}
+        .chart-value {{ color: var(--text-dim); }}
+        .chart-container {{ height: 350px; }}
+        a {{ color: var(--accent); text-decoration: none; }}
+        a:hover {{ text-decoration: underline; }}
+        .theme-toggle {{
+            background: var(--bg-card);
+            border: 1px solid var(--border);
+            border-radius: 6px;
+            padding: 0.5rem;
+            cursor: pointer;
+            color: var(--text);
+            font-size: 1rem;
+            line-height: 1;
+        }}
+        .theme-toggle:hover {{ border-color: var(--accent); }}
+        .header-right {{ display: flex; align-items: center; gap: 1rem; }}
+        .tabs {{
+            display: flex;
+            gap: 0.5rem;
+            margin-bottom: 1.5rem;
+        }}
+        .tab {{
+            background: var(--bg-card);
+            border: 1px solid var(--border);
+            border-radius: 6px;
+            padding: 0.5rem 1rem;
+            cursor: pointer;
+            color: var(--text);
+            font-size: 0.875rem;
+            font-weight: 500;
+        }}
+        .tab:hover {{ border-color: var(--accent); }}
+        .tab.active {{
+            background: var(--accent);
+            border-color: var(--accent);
+            color: white;
+        }}
+        .tab-content {{ display: none; }}
+        .tab-content.active {{ display: block; }}
+        .tab-description {{
+            font-size: 0.875rem;
+            color: var(--text-dim);
+            margin-bottom: 1.5rem;
+            line-height: 1.5;
+        }}
+        .task-grid {{
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(220px, 1fr));
+            gap: 1rem;
+            margin-bottom: 1.5rem;
+        }}
+        .task-card {{
+            background: var(--bg-card);
+            border: 2px solid var(--border);
+            border-radius: 10px;
+            padding: 0.75rem;
+            cursor: pointer;
+            transition: border-color 0.15s, box-shadow 0.15s;
+            text-align: center;
+        }}
+        .task-card:hover {{ border-color: var(--accent); }}
+        .task-card.active {{
+            border-color: var(--accent);
+            box-shadow: 0 0 0 1px var(--accent);
+        }}
+        .task-card img {{
+            width: 100%;
+            aspect-ratio: 16/10;
+            object-fit: cover;
+            border-radius: 6px;
+            margin-bottom: 0.5rem;
+            background: var(--border);
+        }}
+        .task-card .task-name {{
+            font-size: 0.85rem;
+            font-weight: 600;
+        }}
+        .task-card .task-stat {{
+            font-size: 0.75rem;
+            color: var(--text-dim);
+            margin-top: 0.2rem;
+        }}
+        .task-chart-area {{
+            display: none;
+        }}
+        .task-chart-area.active {{
+            display: block;
+        }}
+        footer {{
+            margin-top: 3rem;
+            padding-top: 1.5rem;
+            border-top: 1px solid var(--border);
+            font-size: 0.8rem;
+            color: var(--text-dim);
+            line-height: 1.6;
+        }}
+        @media (max-width: 600px) {{
+            body {{ padding: 1rem; }}
+            h1 {{ font-size: 1.25rem; }}
+            .tabs {{ flex-wrap: wrap; }}
+            .task-grid {{ grid-template-columns: 1fr; }}
+        }}
+    </style>
+</head>
+<body>
+    <header>
+        <div>
+            <h1><a href="{github_repo}" style="color:inherit;text-decoration:none">mjlab</a> Nightly Benchmark</h1>
+            <div class="subtitle">Performance tracking over time</div>
+        </div>
+        <div class="header-right">
+            <span class="timestamp">Updated: {timestamp}</span>
+            <button class="theme-toggle" id="theme-toggle" title="Toggle theme">
+                <span id="theme-icon"></span>
+            </button>
+        </div>
+    </header>
+
+    <div class="tabs">
+        <button class="tab active" data-tab="tracking">Tracking Eval</button>
+        <button class="tab" data-tab="throughput">Throughput</button>
+    </div>
+
+    <div id="tracking" class="tab-content active">
+        <p class="tab-description">Nightly motion imitation training and evaluation on Unitree G1 (1024 trials per run).</p>
+        <div class="charts" id="charts"></div>
+    </div>
+
+    <div id="throughput" class="tab-content">
+        <p class="tab-description">Physics simulation throughput across tasks (4096 parallel envs, NVIDIA RTX 5090).</p>
+        <div class="task-grid" id="task-grid"></div>
+        <div id="task-chart-panels"></div>
+    </div>
+
+    <footer>
+        These benchmarks run nightly using the latest commit.<br>
+        GPU: NVIDIA RTX 5090
+    </footer>
+
+    <script>
+        // Theme toggle logic
+        const themeToggle = document.getElementById('theme-toggle');
+        const themeIcon = document.getElementById('theme-icon');
+        const root = document.documentElement;
+
+        function getSystemTheme() {{
+            return window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light';
+        }}
+
+        function getEffectiveTheme() {{
+            const stored = localStorage.getItem('theme');
+            if (stored === 'dark' || stored === 'light') return stored;
+            return getSystemTheme();
+        }}
+
+        function updateThemeIcon() {{
+            const stored = localStorage.getItem('theme');
+            if (!stored) {{
+                themeIcon.textContent = '\u2699\ufe0f'; // gear for auto
+                themeToggle.title = 'Theme: System (click to toggle)';
+            }} else if (stored === 'dark') {{
+                themeIcon.textContent = '\U0001f319'; // moon
+                themeToggle.title = 'Theme: Dark (click to toggle)';
+            }} else {{
+                themeIcon.textContent = '\u2600\ufe0f'; // sun
+                themeToggle.title = 'Theme: Light (click to toggle)';
+            }}
+        }}
+
+        function applyTheme() {{
+            const stored = localStorage.getItem('theme');
+            if (stored) {{
+                root.setAttribute('data-theme', stored);
+            }} else {{
+                root.removeAttribute('data-theme');
+            }}
+            updateThemeIcon();
+            updateChartColors();
+        }}
+
+        function cycleTheme() {{
+            const stored = localStorage.getItem('theme');
+            if (!stored) {{
+                // auto -> dark
+                localStorage.setItem('theme', 'dark');
+            }} else if (stored === 'dark') {{
+                // dark -> light
+                localStorage.setItem('theme', 'light');
+            }} else {{
+                // light -> auto
+                localStorage.removeItem('theme');
+            }}
+            applyTheme();
+        }}
+
+        themeToggle.addEventListener('click', cycleTheme);
+        window.matchMedia('(prefers-color-scheme: dark)').addEventListener('change', applyTheme);
+
+        const runs = {runs_json};
+        const METRICS = {metrics_json};
+        const GITHUB_REPO = '{github_repo}';
+
+        // Sort by date ascending for charts.
+        runs.sort((a, b) => new Date(a.created_at) - new Date(b.created_at));
+
+        const colors = {{
+            success_rate: '#3fb950',
+            mpkpe: '#58a6ff',
+            r_mpkpe: '#a371f7',
+            ee_pos_error: '#f0883e',
+            ee_ori_error: '#f85149',
+            joint_vel_error: '#79c0ff'
+        }};
+
+        let charts = [];
+
+        function updateChartColors() {{
+            const style = getComputedStyle(root);
+            const textDim = style.getPropertyValue('--text-dim').trim();
+            const border = style.getPropertyValue('--border').trim();
+            const isDark = getEffectiveTheme() === 'dark';
+            const gridColor = isDark ? '#3a424b' : '#d0d7de';
+            Chart.defaults.color = textDim;
+            Chart.defaults.borderColor = gridColor;
+            charts.forEach(c => c.update());
+        }}
+
+        // Initialize theme before creating charts so grid colors are correct.
+        applyTheme();
+
+        // Charts
+        const chartsContainer = document.getElementById('charts');
+
+        // Compute a rolling average over the last `window` points.
+        function rollingAvg(data, window) {{
+            return data.map((d, i) => {{
+                const start = Math.max(0, i - window + 1);
+                const slice = data.slice(start, i + 1);
+                const avg = slice.reduce((s, p) => s + p.y, 0) / slice.length;
+                return {{ x: d.x, y: avg }};
+            }});
+        }}
+
+        const AVG_WINDOW = 7;
+
+        METRICS.forEach(([key, label, unit, scale, higherIsBetter]) => {{
+            const data = runs.map(r => ({{
+                x: new Date(r.created_at),
+                y: r.metrics[key] * scale,
+                commit: r.commit,
+                name: r.name
+            }}));
+
+            const avgData = rollingAvg(data, AVG_WINDOW);
+            const color = colors[key] || '#58a6ff';
+
+            const latestVal = data[data.length - 1]?.y;
+            const arrow = higherIsBetter ? '\u2191' : '\u2193';
+            const tooltip = higherIsBetter ? 'Higher is better' : 'Lower is better';
+
+            const card = document.createElement('div');
+            card.className = 'chart-card';
+            card.innerHTML = `
+                <div class="chart-title">
+                    <span>${{label}} <span title="${{tooltip}}" style="cursor:help;opacity:0.6">${{arrow}}</span></span>
+                    <span class="chart-value">${{latestVal?.toFixed(3)}} ${{unit}}</span>
+                </div>
+                <div class="chart-container"><canvas></canvas></div>
+            `;
+            chartsContainer.appendChild(card);
+
+            charts.push(new Chart(card.querySelector('canvas'), {{
+                type: 'line',
+                data: {{
+                    datasets: [
+                        {{
+                            label: label,
+                            data: data,
+                            borderColor: color,
+                            backgroundColor: color + '20',
+                            borderWidth: 2,
+                            pointRadius: 4,
+                            tension: 0.1,
+                            fill: true
+                        }},
+                        {{
+                            label: `${{AVG_WINDOW}}-run avg`,
+                            data: avgData,
+                            borderColor: color,
+                            borderWidth: 2,
+                            borderDash: [6, 4],
+                            pointRadius: 0,
+                            tension: 0.3,
+                            fill: false
+                        }}
+                    ]
+                }},
+                options: {{
+                    responsive: true,
+                    maintainAspectRatio: false,
+                    onClick: (event, elements) => {{
+                        if (elements.length > 0 && elements[0].datasetIndex === 0) {{
+                            const d = data[elements[0].index];
+                            if (d?.commit && d.commit !== 'unknown') {{
+                                window.open(`${{GITHUB_REPO}}/commit/${{d.commit}}`, '_blank');
+                            }}
+                        }}
+                    }},
+                    plugins: {{
+                        legend: {{
+                            display: true,
+                            position: 'bottom',
+                            labels: {{ usePointStyle: true, pointStyle: 'line', boxHeight: 1 }}
+                        }},
+                        tooltip: {{
+                            filter: (item) => item.datasetIndex === 0,
+                            callbacks: {{
+                                title: (items) => {{
+                                    const d = items[0]?.raw;
+                                    return d ? `${{d.name}} (${{d.commit}})` : '';
+                                }},
+                                label: (item) => {{
+                                    const d = item.raw;
+                                    return `${{label}}: ${{d.y?.toFixed(4)}} ${{unit}}`;
+                                }},
+                                footer: (items) => {{
+                                    const d = items[0]?.raw;
+                                    return d?.commit && d.commit !== 'unknown' ? 'Click to view commit' : '';
+                                }}
+                            }}
+                        }}
+                    }},
+                    scales: {{
+                        x: {{
+                            type: 'time',
+                            time: {{ unit: 'day' }},
+                            ticks: {{ maxTicksLimit: 5 }},
+                            title: {{
+                                display: true,
+                                text: 'Date',
+                                font: {{ size: 11 }}
+                            }}
+                        }},
+                        y: {{
+                            ticks: {{ maxTicksLimit: 5 }},
+                            title: {{
+                                display: true,
+                                text: unit,
+                                font: {{ size: 11 }}
+                            }}
+                        }}
+                    }}
+                }}
+            }}));
+        }});
+
+        // Tab switching
+        document.querySelectorAll('.tab').forEach(tab => {{
+            tab.addEventListener('click', () => {{
+                document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
+                document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active'));
+                tab.classList.add('active');
+                document.getElementById(tab.dataset.tab).classList.add('active');
+                // Update URL hash
+                history.replaceState(null, '', '#' + tab.dataset.tab);
+            }});
+        }});
+
+        // Handle URL hash on load
+        if (window.location.hash) {{
+            const tab = document.querySelector(`.tab[data-tab="${{window.location.hash.slice(1)}}"]`);
+            if (tab) tab.click();
+        }}
+
+        // Throughput data and task cards
+        const throughputData = {throughput_json};
+        const taskGrid = document.getElementById('task-grid');
+        const taskChartPanels = document.getElementById('task-chart-panels');
+
+        // Task metadata: display name and image path (relative to nightly/)
+        const taskMeta = {{
+            'Mjlab-Velocity-Flat-Unitree-Go1': {{ name: 'Velocity \u2014 Go1', img: 'images/velocity_go1.png' }},
+            'Mjlab-Tracking-Flat-Unitree-G1': {{ name: 'Tracking \u2014 G1', img: 'images/tracking_g1.png' }},
+            'Mjlab-Lift-Cube-Yam': {{ name: 'Lift Cube \u2014 Yam', img: 'images/lift_cube_yam.png' }}
+        }};
+
+        const throughputChartInstances = {{}};
+
+        if (throughputData.length > 0) {{
+            const tasks = [...new Set(throughputData.flatMap(d => d.results.map(r => r.task)))];
+            const latestRun = throughputData[throughputData.length - 1];
+
+            tasks.forEach((task, i) => {{
+                const meta = taskMeta[task] || {{ name: task, img: '' }};
+                const latestResult = latestRun?.results.find(r => r.task === task);
+                const latestSps = latestResult ? `${{(latestResult.env_sps / 1000).toFixed(0)}}K env steps/s` : '';
+
+                // Card
+                const card = document.createElement('div');
+                card.className = 'task-card' + (i === 0 ? ' active' : '');
+                card.dataset.task = task;
+                card.innerHTML = `
+                    <img src="${{meta.img}}" alt="${{meta.name}}" onerror="this.style.display='none'">
+                    <div class="task-name">${{meta.name}}</div>
+                    <div class="task-stat">${{latestSps}}</div>
+                `;
+                taskGrid.appendChild(card);
+
+                // Chart panel
+                const panel = document.createElement('div');
+                panel.className = 'task-chart-area' + (i === 0 ? ' active' : '');
+                panel.id = `task-panel-${{i}}`;
+                panel.innerHTML = `
+                    <div class="chart-card">
+                        <div class="chart-title">
+                            <span>${{meta.name}} \u2014 Throughput</span>
+                        </div>
+                        <div class="chart-container"><canvas></canvas></div>
+                    </div>
+                `;
+                taskChartPanels.appendChild(panel);
+
+                // Build datasets
+                const envData = [];
+                const physicsData = [];
+                throughputData.forEach(run => {{
+                    const result = run.results.find(r => r.task === task);
+                    if (!result) return;
+                    const point = {{ x: new Date(run.created_at), commit: run.commit }};
+                    envData.push({{ ...point, y: result.env_sps / 1000 }});
+                    physicsData.push({{ ...point, y: result.physics_sps / 1000 }});
+                }});
+
+                const chart = new Chart(panel.querySelector('canvas'), {{
+                    type: 'line',
+                    data: {{
+                        datasets: [
+                            {{
+                                label: 'Env SPS',
+                                data: envData,
+                                borderColor: '#58a6ff',
+                                backgroundColor: '#58a6ff20',
+                                borderWidth: 2,
+                                pointRadius: 4,
+                                tension: 0.1,
+                                fill: true
+                            }},
+                            {{
+                                label: 'Physics SPS',
+                                data: physicsData,
+                                borderColor: '#3fb950',
+                                backgroundColor: '#3fb95020',
+                                borderWidth: 2,
+                                pointRadius: 4,
+                                tension: 0.1,
+                                fill: true
+                            }}
+                        ]
+                    }},
+                    options: {{
+                        responsive: true,
+                        maintainAspectRatio: false,
+                        onClick: (event, elements) => {{
+                            if (elements.length > 0) {{
+                                const di = elements[0].datasetIndex;
+                                const idx = elements[0].index;
+                                const d = di === 0 ? envData[idx] : physicsData[idx];
+                                if (d?.commit && d.commit !== 'unknown') {{
+                                    window.open(`${{GITHUB_REPO}}/commit/${{d.commit}}`, '_blank');
+                                }}
+                            }}
+                        }},
+                        plugins: {{
+                            legend: {{ display: true, position: 'bottom' }},
+                            tooltip: {{
+                                mode: 'index',
+                                callbacks: {{
+                                    title: (items) => {{
+                                        const d = items[0]?.raw;
+                                        return d ? `Commit: ${{d.commit}}` : '';
+                                    }},
+                                    label: (item) => {{
+                                        return `${{item.dataset.label}}: ${{item.raw.y?.toFixed(0)}}K steps/s`;
+                                    }},
+                                    afterBody: (items) => {{
+                                        if (items.length >= 2) {{
+                                            const envY = items[0]?.raw?.y || 0;
+                                            const physY = items[1]?.raw?.y || 0;
+                                            const overhead = physY > 0 ? ((1 - envY / physY) * 100).toFixed(1) : '?';
+                                            return `Overhead: ${{overhead}}%`;
+                                        }}
+                                        return '';
+                                    }},
+                                    footer: (items) => {{
+                                        const d = items[0]?.raw;
+                                        return d?.commit && d.commit !== 'unknown' ? 'Click to view commit' : '';
+                                    }}
+                                }}
+                            }}
+                        }},
+                        scales: {{
+                            x: {{
+                                type: 'time',
+                                time: {{ unit: 'day' }},
+                                ticks: {{ maxTicksLimit: 5 }},
+                                title: {{ display: true, text: 'Date', font: {{ size: 11 }} }}
+                            }},
+                            y: {{
+                                ticks: {{ maxTicksLimit: 5 }},
+                                title: {{
+                                    display: true,
+                                    text: 'K steps/s',
+                                    font: {{ size: 11 }}
+                                }}
+                            }}
+                        }}
+                    }}
+                }});
+                charts.push(chart);
+                throughputChartInstances[task] = {{ chart, panelId: `task-panel-${{i}}` }};
+
+                // Card click handler
+                card.addEventListener('click', () => {{
+                    document.querySelectorAll('.task-card').forEach(c => c.classList.remove('active'));
+                    document.querySelectorAll('.task-chart-area').forEach(p => p.classList.remove('active'));
+                    card.classList.add('active');
+                    panel.classList.add('active');
+                    // Trigger resize so chart renders at correct size
+                    throughputChartInstances[task].chart.resize();
+                }});
+            }});
+        }} else {{
+            taskGrid.innerHTML = '<p style="color: var(--text-dim)">No throughput data available. Run measure_throughput.py to generate data.</p>';
+        }}
+    </script>
+</body>
+</html>
+"""
+
+
+def load_cached_results(output_dir: Path) -> dict[str, dict]:
+  """Load previously evaluated results from cache."""
+  data_file = output_dir / "data.json"
+  if not data_file.exists():
+    return {}
+
+  with open(data_file) as f:
+    runs = json.load(f)
+
+  return {run["id"]: run for run in runs}
+
+
+def main(
+  run_paths: list[str] | None = None,
+  entity: str = "gcbc_researchers",
+  project: str = "mjlab",
+  tag: str = "nightly",
+  eval_limit: int = 0,
+  num_envs: int = 1024,
+  output_dir: Path = Path("benchmark_results"),
+) -> None:
+  """Generate benchmark report by evaluating nightly runs.
+
+  Args:
+    run_paths: Specific run paths to evaluate (entity/project/run_id).
+    entity: WandB entity.
+    project: WandB project name.
+    tag: Filter runs by tag.
+    eval_limit: Maximum number of NEW runs to evaluate per invocation (0 = no limit).
+    num_envs: Number of envs for evaluation.
+    output_dir: Output directory for generated report.
+  """
+  # Load cached results to avoid re-evaluating old runs.
+  cached = load_cached_results(output_dir)
+  print(f"Loaded {len(cached)} cached evaluation results")
+
+  # Start with all cached results (preserves historical data).
+  eval_results_by_id: dict[str, dict] = dict(cached)
+  new_evals = 0
+
+  if run_paths:
+    for run_path in run_paths:
+      run_id = run_path.split("/")[-1]
+      if run_id in eval_results_by_id:
+        print(f"Using cached result for {run_id}")
+      else:
+        result = evaluate_run(run_path, num_envs)
+        eval_results_by_id[run_id] = result
+        new_evals += 1
+  else:
+    api = wandb.Api()
+    print(f"Fetching runs from {entity}/{project} with tag '{tag}'...")
+    runs = api.runs(f"{entity}/{project}", filters={"tags": tag}, order="-created_at")
+
+    for run in runs:
+      if run.state != "finished":
+        continue
+
+      if run.id in eval_results_by_id:
+        print(f"Using cached result for {run.name} ({run.id})")
+      else:
+        if eval_limit > 0 and new_evals >= eval_limit:
+          print(f"Reached eval limit ({eval_limit}), skipping remaining new runs")
+          break
+        run_path = f"{entity}/{project}/{run.id}"
+        result = evaluate_run(run_path, num_envs)
+        eval_results_by_id[run.id] = result
+        new_evals += 1
+
+  eval_results = list(eval_results_by_id.values())
+  print(f"Total runs: {len(eval_results)} ({new_evals} newly evaluated)")
+  generate_html_report(eval_results, output_dir)
+
+
+if __name__ == "__main__":
+  tyro.cli(main, config=mjlab.TYRO_FLAGS)
@@ -0,0 +1,246 @@
+"""Measure environment throughput for regression tracking.
+
+This script measures physics and environment step throughput across canonical tasks
+to catch performance regressions in the manager-based API.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+import torch
+import tyro
+import wandb
+
+import mjlab
+import mjlab.tasks  # noqa: F401 - registers tasks
+from mjlab.envs import ManagerBasedRlEnv
+from mjlab.tasks.registry import load_env_cfg
+from mjlab.tasks.tracking.mdp.commands import MotionCommandCfg
+
+
+@dataclass
+class BenchmarkResult:
+  """Results from a single benchmark run."""
+
+  task: str
+  num_envs: int
+  num_steps: int
+  decimation: int
+  physics_sps: float
+  env_sps: float
+  overhead_pct: float
+
+  def __str__(self) -> str:
+    return (
+      f"{self.task} (dec={self.decimation}):\n"
+      f"  Physics SPS: {self.physics_sps:,.0f}\n"
+      f"  Env SPS:     {self.env_sps:,.0f}\n"
+      f"  Overhead:    {self.overhead_pct:.1f}%"
+    )
+
+  def to_dict(self) -> dict:
+    return asdict(self)
+
+
+@dataclass
+class ThroughputConfig:
+  """Configuration for throughput benchmarking."""
+
+  num_envs: int = 4096
+  """Number of parallel environments."""
+
+  num_steps: int = 200
+  """Number of steps to measure (after warmup)."""
+
+  warmup_steps: int = 50
+  """Number of warmup steps before measuring."""
+
+  device: str = "cuda:0"
+  """Device to run on."""
+
+  tasks: list[str] = field(
+    default_factory=lambda: [
+      "Mjlab-Velocity-Flat-Unitree-Go1",
+      "Mjlab-Tracking-Flat-Unitree-G1",
+      "Mjlab-Lift-Cube-Yam",
+    ]
+  )
+  """Tasks to benchmark."""
+
+  tracking_motion: str = "rll_humanoid/wandb-registry-Motions/lafan_cartwheel:latest"
+  """W&B artifact path for tracking task motion (entity/project/name:alias)."""
+
+  output_dir: Path | None = None
+  """Output directory for JSON results. If None, results are only printed."""
+
+
+def measure_physics_sps(env: ManagerBasedRlEnv, num_steps: int) -> float:
+  """Measure raw physics stepping in env steps per second.
+
+  Runs num_steps worth of physics (i.e., num_steps * decimation sim.step calls)
+  and reports throughput in env steps/sec for direct comparison with env.step().
+  """
+  decimation = env.cfg.decimation
+  total_physics_steps = num_steps * decimation
+
+  torch.cuda.synchronize()
+  start = time.perf_counter()
+
+  for _ in range(total_physics_steps):
+    env.sim.step()
+
+  torch.cuda.synchronize()
+  elapsed = time.perf_counter() - start
+
+  # Report in env steps/sec (not physics steps/sec) for fair comparison.
+  return (num_steps * env.num_envs) / elapsed
+
+
+def measure_env_sps(env: ManagerBasedRlEnv, num_steps: int) -> float:
+  """Measure full environment step throughput in env steps per second."""
+  action_dim = sum(env.action_manager.action_term_dim)
+  action = torch.zeros(env.num_envs, action_dim, device=env.device)
+
+  torch.cuda.synchronize()
+  start = time.perf_counter()
+
+  for _ in range(num_steps):
+    env.step(action)
+
+  torch.cuda.synchronize()
+  elapsed = time.perf_counter() - start
+
+  return (num_steps * env.num_envs) / elapsed
+
+
+def benchmark_task(task: str, cfg: ThroughputConfig) -> BenchmarkResult:
+  """Benchmark a single task."""
+  print(f"\nBenchmarking {task}...")
+
+  env_cfg = load_env_cfg(task)
+  env_cfg.scene.num_envs = cfg.num_envs
+
+  # Handle tracking task motion file.
+  if len(env_cfg.commands) > 0:
+    motion_cmd = env_cfg.commands.get("motion")
+    if isinstance(motion_cmd, MotionCommandCfg):
+      api = wandb.Api()
+      artifact = api.artifact(cfg.tracking_motion)
+      motion_dir = artifact.download()
+      motion_cmd.motion_file = str(Path(motion_dir) / "motion.npz")
+
+  env = ManagerBasedRlEnv(cfg=env_cfg, device=cfg.device)
+  env.reset()
+
+  # Warmup.
+  action_dim = sum(env.action_manager.action_term_dim)
+  action = torch.zeros(env.num_envs, action_dim, device=env.device)
+  for _ in range(cfg.warmup_steps):
+    env.step(action)
+  torch.cuda.synchronize()
+
+  decimation = env.cfg.decimation
+  physics_sps = measure_physics_sps(env, cfg.num_steps)
+
+  env.reset()
+  torch.cuda.synchronize()
+
+  env_sps = measure_env_sps(env, cfg.num_steps)
+
+  overhead_pct = 100 * (1 - env_sps / physics_sps)
+
+  env.close()
+
+  return BenchmarkResult(
+    task=task,
+    num_envs=cfg.num_envs,
+    num_steps=cfg.num_steps,
+    decimation=decimation,
+    physics_sps=physics_sps,
+    env_sps=env_sps,
+    overhead_pct=overhead_pct,
+  )
+
+
+def get_git_commit() -> str:
+  """Get current git commit SHA."""
+  try:
+    result = subprocess.run(
+      ["git", "rev-parse", "HEAD"],
+      capture_output=True,
+      text=True,
+      check=True,
+    )
+    return result.stdout.strip()[:7]
+  except subprocess.CalledProcessError:
+    return "unknown"
+
+
+def save_results(results: list[BenchmarkResult], output_dir: Path) -> None:
+  """Save benchmark results to JSON, appending to existing data."""
+  output_dir.mkdir(parents=True, exist_ok=True)
+  data_file = output_dir / "throughput_data.json"
+
+  # Load existing data.
+  existing: list[dict] = []
+  if data_file.exists():
+    with open(data_file) as f:
+      existing = json.load(f)
+
+  # Create new run entry.
+  run_entry = {
+    "created_at": datetime.now(timezone.utc).isoformat(),
+    "commit": get_git_commit(),
+    "results": [r.to_dict() for r in results],
+  }
+
+  existing.append(run_entry)
+
+  with open(data_file, "w") as f:
+    json.dump(existing, f, indent=2)
+
+  print(f"\nResults saved to {data_file}")
+
+
+def main(cfg: ThroughputConfig) -> list[BenchmarkResult]:
+  """Run throughput benchmarks on all configured tasks."""
+  print("Throughput Benchmark")
+  print(f"  Envs: {cfg.num_envs}")
+  print(f"  Steps: {cfg.num_steps} (+ {cfg.warmup_steps} warmup)")
+  print(f"  Device: {cfg.device}")
+
+  results = []
+  for task in cfg.tasks:
+    result = benchmark_task(task, cfg)
+    results.append(result)
+    print(result)
+
+  print("\n" + "=" * 74)
+  print("Summary (all values in env steps per second):")
+  print("  Physics SPS: sim.step() only (×decimation per env step)")
+  print("  Env SPS: full env.step() including managers")
+  print("  Overhead: time spent on non-physics work (observations, rewards, etc.)")
+  print("=" * 74)
+  print(f"{'Task':<35} {'Dec':>4} {'Physics SPS':>12} {'Env SPS':>12} {'Overhead':>8}")
+  print("-" * 74)
+  for r in results:
+    task_short = r.task.replace("Mjlab-", "").replace("-Unitree-", "-")
+    print(
+      f"{task_short:<35} {r.decimation:>4} {r.physics_sps:>12,.0f} {r.env_sps:>12,.0f} {r.overhead_pct:>7.1f}%"
+    )
+
+  if cfg.output_dir:
+    save_results(results, cfg.output_dir)
+
+  return results
+
+
+if __name__ == "__main__":
+  cfg = tyro.cli(ThroughputConfig, config=mjlab.TYRO_FLAGS)
+  main(cfg)
@@ -0,0 +1,152 @@
+#!/bin/bash
+# Nightly training script for mjlab benchmarks
+#
+# This script clones mjlab fresh, runs the tracking benchmark, and generates a report.
+# It is designed to be called by a systemd timer or cron job.
+#
+# Usage:
+#   ./scripts/benchmarks/nightly_train.sh
+#
+# Environment variables:
+#   CUDA_DEVICE: GPU device to use (default: 0)
+#   WANDB_TAGS: Comma-separated tags for the run (default: nightly)
+#   SKIP_TRAINING: Set to "1" to skip training and only generate report
+#   SKIP_THROUGHPUT: Set to "1" to skip throughput benchmarking
+
+set -euo pipefail
+
+# Configuration
+CUDA_DEVICE="${CUDA_DEVICE:-0}"
+WANDB_TAGS="${WANDB_TAGS:-(\"nightly\",)}"
+SKIP_TRAINING="${SKIP_TRAINING:-0}"
+SKIP_THROUGHPUT="${SKIP_THROUGHPUT:-0}"
+
+# Training configuration
+TASK="Mjlab-Tracking-Flat-Unitree-G1"
+NUM_ENVS=4096
+MAX_ITERATIONS=6000
+REGISTRY_NAME="rll_humanoid/wandb-registry-Motions/side_kick_test"
+
+REPO_URL="git@github.com:mujocolab/mjlab.git"
+GH_PAGES_BRANCH="gh-pages"
+WORK_DIR="/tmp/mjlab-nightly-$$"
+GH_PAGES_DIR="/tmp/mjlab-gh-pages-$$"
+
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
+}
+
+error() {
+    log "ERROR: $*" >&2
+    exit 1
+}
+
+clear_gpu() {
+    local gpu_device="$1"
+    log "Clearing GPU $gpu_device..."
+    gpu_pids=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader,nounits -i "$gpu_device" 2>/dev/null || true)
+    if [[ -n "$gpu_pids" ]]; then
+        for pid in $gpu_pids; do
+            log "Killing process $pid on GPU $gpu_device"
+            kill -9 "$pid" 2>/dev/null || true
+        done
+        sleep 2  # Wait for processes to fully terminate
+    fi
+}
+
+cleanup() {
+    if [[ -d "$WORK_DIR" ]]; then
+        log "Cleaning up work directory..."
+        rm -rf "$WORK_DIR"
+    fi
+    if [[ -d "$GH_PAGES_DIR" ]]; then
+        log "Cleaning up gh-pages clone..."
+        rm -rf "$GH_PAGES_DIR"
+    fi
+}
+trap cleanup EXIT
+
+export GIT_SSH_COMMAND="ssh -i \"$HOME/.ssh/mjlab_nightly_ed25519\" \
+    -o IdentitiesOnly=yes \
+    -o StrictHostKeyChecking=accept-new"
+
+# Clone fresh copy of mjlab
+log "Cloning mjlab..."
+git clone "$REPO_URL" "$WORK_DIR"
+cd "$WORK_DIR"
+
+log "Starting nightly benchmark run"
+log "Task: $TASK"
+log "GPU: $CUDA_DEVICE"
+log "Commit: $(git rev-parse HEAD)"
+
+# Run training
+if [[ "$SKIP_TRAINING" != "1" ]]; then
+    log "Starting training..."
+
+    clear_gpu "$CUDA_DEVICE"
+
+    CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" uv run train "$TASK" \
+        --env.scene.num-envs "$NUM_ENVS" \
+        --agent.max-iterations "$MAX_ITERATIONS" \
+        --registry-name "$REGISTRY_NAME" \
+        --agent.wandb-tags "$WANDB_TAGS"
+
+    log "Training completed"
+else
+    log "Skipping training (SKIP_TRAINING=1)"
+fi
+
+# Clone gh-pages branch (shallow clone for speed)
+log "Cloning gh-pages branch..."
+if git ls-remote --exit-code --heads origin "$GH_PAGES_BRANCH" > /dev/null 2>&1; then
+    git clone --branch "$GH_PAGES_BRANCH" --depth 1 "$REPO_URL" "$GH_PAGES_DIR"
+else
+    # Create new gh-pages branch
+    mkdir -p "$GH_PAGES_DIR"
+    cd "$GH_PAGES_DIR"
+    git init
+    git remote add origin "$REPO_URL"
+    git checkout -b "$GH_PAGES_BRANCH"
+    cd "$WORK_DIR"
+fi
+
+# Copy cached data if exists
+REPORT_DIR="$GH_PAGES_DIR/nightly"
+mkdir -p "$REPORT_DIR"
+
+# Run throughput benchmark
+if [[ "$SKIP_THROUGHPUT" != "1" ]]; then
+    log "Running throughput benchmark..."
+
+    clear_gpu "$CUDA_DEVICE"
+
+    CUDA_VISIBLE_DEVICES="$CUDA_DEVICE" uv run python scripts/benchmarks/measure_throughput.py \
+        --num-envs "$NUM_ENVS" \
+        --output-dir "$REPORT_DIR"
+    log "Throughput benchmark completed"
+else
+    log "Skipping throughput benchmark (SKIP_THROUGHPUT=1)"
+fi
+
+# Generate report (uses cached data.json if present, only evaluates new runs)
+log "Generating benchmark report..."
+uv run python scripts/benchmarks/generate_report.py \
+    --entity gcbc_researchers \
+    --tag nightly \
+    --output-dir "$REPORT_DIR"
+
+log "Report generated"
+
+# Commit and push
+cd "$GH_PAGES_DIR"
+git add -A
+if git diff --staged --quiet; then
+    log "No changes to commit"
+else
+    git commit -m "Update nightly tracking benchmark $(date '+%Y-%m-%d')"
+    git push origin "$GH_PAGES_BRANCH" || log "Failed to push"
+    log "Deployed to GitHub Pages"
+fi
+
+log "Nightly benchmark complete"
@@ -0,0 +1,65 @@
+# Systemd Setup for Nightly Benchmarks
+
+This directory contains systemd user service and timer files for running nightly
+mjlab benchmarks.
+
+## Setup
+
+```bash
+# 1. Create user systemd directory if it doesn't exist
+mkdir -p ~/.config/systemd/user
+
+# 2. Copy the service and timer files
+cp mjlab-nightly.service ~/.config/systemd/user/
+cp mjlab-nightly.timer ~/.config/systemd/user/
+
+# 3. Edit the service file to set your WANDB_API_KEY
+#    Get your key from: https://wandb.ai/authorize
+nano ~/.config/systemd/user/mjlab-nightly.service
+
+# 4. Reload systemd
+systemctl --user daemon-reload
+
+# 5. Enable and start the timer
+systemctl --user enable mjlab-nightly.timer
+systemctl --user start mjlab-nightly.timer
+
+# 6. Enable lingering (so timer runs even when you're not logged in)
+sudo loginctl enable-linger $USER
+```
+
+## Useful Commands
+
+```bash
+# Check timer status
+systemctl --user status mjlab-nightly.timer
+
+# List all timers and when they'll run next
+systemctl --user list-timers
+
+# Run the benchmark manually (without waiting for timer)
+systemctl --user start mjlab-nightly.service
+
+# View logs
+journalctl --user -u mjlab-nightly.service -f
+
+# View recent logs
+journalctl --user -u mjlab-nightly.service --since "1 hour ago"
+
+# Disable the timer
+systemctl --user disable mjlab-nightly.timer
+```
+
+## Configuration
+
+Edit `~/.config/systemd/user/mjlab-nightly.service` to customize:
+
+- `Environment="CUDA_DEVICE=0"` - Which GPU to use
+- `Environment="WANDB_API_KEY=..."` - Your WandB API key
+- `MemoryMax=32G` - Memory limit for the training job
+
+Edit `~/.config/systemd/user/mjlab-nightly.timer` to change the schedule:
+
+- `OnCalendar=*-*-* 02:00:00` - Default: 2 AM daily
+- `OnCalendar=Mon *-*-* 02:00:00` - Example: Mondays only at 2 AM
+- `OnCalendar=*-*-* 02,14:00:00` - Example: Twice daily at 2 AM and 2 PM
@@ -0,0 +1,38 @@
+# mjlab Nightly Benchmark Service
+#
+# Installation:
+#   1. Copy this file to ~/.config/systemd/user/mjlab-nightly.service
+#   2. Edit the paths below to match your setup
+#   3. Enable with: systemctl --user enable mjlab-nightly.timer
+#   4. Start with: systemctl --user start mjlab-nightly.timer
+#
+# To run manually: systemctl --user start mjlab-nightly.service
+# To check logs: journalctl --user -u mjlab-nightly.service
+
+[Unit]
+Description=mjlab Nightly Training Benchmark
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+
+# Update these paths to match your setup
+Environment="PATH=/home/kevin/.local/bin:/home/kevin/.cargo/bin:/usr/local/bin:/usr/bin:/bin"
+Environment="HOME=/home/kevin"
+Environment="CUDA_DEVICE=0"
+Environment="WANDB_API_KEY="
+
+WorkingDirectory=/home/kevin/dev/mjlab
+ExecStart=/home/kevin/dev/mjlab/scripts/benchmarks/nightly_train.sh
+
+# Logging
+StandardOutput=journal
+StandardError=journal
+
+# Resource limits (adjust as needed)
+Nice=10
+MemoryMax=32G
+
+[Install]
+WantedBy=default.target
@@ -0,0 +1,28 @@
+# mjlab Nightly Benchmark Timer
+#
+# This timer triggers the nightly training run.
+# Default: 2:00 AM daily
+#
+# Installation:
+#   1. Copy to ~/.config/systemd/user/mjlab-nightly.timer
+#   2. Enable: systemctl --user enable mjlab-nightly.timer
+#   3. Start: systemctl --user start mjlab-nightly.timer
+#
+# Check status: systemctl --user list-timers
+# Check next run: systemctl --user status mjlab-nightly.timer
+
+[Unit]
+Description=Run mjlab nightly benchmarks
+
+[Timer]
+# Run at 2:00 AM every day
+OnCalendar=*-*-* 02:00:00
+
+# Add randomized delay up to 30 minutes to avoid thundering herd
+RandomizedDelaySec=1800
+
+# If missed (e.g., machine was off), run on next boot
+Persistent=true
+
+[Install]
+WantedBy=timers.target