From 06ddc59913e0e912124ad0561e3442b3b2d01764 Mon Sep 17 00:00:00 2001 From: Pepijn Date: Tue, 9 Jun 2026 16:55:42 +0200 Subject: [PATCH] feat(streaming): CONDA_ENV knob for the matrix submitter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CONDA_ENV= to run each matrix job via `conda run --no-capture-output -n ` — works inside the dash `sbatch --wrap` without sourcing conda.sh / activating, and streams logs live. Point it at a conda env with a modern torchcodec (>=0.11) + datasets (>=4.7); the default cluster `base` env is often too old to decode AV1. Co-Authored-By: Claude Opus 4.8 (1M context) --- slurm/run_streaming_matrix.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/slurm/run_streaming_matrix.sh b/slurm/run_streaming_matrix.sh index 0958fedab..a33e181fc 100755 --- a/slurm/run_streaming_matrix.sh +++ b/slurm/run_streaming_matrix.sh @@ -38,6 +38,13 @@ CPU_BUFFER=${CPU_BUFFER:-4000} GPU_BUFFER=${GPU_BUFFER:-1000} # smaller buffer bounds on-GPU frame memory BATCH_SIZE=${BATCH_SIZE:-64} RUN=${RUN:-python} +# CONDA_ENV= runs each job via `conda run -n ` (no activation needed inside the dash --wrap; +# --no-capture-output streams logs live). Set this to a conda env that has a MODERN torchcodec (>=0.11) +# + datasets (>=4.7) — the default `base` env on many clusters is too old to decode AV1 / lacks CUDA. +CONDA_ENV=${CONDA_ENV:-} +if [ -n "$CONDA_ENV" ] && [ "$RUN" = "python" ]; then + RUN="conda run --no-capture-output -n $CONDA_ENV python" +fi SOURCES=${SOURCES:-"hub bucket warmed_bucket"} MODES=${MODES:-"single sarm"}