deployment/evaluate_geometry_forcing_8h200.sh

#!/usr/bin/env bash
#SBATCH --job-name=geometry-forcing-re10k-ar12-cfg1-full
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=8
#SBATCH --cpus-per-task=14
#SBATCH --gres=gpu:8
#SBATCH --time=48:00:00
#SBATCH --account=berzelius-2026-196
#SBATCH --chdir=/proj/cvl/users/x_fahkh2/worldmem-baseline-evals/GeometryForcing
#SBATCH --output=slurm_logs/%x-%A_%a.out
#SBATCH --error=slurm_logs/%x-%A_%a.out

set -euo pipefail

module load buildenv-gcccuda/12.9.1-gcc11

ROOT=/proj/cvl/users/x_fahkh2/worldmem-baseline-evals
ENV_ROOT=/proj/cvl/users/x_fahkh2/envs
REPO=$ROOT/GeometryForcing
PYTHON=$ENV_ROOT/geometry-forcing-eval/bin/python
METRIC_PYTHON=$ENV_ROOT/geometry-forcing-eval/bin/python
DATA_ROOT=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/re10k_dfot/real-estate-10k
MANIFEST=$ROOT/deployment/manifests/re10k_real_trajectory_h100_f399_scopes_v1.csv
SEED=0
JOB=$SLURM_JOB_ID
RUN=geometry-forcing_ar12_cfg1_re10k_100cases_seed${SEED}_job${JOB}
OUTPUT=$REPO/outputs/$RUN
PREFLIGHT_OUTPUT=$REPO/outputs/${RUN}_preflight

export TMPDIR=$ROOT/cache
export HF_HOME=$ROOT/cache/huggingface
export TORCH_HOME=$ROOT/cache/torch
export XDG_CACHE_HOME=$ROOT/cache
export MPLCONFIGDIR=$ROOT/cache/matplotlib
export TRITON_CACHE_DIR=$ROOT/cache
export PYTHONNOUSERSITE=1
export PYTHONUNBUFFERED=1
export OMP_NUM_THREADS=4
export MKL_NUM_THREADS=4
export WANDB_MODE=disabled
export HF_HUB_OFFLINE=1
export TRANSFORMERS_OFFLINE=1
mkdir -p "$TMPDIR" "$OUTPUT"
cd "$REPO"

"$PYTHON" "$ROOT/deployment/verify_node.py" --output "$OUTPUT"
cp "$0" "$OUTPUT/launcher.sh"
cp "$ROOT/deployment/geometry_causal_controls.py" "$OUTPUT/probe.py"

# Check generation and scoring across the complete loop, then continue automatically.
srun --exclusive --exact --nodes=1 --ntasks=1 --gpus-per-task=1 --cpus-per-task=14 --cpu-bind=cores \
    "$PYTHON" "$ROOT/deployment/geometry_causal_controls.py" \
    --mode ar12_cfg1 \
    --data-root "$DATA_ROOT" \
    --output "$PREFLIGHT_OUTPUT" \
    --scope monitor8 \
    --limit 1 \
    --frames 300 \
    --rank 0 \
    --world-size 1

srun --exclusive --exact --nodes=1 --ntasks=1 --gpus-per-task=1 --cpus-per-task=14 \
    "$METRIC_PYTHON" "$ROOT/shared/aggregate.py" \
    --output "$PREFLIGHT_OUTPUT" \
    --manifest "$MANIFEST" \
    --dataset re10k \
    --scope monitor8 \
    --seed "$SEED" \
    --expected-frames 300 \
    --limit 1

srun --ntasks=8 --gpus-per-task=1 --cpus-per-task=14 --cpu-bind=cores --kill-on-bad-exit=1 \
    "$PYTHON" "$ROOT/deployment/geometry_causal_controls.py" \
    --mode ar12_cfg1 \
    --data-root "$DATA_ROOT" \
    --output "$OUTPUT" \
    --scope final100 \
    --limit 100 \
    --frames 300 \
    --world-size 8

srun --exclusive --exact --nodes=1 --ntasks=1 --gpus-per-task=1 --cpus-per-task=14 \
    "$METRIC_PYTHON" "$ROOT/shared/aggregate.py" \
    --output "$OUTPUT" \
    --manifest "$MANIFEST" \
    --dataset re10k \
    --scope final100 \
    --seed "$SEED" \
    --expected-frames 300