#!/usr/bin/env bash
# Single-node experiment wrapper; original smoke lock and memory cap are retained.
# Usage: bash scripts/run_experiment.sh train|finalize [experiment.py arguments]
# Pilot timeout defaults to 25 minutes. Longer campaigns use launch_24h.py.
set -euo pipefail
cd "$(dirname "$0")/.."
RUN_COMMAND="$1"
shift
case "$RUN_COMMAND" in train|finalize) ;; *) echo 'Expected train or finalize' >&2; exit 2 ;; esac
RUN_PYTHON="${OPENSYSONE_PYTHON:-$HOME/ai/envs/opensysone/bin/python}"
RUN_ROOT="${OPENSYSONE_RUN_ROOT:-$HOME/ai/opensysone/runs}"
mkdir -p "$RUN_ROOT"
exec 9>"$RUN_ROOT/.smoke.lock"
flock -n 9 || { echo 'Another opensysone run holds the lock.' >&2; exit 1; }
RUN_DIR="$RUN_ROOT/$(date -u +%Y%m%dT%H%M%SZ)-$RUN_COMMAND"
mkdir "$RUN_DIR"
export PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_XET=1
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
set +e
timeout --signal=TERM --kill-after=30s 25m "$RUN_PYTHON" experiment.py "$RUN_COMMAND" \
  --output "$RUN_DIR/artifacts" "$@" 2>&1 | tee "$RUN_DIR/run.log"
RUN_EXIT=${PIPESTATUS[0]}
set -e
printf '%s\n' "$RUN_EXIT" > "$RUN_DIR/exit_code"
printf '%s\n' "$RUN_DIR" > "$RUN_ROOT/LAST_EXPERIMENT_RUN"
exit "$RUN_EXIT"