#!/usr/bin/env bash
# Run on GX10 from a login shell. Bounds runtime, prevents overlapping opensysone
# jobs, and keeps logs/checkpoints off the source tree. No service is modified.
# Usage: bash scripts/run_smoke.sh [arguments forwarded to smoke_train.py]
set -euo pipefail
cd "$(dirname "$0")/.."
RUN_PYTHON="${OPENSYSONE_PYTHON:-$HOME/ai/envs/comfy/bin/python}"
RUN_ROOT="${OPENSYSONE_RUN_ROOT:-$HOME/ai/opensysone/runs}"
mkdir -p "$RUN_ROOT"
exec 9>"$RUN_ROOT/.smoke.lock"
flock -n 9 || { echo 'Another opensysone smoke run holds the lock.' >&2; exit 1; }
RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)"
RUN_DIR="$RUN_ROOT/$RUN_ID"
mkdir "$RUN_DIR"
export PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_XET=1
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
set +e
timeout --signal=TERM --kill-after=30s 25m "$RUN_PYTHON" smoke_train.py --output "$RUN_DIR" "$@" 2>&1 | tee "$RUN_DIR/run.log"
RUN_EXIT=${PIPESTATUS[0]}
set -e
printf '%s\n' "$RUN_EXIT" > "$RUN_DIR/exit_code"
printf '%s\n' "$RUN_DIR" > "$RUN_ROOT/LAST_RUN"
exit "$RUN_EXIT"