File size: 998 Bytes
2d5c26a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | #!/usr/bin/env bash
# Bounded read-only precision diagnosis. Same shared lock and cap as the smoke.
# Usage: bash scripts/run_precision.sh --run /path/to/trusted/smoke/run
set -euo pipefail
cd "$(dirname "$0")/.."
RUN_PYTHON="${OPENSYSONE_PYTHON:-$HOME/ai/envs/comfy/bin/python}"
RUN_ROOT="${OPENSYSONE_RUN_ROOT:-$HOME/ai/opensysone/runs}"
mkdir -p "$RUN_ROOT"
exec 9>"$RUN_ROOT/.smoke.lock"
flock -n 9 || { echo 'Another opensysone run holds the lock.' >&2; exit 1; }
RUN_DIR="$RUN_ROOT/$(date -u +%Y%m%dT%H%M%SZ)-precision"
mkdir "$RUN_DIR"
export PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false HF_HUB_DISABLE_XET=1
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
set +e
timeout --signal=TERM --kill-after=30s 25m "$RUN_PYTHON" scripts/investigate_precision.py \
--output "$RUN_DIR/artifacts" "$@" 2>&1 | tee "$RUN_DIR/run.log"
RUN_EXIT=${PIPESTATUS[0]}
set -e
printf '%s\n' "$RUN_EXIT" > "$RUN_DIR/exit_code"
printf '%s\n' "$RUN_DIR" > "$RUN_ROOT/LAST_PRECISION_RUN"
exit "$RUN_EXIT"
|