recipe/pipeline/upload_watchdog.sh

#!/bin/bash
# upload_watchdog.sh <name> <workdir> <command...>
# Runs a Hugging Face upload command in a 40G memory scope (niced) and kills + retries it (3 attempts) when its progress
# output — upload_large_folder's minute-by-minute "Files:" status and the xet progress bars — has not changed for
# 20 minutes. Reason: on 2026-09-16 the Agnes imatrix upload hung silently for 51 min (both HTTPS sockets in
# CLOSE-WAIT, 0 B/s, no error line) while holding the quiet-box lock. upload_large_folder resumes from the folder's
# .cache, so a retry only sends what is missing. Exit 0 once an attempt prints UPLOAD_DONE.
# The caller holds the quiet-box lock on fd 9; the upload process itself does not inherit it.
set -uo pipefail
name=$1; dir=$2; shift 2
cd "$dir" || exit 2
mkdir -p logs
log(){ echo "[$(date -u +%FT%TZ)] $*"; }
for try in 1 2 3; do
  L=logs/upload_$name.r$try.log; U=$name-r$try-$(date +%s)
  log "$name attempt $try -> $dir/$L (scope $U, MemoryMax=40G)"
  systemd-run --scope --quiet --unit="$U" -p MemoryMax=40G -p MemorySwapMax=0 nice -n 10 "$@" > "$L" 2>&1 9>&- &
  pid=$!; last=""; same=0
  while kill -0 $pid 2>/dev/null; do
    sleep 60
    cur=$(tr '\r' '\n' < "$L" | grep -E '^Files:|B / ' | tail -3 | md5sum)
    if [ "$cur" = "$last" ]; then same=$((same + 1)); else same=0; last=$cur; fi
    if [ $same -ge 20 ]; then
      log "  $name attempt $try: no progress for 20 min -> kill ($(tr '\r' '\n' < "$L" | grep -E '^Files:' | tail -1))"
      systemctl kill --signal=KILL "$U"; same=0
    fi
  done
  wait $pid; rc=$?
  if [ $rc -eq 0 ] && grep -q UPLOAD_DONE "$L"; then log "  $name attempt $try done"; exit 0; fi
  log "  $name attempt $try failed rc=$rc: $(tr '\r' '\n' < "$L" | grep -v '^\s*$' | tail -1 | cut -c1-200)"
  sleep 30
done
exit 1