#!/usr/bin/env bash
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
# Kills the ACTIVE central node (docker kill = SIGKILL, the hard-crash path that
# review 01 found had NO automatic recovery before the SBR downing provider was
# enabled) and measures how long Traefik takes to route to the survivor.
set -euo pipefail

TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
TIMEOUT_S="${TIMEOUT_S:-90}"

active_container() {
  if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a
  elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b
  else echo "ERROR: no active central node found" >&2; exit 1; fi
}

VICTIM=$(active_container)
echo "Active central node: ${VICTIM} — killing it (SIGKILL, crash path)"
docker kill "${VICTIM}" > /dev/null
START=$(date +%s)

echo "Waiting for the survivor to become active through Traefik (${TRAEFIK_URL}/health/active)..."
while true; do
  ELAPSED=$(( $(date +%s) - START ))
  if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then
    echo "PASS: failover complete in ${ELAPSED}s (design budget ~25s + Traefik health interval)"
    break
  fi
  if (( ELAPSED > TIMEOUT_S )); then
    echo "FAIL: no active central node after ${ELAPSED}s — SBR/singleton handover did not recover" >&2
    docker start "${VICTIM}" > /dev/null
    exit 1
  fi
  sleep 1
done

SURVIVOR=$([ "${VICTIM}" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a)
echo "Survivor singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|Downing|Removed" | tail -20 || true

echo "Restarting ${VICTIM} and waiting for it to rejoin..."
docker start "${VICTIM}" > /dev/null
sleep 5
echo "Drill complete. Verify on the Health dashboard that both nodes show Up and the survivor is Primary."