Joseph Doherty
45 lines
1.9 KiB
Bash
Executable File
45 lines
1.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
|
|
# Kills the ACTIVE central node (docker kill = SIGKILL, the hard-crash path that
|
|
# review 01 found had NO automatic recovery before the SBR downing provider was
|
|
# enabled) and measures how long Traefik takes to route to the survivor.
|
|
set -euo pipefail
|
|
|
|
TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
|
|
TIMEOUT_S="${TIMEOUT_S:-90}"
|
|
|
|
active_container() {
|
|
if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a
|
|
elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b
|
|
else echo "ERROR: no active central node found" >&2; exit 1; fi
|
|
}
|
|
|
|
VICTIM=$(active_container)
|
|
echo "Active central node: ${VICTIM} — killing it (SIGKILL, crash path)"
|
|
docker kill "${VICTIM}" > /dev/null
|
|
START=$(date +%s)
|
|
|
|
echo "Waiting for the survivor to become active through Traefik (${TRAEFIK_URL}/health/active)..."
|
|
while true; do
|
|
ELAPSED=$(( $(date +%s) - START ))
|
|
if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then
|
|
echo "PASS: failover complete in ${ELAPSED}s (design budget ~25s + Traefik health interval)"
|
|
break
|
|
fi
|
|
if (( ELAPSED > TIMEOUT_S )); then
|
|
echo "FAIL: no active central node after ${ELAPSED}s — SBR/singleton handover did not recover" >&2
|
|
docker start "${VICTIM}" > /dev/null
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
SURVIVOR=$([ "${VICTIM}" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a)
|
|
echo "Survivor singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
|
|
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|Downing|Removed" | tail -20 || true
|
|
|
|
echo "Restarting ${VICTIM} and waiting for it to rejoin..."
|
|
docker start "${VICTIM}" > /dev/null
|
|
sleep 5
|
|
echo "Drill complete. Verify on the Health dashboard that both nodes show Up and the survivor is Primary."
|