test(docker): failover drill script — SIGKILL the active central node, assert Traefik recovery

2026-07-08 17:05:31 -04:00
parent 1f1dbd916d
commit 5117fa97b9
3 changed files with 62 additions and 0 deletions
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Failover drill against the running docker cluster (bash docker/deploy.sh first).
+# Kills the ACTIVE central node (docker kill = SIGKILL, the hard-crash path that
+# review 01 found had NO automatic recovery before the SBR downing provider was
+# enabled) and measures how long Traefik takes to route to the survivor.
+set -euo pipefail
+
+TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
+TIMEOUT_S="${TIMEOUT_S:-90}"
+
+active_container() {
+  if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a
+  elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b
+  else echo "ERROR: no active central node found" >&2; exit 1; fi
+}
+
+VICTIM=$(active_container)
+echo "Active central node: ${VICTIM} — killing it (SIGKILL, crash path)"
+docker kill "${VICTIM}" > /dev/null
+START=$(date +%s)
+
+echo "Waiting for the survivor to become active through Traefik (${TRAEFIK_URL}/health/active)..."
+while true; do
+  ELAPSED=$(( $(date +%s) - START ))
+  if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then
+    echo "PASS: failover complete in ${ELAPSED}s (design budget ~25s + Traefik health interval)"
+    break
+  fi
+  if (( ELAPSED > TIMEOUT_S )); then
+    echo "FAIL: no active central node after ${ELAPSED}s — SBR/singleton handover did not recover" >&2
+    docker start "${VICTIM}" > /dev/null
+    exit 1
+  fi
+  sleep 1
+done
+
+SURVIVOR=$([ "${VICTIM}" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a)
+echo "Survivor singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
+docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|Downing|Removed" | tail -20 || true
+
+echo "Restarting ${VICTIM} and waiting for it to rejoin..."
+docker start "${VICTIM}" > /dev/null
+sleep 5
+echo "Drill complete. Verify on the Health dashboard that both nodes show Up and the survivor is Primary."