ScadaBridge/docker/failover-drill.sh

#!/usr/bin/env bash
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
#
# AUTO-DOWN REWRITE (decision 2026-07-21). The cluster now runs the 'auto-down'
# downing strategy (availability-first): the leader among the REACHABLE members
# downs the unreachable peer after StableAfter, so a hard crash of EITHER
# central node — the active/oldest included — fails over to the survivor. The
# accepted trade (made explicitly by the owner) is dual-active during a real
# network partition. Both drill directions therefore expect RECOVERY:
#
#   DRILL_MODE=standby (default) — kills the STANDBY (younger) central node.
#     The active node is untouched: expect zero /health/active routing blips
#     and member removal on the survivor within ~25s (10s failure detection +
#     15s auto-down-unreachable-after).
#
#   DRILL_MODE=active — kills the ACTIVE (oldest) central node. THE SURVIVOR
#     MUST TAKE OVER: it downs the dead oldest, becomes oldest itself, hosts
#     the singletons, and /health/active goes 200 on the survivor WHILE THE
#     VICTIM IS STILL DOWN. Budget ~25s + singleton hand-over + health-probe
#     margin. (Under the pre-2026-07-21 keep-oldest strategy this direction
#     was a total outage — the younger survivor downed ITSELF, verified live;
#     Akka's down-if-alone only rescues a side with >= 2 members.)
#
# Both modes finish by restarting the victim and confirming it rejoins as a
# fresh incarnation (standby).
set -euo pipefail

TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
TIMEOUT_S="${TIMEOUT_S:-90}"
DRILL_MODE="${DRILL_MODE:-standby}"

active_container() {
  if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a
  elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b
  else echo "ERROR: no active central node found" >&2; exit 1; fi
}
peer_of() { [ "$1" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a; }
port_of() { [ "$1" = scadabridge-central-a ] && echo 9001 || echo 9002; }

case "$DRILL_MODE" in
  standby|active) ;;
  *) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;;
esac

ACTIVE=$(active_container)
if [ "$DRILL_MODE" = standby ]; then
  VICTIM=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE"
else
  VICTIM="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
fi
SURVIVOR_PORT=$(port_of "$SURVIVOR")

echo "mode=${DRILL_MODE}  active=${ACTIVE}  victim=${VICTIM}  survivor=${SURVIVOR}"
KILL_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
docker kill "${VICTIM}" > /dev/null
START=$(date +%s)

if [ "$DRILL_MODE" = standby ]; then
  echo "Standby crash: waiting for ${SURVIVOR} to DOWN+REMOVE the dead member (budget ~25s)..."
  BLIPS=0
  while true; do
    ELAPSED=$(( $(date +%s) - START ))
    curl -sf -o /dev/null "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1))
    if docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "auto-downing|marking.*node.*down|member removed|is removed"; then
      echo "PASS: survivor downed/removed the crashed member in ${ELAPSED}s (budget ~25s: 10s detection + 15s auto-down)."
      echo "Active-node routing blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)."
      break
    fi
    if (( ELAPSED > TIMEOUT_S )); then
      echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — auto-down did not act" >&2
      docker start "${VICTIM}" > /dev/null
      exit 1
    fi
    sleep 1
  done
else
  echo "Active crash: waiting for ${SURVIVOR} to take over as the active node (victim stays DOWN; budget ~25s + hand-over)..."
  while true; do
    ELAPSED=$(( $(date +%s) - START ))
    if curl -sf -o /dev/null "http://localhost:${SURVIVOR_PORT}/health/active"; then
      echo "PASS: ${SURVIVOR} took over as active in ${ELAPSED}s with the victim still down"
      echo "(downed the dead oldest via auto-down, assumed Oldest, re-hosted the singletons)."
      break
    fi
    if (( ELAPSED > TIMEOUT_S )); then
      echo "FAIL: ${SURVIVOR} never became active within ${ELAPSED}s of killing the oldest — takeover did not happen." >&2
      docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "sbr|downing|oldest|shutting down|terminated" | tail -20 >&2 || true
      docker start "${VICTIM}" > /dev/null
      exit 1
    fi
    sleep 1
  done
  echo "Confirming Traefik routes to the new active node..."
  TR_START=$(date +%s)
  while ! curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; do
    if (( $(date +%s) - TR_START > 60 )); then
      echo "FAIL: survivor is active but not routable through Traefik after 60s" >&2
      docker start "${VICTIM}" > /dev/null
      exit 1
    fi
    sleep 1
  done
  echo "Traefik routing recovered $(( $(date +%s) - START ))s after the kill."
fi

echo "Restarting ${VICTIM}..."
docker start "${VICTIM}" > /dev/null
RESTART_AT=$(date +%s)
echo "Waiting for the restarted victim to rejoin as a ready standby (${VICTIM} /health/ready)..."
VICTIM_PORT=$(port_of "$VICTIM")
while true; do
  ELAPSED=$(( $(date +%s) - RESTART_AT ))
  if curl -sf -o /dev/null "http://localhost:${VICTIM_PORT}/health/ready"; then
    echo "Recovered: ${VICTIM} is ready (rejoined as a fresh incarnation) ${ELAPSED}s after restart."
    break
  fi
  if (( ELAPSED > 120 )); then
    echo "FAIL: ${VICTIM} not ready 120s after restart" >&2
    exit 1
  fi
  sleep 1
done

echo "Survivor downing/singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "auto-downing|singleton|oldest|downing|removed" | tail -20 || true
echo "Drill complete (${DRILL_MODE}). Verify on the Health dashboard that both nodes show Up and exactly one is Primary."