fix(docker): failover drill kills the STANDBY by default; active mode measures the registered keep-oldest outage instead of pretending recovery (plan R2-01 T1)
This commit is contained in:
Joseph Doherty
+88
-19
@@ -1,44 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
|
||||
# Kills the ACTIVE central node (docker kill = SIGKILL, the hard-crash path that
|
||||
# review 01 found had NO automatic recovery before the SBR downing provider was
|
||||
# enabled) and measures how long Traefik takes to route to the survivor.
|
||||
#
|
||||
# ROUND-2 REWRITE (arch-review 01 round 2, N1). The original drill killed the
|
||||
# ACTIVE central node — but under the unified oldest-member semantics the
|
||||
# active node IS the oldest, i.e. the one crash two-node keep-oldest CANNOT
|
||||
# survive (registered deferred user decision, master tracker 2026-07-08;
|
||||
# SbrFailoverTests.cs XML doc). Two modes:
|
||||
#
|
||||
# DRILL_MODE=standby (default) — kills the STANDBY (younger) central node.
|
||||
# The survivable direction: SBR downs the crashed member, the active node
|
||||
# keeps its singletons, and Traefik routing never goes dark. PASS = the
|
||||
# survivor logs the member removal within TIMEOUT_S (budget ~25s+: 10s
|
||||
# failure detection + 15s stable-after) while /health/active stays up.
|
||||
#
|
||||
# DRILL_MODE=active — kills the ACTIVE (oldest) central node. THE EXPECTED
|
||||
# OUTCOME IS A TOTAL CENTRAL OUTAGE: keep-oldest downs the partition
|
||||
# without the oldest, so the younger survivor downs ITSELF (down-if-alone
|
||||
# cannot help — the alone-oldest is dead and cannot down itself), and the
|
||||
# self-downed survivor cannot re-form a cluster alone unless it is the
|
||||
# FIRST seed (both nodes list central-a first; only the first seed may
|
||||
# self-join). This mode measures the dark window and PASSes only when
|
||||
# central recovers AFTER the victim container is restarted. It exists to
|
||||
# make the registered gap observable — not to pretend it is covered.
|
||||
set -euo pipefail
|
||||
|
||||
TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
|
||||
TIMEOUT_S="${TIMEOUT_S:-90}"
|
||||
DRILL_MODE="${DRILL_MODE:-standby}"
|
||||
OUTAGE_CONFIRM_S="${OUTAGE_CONFIRM_S:-60}"
|
||||
|
||||
active_container() {
|
||||
if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a
|
||||
elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b
|
||||
else echo "ERROR: no active central node found" >&2; exit 1; fi
|
||||
}
|
||||
peer_of() { [ "$1" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a; }
|
||||
|
||||
VICTIM=$(active_container)
|
||||
echo "Active central node: ${VICTIM} — killing it (SIGKILL, crash path)"
|
||||
case "$DRILL_MODE" in
|
||||
standby|active) ;;
|
||||
*) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
ACTIVE=$(active_container)
|
||||
if [ "$DRILL_MODE" = standby ]; then
|
||||
VICTIM=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE"
|
||||
else
|
||||
VICTIM="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
|
||||
fi
|
||||
|
||||
echo "mode=${DRILL_MODE} active=${ACTIVE} victim=${VICTIM} survivor=${SURVIVOR}"
|
||||
KILL_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
docker kill "${VICTIM}" > /dev/null
|
||||
START=$(date +%s)
|
||||
|
||||
echo "Waiting for the survivor to become active through Traefik (${TRAEFIK_URL}/health/active)..."
|
||||
if [ "$DRILL_MODE" = standby ]; then
|
||||
echo "Standby crash: waiting for ${SURVIVOR} to DOWN+REMOVE the dead member (SBR budget ~25s)..."
|
||||
BLIPS=0
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
curl -sf -o /dev/null "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1))
|
||||
if docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "marking.*node.*down|member removed|is removed"; then
|
||||
echo "PASS: survivor removed the crashed member in ${ELAPSED}s (budget ~25s: 10s detection + 15s stable-after)."
|
||||
echo "Active-node routing blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > TIMEOUT_S )); then
|
||||
echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — SBR did not act" >&2
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
echo "Active crash: EXPECTING a central outage (registered keep-oldest gap). Watching /health/active..."
|
||||
DARK_STREAK=0
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then DARK_STREAK=0; else DARK_STREAK=$((DARK_STREAK + 1)); fi
|
||||
if (( DARK_STREAK >= 10 )); then
|
||||
echo "Outage confirmed at ${ELAPSED}s: no active central node — the younger survivor self-downed"
|
||||
echo "(keep-oldest downs the partition WITHOUT the oldest; this is the registered deferred gap)."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > OUTAGE_CONFIRM_S )); then
|
||||
echo "NOTE: /health/active stayed reachable ${ELAPSED}s after killing the oldest — better than the"
|
||||
echo "registered gap predicts. Do NOT celebrate: capture both nodes' logs and investigate before trusting it."
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
echo "Restarting ${VICTIM}..."
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
RESTART_AT=$(date +%s)
|
||||
echo "Waiting for central to be routable again through Traefik (${TRAEFIK_URL}/health/active)..."
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
ELAPSED=$(( $(date +%s) - RESTART_AT ))
|
||||
if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then
|
||||
echo "PASS: failover complete in ${ELAPSED}s (design budget ~25s + Traefik health interval)"
|
||||
echo "Recovered: an active central node is routable ${ELAPSED}s after the victim restart."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > TIMEOUT_S )); then
|
||||
echo "FAIL: no active central node after ${ELAPSED}s — SBR/singleton handover did not recover" >&2
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
if (( ELAPSED > 120 )); then
|
||||
echo "FAIL: central not routable 120s after restarting ${VICTIM}" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
SURVIVOR=$([ "${VICTIM}" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a)
|
||||
echo "Survivor singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
|
||||
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|Downing|Removed" | tail -20 || true
|
||||
|
||||
echo "Restarting ${VICTIM} and waiting for it to rejoin..."
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
sleep 5
|
||||
echo "Drill complete. Verify on the Health dashboard that both nodes show Up and the survivor is Primary."
|
||||
echo "Survivor singleton/downing evidence (last 20 matching log lines from ${SURVIVOR}):"
|
||||
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|downing|removed" | tail -20 || true
|
||||
echo "Drill complete (${DRILL_MODE}). Verify on the Health dashboard that both nodes show Up and exactly one is Primary."
|
||||
|
||||
Reference in New Issue
Block a user