From badc97afe85d1c0773b174f417ce6ee1f791a45a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 13 Jul 2026 10:32:54 -0400 Subject: [PATCH] fix(docker): failover drill kills the STANDBY by default; active mode measures the registered keep-oldest outage instead of pretending recovery (plan R2-01 T1) --- docker/failover-drill.sh | 107 ++++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 19 deletions(-) diff --git a/docker/failover-drill.sh b/docker/failover-drill.sh index d600eb72..ca4c238a 100755 --- a/docker/failover-drill.sh +++ b/docker/failover-drill.sh @@ -1,44 +1,113 @@ #!/usr/bin/env bash # Failover drill against the running docker cluster (bash docker/deploy.sh first). -# Kills the ACTIVE central node (docker kill = SIGKILL, the hard-crash path that -# review 01 found had NO automatic recovery before the SBR downing provider was -# enabled) and measures how long Traefik takes to route to the survivor. +# +# ROUND-2 REWRITE (arch-review 01 round 2, N1). The original drill killed the +# ACTIVE central node — but under the unified oldest-member semantics the +# active node IS the oldest, i.e. the one crash two-node keep-oldest CANNOT +# survive (registered deferred user decision, master tracker 2026-07-08; +# SbrFailoverTests.cs XML doc). Two modes: +# +# DRILL_MODE=standby (default) — kills the STANDBY (younger) central node. +# The survivable direction: SBR downs the crashed member, the active node +# keeps its singletons, and Traefik routing never goes dark. PASS = the +# survivor logs the member removal within TIMEOUT_S (budget ~25s+: 10s +# failure detection + 15s stable-after) while /health/active stays up. +# +# DRILL_MODE=active — kills the ACTIVE (oldest) central node. THE EXPECTED +# OUTCOME IS A TOTAL CENTRAL OUTAGE: keep-oldest downs the partition +# without the oldest, so the younger survivor downs ITSELF (down-if-alone +# cannot help — the alone-oldest is dead and cannot down itself), and the +# self-downed survivor cannot re-form a cluster alone unless it is the +# FIRST seed (both nodes list central-a first; only the first seed may +# self-join). This mode measures the dark window and PASSes only when +# central recovers AFTER the victim container is restarted. It exists to +# make the registered gap observable — not to pretend it is covered. set -euo pipefail TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}" TIMEOUT_S="${TIMEOUT_S:-90}" +DRILL_MODE="${DRILL_MODE:-standby}" +OUTAGE_CONFIRM_S="${OUTAGE_CONFIRM_S:-60}" active_container() { if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b else echo "ERROR: no active central node found" >&2; exit 1; fi } +peer_of() { [ "$1" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a; } -VICTIM=$(active_container) -echo "Active central node: ${VICTIM} — killing it (SIGKILL, crash path)" +case "$DRILL_MODE" in + standby|active) ;; + *) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;; +esac + +ACTIVE=$(active_container) +if [ "$DRILL_MODE" = standby ]; then + VICTIM=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE" +else + VICTIM="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE") +fi + +echo "mode=${DRILL_MODE} active=${ACTIVE} victim=${VICTIM} survivor=${SURVIVOR}" +KILL_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) docker kill "${VICTIM}" > /dev/null START=$(date +%s) -echo "Waiting for the survivor to become active through Traefik (${TRAEFIK_URL}/health/active)..." +if [ "$DRILL_MODE" = standby ]; then + echo "Standby crash: waiting for ${SURVIVOR} to DOWN+REMOVE the dead member (SBR budget ~25s)..." + BLIPS=0 + while true; do + ELAPSED=$(( $(date +%s) - START )) + curl -sf -o /dev/null "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1)) + if docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "marking.*node.*down|member removed|is removed"; then + echo "PASS: survivor removed the crashed member in ${ELAPSED}s (budget ~25s: 10s detection + 15s stable-after)." + echo "Active-node routing blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)." + break + fi + if (( ELAPSED > TIMEOUT_S )); then + echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — SBR did not act" >&2 + docker start "${VICTIM}" > /dev/null + exit 1 + fi + sleep 1 + done +else + echo "Active crash: EXPECTING a central outage (registered keep-oldest gap). Watching /health/active..." + DARK_STREAK=0 + while true; do + ELAPSED=$(( $(date +%s) - START )) + if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then DARK_STREAK=0; else DARK_STREAK=$((DARK_STREAK + 1)); fi + if (( DARK_STREAK >= 10 )); then + echo "Outage confirmed at ${ELAPSED}s: no active central node — the younger survivor self-downed" + echo "(keep-oldest downs the partition WITHOUT the oldest; this is the registered deferred gap)." + break + fi + if (( ELAPSED > OUTAGE_CONFIRM_S )); then + echo "NOTE: /health/active stayed reachable ${ELAPSED}s after killing the oldest — better than the" + echo "registered gap predicts. Do NOT celebrate: capture both nodes' logs and investigate before trusting it." + break + fi + sleep 1 + done +fi + +echo "Restarting ${VICTIM}..." +docker start "${VICTIM}" > /dev/null +RESTART_AT=$(date +%s) +echo "Waiting for central to be routable again through Traefik (${TRAEFIK_URL}/health/active)..." while true; do - ELAPSED=$(( $(date +%s) - START )) + ELAPSED=$(( $(date +%s) - RESTART_AT )) if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then - echo "PASS: failover complete in ${ELAPSED}s (design budget ~25s + Traefik health interval)" + echo "Recovered: an active central node is routable ${ELAPSED}s after the victim restart." break fi - if (( ELAPSED > TIMEOUT_S )); then - echo "FAIL: no active central node after ${ELAPSED}s — SBR/singleton handover did not recover" >&2 - docker start "${VICTIM}" > /dev/null + if (( ELAPSED > 120 )); then + echo "FAIL: central not routable 120s after restarting ${VICTIM}" >&2 exit 1 fi sleep 1 done -SURVIVOR=$([ "${VICTIM}" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a) -echo "Survivor singleton evidence (last 20 matching log lines from ${SURVIVOR}):" -docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|Downing|Removed" | tail -20 || true - -echo "Restarting ${VICTIM} and waiting for it to rejoin..." -docker start "${VICTIM}" > /dev/null -sleep 5 -echo "Drill complete. Verify on the Health dashboard that both nodes show Up and the survivor is Primary." +echo "Survivor singleton/downing evidence (last 20 matching log lines from ${SURVIVOR}):" +docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|downing|removed" | tail -20 || true +echo "Drill complete (${DRILL_MODE}). Verify on the Health dashboard that both nodes show Up and exactly one is Primary."