test(docker): leader-auto-down rig failover drill — #116 rig gate PASS on central + site-a
Ran every #116 gate on the local docker rig (main 2d2e9f35), with an auto-down baseline
on the same build:
- active kill: takeover 29s central / 26s site-a; standby kill: downed 29s / 26s, 0 blips;
- boot logs: no deprecation warning, [leader-auto-down] started/unreachable/DOWNING lines;
- partition past the window: accepted dual-active, restart of one side recovers;
- cold start of both pairs together: one cluster each; site-a S&F drains on the survivor.
Extra: docker pause of the active node leaves a permanent second active after unpause on
BOTH strategies (Akka.Cluster ignores ThisActorSystemQuarantinedEvent) — filed as #132.
- docker/docker-compose.leader-auto-down.yml: opt-in overlay (env override) for the central
and site-a pairs; shipped default stays auto-down.
- docker/failover-drill.sh: DRILL_PAIR (central|site-a|site-b|site-c), DRILL_ACTION
(kill|pause), sidecar probes for unpublished site health, strict rejoin + dual-active check.
- docs/plans/2026-11-12-leader-auto-down-rig-drill.md (+ raw logs), docker/README.md.
Refs #116, #106, #132
Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
+147
-67
@@ -1,126 +1,206 @@
|
||||
#!/usr/bin/env bash
|
||||
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
|
||||
#
|
||||
# AUTO-DOWN REWRITE (decision 2026-07-21). The cluster now runs the 'auto-down'
|
||||
# downing strategy (availability-first): the leader among the REACHABLE members
|
||||
# downs the unreachable peer after StableAfter, so a hard crash of EITHER
|
||||
# central node — the active/oldest included — fails over to the survivor. The
|
||||
# accepted trade (made explicitly by the owner) is dual-active during a real
|
||||
# network partition. Both drill directions therefore expect RECOVERY:
|
||||
# AUTO-DOWN REWRITE (decision 2026-07-21). The cluster runs an availability-first
|
||||
# downing strategy — `auto-down` (default) or the in-repo `leader-auto-down`
|
||||
# (ScadaBridge#106; opt-in overlay docker/docker-compose.leader-auto-down.yml) —
|
||||
# under which the leader among the REACHABLE members downs the unreachable peer
|
||||
# after StableAfter, so a hard crash of EITHER node of a pair — the active/oldest
|
||||
# included — fails over to the survivor. The accepted trade (made explicitly by
|
||||
# the owner) is dual-active during a real network partition. Both drill
|
||||
# directions therefore expect RECOVERY:
|
||||
#
|
||||
# DRILL_MODE=standby (default) — kills the STANDBY (younger) central node.
|
||||
# The active node is untouched: expect zero /health/active routing blips
|
||||
# and member removal on the survivor within ~25s (10s failure detection +
|
||||
# 15s auto-down-unreachable-after).
|
||||
# DRILL_MODE=standby (default) — hits the STANDBY (younger) node.
|
||||
# The active node is untouched: expect zero /health/active blips on it and
|
||||
# member removal on the survivor within ~25s (10s failure detection +
|
||||
# 15s down window).
|
||||
#
|
||||
# DRILL_MODE=active — kills the ACTIVE (oldest) central node. THE SURVIVOR
|
||||
# MUST TAKE OVER: it downs the dead oldest, becomes oldest itself, hosts
|
||||
# the singletons, and /health/active goes 200 on the survivor WHILE THE
|
||||
# VICTIM IS STILL DOWN. Budget ~25s + singleton hand-over + health-probe
|
||||
# margin. (Under the pre-2026-07-21 keep-oldest strategy this direction
|
||||
# was a total outage — the younger survivor downed ITSELF, verified live;
|
||||
# Akka's down-if-alone only rescues a side with >= 2 members.)
|
||||
# DRILL_MODE=active — hits the ACTIVE (oldest) node. THE SURVIVOR MUST TAKE
|
||||
# OVER: it downs the dead oldest, becomes oldest itself, hosts the
|
||||
# singletons, and /health/active goes 200 on the survivor WHILE THE VICTIM
|
||||
# IS STILL DOWN. Budget ~25s + singleton hand-over + health-probe margin.
|
||||
#
|
||||
# Both modes finish by restarting the victim and confirming it rejoins as a
|
||||
# fresh incarnation (standby).
|
||||
# DRILL_PAIR=central (default) | site-a | site-b | site-c — which 2-node pair.
|
||||
# Central health is on container port 5000 (also published as 9001/9002 and
|
||||
# routed by Traefik on 9000); a site's /health/* is on container port 8084 and
|
||||
# is NOT published, so the script probes every pair from a throwaway sidecar
|
||||
# container on scadabridge-net (the app image ships no curl/wget).
|
||||
#
|
||||
# DRILL_ACTION=kill (default) | pause — `docker kill` (SIGKILL: the hard-crash
|
||||
# path; a `docker stop` would take the graceful CoordinatedShutdown path and
|
||||
# prove nothing) or `docker pause` (a frozen process: sockets stay open but
|
||||
# nothing answers — the GC-pause / hung-VM shape). A paused victim is
|
||||
# unpaused at the end instead of started; it finds it was downed, terminates
|
||||
# its actor system and comes back through the container restart policy.
|
||||
#
|
||||
# Both modes finish by recovering the victim and confirming it rejoins as a
|
||||
# fresh incarnation (standby), then that exactly one node of the pair is active.
|
||||
set -euo pipefail
|
||||
|
||||
TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
|
||||
TIMEOUT_S="${TIMEOUT_S:-90}"
|
||||
REJOIN_TIMEOUT_S="${REJOIN_TIMEOUT_S:-180}"
|
||||
DRILL_MODE="${DRILL_MODE:-standby}"
|
||||
|
||||
active_container() {
|
||||
if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a
|
||||
elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b
|
||||
else echo "ERROR: no active central node found" >&2; exit 1; fi
|
||||
}
|
||||
peer_of() { [ "$1" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a; }
|
||||
port_of() { [ "$1" = scadabridge-central-a ] && echo 9001 || echo 9002; }
|
||||
DRILL_PAIR="${DRILL_PAIR:-central}"
|
||||
DRILL_ACTION="${DRILL_ACTION:-kill}"
|
||||
PROBE_IMAGE="${PROBE_IMAGE:-alpine:latest}"
|
||||
|
||||
case "$DRILL_MODE" in
|
||||
standby|active) ;;
|
||||
*) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$DRILL_PAIR" in
|
||||
central) HEALTH_PORT=5000 ;;
|
||||
site-a|site-b|site-c) HEALTH_PORT=8084 ;;
|
||||
*) echo "ERROR: DRILL_PAIR must be central|site-a|site-b|site-c (was '$DRILL_PAIR')" >&2; exit 2 ;;
|
||||
esac
|
||||
case "$DRILL_ACTION" in
|
||||
kill|pause) ;;
|
||||
*) echo "ERROR: DRILL_ACTION must be 'kill' or 'pause' (was '$DRILL_ACTION')" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
NODE_A="scadabridge-${DRILL_PAIR}-a"
|
||||
NODE_B="scadabridge-${DRILL_PAIR}-b"
|
||||
|
||||
PROBE="scadabridge-drill-probe-$$"
|
||||
docker run -d --rm --name "$PROBE" --network scadabridge-net "$PROBE_IMAGE" sleep 3600 > /dev/null
|
||||
VICTIM=""
|
||||
cleanup() {
|
||||
# Never leave the victim paused/dead if the script dies early.
|
||||
if [ -n "$VICTIM" ]; then
|
||||
state=$(docker inspect -f '{{.State.Status}}' "$VICTIM" 2>/dev/null || true)
|
||||
[ "$state" = paused ] && docker unpause "$VICTIM" > /dev/null || true
|
||||
[ "$state" = exited ] && docker start "$VICTIM" > /dev/null || true
|
||||
fi
|
||||
docker rm -f "$PROBE" > /dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# probe <container> <path> -> exit 0 on 2xx. Bounded so a frozen target cannot hang us.
|
||||
probe() { docker exec "$PROBE" wget -q -T 2 -O /dev/null "http://$1:${HEALTH_PORT}$2" 2>/dev/null; }
|
||||
active_container() {
|
||||
if probe "$NODE_A" /health/active; then echo "$NODE_A"
|
||||
elif probe "$NODE_B" /health/active; then echo "$NODE_B"
|
||||
else echo "ERROR: no active ${DRILL_PAIR} node found" >&2; exit 1; fi
|
||||
}
|
||||
peer_of() { [ "$1" = "$NODE_A" ] && echo "$NODE_B" || echo "$NODE_A"; }
|
||||
recover_victim() {
|
||||
if [ "$DRILL_ACTION" = pause ]; then docker unpause "$VICTIM" > /dev/null
|
||||
else docker start "$VICTIM" > /dev/null; fi
|
||||
}
|
||||
|
||||
ACTIVE=$(active_container)
|
||||
if [ "$DRILL_MODE" = standby ]; then
|
||||
VICTIM=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE"
|
||||
VICTIM_SEL=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE"
|
||||
else
|
||||
VICTIM="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
|
||||
VICTIM_SEL="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
|
||||
fi
|
||||
SURVIVOR_PORT=$(port_of "$SURVIVOR")
|
||||
|
||||
echo "mode=${DRILL_MODE} active=${ACTIVE} victim=${VICTIM} survivor=${SURVIVOR}"
|
||||
KILL_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
docker kill "${VICTIM}" > /dev/null
|
||||
echo "pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} active=${ACTIVE} victim=${VICTIM_SEL} survivor=${SURVIVOR}"
|
||||
HIT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
VICTIM="$VICTIM_SEL"
|
||||
docker "$DRILL_ACTION" "${VICTIM}" > /dev/null
|
||||
START=$(date +%s)
|
||||
echo "hit at ${HIT_AT}"
|
||||
|
||||
# Downing evidence on the survivor. Matches both providers: Akka's own
|
||||
# "Marking unreachable node ... as [Down]" / "Leader is auto-downing" and the
|
||||
# in-repo "[leader-auto-down] DOWNING" line.
|
||||
DOWN_RE="leader-auto-down\] DOWNING|auto-downing|marking.*node.*down|member removed|is removed"
|
||||
DOWN_S=""
|
||||
if [ "$DRILL_MODE" = standby ]; then
|
||||
echo "Standby crash: waiting for ${SURVIVOR} to DOWN+REMOVE the dead member (budget ~25s)..."
|
||||
echo "Standby ${DRILL_ACTION}: waiting for ${SURVIVOR} to DOWN+REMOVE the victim (budget ~25s)..."
|
||||
BLIPS=0
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
curl -sf -o /dev/null "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1))
|
||||
if docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "auto-downing|marking.*node.*down|member removed|is removed"; then
|
||||
echo "PASS: survivor downed/removed the crashed member in ${ELAPSED}s (budget ~25s: 10s detection + 15s auto-down)."
|
||||
echo "Active-node routing blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)."
|
||||
probe "$SURVIVOR" /health/active || BLIPS=$((BLIPS + 1))
|
||||
if [ "$DRILL_PAIR" = central ]; then curl -sf -o /dev/null -m 2 "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1)); fi
|
||||
if docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
|
||||
DOWN_S=$ELAPSED
|
||||
echo "PASS: survivor downed/removed the victim in ${ELAPSED}s (budget ~25s: 10s detection + 15s down window)."
|
||||
echo "Active-node /health/active blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > TIMEOUT_S )); then
|
||||
echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — auto-down did not act" >&2
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — downing did not act" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
echo "Active crash: waiting for ${SURVIVOR} to take over as the active node (victim stays DOWN; budget ~25s + hand-over)..."
|
||||
echo "Active ${DRILL_ACTION}: waiting for ${SURVIVOR} to take over as the active node (victim stays down; budget ~25s + hand-over)..."
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
if curl -sf -o /dev/null "http://localhost:${SURVIVOR_PORT}/health/active"; then
|
||||
if [ -z "$DOWN_S" ] && docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
|
||||
DOWN_S=$ELAPSED; echo " survivor logged the Down at +${DOWN_S}s"
|
||||
fi
|
||||
if probe "$SURVIVOR" /health/active; then
|
||||
echo "PASS: ${SURVIVOR} took over as active in ${ELAPSED}s with the victim still down"
|
||||
echo "(downed the dead oldest via auto-down, assumed Oldest, re-hosted the singletons)."
|
||||
echo "(downed the dead oldest, assumed Oldest, re-hosted the singletons)."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > TIMEOUT_S )); then
|
||||
echo "FAIL: ${SURVIVOR} never became active within ${ELAPSED}s of killing the oldest — takeover did not happen." >&2
|
||||
docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "sbr|downing|oldest|shutting down|terminated" | tail -20 >&2 || true
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
echo "FAIL: ${SURVIVOR} never became active within ${ELAPSED}s of hitting the oldest — takeover did not happen." >&2
|
||||
docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "sbr|downing|leader-auto-down|oldest|shutting down|terminated" | tail -20 >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "Confirming Traefik routes to the new active node..."
|
||||
TR_START=$(date +%s)
|
||||
while ! curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; do
|
||||
if (( $(date +%s) - TR_START > 60 )); then
|
||||
echo "FAIL: survivor is active but not routable through Traefik after 60s" >&2
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "Traefik routing recovered $(( $(date +%s) - START ))s after the kill."
|
||||
if [ -z "$DOWN_S" ] && docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
|
||||
DOWN_S="<=${ELAPSED}"; echo " survivor logged the Down at ${DOWN_S}s (seen on the same poll as the takeover)"
|
||||
fi
|
||||
if [ "$DRILL_PAIR" = central ]; then
|
||||
echo "Confirming Traefik routes to the new active node..."
|
||||
TR_START=$(date +%s)
|
||||
while ! curl -sf -o /dev/null -m 2 "${TRAEFIK_URL}/health/active"; do
|
||||
if (( $(date +%s) - TR_START > 60 )); then
|
||||
echo "FAIL: survivor is active but not routable through Traefik after 60s" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "Traefik routing recovered $(( $(date +%s) - START ))s after the hit."
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Restarting ${VICTIM}..."
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
echo "Recovering ${VICTIM} (${DRILL_ACTION} -> $([ "$DRILL_ACTION" = pause ] && echo unpause || echo start))..."
|
||||
recover_victim
|
||||
RESTART_AT=$(date +%s)
|
||||
echo "Waiting for the restarted victim to rejoin as a ready standby (${VICTIM} /health/ready)..."
|
||||
VICTIM_PORT=$(port_of "$VICTIM")
|
||||
echo "Waiting for the victim to rejoin as a ready standby (${VICTIM} /health/ready)..."
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - RESTART_AT ))
|
||||
if curl -sf -o /dev/null "http://localhost:${VICTIM_PORT}/health/ready"; then
|
||||
echo "Recovered: ${VICTIM} is ready (rejoined as a fresh incarnation) ${ELAPSED}s after restart."
|
||||
# A paused-then-unpaused victim was downed while frozen: it must NOT be counted
|
||||
# as recovered on its stale pre-pause readiness — require the survivor to see a
|
||||
# 2-member cluster again (the victim's /health/ready alone can be a zombie).
|
||||
if probe "$VICTIM" /health/ready && ! probe "$VICTIM" /health/active \
|
||||
&& docker exec "$PROBE" wget -q -T 2 -O - "http://${SURVIVOR}:${HEALTH_PORT}/health/ready" 2>/dev/null | grep -q '"memberCount":2,"unreachableCount":0'; then
|
||||
echo "Recovered: ${VICTIM} is ready as standby (rejoined) ${ELAPSED}s after recovery."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > 120 )); then
|
||||
echo "FAIL: ${VICTIM} not ready 120s after restart" >&2
|
||||
if (( ELAPSED > REJOIN_TIMEOUT_S )); then
|
||||
echo "FAIL: ${VICTIM} not a ready, rejoined standby ${REJOIN_TIMEOUT_S}s after recovery" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
VICTIM="" # recovered; nothing for the trap to undo
|
||||
|
||||
echo "Checking for a lingering dual-active (10 samples, 1s apart)..."
|
||||
DUAL=0
|
||||
for _ in $(seq 1 10); do
|
||||
a=0; b=0
|
||||
probe "$NODE_A" /health/active && a=1
|
||||
probe "$NODE_B" /health/active && b=1
|
||||
(( a + b != 1 )) && DUAL=$((DUAL + 1))
|
||||
sleep 1
|
||||
done
|
||||
if (( DUAL > 0 )); then
|
||||
echo "FAIL: ${DUAL}/10 samples did not show exactly one active node" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Exactly one active node in 10/10 samples."
|
||||
|
||||
echo "Survivor downing/singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
|
||||
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "auto-downing|singleton|oldest|downing|removed" | tail -20 || true
|
||||
echo "Drill complete (${DRILL_MODE}). Verify on the Health dashboard that both nodes show Up and exactly one is Primary."
|
||||
docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "leader-auto-down|auto-downing|-> Oldest|downing|marking.*down|removed" | tail -20 || true
|
||||
echo "SUMMARY pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} down_logged_s=${DOWN_S:-n/a} rejoin_s=${ELAPSED}"
|
||||
echo "Drill complete. Verify on the Health dashboard / Overview that both nodes show Up and exactly one is Primary."
|
||||
|
||||
Reference in New Issue
Block a user