Joseph Doherty
139a43fb25
Ran every #116 gate on the local docker rig (main 2d2e9f35), with an auto-down baseline
on the same build:
- active kill: takeover 29s central / 26s site-a; standby kill: downed 29s / 26s, 0 blips;
- boot logs: no deprecation warning, [leader-auto-down] started/unreachable/DOWNING lines;
- partition past the window: accepted dual-active, restart of one side recovers;
- cold start of both pairs together: one cluster each; site-a S&F drains on the survivor.
Extra: docker pause of the active node leaves a permanent second active after unpause on
BOTH strategies (Akka.Cluster ignores ThisActorSystemQuarantinedEvent) — filed as #132.
- docker/docker-compose.leader-auto-down.yml: opt-in overlay (env override) for the central
and site-a pairs; shipped default stays auto-down.
- docker/failover-drill.sh: DRILL_PAIR (central|site-a|site-b|site-c), DRILL_ACTION
(kill|pause), sidecar probes for unpublished site health, strict rejoin + dual-active check.
- docs/plans/2026-11-12-leader-auto-down-rig-drill.md (+ raw logs), docker/README.md.
Refs #116, #106, #132
Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
207 lines
9.5 KiB
Bash
Executable File
207 lines
9.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
|
|
#
|
|
# AUTO-DOWN REWRITE (decision 2026-07-21). The cluster runs an availability-first
|
|
# downing strategy — `auto-down` (default) or the in-repo `leader-auto-down`
|
|
# (ScadaBridge#106; opt-in overlay docker/docker-compose.leader-auto-down.yml) —
|
|
# under which the leader among the REACHABLE members downs the unreachable peer
|
|
# after StableAfter, so a hard crash of EITHER node of a pair — the active/oldest
|
|
# included — fails over to the survivor. The accepted trade (made explicitly by
|
|
# the owner) is dual-active during a real network partition. Both drill
|
|
# directions therefore expect RECOVERY:
|
|
#
|
|
# DRILL_MODE=standby (default) — hits the STANDBY (younger) node.
|
|
# The active node is untouched: expect zero /health/active blips on it and
|
|
# member removal on the survivor within ~25s (10s failure detection +
|
|
# 15s down window).
|
|
#
|
|
# DRILL_MODE=active — hits the ACTIVE (oldest) node. THE SURVIVOR MUST TAKE
|
|
# OVER: it downs the dead oldest, becomes oldest itself, hosts the
|
|
# singletons, and /health/active goes 200 on the survivor WHILE THE VICTIM
|
|
# IS STILL DOWN. Budget ~25s + singleton hand-over + health-probe margin.
|
|
#
|
|
# DRILL_PAIR=central (default) | site-a | site-b | site-c — which 2-node pair.
|
|
# Central health is on container port 5000 (also published as 9001/9002 and
|
|
# routed by Traefik on 9000); a site's /health/* is on container port 8084 and
|
|
# is NOT published, so the script probes every pair from a throwaway sidecar
|
|
# container on scadabridge-net (the app image ships no curl/wget).
|
|
#
|
|
# DRILL_ACTION=kill (default) | pause — `docker kill` (SIGKILL: the hard-crash
|
|
# path; a `docker stop` would take the graceful CoordinatedShutdown path and
|
|
# prove nothing) or `docker pause` (a frozen process: sockets stay open but
|
|
# nothing answers — the GC-pause / hung-VM shape). A paused victim is
|
|
# unpaused at the end instead of started; it finds it was downed, terminates
|
|
# its actor system and comes back through the container restart policy.
|
|
#
|
|
# Both modes finish by recovering the victim and confirming it rejoins as a
|
|
# fresh incarnation (standby), then that exactly one node of the pair is active.
|
|
set -euo pipefail
|
|
|
|
TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
|
|
TIMEOUT_S="${TIMEOUT_S:-90}"
|
|
REJOIN_TIMEOUT_S="${REJOIN_TIMEOUT_S:-180}"
|
|
DRILL_MODE="${DRILL_MODE:-standby}"
|
|
DRILL_PAIR="${DRILL_PAIR:-central}"
|
|
DRILL_ACTION="${DRILL_ACTION:-kill}"
|
|
PROBE_IMAGE="${PROBE_IMAGE:-alpine:latest}"
|
|
|
|
case "$DRILL_MODE" in
|
|
standby|active) ;;
|
|
*) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;;
|
|
esac
|
|
case "$DRILL_PAIR" in
|
|
central) HEALTH_PORT=5000 ;;
|
|
site-a|site-b|site-c) HEALTH_PORT=8084 ;;
|
|
*) echo "ERROR: DRILL_PAIR must be central|site-a|site-b|site-c (was '$DRILL_PAIR')" >&2; exit 2 ;;
|
|
esac
|
|
case "$DRILL_ACTION" in
|
|
kill|pause) ;;
|
|
*) echo "ERROR: DRILL_ACTION must be 'kill' or 'pause' (was '$DRILL_ACTION')" >&2; exit 2 ;;
|
|
esac
|
|
|
|
NODE_A="scadabridge-${DRILL_PAIR}-a"
|
|
NODE_B="scadabridge-${DRILL_PAIR}-b"
|
|
|
|
PROBE="scadabridge-drill-probe-$$"
|
|
docker run -d --rm --name "$PROBE" --network scadabridge-net "$PROBE_IMAGE" sleep 3600 > /dev/null
|
|
VICTIM=""
|
|
cleanup() {
|
|
# Never leave the victim paused/dead if the script dies early.
|
|
if [ -n "$VICTIM" ]; then
|
|
state=$(docker inspect -f '{{.State.Status}}' "$VICTIM" 2>/dev/null || true)
|
|
[ "$state" = paused ] && docker unpause "$VICTIM" > /dev/null || true
|
|
[ "$state" = exited ] && docker start "$VICTIM" > /dev/null || true
|
|
fi
|
|
docker rm -f "$PROBE" > /dev/null 2>&1 || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
# probe <container> <path> -> exit 0 on 2xx. Bounded so a frozen target cannot hang us.
|
|
probe() { docker exec "$PROBE" wget -q -T 2 -O /dev/null "http://$1:${HEALTH_PORT}$2" 2>/dev/null; }
|
|
active_container() {
|
|
if probe "$NODE_A" /health/active; then echo "$NODE_A"
|
|
elif probe "$NODE_B" /health/active; then echo "$NODE_B"
|
|
else echo "ERROR: no active ${DRILL_PAIR} node found" >&2; exit 1; fi
|
|
}
|
|
peer_of() { [ "$1" = "$NODE_A" ] && echo "$NODE_B" || echo "$NODE_A"; }
|
|
recover_victim() {
|
|
if [ "$DRILL_ACTION" = pause ]; then docker unpause "$VICTIM" > /dev/null
|
|
else docker start "$VICTIM" > /dev/null; fi
|
|
}
|
|
|
|
ACTIVE=$(active_container)
|
|
if [ "$DRILL_MODE" = standby ]; then
|
|
VICTIM_SEL=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE"
|
|
else
|
|
VICTIM_SEL="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
|
|
fi
|
|
|
|
echo "pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} active=${ACTIVE} victim=${VICTIM_SEL} survivor=${SURVIVOR}"
|
|
HIT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
VICTIM="$VICTIM_SEL"
|
|
docker "$DRILL_ACTION" "${VICTIM}" > /dev/null
|
|
START=$(date +%s)
|
|
echo "hit at ${HIT_AT}"
|
|
|
|
# Downing evidence on the survivor. Matches both providers: Akka's own
|
|
# "Marking unreachable node ... as [Down]" / "Leader is auto-downing" and the
|
|
# in-repo "[leader-auto-down] DOWNING" line.
|
|
DOWN_RE="leader-auto-down\] DOWNING|auto-downing|marking.*node.*down|member removed|is removed"
|
|
DOWN_S=""
|
|
if [ "$DRILL_MODE" = standby ]; then
|
|
echo "Standby ${DRILL_ACTION}: waiting for ${SURVIVOR} to DOWN+REMOVE the victim (budget ~25s)..."
|
|
BLIPS=0
|
|
while true; do
|
|
ELAPSED=$(( $(date +%s) - START ))
|
|
probe "$SURVIVOR" /health/active || BLIPS=$((BLIPS + 1))
|
|
if [ "$DRILL_PAIR" = central ]; then curl -sf -o /dev/null -m 2 "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1)); fi
|
|
if docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
|
|
DOWN_S=$ELAPSED
|
|
echo "PASS: survivor downed/removed the victim in ${ELAPSED}s (budget ~25s: 10s detection + 15s down window)."
|
|
echo "Active-node /health/active blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)."
|
|
break
|
|
fi
|
|
if (( ELAPSED > TIMEOUT_S )); then
|
|
echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — downing did not act" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
else
|
|
echo "Active ${DRILL_ACTION}: waiting for ${SURVIVOR} to take over as the active node (victim stays down; budget ~25s + hand-over)..."
|
|
while true; do
|
|
ELAPSED=$(( $(date +%s) - START ))
|
|
if [ -z "$DOWN_S" ] && docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
|
|
DOWN_S=$ELAPSED; echo " survivor logged the Down at +${DOWN_S}s"
|
|
fi
|
|
if probe "$SURVIVOR" /health/active; then
|
|
echo "PASS: ${SURVIVOR} took over as active in ${ELAPSED}s with the victim still down"
|
|
echo "(downed the dead oldest, assumed Oldest, re-hosted the singletons)."
|
|
break
|
|
fi
|
|
if (( ELAPSED > TIMEOUT_S )); then
|
|
echo "FAIL: ${SURVIVOR} never became active within ${ELAPSED}s of hitting the oldest — takeover did not happen." >&2
|
|
docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "sbr|downing|leader-auto-down|oldest|shutting down|terminated" | tail -20 >&2 || true
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ -z "$DOWN_S" ] && docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
|
|
DOWN_S="<=${ELAPSED}"; echo " survivor logged the Down at ${DOWN_S}s (seen on the same poll as the takeover)"
|
|
fi
|
|
if [ "$DRILL_PAIR" = central ]; then
|
|
echo "Confirming Traefik routes to the new active node..."
|
|
TR_START=$(date +%s)
|
|
while ! curl -sf -o /dev/null -m 2 "${TRAEFIK_URL}/health/active"; do
|
|
if (( $(date +%s) - TR_START > 60 )); then
|
|
echo "FAIL: survivor is active but not routable through Traefik after 60s" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "Traefik routing recovered $(( $(date +%s) - START ))s after the hit."
|
|
fi
|
|
fi
|
|
|
|
echo "Recovering ${VICTIM} (${DRILL_ACTION} -> $([ "$DRILL_ACTION" = pause ] && echo unpause || echo start))..."
|
|
recover_victim
|
|
RESTART_AT=$(date +%s)
|
|
echo "Waiting for the victim to rejoin as a ready standby (${VICTIM} /health/ready)..."
|
|
while true; do
|
|
ELAPSED=$(( $(date +%s) - RESTART_AT ))
|
|
# A paused-then-unpaused victim was downed while frozen: it must NOT be counted
|
|
# as recovered on its stale pre-pause readiness — require the survivor to see a
|
|
# 2-member cluster again (the victim's /health/ready alone can be a zombie).
|
|
if probe "$VICTIM" /health/ready && ! probe "$VICTIM" /health/active \
|
|
&& docker exec "$PROBE" wget -q -T 2 -O - "http://${SURVIVOR}:${HEALTH_PORT}/health/ready" 2>/dev/null | grep -q '"memberCount":2,"unreachableCount":0'; then
|
|
echo "Recovered: ${VICTIM} is ready as standby (rejoined) ${ELAPSED}s after recovery."
|
|
break
|
|
fi
|
|
if (( ELAPSED > REJOIN_TIMEOUT_S )); then
|
|
echo "FAIL: ${VICTIM} not a ready, rejoined standby ${REJOIN_TIMEOUT_S}s after recovery" >&2
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
VICTIM="" # recovered; nothing for the trap to undo
|
|
|
|
echo "Checking for a lingering dual-active (10 samples, 1s apart)..."
|
|
DUAL=0
|
|
for _ in $(seq 1 10); do
|
|
a=0; b=0
|
|
probe "$NODE_A" /health/active && a=1
|
|
probe "$NODE_B" /health/active && b=1
|
|
(( a + b != 1 )) && DUAL=$((DUAL + 1))
|
|
sleep 1
|
|
done
|
|
if (( DUAL > 0 )); then
|
|
echo "FAIL: ${DUAL}/10 samples did not show exactly one active node" >&2
|
|
exit 1
|
|
fi
|
|
echo "Exactly one active node in 10/10 samples."
|
|
|
|
echo "Survivor downing/singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
|
|
docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "leader-auto-down|auto-downing|-> Oldest|downing|marking.*down|removed" | tail -20 || true
|
|
echo "SUMMARY pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} down_logged_s=${DOWN_S:-n/a} rejoin_s=${ELAPSED}"
|
|
echo "Drill complete. Verify on the Health dashboard / Overview that both nodes show Up and exactly one is Primary."
|