cafdc0ce60
Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
52 lines
3.6 KiB
Bash
52 lines
3.6 KiB
Bash
#!/usr/bin/env bash
|
|
# ScadaBridge#132 rig gate: partition a pair past the leader-auto-down window, heal it, and
|
|
# watch the shut-out restart converge the pair WITHOUT operator action.
|
|
# Usage: partition-heal.sh <pair> <hold_s> [watch_s]
|
|
# Needs: sb-drill-net:local (alpine + iptables + curl) and a probe container on scadabridge-net.
|
|
set -uo pipefail
|
|
PAIR=$1; HOLD=$2; WATCH=${3:-300}
|
|
A=${PREFIX:-scadabridge-}$PAIR-a; B=${PREFIX:-scadabridge-}$PAIR-b
|
|
[ "$PAIR" = central ] && HP=5000 || HP=8084
|
|
PROBE=sb116env2-probe
|
|
docker rm -f $PROBE >/dev/null 2>&1
|
|
docker run -d --name $PROBE --network scadabridge-net sb-drill-net:local sleep infinity >/dev/null
|
|
AIP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $A)
|
|
c() { docker exec $PROBE curl -s -m 2 -o /dev/null -w %{http_code} "http://$1:$HP/health/$2" 2>/dev/null || echo ERR; }
|
|
mem() { docker exec $PROBE curl -s -m 2 "http://$1:$HP/health/ready" 2>/dev/null | grep -o '"memberCount":[0-9]*' | head -1; }
|
|
rc() { docker inspect -f '{{.RestartCount}}/{{.State.StartedAt}}' $1; }
|
|
RA0=$(rc $A); RB0=$(rc $B)
|
|
echo "$(date -u +%T) before: $A active=$(c $A active) $B active=$(c $B active) restarts A=$RA0 B=$RB0 (A ip $AIP)"
|
|
CUT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ); T0=$(date +%s)
|
|
docker run --rm --net container:$B --cap-add NET_ADMIN sb-drill-net:local sh -c "iptables -I INPUT -s $AIP -j DROP && iptables -I OUTPUT -d $AIP -j DROP"
|
|
echo "$(date -u +%T) partition ON (B drops all traffic to/from A)"
|
|
DUAL_AT=""
|
|
while (( $(date +%s) - T0 < HOLD )); do
|
|
a=$(c $A active); b=$(c $B active)
|
|
if [ -z "$DUAL_AT" ] && [ "$a" = 200 ] && [ "$b" = 200 ]; then DUAL_AT=$(( $(date +%s) - T0 )); echo "$(date -u +%T) DUAL-ACTIVE reached at +${DUAL_AT}s"; fi
|
|
sleep 1
|
|
done
|
|
echo "$(date -u +%T) end of hold (+$(( $(date +%s)-T0 ))s): $A active=$(c $A active) $B active=$(c $B active)"
|
|
docker run --rm --net container:$B --cap-add NET_ADMIN sb-drill-net:local sh -c "iptables -D INPUT -s $AIP -j DROP && iptables -D OUTPUT -d $AIP -j DROP"
|
|
H0=$(date +%s); echo "$(date -u +%T) partition HEALED (+$(( H0-T0 ))s)"
|
|
RESTARTED=""; CONVERGED=""; STABLE=0
|
|
while (( $(date +%s) - H0 < WATCH )); do
|
|
sleep 2
|
|
ra=$(rc $A); rb=$(rc $B); a=$(c $A active); b=$(c $B active)
|
|
if [ -z "$RESTARTED" ] && { [ "$ra" != "$RA0" ] || [ "$rb" != "$RB0" ]; }; then
|
|
RESTARTED="+$(( $(date +%s)-H0 ))s after heal: A $([ "$ra" != "$RA0" ] && echo RESTARTED || echo stayed), B $([ "$rb" != "$RB0" ] && echo RESTARTED || echo stayed)"
|
|
echo "$(date -u +%T) $RESTARTED"
|
|
fi
|
|
if [ -n "$RESTARTED" ] && [ "$(c $A ready)" = 200 ] && [ "$(c $B ready)" = 200 ] && { [ "$a$b" = 200503 ] || [ "$a$b" = 503200 ]; }; then
|
|
STABLE=$((STABLE+1))
|
|
if [ -z "$CONVERGED" ]; then CONVERGED="+$(( $(date +%s)-H0 ))s after heal"; echo "$(date -u +%T) one active ($A=$a $B=$b), $(mem $A) $(mem $B) — converged $CONVERGED"; fi
|
|
(( STABLE >= 10 )) && break
|
|
else
|
|
[ -n "$CONVERGED" ] && { echo "$(date -u +%T) lost convergence: $A=$a $B=$b"; CONVERGED=""; STABLE=0; }
|
|
fi
|
|
done
|
|
echo "$(date -u +%T) final: $A active=$(c $A active) ready=$(c $A ready) restarts=$(rc $A) | $B active=$(c $B active) ready=$(c $B ready) restarts=$(rc $B)"
|
|
echo "result: restart=[$RESTARTED] converged=[$CONVERGED] stable-samples=$STABLE"
|
|
echo "--- $A [leader-auto-down] + quarantine lines"; docker logs --since $CUT_AT $A 2>&1 | grep -E "leader-auto-down\]|quarantined this system|terminated outside host" | cut -c1-330
|
|
echo "--- $B [leader-auto-down] + quarantine lines"; docker logs --since $CUT_AT $B 2>&1 | grep -E "leader-auto-down\]|quarantined this system|terminated outside host" | cut -c1-330
|
|
docker rm -f $PROBE >/dev/null 2>&1
|