#!/usr/bin/env bash
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
#
# AUTO-DOWN REWRITE (decision 2026-07-21). The cluster runs an availability-first
# downing strategy — `auto-down` (default) or the in-repo `leader-auto-down`
# (ScadaBridge#106; opt-in overlay docker/docker-compose.leader-auto-down.yml) —
# under which the leader among the REACHABLE members downs the unreachable peer
# after StableAfter, so a hard crash of EITHER node of a pair — the active/oldest
# included — fails over to the survivor. The accepted trade (made explicitly by
# the owner) is dual-active during a real network partition. Both drill
# directions therefore expect RECOVERY:
#
#   DRILL_MODE=standby (default) — hits the STANDBY (younger) node.
#     The active node is untouched: expect zero /health/active blips on it and
#     member removal on the survivor within ~25s (10s failure detection +
#     15s down window).
#
#   DRILL_MODE=active — hits the ACTIVE (oldest) node. THE SURVIVOR MUST TAKE
#     OVER: it downs the dead oldest, becomes oldest itself, hosts the
#     singletons, and /health/active goes 200 on the survivor WHILE THE VICTIM
#     IS STILL DOWN. Budget ~25s + singleton hand-over + health-probe margin.
#
#   DRILL_PAIR=central (default) | site-a | site-b | site-c — which 2-node pair.
#     Central health is on container port 5000 (also published as 9001/9002 and
#     routed by Traefik on 9000); a site's /health/* is on container port 8084 and
#     is NOT published, so the script probes every pair from a throwaway sidecar
#     container on scadabridge-net (the app image ships no curl/wget).
#
#   DRILL_ACTION=kill (default) | pause — `docker kill` (SIGKILL: the hard-crash
#     path; a `docker stop` would take the graceful CoordinatedShutdown path and
#     prove nothing) or `docker pause` (a frozen process: sockets stay open but
#     nothing answers — the GC-pause / hung-VM shape). A paused victim is
#     unpaused at the end instead of started; it finds it was downed, terminates
#     its actor system and comes back through the container restart policy.
#
# Both modes finish by recovering the victim and confirming it rejoins as a
# fresh incarnation (standby), then that exactly one node of the pair is active.
set -euo pipefail

TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
TIMEOUT_S="${TIMEOUT_S:-90}"
REJOIN_TIMEOUT_S="${REJOIN_TIMEOUT_S:-180}"
DRILL_MODE="${DRILL_MODE:-standby}"
DRILL_PAIR="${DRILL_PAIR:-central}"
DRILL_ACTION="${DRILL_ACTION:-kill}"
PROBE_IMAGE="${PROBE_IMAGE:-alpine:latest}"

case "$DRILL_MODE" in
  standby|active) ;;
  *) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;;
esac
case "$DRILL_PAIR" in
  central) HEALTH_PORT=5000 ;;
  site-a|site-b|site-c) HEALTH_PORT=8084 ;;
  *) echo "ERROR: DRILL_PAIR must be central|site-a|site-b|site-c (was '$DRILL_PAIR')" >&2; exit 2 ;;
esac
case "$DRILL_ACTION" in
  kill|pause) ;;
  *) echo "ERROR: DRILL_ACTION must be 'kill' or 'pause' (was '$DRILL_ACTION')" >&2; exit 2 ;;
esac

NODE_A="scadabridge-${DRILL_PAIR}-a"
NODE_B="scadabridge-${DRILL_PAIR}-b"

PROBE="scadabridge-drill-probe-$$"
docker run -d --rm --name "$PROBE" --network scadabridge-net "$PROBE_IMAGE" sleep 3600 > /dev/null
VICTIM=""
cleanup() {
  # Never leave the victim paused/dead if the script dies early.
  if [ -n "$VICTIM" ]; then
    state=$(docker inspect -f '{{.State.Status}}' "$VICTIM" 2>/dev/null || true)
    [ "$state" = paused ] && docker unpause "$VICTIM" > /dev/null || true
    [ "$state" = exited ] && docker start "$VICTIM" > /dev/null || true
  fi
  docker rm -f "$PROBE" > /dev/null 2>&1 || true
}
trap cleanup EXIT

# probe <container> <path> -> exit 0 on 2xx. Bounded so a frozen target cannot hang us.
probe() { docker exec "$PROBE" wget -q -T 2 -O /dev/null "http://$1:${HEALTH_PORT}$2" 2>/dev/null; }
active_container() {
  if probe "$NODE_A" /health/active; then echo "$NODE_A"
  elif probe "$NODE_B" /health/active; then echo "$NODE_B"
  else echo "ERROR: no active ${DRILL_PAIR} node found" >&2; exit 1; fi
}
peer_of() { [ "$1" = "$NODE_A" ] && echo "$NODE_B" || echo "$NODE_A"; }
recover_victim() {
  if [ "$DRILL_ACTION" = pause ]; then docker unpause "$VICTIM" > /dev/null
  else docker start "$VICTIM" > /dev/null; fi
}

ACTIVE=$(active_container)
if [ "$DRILL_MODE" = standby ]; then
  VICTIM_SEL=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE"
else
  VICTIM_SEL="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
fi

echo "pair=${DRILL_PAIR}  mode=${DRILL_MODE}  action=${DRILL_ACTION}  active=${ACTIVE}  victim=${VICTIM_SEL}  survivor=${SURVIVOR}"
HIT_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
VICTIM="$VICTIM_SEL"
docker "$DRILL_ACTION" "${VICTIM}" > /dev/null
START=$(date +%s)
echo "hit at ${HIT_AT}"

# Downing evidence on the survivor. Matches both providers: Akka's own
# "Marking unreachable node ... as [Down]" / "Leader is auto-downing" and the
# in-repo "[leader-auto-down] DOWNING" line.
DOWN_RE="leader-auto-down\] DOWNING|auto-downing|marking.*node.*down|member removed|is removed"
DOWN_S=""
if [ "$DRILL_MODE" = standby ]; then
  echo "Standby ${DRILL_ACTION}: waiting for ${SURVIVOR} to DOWN+REMOVE the victim (budget ~25s)..."
  BLIPS=0
  while true; do
    ELAPSED=$(( $(date +%s) - START ))
    probe "$SURVIVOR" /health/active || BLIPS=$((BLIPS + 1))
    if [ "$DRILL_PAIR" = central ]; then curl -sf -o /dev/null -m 2 "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1)); fi
    if docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
      DOWN_S=$ELAPSED
      echo "PASS: survivor downed/removed the victim in ${ELAPSED}s (budget ~25s: 10s detection + 15s down window)."
      echo "Active-node /health/active blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)."
      break
    fi
    if (( ELAPSED > TIMEOUT_S )); then
      echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — downing did not act" >&2
      exit 1
    fi
    sleep 1
  done
else
  echo "Active ${DRILL_ACTION}: waiting for ${SURVIVOR} to take over as the active node (victim stays down; budget ~25s + hand-over)..."
  while true; do
    ELAPSED=$(( $(date +%s) - START ))
    if [ -z "$DOWN_S" ] && docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
      DOWN_S=$ELAPSED; echo "  survivor logged the Down at +${DOWN_S}s"
    fi
    if probe "$SURVIVOR" /health/active; then
      echo "PASS: ${SURVIVOR} took over as active in ${ELAPSED}s with the victim still down"
      echo "(downed the dead oldest, assumed Oldest, re-hosted the singletons)."
      break
    fi
    if (( ELAPSED > TIMEOUT_S )); then
      echo "FAIL: ${SURVIVOR} never became active within ${ELAPSED}s of hitting the oldest — takeover did not happen." >&2
      docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "sbr|downing|leader-auto-down|oldest|shutting down|terminated" | tail -20 >&2 || true
      exit 1
    fi
    sleep 1
  done
  if [ -z "$DOWN_S" ] && docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "$DOWN_RE"; then
    DOWN_S="<=${ELAPSED}"; echo "  survivor logged the Down at ${DOWN_S}s (seen on the same poll as the takeover)"
  fi
  if [ "$DRILL_PAIR" = central ]; then
    echo "Confirming Traefik routes to the new active node..."
    TR_START=$(date +%s)
    while ! curl -sf -o /dev/null -m 2 "${TRAEFIK_URL}/health/active"; do
      if (( $(date +%s) - TR_START > 60 )); then
        echo "FAIL: survivor is active but not routable through Traefik after 60s" >&2
        exit 1
      fi
      sleep 1
    done
    echo "Traefik routing recovered $(( $(date +%s) - START ))s after the hit."
  fi
fi

echo "Recovering ${VICTIM} (${DRILL_ACTION} -> $([ "$DRILL_ACTION" = pause ] && echo unpause || echo start))..."
recover_victim
RESTART_AT=$(date +%s)
echo "Waiting for the victim to rejoin as a ready standby (${VICTIM} /health/ready)..."
while true; do
  ELAPSED=$(( $(date +%s) - RESTART_AT ))
  # A paused-then-unpaused victim was downed while frozen: it must NOT be counted
  # as recovered on its stale pre-pause readiness — require the survivor to see a
  # 2-member cluster again (the victim's /health/ready alone can be a zombie).
  if probe "$VICTIM" /health/ready && ! probe "$VICTIM" /health/active \
     && docker exec "$PROBE" wget -q -T 2 -O - "http://${SURVIVOR}:${HEALTH_PORT}/health/ready" 2>/dev/null | grep -q '"memberCount":2,"unreachableCount":0'; then
    echo "Recovered: ${VICTIM} is ready as standby (rejoined) ${ELAPSED}s after recovery."
    break
  fi
  if (( ELAPSED > REJOIN_TIMEOUT_S )); then
    echo "FAIL: ${VICTIM} not a ready, rejoined standby ${REJOIN_TIMEOUT_S}s after recovery" >&2
    exit 1
  fi
  sleep 1
done
VICTIM=""   # recovered; nothing for the trap to undo

echo "Checking for a lingering dual-active (10 samples, 1s apart)..."
DUAL=0
for _ in $(seq 1 10); do
  a=0; b=0
  probe "$NODE_A" /health/active && a=1
  probe "$NODE_B" /health/active && b=1
  (( a + b != 1 )) && DUAL=$((DUAL + 1))
  sleep 1
done
if (( DUAL > 0 )); then
  echo "FAIL: ${DUAL}/10 samples did not show exactly one active node" >&2
  exit 1
fi
echo "Exactly one active node in 10/10 samples."

echo "Survivor downing/singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
docker logs --since "${HIT_AT}" "${SURVIVOR}" 2>&1 | grep -Ei "leader-auto-down|auto-downing|-> Oldest|downing|marking.*down|removed" | tail -20 || true
echo "SUMMARY pair=${DRILL_PAIR} mode=${DRILL_MODE} action=${DRILL_ACTION} down_logged_s=${DOWN_S:-n/a} rejoin_s=${ELAPSED}"
echo "Drill complete. Verify on the Health dashboard / Overview that both nodes show Up and exactly one is Primary."