#!/usr/bin/env bash
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
#
# ROUND-2 REWRITE (arch-review 01 round 2, N1). The original drill killed the
# ACTIVE central node — but under the unified oldest-member semantics the
# active node IS the oldest, i.e. the one crash two-node keep-oldest CANNOT
# survive (registered deferred user decision, master tracker 2026-07-08;
# SbrFailoverTests.cs XML doc). Two modes:
#
#   DRILL_MODE=standby (default) — kills the STANDBY (younger) central node.
#     The survivable direction: SBR downs the crashed member, the active node
#     keeps its singletons, and Traefik routing never goes dark. PASS = the
#     survivor logs the member removal within TIMEOUT_S (budget ~25s+: 10s
#     failure detection + 15s stable-after) while /health/active stays up.
#
#   DRILL_MODE=active — kills the ACTIVE (oldest) central node. THE EXPECTED
#     OUTCOME IS A TOTAL CENTRAL OUTAGE: keep-oldest downs the partition
#     without the oldest, so the younger survivor downs ITSELF (down-if-alone
#     cannot help — the alone-oldest is dead and cannot down itself), and the
#     self-downed survivor cannot re-form a cluster alone unless it is the
#     FIRST seed (both nodes list central-a first; only the first seed may
#     self-join). This mode measures the dark window and PASSes only when
#     central recovers AFTER the victim container is restarted. It exists to
#     make the registered gap observable — not to pretend it is covered.
set -euo pipefail

TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
TIMEOUT_S="${TIMEOUT_S:-90}"
DRILL_MODE="${DRILL_MODE:-standby}"
OUTAGE_CONFIRM_S="${OUTAGE_CONFIRM_S:-60}"

active_container() {
  if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a
  elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b
  else echo "ERROR: no active central node found" >&2; exit 1; fi
}
peer_of() { [ "$1" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a; }

case "$DRILL_MODE" in
  standby|active) ;;
  *) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;;
esac

ACTIVE=$(active_container)
if [ "$DRILL_MODE" = standby ]; then
  VICTIM=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE"
else
  VICTIM="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
fi

echo "mode=${DRILL_MODE}  active=${ACTIVE}  victim=${VICTIM}  survivor=${SURVIVOR}"
KILL_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
docker kill "${VICTIM}" > /dev/null
START=$(date +%s)

if [ "$DRILL_MODE" = standby ]; then
  echo "Standby crash: waiting for ${SURVIVOR} to DOWN+REMOVE the dead member (SBR budget ~25s)..."
  BLIPS=0
  while true; do
    ELAPSED=$(( $(date +%s) - START ))
    curl -sf -o /dev/null "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1))
    if docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "marking.*node.*down|member removed|is removed"; then
      echo "PASS: survivor removed the crashed member in ${ELAPSED}s (budget ~25s: 10s detection + 15s stable-after)."
      echo "Active-node routing blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)."
      break
    fi
    if (( ELAPSED > TIMEOUT_S )); then
      echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — SBR did not act" >&2
      docker start "${VICTIM}" > /dev/null
      exit 1
    fi
    sleep 1
  done
else
  echo "Active crash: EXPECTING a central outage (registered keep-oldest gap). Watching /health/active..."
  DARK_STREAK=0
  while true; do
    ELAPSED=$(( $(date +%s) - START ))
    if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then DARK_STREAK=0; else DARK_STREAK=$((DARK_STREAK + 1)); fi
    if (( DARK_STREAK >= 10 )); then
      echo "Outage confirmed at ${ELAPSED}s: no active central node — the younger survivor self-downed"
      echo "(keep-oldest downs the partition WITHOUT the oldest; this is the registered deferred gap)."
      break
    fi
    if (( ELAPSED > OUTAGE_CONFIRM_S )); then
      echo "NOTE: /health/active stayed reachable ${ELAPSED}s after killing the oldest — better than the"
      echo "registered gap predicts. Do NOT celebrate: capture both nodes' logs and investigate before trusting it."
      break
    fi
    sleep 1
  done
fi

echo "Restarting ${VICTIM}..."
docker start "${VICTIM}" > /dev/null
RESTART_AT=$(date +%s)
echo "Waiting for central to be routable again through Traefik (${TRAEFIK_URL}/health/active)..."
while true; do
  ELAPSED=$(( $(date +%s) - RESTART_AT ))
  if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then
    echo "Recovered: an active central node is routable ${ELAPSED}s after the victim restart."
    break
  fi
  if (( ELAPSED > 120 )); then
    echo "FAIL: central not routable 120s after restarting ${VICTIM}" >&2
    exit 1
  fi
  sleep 1
done

echo "Survivor singleton/downing evidence (last 20 matching log lines from ${SURVIVOR}):"
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|downing|removed" | tail -20 || true
echo "Drill complete (${DRILL_MODE}). Verify on the Health dashboard that both nodes show Up and exactly one is Primary."