ScadaBridge/docker/verify-delmia-notifier.sh

#!/bin/bash
set -euo pipefail

# End-to-end DelmiaNotifier verification against the docker rig, including an ACTIVE central
# crash (docker kill = SIGKILL, so auto-down failover is exercised, not CoordinatedShutdown's
# graceful handover) + rejoin. Requires: seed-delmia-sim.sh + seed-delmia-api.sh already run, and
# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh).
#
# PREREQUISITE the first run of this script found the hard way (2026-09-02): inbound API keys are a
# PER-NODE SQLite store, so the key minted through the active node does not exist on the standby
# and every call after a takeover answers 401 KeyNotFound. Run `bash docker/sync-inbound-api-keys.sh`
# after seed-delmia-api.sh (and again after re-minting a key). The preflight below lists the key
# via each node's own management API and refuses to start the drill if either node lacks it.
#
# The notifier is run as the managed build (dotnet WWNotifier.dll) from a scratch copy so the
# API key is written to THAT copy's appsettings.json (ScadaBridge:ApiKey) and never into the
# repo. BaseUrls lists the two central nodes DIRECTLY (not Traefik) so the notifier's own
# connect-failure failover is what gets exercised.
#
# Expected outcomes:
#   A  baseline (active listed first)            -> YES
#   B  standby listed first (standby relays)     -> YES   (StandbyForwardPeerEndpoints)
#   C  bad key                                   -> NO    (HTTP 401, no failover: 401 is an answer)
#   D  active KILLED, before takeover completes  -> NO or YES — informational only. A NO here is the
#      documented connect-failure-only design (standby answers 503 ACTIVE_NODE_UNREACHABLE).
#   E  active KILLED, after survivor /health/active=200 -> YES (first URL refused -> failover)
#   F  victim restarted and rejoined (memberCount=2) -> YES
#   G  reject path (DownloadPath *.bad)           -> NO + "Simulated reject"

: "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
RUN_DIR="${RUN_DIR:-${TMPDIR:-/tmp}/delmia-notifier-run}"
MACHINE="${MACHINE:-DelmiaSim01}"
A_URL=http://localhost:9001; B_URL=http://localhost:9002

echo "== build notifier + stage a runnable copy in $RUN_DIR"
dotnet build "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier" -c Debug -nologo -v q
rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR"; cp -R "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier/bin/Debug/net10.0/." "$RUN_DIR/"

write_cfg() { # $1 = BaseUrls, $2 = ApiKey
  printf '{"ScadaBridge":{"BaseUrls":"%s","TimeoutSeconds":30,"LogPath":"delmia-notifier.log","ApiKey":"%s"}}\n' "$1" "$2" > "$RUN_DIR/appsettings.json"
}
notify() { # $1 = label, $2 = expected YES|NO, rest = extra args; prints stdout + exit code
  local label=$1 expect=$2; shift 2
  local out rc; set +e; out=$(env -u SCADABRIDGE_API_KEY dotnet "$RUN_DIR/WWNotifier.dll" -m "$MACHINE" -d 'C:\recipes\wo1.nc' -w "WO-$label" -p PN-1 -s 0100 -u drill "$@" 2>/dev/null); rc=$?; set -e
  local first; first=$(echo "$out" | head -1)
  printf '%-3s stdout=%-4s exit=%-3s | %s\n' "$label" "$first" "$rc" "$(echo "$out" | sed -n 2p)"
  [ "$first" = "$expect" ] || { echo "   ^^ EXPECTED $expect"; return 1; }
}
active() { curl -sf -o /dev/null "$A_URL/health/active" && echo a || { curl -sf -o /dev/null "$B_URL/health/active" && echo b; }; }
url_of() { [ "$1" = a ] && echo "$A_URL" || echo "$B_URL"; }
container_of() { [ "$1" = a ] && echo scadabridge-central-a || echo scadabridge-central-b; }
other() { [ "$1" = a ] && echo b || echo a; }
wait_for() { # $1 url $2 path $3 seconds
  local i; for i in $(seq 1 "$3"); do curl -sf -o /dev/null "$1$2" && return 0; sleep 1; done; return 1; }

ACT=$(active); STB=$(other "$ACT")
echo "== active=central-$ACT standby=central-$STB"

echo "== preflight: the notifier's key must be present in BOTH nodes' key stores"
CLI="$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge"
KEY_ID="${DELMIA_API_KEY#sbk_}"; KEY_ID="${KEY_ID%%_*}"
for n in a b; do
  if "$CLI" security api-key list --url "$(url_of "$n")" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}"       | grep -q "\"keyId\":\"$KEY_ID\""; then echo "   central-$n: key $KEY_ID present"
  else echo "FAIL: central-$n does not hold key $KEY_ID — run: bash docker/sync-inbound-api-keys.sh $ACT $STB" >&2; exit 1; fi
done

echo "== A baseline (active first)";        write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"; notify A YES
echo "== B standby first (relay)";           write_cfg "$(url_of "$STB"),$(url_of "$ACT")" "$DELMIA_API_KEY"; notify B YES
echo "== C bad key";                         write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "sbk_bogus_bogus"; notify C NO || true
write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"

echo "== D/E KILL the ACTIVE central ($(container_of "$ACT"))"; docker kill "$(container_of "$ACT")" > /dev/null
T0=$(date +%s)
echo "-- D immediate attempt (informational):"; notify D NO || notify D YES || true
echo "-- waiting for survivor $(url_of "$STB")/health/active (auto-down ~25-30 s)..."
wait_for "$(url_of "$STB")" /health/active 90 || { echo "FAIL: survivor never became active"; exit 1; }
echo "   takeover after $(( $(date +%s) - T0 )) s"
echo "== E after takeover (first URL refused -> failover)"; notify E YES

echo "== F restart victim + wait for rejoin"; docker start "$(container_of "$ACT")" > /dev/null
T1=$(date +%s)
wait_for "$(url_of "$ACT")" /health/ready 120 || { echo "FAIL: victim never ready"; exit 1; }
for i in $(seq 1 60); do
  curl -s "$(url_of "$STB")/health/ready" | grep -q '"memberCount":2,"unreachableCount":0' && break; sleep 2
done
echo "   victim ready + memberCount=2 on survivor after $(( $(date +%s) - T1 )) s; victim /health/active=$(curl -s -o /dev/null -w '%{http_code}' "$(url_of "$ACT")/health/active") (503 = rejoined as standby)"
notify F YES

echo "== G reject path"; notify G NO -d 'C:\recipes\x.bad' || true
echo "== done. Notifier log: $RUN_DIR/delmia-notifier.log"