diff --git a/docker/README.md b/docker/README.md index 0a01ed7b..4a3247bd 100644 --- a/docker/README.md +++ b/docker/README.md @@ -97,6 +97,8 @@ docker/ ├── seed-live-probe.sh # Seed one deployable live-data instance on site-a (stream traffic) ├── seed-delmia-sim.sh # Seed the in-memory Delmia machine stand-in (DelmiaSim01) on site-a ├── seed-delmia-api.sh # Seed the DelmiaRecipeDownload inbound method + a scoped API key (token printed once) +├── verify-delmia-notifier.sh # DelmiaNotifier end-to-end vs the rig: standby relay, active-central crash/takeover, rejoin +├── sync-inbound-api-keys.sh # RIG-ONLY: mirror the per-node inbound API key store a->b (needed before any central failover test) ├── teardown.sh # Stop and remove containers ├── central-node-a/ │ ├── appsettings.Central.json # Central node A configuration diff --git a/docker/sync-inbound-api-keys.sh b/docker/sync-inbound-api-keys.sh new file mode 100755 index 00000000..c8a6e64f --- /dev/null +++ b/docker/sync-inbound-api-keys.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -euo pipefail + +# RIG-ONLY workaround: copy the inbound API key store from one central node to the other. +# +# Why this exists: inbound API keys live in a PER-NODE SQLite file +# (ScadaBridge:InboundApi:ApiKeyStore:SqlitePath, /app/data/inbound-api-keys.sqlite on each +# central's own data volume) and key admin writes only the node that serves the management call +# (the active one, via Traefik). A key therefore exists on exactly one central node, and after a +# central failover the survivor answers 401 to every inbound client — found live 2026-09-02 by +# docker/verify-delmia-notifier.sh (deferred-work register). Until the store is shared, a 2-node +# central needs the file mirrored by hand; this script does that for the docker rig. +# +# Usage: bash docker/sync-inbound-api-keys.sh [from a|b] [to b|a] (default: a -> b) +# - The SOURCE stays up (its WAL is checkpointed into the copy on the host with sqlite3). +# Quiesce key admin while it runs — a create racing the copy may be missed. +# - The TARGET is stopped for the copy and restarted; if it is the ACTIVE node the peer takes over +# (graceful CoordinatedShutdown handover) and the target rejoins as standby. Its own key store is +# REPLACED (both rig nodes share one pepper, so the copied hashes verify on either node). + +FROM="${1:-a}"; TO="${2:-b}" +[ "$FROM" != "$TO" ] || { echo "from and to must differ" >&2; exit 2; } +SRC="scadabridge-central-$FROM"; DST="scadabridge-central-$TO"; VOL="scadabridge-central-node-$TO-data" +port_of() { [ "$1" = a ] && echo 9001 || echo 9002; } +WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT + +echo "== copying key store out of $SRC (main + WAL + SHM) and checkpointing on the host" +for f in inbound-api-keys.sqlite inbound-api-keys.sqlite-wal inbound-api-keys.sqlite-shm; do + docker cp "$SRC:/app/data/$f" "$WORK/$f" 2>/dev/null || true +done +[ -s "$WORK/inbound-api-keys.sqlite" ] || { echo "FAIL: no key store on $SRC" >&2; exit 1; } +sqlite3 "$WORK/inbound-api-keys.sqlite" "PRAGMA wal_checkpoint(TRUNCATE);" > /dev/null +[ "$(sqlite3 "$WORK/inbound-api-keys.sqlite" "PRAGMA integrity_check;")" = ok ] || { echo "FAIL: copy failed integrity_check" >&2; exit 1; } +echo " api_keys rows in copy: $(sqlite3 "$WORK/inbound-api-keys.sqlite" "select count(*) from api_keys")" + +echo "== stopping $DST, replacing its key store on volume $VOL, restarting" +docker stop "$DST" > /dev/null +docker run --rm -v "$VOL:/d" -v "$WORK:/src:ro" alpine:3.20 sh -c \ + 'cp /src/inbound-api-keys.sqlite /d/inbound-api-keys.sqlite && rm -f /d/inbound-api-keys.sqlite-wal /d/inbound-api-keys.sqlite-shm && chmod 644 /d/inbound-api-keys.sqlite' +docker start "$DST" > /dev/null + +P=$(port_of "$TO"); Q=$(port_of "$FROM") +for i in $(seq 1 120); do curl -sf -o /dev/null "http://localhost:$P/health/ready" && break; sleep 1; done +for i in $(seq 1 60); do curl -s "http://localhost:$Q/health/ready" | grep -q '"memberCount":2,"unreachableCount":0' && break; sleep 2; done +echo "== $DST ready; members on $SRC: $(curl -s "http://localhost:$Q/health/ready" | grep -o '"memberCount":[0-9]*,"unreachableCount":[0-9]*')" +echo " active: central-a=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:9001/health/active) central-b=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:9002/health/active)" diff --git a/docker/verify-delmia-notifier.sh b/docker/verify-delmia-notifier.sh new file mode 100755 index 00000000..d541f833 --- /dev/null +++ b/docker/verify-delmia-notifier.sh @@ -0,0 +1,91 @@ +#!/bin/bash +set -euo pipefail + +# End-to-end DelmiaNotifier verification against the docker rig, including an ACTIVE central +# crash (docker kill = SIGKILL, so auto-down failover is exercised, not CoordinatedShutdown's +# graceful handover) + rejoin. Requires: seed-delmia-sim.sh + seed-delmia-api.sh already run, and +# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh). +# +# PREREQUISITE the first run of this script found the hard way (2026-09-02): inbound API keys are a +# PER-NODE SQLite store, so the key minted through the active node does not exist on the standby +# and every call after a takeover answers 401 KeyNotFound. Run `bash docker/sync-inbound-api-keys.sh` +# after seed-delmia-api.sh (and again after re-minting a key). The preflight below lists the key +# via each node's own management API and refuses to start the drill if either node lacks it. +# +# The notifier is run as the managed build (dotnet WWNotifier.dll) from a scratch copy so the +# API key is written to THAT copy's appsettings.json (ScadaBridge:ApiKey) and never into the +# repo. BaseUrls lists the two central nodes DIRECTLY (not Traefik) so the notifier's own +# connect-failure failover is what gets exercised. +# +# Expected outcomes: +# A baseline (active listed first) -> YES +# B standby listed first (standby relays) -> YES (StandbyForwardPeerEndpoints) +# C bad key -> NO (HTTP 401, no failover: 401 is an answer) +# D active KILLED, before takeover completes -> NO or YES — informational only. A NO here is the +# documented connect-failure-only design (standby answers 503 ACTIVE_NODE_UNREACHABLE). +# E active KILLED, after survivor /health/active=200 -> YES (first URL refused -> failover) +# F victim restarted and rejoined (memberCount=2) -> YES +# G reject path (DownloadPath *.bad) -> NO + "Simulated reject" + +: "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +RUN_DIR="${RUN_DIR:-${TMPDIR:-/tmp}/delmia-notifier-run}" +MACHINE="${MACHINE:-DelmiaSim01}" +A_URL=http://localhost:9001; B_URL=http://localhost:9002 + +echo "== build notifier + stage a runnable copy in $RUN_DIR" +dotnet build "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier" -c Debug -nologo -v q +rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR"; cp -R "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier/bin/Debug/net10.0/." "$RUN_DIR/" + +write_cfg() { # $1 = BaseUrls, $2 = ApiKey + printf '{"ScadaBridge":{"BaseUrls":"%s","TimeoutSeconds":30,"LogPath":"delmia-notifier.log","ApiKey":"%s"}}\n' "$1" "$2" > "$RUN_DIR/appsettings.json" +} +notify() { # $1 = label, $2 = expected YES|NO, rest = extra args; prints stdout + exit code + local label=$1 expect=$2; shift 2 + local out rc; set +e; out=$(env -u SCADABRIDGE_API_KEY dotnet "$RUN_DIR/WWNotifier.dll" -m "$MACHINE" -d 'C:\recipes\wo1.nc' -w "WO-$label" -p PN-1 -s 0100 -u drill "$@" 2>/dev/null); rc=$?; set -e + local first; first=$(echo "$out" | head -1) + printf '%-3s stdout=%-4s exit=%-3s | %s\n' "$label" "$first" "$rc" "$(echo "$out" | sed -n 2p)" + [ "$first" = "$expect" ] || { echo " ^^ EXPECTED $expect"; return 1; } +} +active() { curl -sf -o /dev/null "$A_URL/health/active" && echo a || { curl -sf -o /dev/null "$B_URL/health/active" && echo b; }; } +url_of() { [ "$1" = a ] && echo "$A_URL" || echo "$B_URL"; } +container_of() { [ "$1" = a ] && echo scadabridge-central-a || echo scadabridge-central-b; } +other() { [ "$1" = a ] && echo b || echo a; } +wait_for() { # $1 url $2 path $3 seconds + local i; for i in $(seq 1 "$3"); do curl -sf -o /dev/null "$1$2" && return 0; sleep 1; done; return 1; } + +ACT=$(active); STB=$(other "$ACT") +echo "== active=central-$ACT standby=central-$STB" + +echo "== preflight: the notifier's key must be present in BOTH nodes' key stores" +CLI="$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge" +KEY_ID="${DELMIA_API_KEY#sbk_}"; KEY_ID="${KEY_ID%%_*}" +for n in a b; do + if "$CLI" security api-key list --url "$(url_of "$n")" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}" | grep -q "\"keyId\":\"$KEY_ID\""; then echo " central-$n: key $KEY_ID present" + else echo "FAIL: central-$n does not hold key $KEY_ID — run: bash docker/sync-inbound-api-keys.sh $ACT $STB" >&2; exit 1; fi +done + +echo "== A baseline (active first)"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"; notify A YES +echo "== B standby first (relay)"; write_cfg "$(url_of "$STB"),$(url_of "$ACT")" "$DELMIA_API_KEY"; notify B YES +echo "== C bad key"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "sbk_bogus_bogus"; notify C NO || true +write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY" + +echo "== D/E KILL the ACTIVE central ($(container_of "$ACT"))"; docker kill "$(container_of "$ACT")" > /dev/null +T0=$(date +%s) +echo "-- D immediate attempt (informational):"; notify D NO || notify D YES || true +echo "-- waiting for survivor $(url_of "$STB")/health/active (auto-down ~25-30 s)..." +wait_for "$(url_of "$STB")" /health/active 90 || { echo "FAIL: survivor never became active"; exit 1; } +echo " takeover after $(( $(date +%s) - T0 )) s" +echo "== E after takeover (first URL refused -> failover)"; notify E YES + +echo "== F restart victim + wait for rejoin"; docker start "$(container_of "$ACT")" > /dev/null +T1=$(date +%s) +wait_for "$(url_of "$ACT")" /health/ready 120 || { echo "FAIL: victim never ready"; exit 1; } +for i in $(seq 1 60); do + curl -s "$(url_of "$STB")/health/ready" | grep -q '"memberCount":2,"unreachableCount":0' && break; sleep 2 +done +echo " victim ready + memberCount=2 on survivor after $(( $(date +%s) - T1 )) s; victim /health/active=$(curl -s -o /dev/null -w '%{http_code}' "$(url_of "$ACT")/health/active") (503 = rejoined as standby)" +notify F YES + +echo "== G reject path"; notify G NO -d 'C:\recipes\x.bad' || true +echo "== done. Notifier log: $RUN_DIR/delmia-notifier.log"