test(docker): verify-delmia-notifier.sh — notifier end-to-end on the rig incl. standby relay, active-central kill/takeover and rejoin; sync-inbound-api-keys.sh mirrors the per-node key store
First run (docker stop) found two things: SIGTERM is a graceful CoordinatedShutdown handover (takeover 0 s), so the drill now uses docker kill; and inbound API keys are a PER-NODE SQLite store (/app/data/inbound-api-keys.sqlite on each central's own volume, written by whichever node serves the management call), so the key minted through the active node did not exist on the survivor and every post-takeover call answered 401 KeyNotFound. The sync script is a rig-only workaround; the verify script's preflight refuses to start unless both nodes list the key. Second run: A/B/E/F YES, C/G NO, D 503 in the takeover window, takeover 27 s, victim rejoined as standby. Claude-Session: https://claude.ai/code/session_013L3XdmVii9i5sY4FChU8pB
This commit is contained in:
Joseph Doherty
@@ -97,6 +97,8 @@ docker/
|
||||
├── seed-live-probe.sh # Seed one deployable live-data instance on site-a (stream traffic)
|
||||
├── seed-delmia-sim.sh # Seed the in-memory Delmia machine stand-in (DelmiaSim01) on site-a
|
||||
├── seed-delmia-api.sh # Seed the DelmiaRecipeDownload inbound method + a scoped API key (token printed once)
|
||||
├── verify-delmia-notifier.sh # DelmiaNotifier end-to-end vs the rig: standby relay, active-central crash/takeover, rejoin
|
||||
├── sync-inbound-api-keys.sh # RIG-ONLY: mirror the per-node inbound API key store a->b (needed before any central failover test)
|
||||
├── teardown.sh # Stop and remove containers
|
||||
├── central-node-a/
|
||||
│ ├── appsettings.Central.json # Central node A configuration
|
||||
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# RIG-ONLY workaround: copy the inbound API key store from one central node to the other.
|
||||
#
|
||||
# Why this exists: inbound API keys live in a PER-NODE SQLite file
|
||||
# (ScadaBridge:InboundApi:ApiKeyStore:SqlitePath, /app/data/inbound-api-keys.sqlite on each
|
||||
# central's own data volume) and key admin writes only the node that serves the management call
|
||||
# (the active one, via Traefik). A key therefore exists on exactly one central node, and after a
|
||||
# central failover the survivor answers 401 to every inbound client — found live 2026-09-02 by
|
||||
# docker/verify-delmia-notifier.sh (deferred-work register). Until the store is shared, a 2-node
|
||||
# central needs the file mirrored by hand; this script does that for the docker rig.
|
||||
#
|
||||
# Usage: bash docker/sync-inbound-api-keys.sh [from a|b] [to b|a] (default: a -> b)
|
||||
# - The SOURCE stays up (its WAL is checkpointed into the copy on the host with sqlite3).
|
||||
# Quiesce key admin while it runs — a create racing the copy may be missed.
|
||||
# - The TARGET is stopped for the copy and restarted; if it is the ACTIVE node the peer takes over
|
||||
# (graceful CoordinatedShutdown handover) and the target rejoins as standby. Its own key store is
|
||||
# REPLACED (both rig nodes share one pepper, so the copied hashes verify on either node).
|
||||
|
||||
FROM="${1:-a}"; TO="${2:-b}"
|
||||
[ "$FROM" != "$TO" ] || { echo "from and to must differ" >&2; exit 2; }
|
||||
SRC="scadabridge-central-$FROM"; DST="scadabridge-central-$TO"; VOL="scadabridge-central-node-$TO-data"
|
||||
port_of() { [ "$1" = a ] && echo 9001 || echo 9002; }
|
||||
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
echo "== copying key store out of $SRC (main + WAL + SHM) and checkpointing on the host"
|
||||
for f in inbound-api-keys.sqlite inbound-api-keys.sqlite-wal inbound-api-keys.sqlite-shm; do
|
||||
docker cp "$SRC:/app/data/$f" "$WORK/$f" 2>/dev/null || true
|
||||
done
|
||||
[ -s "$WORK/inbound-api-keys.sqlite" ] || { echo "FAIL: no key store on $SRC" >&2; exit 1; }
|
||||
sqlite3 "$WORK/inbound-api-keys.sqlite" "PRAGMA wal_checkpoint(TRUNCATE);" > /dev/null
|
||||
[ "$(sqlite3 "$WORK/inbound-api-keys.sqlite" "PRAGMA integrity_check;")" = ok ] || { echo "FAIL: copy failed integrity_check" >&2; exit 1; }
|
||||
echo " api_keys rows in copy: $(sqlite3 "$WORK/inbound-api-keys.sqlite" "select count(*) from api_keys")"
|
||||
|
||||
echo "== stopping $DST, replacing its key store on volume $VOL, restarting"
|
||||
docker stop "$DST" > /dev/null
|
||||
docker run --rm -v "$VOL:/d" -v "$WORK:/src:ro" alpine:3.20 sh -c \
|
||||
'cp /src/inbound-api-keys.sqlite /d/inbound-api-keys.sqlite && rm -f /d/inbound-api-keys.sqlite-wal /d/inbound-api-keys.sqlite-shm && chmod 644 /d/inbound-api-keys.sqlite'
|
||||
docker start "$DST" > /dev/null
|
||||
|
||||
P=$(port_of "$TO"); Q=$(port_of "$FROM")
|
||||
for i in $(seq 1 120); do curl -sf -o /dev/null "http://localhost:$P/health/ready" && break; sleep 1; done
|
||||
for i in $(seq 1 60); do curl -s "http://localhost:$Q/health/ready" | grep -q '"memberCount":2,"unreachableCount":0' && break; sleep 2; done
|
||||
echo "== $DST ready; members on $SRC: $(curl -s "http://localhost:$Q/health/ready" | grep -o '"memberCount":[0-9]*,"unreachableCount":[0-9]*')"
|
||||
echo " active: central-a=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:9001/health/active) central-b=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:9002/health/active)"
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# End-to-end DelmiaNotifier verification against the docker rig, including an ACTIVE central
|
||||
# crash (docker kill = SIGKILL, so auto-down failover is exercised, not CoordinatedShutdown's
|
||||
# graceful handover) + rejoin. Requires: seed-delmia-sim.sh + seed-delmia-api.sh already run, and
|
||||
# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh).
|
||||
#
|
||||
# PREREQUISITE the first run of this script found the hard way (2026-09-02): inbound API keys are a
|
||||
# PER-NODE SQLite store, so the key minted through the active node does not exist on the standby
|
||||
# and every call after a takeover answers 401 KeyNotFound. Run `bash docker/sync-inbound-api-keys.sh`
|
||||
# after seed-delmia-api.sh (and again after re-minting a key). The preflight below lists the key
|
||||
# via each node's own management API and refuses to start the drill if either node lacks it.
|
||||
#
|
||||
# The notifier is run as the managed build (dotnet WWNotifier.dll) from a scratch copy so the
|
||||
# API key is written to THAT copy's appsettings.json (ScadaBridge:ApiKey) and never into the
|
||||
# repo. BaseUrls lists the two central nodes DIRECTLY (not Traefik) so the notifier's own
|
||||
# connect-failure failover is what gets exercised.
|
||||
#
|
||||
# Expected outcomes:
|
||||
# A baseline (active listed first) -> YES
|
||||
# B standby listed first (standby relays) -> YES (StandbyForwardPeerEndpoints)
|
||||
# C bad key -> NO (HTTP 401, no failover: 401 is an answer)
|
||||
# D active KILLED, before takeover completes -> NO or YES — informational only. A NO here is the
|
||||
# documented connect-failure-only design (standby answers 503 ACTIVE_NODE_UNREACHABLE).
|
||||
# E active KILLED, after survivor /health/active=200 -> YES (first URL refused -> failover)
|
||||
# F victim restarted and rejoined (memberCount=2) -> YES
|
||||
# G reject path (DownloadPath *.bad) -> NO + "Simulated reject"
|
||||
|
||||
: "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
RUN_DIR="${RUN_DIR:-${TMPDIR:-/tmp}/delmia-notifier-run}"
|
||||
MACHINE="${MACHINE:-DelmiaSim01}"
|
||||
A_URL=http://localhost:9001; B_URL=http://localhost:9002
|
||||
|
||||
echo "== build notifier + stage a runnable copy in $RUN_DIR"
|
||||
dotnet build "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier" -c Debug -nologo -v q
|
||||
rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR"; cp -R "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier/bin/Debug/net10.0/." "$RUN_DIR/"
|
||||
|
||||
write_cfg() { # $1 = BaseUrls, $2 = ApiKey
|
||||
printf '{"ScadaBridge":{"BaseUrls":"%s","TimeoutSeconds":30,"LogPath":"delmia-notifier.log","ApiKey":"%s"}}\n' "$1" "$2" > "$RUN_DIR/appsettings.json"
|
||||
}
|
||||
notify() { # $1 = label, $2 = expected YES|NO, rest = extra args; prints stdout + exit code
|
||||
local label=$1 expect=$2; shift 2
|
||||
local out rc; set +e; out=$(env -u SCADABRIDGE_API_KEY dotnet "$RUN_DIR/WWNotifier.dll" -m "$MACHINE" -d 'C:\recipes\wo1.nc' -w "WO-$label" -p PN-1 -s 0100 -u drill "$@" 2>/dev/null); rc=$?; set -e
|
||||
local first; first=$(echo "$out" | head -1)
|
||||
printf '%-3s stdout=%-4s exit=%-3s | %s\n' "$label" "$first" "$rc" "$(echo "$out" | sed -n 2p)"
|
||||
[ "$first" = "$expect" ] || { echo " ^^ EXPECTED $expect"; return 1; }
|
||||
}
|
||||
active() { curl -sf -o /dev/null "$A_URL/health/active" && echo a || { curl -sf -o /dev/null "$B_URL/health/active" && echo b; }; }
|
||||
url_of() { [ "$1" = a ] && echo "$A_URL" || echo "$B_URL"; }
|
||||
container_of() { [ "$1" = a ] && echo scadabridge-central-a || echo scadabridge-central-b; }
|
||||
other() { [ "$1" = a ] && echo b || echo a; }
|
||||
wait_for() { # $1 url $2 path $3 seconds
|
||||
local i; for i in $(seq 1 "$3"); do curl -sf -o /dev/null "$1$2" && return 0; sleep 1; done; return 1; }
|
||||
|
||||
ACT=$(active); STB=$(other "$ACT")
|
||||
echo "== active=central-$ACT standby=central-$STB"
|
||||
|
||||
echo "== preflight: the notifier's key must be present in BOTH nodes' key stores"
|
||||
CLI="$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge"
|
||||
KEY_ID="${DELMIA_API_KEY#sbk_}"; KEY_ID="${KEY_ID%%_*}"
|
||||
for n in a b; do
|
||||
if "$CLI" security api-key list --url "$(url_of "$n")" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}" | grep -q "\"keyId\":\"$KEY_ID\""; then echo " central-$n: key $KEY_ID present"
|
||||
else echo "FAIL: central-$n does not hold key $KEY_ID — run: bash docker/sync-inbound-api-keys.sh $ACT $STB" >&2; exit 1; fi
|
||||
done
|
||||
|
||||
echo "== A baseline (active first)"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"; notify A YES
|
||||
echo "== B standby first (relay)"; write_cfg "$(url_of "$STB"),$(url_of "$ACT")" "$DELMIA_API_KEY"; notify B YES
|
||||
echo "== C bad key"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "sbk_bogus_bogus"; notify C NO || true
|
||||
write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"
|
||||
|
||||
echo "== D/E KILL the ACTIVE central ($(container_of "$ACT"))"; docker kill "$(container_of "$ACT")" > /dev/null
|
||||
T0=$(date +%s)
|
||||
echo "-- D immediate attempt (informational):"; notify D NO || notify D YES || true
|
||||
echo "-- waiting for survivor $(url_of "$STB")/health/active (auto-down ~25-30 s)..."
|
||||
wait_for "$(url_of "$STB")" /health/active 90 || { echo "FAIL: survivor never became active"; exit 1; }
|
||||
echo " takeover after $(( $(date +%s) - T0 )) s"
|
||||
echo "== E after takeover (first URL refused -> failover)"; notify E YES
|
||||
|
||||
echo "== F restart victim + wait for rejoin"; docker start "$(container_of "$ACT")" > /dev/null
|
||||
T1=$(date +%s)
|
||||
wait_for "$(url_of "$ACT")" /health/ready 120 || { echo "FAIL: victim never ready"; exit 1; }
|
||||
for i in $(seq 1 60); do
|
||||
curl -s "$(url_of "$STB")/health/ready" | grep -q '"memberCount":2,"unreachableCount":0' && break; sleep 2
|
||||
done
|
||||
echo " victim ready + memberCount=2 on survivor after $(( $(date +%s) - T1 )) s; victim /health/active=$(curl -s -o /dev/null -w '%{http_code}' "$(url_of "$ACT")/health/active") (503 = rejoined as standby)"
|
||||
notify F YES
|
||||
|
||||
echo "== G reject path"; notify G NO -d 'C:\recipes\x.bad' || true
|
||||
echo "== done. Notifier log: $RUN_DIR/delmia-notifier.log"
|
||||
Reference in New Issue
Block a user