Joseph Doherty
a300f35f9d
Both central nodes gain a root AuditLog:PerChannelRetentionDays:ScriptTrace=30 (validator
floor). verify-delmia-notifier.sh step H: finds DelmiaRecipeDownload via api-method list,
runs flow-debug run-method with the drill parameters and asserts completed=true plus the
kind set {InboundRequest, RouteCall, ScriptRun, AttributeWrite, AttributeWait} from each
event's detailsJson, then flow-debug replay on the root ExecutionId with the same check,
retried up to 40 s for the 5 s site telemetry push; prints 'H live+replay -> YES'.
VERIFICATION PENDING: the image scadabridge:latest was built from this branch
(docker/build.sh, exit 0) but the containers have not been recreated and step H has not
run yet — the next step is a recreate through the MAIN checkout's compose file (deploy.sh
refuses a worktree, register row 68: bind-mounted appsettings would rehome), a fresh
DelmiaRecipeDownload-scoped key, and the drill.
Claude-Session: https://claude.ai/code/session_01GSLkkHKz9JrAtmJjmQK3cP
135 lines
8.9 KiB
Bash
Executable File
135 lines
8.9 KiB
Bash
Executable File
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
# End-to-end DelmiaNotifier verification against the docker rig, including an ACTIVE central
|
|
# crash (docker kill = SIGKILL, so auto-down failover is exercised, not CoordinatedShutdown's
|
|
# graceful handover) + rejoin. Requires: seed-delmia-sim.sh + seed-delmia-api.sh already run, and
|
|
# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh).
|
|
#
|
|
# HISTORY (2026-09-02): the first run of this script found that inbound API keys were a PER-NODE
|
|
# SQLite store, so the key minted through the active node did not exist on the standby and every
|
|
# call after a takeover answered 401 KeyNotFound (deferred-work register row 108; the rig carried
|
|
# a sync-inbound-api-keys.sh file-copy workaround). Both rig centrals now run
|
|
# ScadaBridge__InboundApi__ApiKeyStore__Provider=SqlServer (one shared zbauth schema in the config
|
|
# DB), so the preflight below — which lists the key via EACH node's own management API — is no
|
|
# longer a gate that needs a script to satisfy: it is the proof that a key minted on one node is
|
|
# visible on the other without any file copy.
|
|
#
|
|
# The notifier is run as the managed build (dotnet WWNotifier.dll) from a scratch copy so the
|
|
# API key is written to THAT copy's appsettings.json (ScadaBridge:ApiKey) and never into the
|
|
# repo. BaseUrls lists the two central nodes DIRECTLY (not Traefik) so the notifier's own
|
|
# connect-failure failover is what gets exercised.
|
|
#
|
|
# Expected outcomes:
|
|
# A baseline (active listed first) -> YES
|
|
# B standby listed first (standby relays) -> YES (StandbyForwardPeerEndpoints)
|
|
# C bad key -> NO (HTTP 401, no failover: 401 is an answer)
|
|
# D active KILLED, before takeover completes -> NO or YES — informational only. A NO here is the
|
|
# documented connect-failure-only design (standby answers 503 ACTIVE_NODE_UNREACHABLE).
|
|
# E active KILLED, after survivor /health/active=200 -> YES (first URL refused -> failover)
|
|
# F victim restarted and rejoined (memberCount=2) -> YES
|
|
# G reject path (DownloadPath *.bad) -> NO + "Simulated reject"
|
|
# H Flow Debugger live run + replay -> YES (flow-debug run-method returns the traced
|
|
# site steps inline; flow-debug replay rebuilds the same chain from the central audit store)
|
|
|
|
: "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}"
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
RUN_DIR="${RUN_DIR:-${TMPDIR:-/tmp}/delmia-notifier-run}"
|
|
MACHINE="${MACHINE:-DelmiaSim01}"
|
|
A_URL=http://localhost:9001; B_URL=http://localhost:9002
|
|
|
|
echo "== build notifier + stage a runnable copy in $RUN_DIR"
|
|
dotnet build "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier" -c Debug -nologo -v q
|
|
rm -rf "$RUN_DIR"; mkdir -p "$RUN_DIR"; cp -R "$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier/bin/Debug/net10.0/." "$RUN_DIR/"
|
|
|
|
write_cfg() { # $1 = BaseUrls, $2 = ApiKey
|
|
printf '{"ScadaBridge":{"BaseUrls":"%s","TimeoutSeconds":30,"LogPath":"delmia-notifier.log","ApiKey":"%s"}}\n' "$1" "$2" > "$RUN_DIR/appsettings.json"
|
|
}
|
|
notify() { # $1 = label, $2 = expected YES|NO, rest = extra args; prints stdout + exit code
|
|
local label=$1 expect=$2; shift 2
|
|
local out rc; set +e; out=$(env -u SCADABRIDGE_API_KEY dotnet "$RUN_DIR/WWNotifier.dll" -m "$MACHINE" -d 'C:\recipes\wo1.nc' -w "WO-$label" -p PN-1 -s 0100 -u drill "$@" 2>/dev/null); rc=$?; set -e
|
|
local first; first=$(echo "$out" | head -1)
|
|
printf '%-3s stdout=%-4s exit=%-3s | %s\n' "$label" "$first" "$rc" "$(echo "$out" | sed -n 2p)"
|
|
[ "$first" = "$expect" ] || { echo " ^^ EXPECTED $expect"; return 1; }
|
|
}
|
|
active() { curl -sf -o /dev/null "$A_URL/health/active" && echo a || { curl -sf -o /dev/null "$B_URL/health/active" && echo b; }; }
|
|
url_of() { [ "$1" = a ] && echo "$A_URL" || echo "$B_URL"; }
|
|
container_of() { [ "$1" = a ] && echo scadabridge-central-a || echo scadabridge-central-b; }
|
|
other() { [ "$1" = a ] && echo b || echo a; }
|
|
wait_for() { # $1 url $2 path $3 seconds
|
|
local i; for i in $(seq 1 "$3"); do curl -sf -o /dev/null "$1$2" && return 0; sleep 1; done; return 1; }
|
|
|
|
ACT=$(active); STB=$(other "$ACT")
|
|
echo "== active=central-$ACT standby=central-$STB"
|
|
|
|
echo "== preflight/proof: the key minted on the active node must be visible from BOTH nodes (shared store, no file copy)"
|
|
CLI="$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge"
|
|
KEY_ID="${DELMIA_API_KEY#sbk_}"; KEY_ID="${KEY_ID%%_*}"
|
|
for n in a b; do
|
|
if "$CLI" security api-key list --url "$(url_of "$n")" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}" | grep -q "\"keyId\":\"$KEY_ID\""; then echo " central-$n: key $KEY_ID present"
|
|
else echo "FAIL: central-$n does not see key $KEY_ID — is ScadaBridge__InboundApi__ApiKeyStore__Provider=SqlServer set on both centrals (docker-compose.yml)?" >&2; exit 1; fi
|
|
done
|
|
|
|
echo "== A baseline (active first)"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"; notify A YES
|
|
echo "== B standby first (relay)"; write_cfg "$(url_of "$STB"),$(url_of "$ACT")" "$DELMIA_API_KEY"; notify B YES
|
|
echo "== C bad key"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "sbk_bogus_bogus"; notify C NO || true
|
|
write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"
|
|
|
|
echo "== D/E KILL the ACTIVE central ($(container_of "$ACT"))"; docker kill "$(container_of "$ACT")" > /dev/null
|
|
T0=$(date +%s)
|
|
echo "-- D immediate attempt (informational):"; notify D NO || notify D YES || true
|
|
echo "-- waiting for survivor $(url_of "$STB")/health/active (auto-down ~25-30 s)..."
|
|
wait_for "$(url_of "$STB")" /health/active 90 || { echo "FAIL: survivor never became active"; exit 1; }
|
|
echo " takeover after $(( $(date +%s) - T0 )) s"
|
|
echo "== E after takeover (first URL refused -> failover)"; notify E YES
|
|
|
|
echo "== F restart victim + wait for rejoin"; docker start "$(container_of "$ACT")" > /dev/null
|
|
T1=$(date +%s)
|
|
wait_for "$(url_of "$ACT")" /health/ready 120 || { echo "FAIL: victim never ready"; exit 1; }
|
|
for i in $(seq 1 60); do
|
|
curl -s "$(url_of "$STB")/health/ready" | grep -q '"memberCount":2,"unreachableCount":0' && break; sleep 2
|
|
done
|
|
echo " victim ready + memberCount=2 on survivor after $(( $(date +%s) - T1 )) s; victim /health/active=$(curl -s -o /dev/null -w '%{http_code}' "$(url_of "$ACT")/health/active") (503 = rejoined as standby)"
|
|
notify F YES
|
|
|
|
echo "== G reject path"; notify G NO -d 'C:\recipes\x.bad' || true
|
|
|
|
# H — Flow Debugger (design 2026-09-05). A Live run executes the real method on the active node
|
|
# with tracing on, so the site's ScriptRun / AttributeWrite / AttributeWait rows come back inline
|
|
# in the same reply; Replay reads the same chain back from the central AuditLog, which the site
|
|
# feeds through the 5 s telemetry push — hence the bounded retry. Both must show the full kind set.
|
|
echo "== H Flow Debugger live run + replay"
|
|
ACT=$(active)
|
|
H_ARGS=(--url "$(url_of "$ACT")" --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}" --format json)
|
|
h_check() { # stdin = FlowDebugRunResult JSON; prints a one-line verdict, exits 1 when incomplete or kinds missing
|
|
python3 -c '
|
|
import json, sys
|
|
r = json.load(sys.stdin)
|
|
kinds = sorted(json.loads(e["detailsJson"]).get("kind", "?") for e in r.get("events", []))
|
|
need = ["AttributeWait", "AttributeWrite", "InboundRequest", "RouteCall", "ScriptRun"]
|
|
missing = [k for k in need if k not in kinds]
|
|
ok = r.get("completed") is True and not missing
|
|
print(("YES" if ok else "NO") + " root=" + str(r.get("rootExecutionId")) + " events=" + str(len(kinds)) + " kinds=" + ",".join(kinds)
|
|
+ (" missing=" + ",".join(missing) if missing else "") + (" error=" + str(r.get("errorCode")) if r.get("errorCode") else ""))
|
|
sys.exit(0 if ok else 1)'
|
|
}
|
|
METHOD_ID=$("$CLI" api-method list "${H_ARGS[@]}" | python3 -c 'import json,sys; ms=json.load(sys.stdin); print(next(m["id"] for m in ms if m["name"]=="DelmiaRecipeDownload"))')
|
|
H_PARAMS=$(python3 -c 'import json,sys; print(json.dumps({"MachineCode":sys.argv[1],"DownloadPath":"C:\\recipes\\wo1.nc","WorkOrderNumber":"WO-H","PartNumber":"PN-1","JobStepNumber":"0100","Username":"drill"}))' "$MACHINE")
|
|
set +e
|
|
H_RUN=$("$CLI" flow-debug run-method --method-id "$METHOD_ID" --params "$H_PARAMS" "${H_ARGS[@]}")
|
|
H_RC=$?; set -e
|
|
[ "$H_RC" -eq 0 ] || { echo "FAIL: flow-debug run-method exited $H_RC: $H_RUN"; exit 1; }
|
|
H_LIVE=$(echo "$H_RUN" | h_check) || { echo "FAIL: live run incomplete: $H_LIVE"; exit 1; }
|
|
echo " live: $H_LIVE"
|
|
H_ROOT=$(echo "$H_RUN" | python3 -c 'import json,sys; print(json.load(sys.stdin)["rootExecutionId"])')
|
|
H_REPLAY=""
|
|
for i in $(seq 1 20); do
|
|
set +e; H_REPLAY=$("$CLI" flow-debug replay --execution-id "$H_ROOT" "${H_ARGS[@]}" | h_check); H_RRC=$?; set -e
|
|
[ "$H_RRC" -eq 0 ] && break
|
|
sleep 2
|
|
done
|
|
[ "${H_RRC:-1}" -eq 0 ] || { echo "FAIL: replay never showed the full chain after 40 s: $H_REPLAY"; exit 1; }
|
|
echo " replay: $H_REPLAY"
|
|
echo "H live+replay -> YES"
|
|
echo "== done. Notifier log: $RUN_DIR/delmia-notifier.log"
|