From a3f8eb8bd4174dbd4d1d1804e77da43ccd5a3840 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 6 Sep 2026 02:17:31 -0400 Subject: [PATCH] =?UTF-8?q?chore(rig):=20Task=2010=20verified=20=E2=80=94?= =?UTF-8?q?=20verify-delmia-notifier=20step=20I=20passes=20on=20the=20rig?= =?UTF-8?q?=20(outbound=20API/DB/notify=20nodes=20live=20+=20replay)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rig run 2026-09-06 (image built from this branch, containers recreated from main's compose file): full drill A-H YES (active-central kill takeover 27 s, rejoin as standby in 3 s), then with FLOW_ONLY=1: H live+replay -> YES live: YES root=8e628d0a-428b-48df-b937-f4c2ac631fc3 events=8 api=Delivered db=Delivered notify=Submitted replay: YES root=8e628d0a-428b-48df-b937-f4c2ac631fc3 events=10 api=Delivered db=Delivered notify=Delivered I outbound live+replay -> YES Two rig facts the drill established: (1) with the DB reachable the site's immediate-completion path resolves a CachedWrite INSIDE the run, so the DbCall node is already Delivered inline (Submit/Attempted/Resolve share a tick — the check now ranks statuses instead of taking list order); (2) cached-call telemetry reaches central on its own ~20-30 s cadence, later than the 5 s audit push, so replay needs the bounded retry. New FLOW_ONLY=1 switch runs H + I without the notifier build or the docker kill. Claude-Session: https://claude.ai/code/session_01NcMG6xXdsJbmD2wYAgkK29 --- docker/verify-delmia-notifier.sh | 25 ++++++++++++++----- ...flow-debugger-outbound-calls.md.tasks.json | 4 +-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/docker/verify-delmia-notifier.sh b/docker/verify-delmia-notifier.sh index 71b95dfd..219d7990 100755 --- a/docker/verify-delmia-notifier.sh +++ b/docker/verify-delmia-notifier.sh @@ -4,7 +4,8 @@ set -euo pipefail # End-to-end DelmiaNotifier verification against the docker rig, including an ACTIVE central # crash (docker kill = SIGKILL, so auto-down failover is exercised, not CoordinatedShutdown's # graceful handover) + rejoin. Requires: seed-delmia-sim.sh + seed-delmia-api.sh already run, and -# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh). +# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh). FLOW_ONLY=1 skips A-G (no +# notifier build, no docker kill) and runs the Flow Debugger steps H and I alone. # # HISTORY (2026-09-02): the first run of this script found that inbound API keys were a PER-NODE # SQLite store, so the key minted through the active node did not exist on the standby and every @@ -32,9 +33,10 @@ set -euo pipefail # H Flow Debugger live run + replay -> YES (flow-debug run-method returns the traced # site steps inline; flow-debug replay rebuilds the same chain from the central audit store) # I Flow Debugger OUTBOUND calls (2026-09-06) -> YES (FlowOutboundDemo — seed-flow-outbound-demo.sh — -# returns ApiOutbound + DbOutbound + Notification rows inline, the two async ones still Submitted; -# replay is retried <= 90 s until both have reached a terminal lifecycle row). Skipped with a -# note when the demo method is not seeded. +# returns ApiOutbound + DbOutbound + Notification rows inline: the sync ApiCall Delivered, the cached +# DB write Submitted or — DB reachable, immediate-completion path — already Delivered, the +# notification Submitted; replay is retried <= 90 s until the write is Delivered and the +# notification has a terminal NotifyDeliver row). Skipped with a note when the demo is not seeded. : "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" @@ -74,6 +76,9 @@ for n in a b; do else echo "FAIL: central-$n does not see key $KEY_ID — is ScadaBridge__InboundApi__ApiKeyStore__Provider=SqlServer set on both centrals (docker-compose.yml)?" >&2; exit 1; fi done +if [ "${FLOW_ONLY:-}" = 1 ]; then + echo "== FLOW_ONLY=1: skipping A-G (notifier + failover drill); running the Flow Debugger steps H and I only" +else echo "== A baseline (active first)"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"; notify A YES echo "== B standby first (relay)"; write_cfg "$(url_of "$STB"),$(url_of "$ACT")" "$DELMIA_API_KEY"; notify B YES echo "== C bad key"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "sbk_bogus_bogus"; notify C NO || true @@ -97,6 +102,7 @@ echo " victim ready + memberCount=2 on survivor after $(( $(date +%s) - T1 )) notify F YES echo "== G reject path"; notify G NO -d 'C:\recipes\x.bad' || true +fi # H — Flow Debugger (design 2026-09-05). A Live run executes the real method on the active node # with tracing on, so the site's ScriptRun / AttributeWrite / AttributeWait rows come back inline @@ -155,14 +161,21 @@ rows = [json.loads(e["detailsJson"]) for e in r.get("events", [])] by = lambda ch: [d for d in rows if d.get("channel") == ch] api, db, nt = by("ApiOutbound"), by("DbOutbound"), by("Notification") terminal = {"Delivered", "Failed", "Parked", "Discarded"} -last = lambda ds: (ds[-1].get("status") if ds else None) +rank = {"Submitted": 0, "Forwarded": 1, "Attempted": 2} +# The operation status is the furthest-along lifecycle row, not the last in list order: a same-tick +# Attempted + Resolve pair (the immediate-completion path) can arrive in either order. +last = lambda ds: (max((d.get("status") for d in ds), key=lambda st: 3 if st in terminal else rank.get(st, -1)) if ds else None) problems = [] if r.get("completed") is not True: problems.append("incomplete") if not any(d.get("kind") == "ApiCall" and d.get("status") == "Delivered" for d in api): problems.append("no Delivered ApiCall") if not db: problems.append("no DbOutbound rows") if not nt: problems.append("no Notification rows") if mode == "live": - if last(db) != "Submitted": problems.append("db not pending inline") + # The cached write is Submitted inline when the S&F engine buffers it, but with the DB reachable the + # site resolves it INSIDE the run on the immediate-completion path (Submit, Attempted, Delivered all + # inline). Either is a truthful live picture; only a Failed/Parked write is wrong here. + if last(db) not in ("Submitted", "Delivered"): problems.append("db inline ended " + str(last(db))) + # The notification is delivered by the central Notification Outbox after the run: always pending inline. if last(nt) != "Submitted": problems.append("notify not pending inline") else: if last(db) not in terminal: problems.append("db still " + str(last(db))) diff --git a/docs/plans/2026-09-06-flow-debugger-outbound-calls.md.tasks.json b/docs/plans/2026-09-06-flow-debugger-outbound-calls.md.tasks.json index 8c5d1c02..9109196d 100644 --- a/docs/plans/2026-09-06-flow-debugger-outbound-calls.md.tasks.json +++ b/docs/plans/2026-09-06-flow-debugger-outbound-calls.md.tasks.json @@ -86,7 +86,7 @@ { "id": 10, "subject": "Task 10: CLI projection, docs, rig demo + verification", - "status": "in_progress", + "status": "completed", "blockedBy": [ 3, 4, @@ -96,5 +96,5 @@ ] } ], - "lastUpdated": "2026-09-06T06:03:16.400002+00:00" + "lastUpdated": "2026-09-06T06:17:31.653045+00:00" } \ No newline at end of file