chore(rig): Task 10 verified — verify-delmia-notifier step I passes on the rig (outbound API/DB/notify nodes live + replay)
Rig run 2026-09-06 (image built from this branch, containers recreated from main's compose file): full drill A-H YES (active-central kill takeover 27 s, rejoin as standby in 3 s), then with FLOW_ONLY=1: H live+replay -> YES live: YES root=8e628d0a-428b-48df-b937-f4c2ac631fc3 events=8 api=Delivered db=Delivered notify=Submitted replay: YES root=8e628d0a-428b-48df-b937-f4c2ac631fc3 events=10 api=Delivered db=Delivered notify=Delivered I outbound live+replay -> YES Two rig facts the drill established: (1) with the DB reachable the site's immediate-completion path resolves a CachedWrite INSIDE the run, so the DbCall node is already Delivered inline (Submit/Attempted/Resolve share a tick — the check now ranks statuses instead of taking list order); (2) cached-call telemetry reaches central on its own ~20-30 s cadence, later than the 5 s audit push, so replay needs the bounded retry. New FLOW_ONLY=1 switch runs H + I without the notifier build or the docker kill. Claude-Session: https://claude.ai/code/session_01NcMG6xXdsJbmD2wYAgkK29
This commit is contained in:
@@ -4,7 +4,8 @@ set -euo pipefail
|
|||||||
# End-to-end DelmiaNotifier verification against the docker rig, including an ACTIVE central
|
# End-to-end DelmiaNotifier verification against the docker rig, including an ACTIVE central
|
||||||
# crash (docker kill = SIGKILL, so auto-down failover is exercised, not CoordinatedShutdown's
|
# crash (docker kill = SIGKILL, so auto-down failover is exercised, not CoordinatedShutdown's
|
||||||
# graceful handover) + rejoin. Requires: seed-delmia-sim.sh + seed-delmia-api.sh already run, and
|
# graceful handover) + rejoin. Requires: seed-delmia-sim.sh + seed-delmia-api.sh already run, and
|
||||||
# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh).
|
# DELMIA_API_KEY exported (the token printed by seed-delmia-api.sh). FLOW_ONLY=1 skips A-G (no
|
||||||
|
# notifier build, no docker kill) and runs the Flow Debugger steps H and I alone.
|
||||||
#
|
#
|
||||||
# HISTORY (2026-09-02): the first run of this script found that inbound API keys were a PER-NODE
|
# HISTORY (2026-09-02): the first run of this script found that inbound API keys were a PER-NODE
|
||||||
# SQLite store, so the key minted through the active node did not exist on the standby and every
|
# SQLite store, so the key minted through the active node did not exist on the standby and every
|
||||||
@@ -32,9 +33,10 @@ set -euo pipefail
|
|||||||
# H Flow Debugger live run + replay -> YES (flow-debug run-method returns the traced
|
# H Flow Debugger live run + replay -> YES (flow-debug run-method returns the traced
|
||||||
# site steps inline; flow-debug replay rebuilds the same chain from the central audit store)
|
# site steps inline; flow-debug replay rebuilds the same chain from the central audit store)
|
||||||
# I Flow Debugger OUTBOUND calls (2026-09-06) -> YES (FlowOutboundDemo — seed-flow-outbound-demo.sh —
|
# I Flow Debugger OUTBOUND calls (2026-09-06) -> YES (FlowOutboundDemo — seed-flow-outbound-demo.sh —
|
||||||
# returns ApiOutbound + DbOutbound + Notification rows inline, the two async ones still Submitted;
|
# returns ApiOutbound + DbOutbound + Notification rows inline: the sync ApiCall Delivered, the cached
|
||||||
# replay is retried <= 90 s until both have reached a terminal lifecycle row). Skipped with a
|
# DB write Submitted or — DB reachable, immediate-completion path — already Delivered, the
|
||||||
# note when the demo method is not seeded.
|
# notification Submitted; replay is retried <= 90 s until the write is Delivered and the
|
||||||
|
# notification has a terminal NotifyDeliver row). Skipped with a note when the demo is not seeded.
|
||||||
|
|
||||||
: "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}"
|
: "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}"
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
@@ -74,6 +76,9 @@ for n in a b; do
|
|||||||
else echo "FAIL: central-$n does not see key $KEY_ID — is ScadaBridge__InboundApi__ApiKeyStore__Provider=SqlServer set on both centrals (docker-compose.yml)?" >&2; exit 1; fi
|
else echo "FAIL: central-$n does not see key $KEY_ID — is ScadaBridge__InboundApi__ApiKeyStore__Provider=SqlServer set on both centrals (docker-compose.yml)?" >&2; exit 1; fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
if [ "${FLOW_ONLY:-}" = 1 ]; then
|
||||||
|
echo "== FLOW_ONLY=1: skipping A-G (notifier + failover drill); running the Flow Debugger steps H and I only"
|
||||||
|
else
|
||||||
echo "== A baseline (active first)"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"; notify A YES
|
echo "== A baseline (active first)"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "$DELMIA_API_KEY"; notify A YES
|
||||||
echo "== B standby first (relay)"; write_cfg "$(url_of "$STB"),$(url_of "$ACT")" "$DELMIA_API_KEY"; notify B YES
|
echo "== B standby first (relay)"; write_cfg "$(url_of "$STB"),$(url_of "$ACT")" "$DELMIA_API_KEY"; notify B YES
|
||||||
echo "== C bad key"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "sbk_bogus_bogus"; notify C NO || true
|
echo "== C bad key"; write_cfg "$(url_of "$ACT"),$(url_of "$STB")" "sbk_bogus_bogus"; notify C NO || true
|
||||||
@@ -97,6 +102,7 @@ echo " victim ready + memberCount=2 on survivor after $(( $(date +%s) - T1 ))
|
|||||||
notify F YES
|
notify F YES
|
||||||
|
|
||||||
echo "== G reject path"; notify G NO -d 'C:\recipes\x.bad' || true
|
echo "== G reject path"; notify G NO -d 'C:\recipes\x.bad' || true
|
||||||
|
fi
|
||||||
|
|
||||||
# H — Flow Debugger (design 2026-09-05). A Live run executes the real method on the active node
|
# H — Flow Debugger (design 2026-09-05). A Live run executes the real method on the active node
|
||||||
# with tracing on, so the site's ScriptRun / AttributeWrite / AttributeWait rows come back inline
|
# with tracing on, so the site's ScriptRun / AttributeWrite / AttributeWait rows come back inline
|
||||||
@@ -155,14 +161,21 @@ rows = [json.loads(e["detailsJson"]) for e in r.get("events", [])]
|
|||||||
by = lambda ch: [d for d in rows if d.get("channel") == ch]
|
by = lambda ch: [d for d in rows if d.get("channel") == ch]
|
||||||
api, db, nt = by("ApiOutbound"), by("DbOutbound"), by("Notification")
|
api, db, nt = by("ApiOutbound"), by("DbOutbound"), by("Notification")
|
||||||
terminal = {"Delivered", "Failed", "Parked", "Discarded"}
|
terminal = {"Delivered", "Failed", "Parked", "Discarded"}
|
||||||
last = lambda ds: (ds[-1].get("status") if ds else None)
|
rank = {"Submitted": 0, "Forwarded": 1, "Attempted": 2}
|
||||||
|
# The operation status is the furthest-along lifecycle row, not the last in list order: a same-tick
|
||||||
|
# Attempted + Resolve pair (the immediate-completion path) can arrive in either order.
|
||||||
|
last = lambda ds: (max((d.get("status") for d in ds), key=lambda st: 3 if st in terminal else rank.get(st, -1)) if ds else None)
|
||||||
problems = []
|
problems = []
|
||||||
if r.get("completed") is not True: problems.append("incomplete")
|
if r.get("completed") is not True: problems.append("incomplete")
|
||||||
if not any(d.get("kind") == "ApiCall" and d.get("status") == "Delivered" for d in api): problems.append("no Delivered ApiCall")
|
if not any(d.get("kind") == "ApiCall" and d.get("status") == "Delivered" for d in api): problems.append("no Delivered ApiCall")
|
||||||
if not db: problems.append("no DbOutbound rows")
|
if not db: problems.append("no DbOutbound rows")
|
||||||
if not nt: problems.append("no Notification rows")
|
if not nt: problems.append("no Notification rows")
|
||||||
if mode == "live":
|
if mode == "live":
|
||||||
if last(db) != "Submitted": problems.append("db not pending inline")
|
# The cached write is Submitted inline when the S&F engine buffers it, but with the DB reachable the
|
||||||
|
# site resolves it INSIDE the run on the immediate-completion path (Submit, Attempted, Delivered all
|
||||||
|
# inline). Either is a truthful live picture; only a Failed/Parked write is wrong here.
|
||||||
|
if last(db) not in ("Submitted", "Delivered"): problems.append("db inline ended " + str(last(db)))
|
||||||
|
# The notification is delivered by the central Notification Outbox after the run: always pending inline.
|
||||||
if last(nt) != "Submitted": problems.append("notify not pending inline")
|
if last(nt) != "Submitted": problems.append("notify not pending inline")
|
||||||
else:
|
else:
|
||||||
if last(db) not in terminal: problems.append("db still " + str(last(db)))
|
if last(db) not in terminal: problems.append("db still " + str(last(db)))
|
||||||
|
|||||||
@@ -86,7 +86,7 @@
|
|||||||
{
|
{
|
||||||
"id": 10,
|
"id": 10,
|
||||||
"subject": "Task 10: CLI projection, docs, rig demo + verification",
|
"subject": "Task 10: CLI projection, docs, rig demo + verification",
|
||||||
"status": "in_progress",
|
"status": "completed",
|
||||||
"blockedBy": [
|
"blockedBy": [
|
||||||
3,
|
3,
|
||||||
4,
|
4,
|
||||||
@@ -96,5 +96,5 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"lastUpdated": "2026-09-06T06:03:16.400002+00:00"
|
"lastUpdated": "2026-09-06T06:17:31.653045+00:00"
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user