diff --git a/CLAUDE.md b/CLAUDE.md index cbdcde0d..ed4bcd86 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -196,7 +196,7 @@ spec for each is `docs/requirements/Component-.md`, and `README.md` carrie - Central UI: Blazor Server (ASP.NET Core + SignalR) with Bootstrap CSS. No third-party component frameworks (no Blazorise, MudBlazor, Radzen, etc.). Build custom Blazor components for tables, grids, forms, etc. - UI design: Clean, corporate, internal-use aesthetic. Not flashy. Use the `frontend-design` skill when designing UI pages/components. - Debug view: real-time streaming via DebugStreamBridgeActor + gRPC (events via `SiteStreamGrpcClient`, snapshot via the `SiteCommandService` query surface — `QueryReply.DebugViewSnapshot`). Health dashboard: 10s polling timer. Deployment status: real-time push via SignalR. -- **Flow Debugger (2026-09-05, Deployment → `/deployment/flow-debugger` + CLI `flow-debug run-method|run-value-change|replay`).** Runs one entry point (an inbound API method, or a value change on an instance attribute) and draws every step as a Central / Site / Tags swimlane; *Live* really runs it, *Replay* rebuilds a past execution. **Trace steps ARE audit events** — `AuditChannel.ScriptTrace` with kinds `RouteCall` (central, **always-on** per `Route.To().Call`, joins the two halves of a Replay) and `ScriptRun` / `AttributeWrite` / `AttributeWait` (site, **gated** by a `bool Trace` riding beside `ParentExecutionId` on `RouteToCallRequest` / `ScriptCallRequest` / `SetStaticAttributeCommand` / `AttributeValueChanged`; proto fields additive, `site_command.proto` imports `sitestream.proto`). A Live run gets the site rows **inline** on `RouteToCallResponse.TraceEvents` (+ `ChildExecutionId`) because the 5 s telemetry cadence is too slow; Replay reads them from `AuditLog` (`GetTraceEventsAsync`). A value-change on a data-sourced attribute is a REAL tag write; the Instance Actor's **trace arm** (`SiteRuntime:TraceArmWindowSeconds`, 30) stamps the device echo with the writer's execution id so the triggered run is parented and traced, and central polls the site's new `PullTraceEvents` RPC (point read, not reconciliation) for it. Execution path is three management commands on the **per-node `ManagementActor`** (`FlowDebug*`, Deployer role) — a standby refuses the two run commands with **`STANDBY_NODE`** (shown verbatim); `ApiMethod.TraceAllCalls` (migration `AddApiMethodTraceAllCalls`, CLI `--trace-all-calls`) traces every real call to a method for later Replay (~7 rows/call). Prediction before a run is a Roslyn syntax walk (`FlowShapePredictor`, REQ-SA-7, DB-free); the graph (`FlowStep`/`FlowGraphBuilder`, Commons) is a central projection, never a wire type; the renderer is custom SVG (`FlowGraphView`). Rig: `AuditLog:PerChannelRetentionDays:ScriptTrace=30` on both centrals; `verify-delmia-notifier.sh` step H proves live + replay. **Mixed-version caveat:** an older site ignores the flag → central `RouteCall` only, page says *site not traced*. Design `docs/plans/2026-09-05-flow-debugger-design.md`; plan `docs/plans/2026-09-05-flow-debugger.md`. +- **Flow Debugger (2026-09-05, Deployment → `/deployment/flow-debugger` + CLI `flow-debug run-method|run-value-change|replay`).** Runs one entry point (an inbound API method, or a value change on an instance attribute) and draws every step as a Central / Site / Tags / **External** swimlane; *Live* really runs it, *Replay* rebuilds a past execution. **v1.1 (2026-09-06, `docs/plans/2026-09-06-flow-debugger-outbound-calls.md`):** every `ExternalSystem.*` / `Database.*` / `Notify.Send` a traced run makes is an `ApiCall` / `DbCall` / `Notify` node on the External lane — the site routes those already-existing `ApiOutbound` / `DbOutbound` / `Notification` rows through the same inline trace collector, the replay query widens to those channels, and `FlowGraphBuilder` folds a cached operation's rows by `(kind, CorrelationId)` into ONE node with a `Lifecycle` (sync calls in-chain, cached/notify on dashed `Async` edges, trigger cascades on `Spawn` edges, the Response linked from the inbound execution's last step). Later lifecycle rows are not inline, so after a Live run the page polls the Replay command (5 s, ≤ 2 min) while any node is `Pending`, plus a *Refresh outcomes* button. Central method-script `Database.*` calls (`InboundDatabaseHelper`) are now audited (`DbOutbound`/`DbWrite`) — they were not. Rig demo: `docker/seed-flow-outbound-demo.sh` + `verify-delmia-notifier.sh` step I. **Trace steps ARE audit events** — `AuditChannel.ScriptTrace` with kinds `RouteCall` (central, **always-on** per `Route.To().Call`, joins the two halves of a Replay) and `ScriptRun` / `AttributeWrite` / `AttributeWait` (site, **gated** by a `bool Trace` riding beside `ParentExecutionId` on `RouteToCallRequest` / `ScriptCallRequest` / `SetStaticAttributeCommand` / `AttributeValueChanged`; proto fields additive, `site_command.proto` imports `sitestream.proto`). A Live run gets the site rows **inline** on `RouteToCallResponse.TraceEvents` (+ `ChildExecutionId`) because the 5 s telemetry cadence is too slow; Replay reads them from `AuditLog` (`GetTraceEventsAsync`). A value-change on a data-sourced attribute is a REAL tag write; the Instance Actor's **trace arm** (`SiteRuntime:TraceArmWindowSeconds`, 30) stamps the device echo with the writer's execution id so the triggered run is parented and traced, and central polls the site's new `PullTraceEvents` RPC (point read, not reconciliation) for it. Execution path is three management commands on the **per-node `ManagementActor`** (`FlowDebug*`, Deployer role) — a standby refuses the two run commands with **`STANDBY_NODE`** (shown verbatim); `ApiMethod.TraceAllCalls` (migration `AddApiMethodTraceAllCalls`, CLI `--trace-all-calls`) traces every real call to a method for later Replay (~7 rows/call). Prediction before a run is a Roslyn syntax walk (`FlowShapePredictor`, REQ-SA-7, DB-free); the graph (`FlowStep`/`FlowGraphBuilder`, Commons) is a central projection, never a wire type; the renderer is custom SVG (`FlowGraphView`). Rig: `AuditLog:PerChannelRetentionDays:ScriptTrace=30` on both centrals; `verify-delmia-notifier.sh` step H proves live + replay. **Mixed-version caveat:** an older site ignores the flag → central `RouteCall` only, page says *site not traced*. Design `docs/plans/2026-09-05-flow-debugger-design.md`; plan `docs/plans/2026-09-05-flow-debugger.md`. - Health reports: 30s interval, 60s offline threshold, monotonic sequence numbers, raw error counts per interval. - Dead letter monitoring as a health metric. - Site Event Logging: 30-day retention, 1GB storage cap, daily purge, paginated queries with keyword search. **The cap measures `site_events` + its four indexes, NOT the consolidated LocalDb file** (register row 81, 2026-08-17): whole-file measurement made the trim unwinnable on a replicated node — each batch freed event pages while adding an equal count of oplog rows + tombstones to the same file, so it stopped 2.5x over a 64 MB cap and a node with a down peer could not enforce the cap at all. `dbstat` (the exact per-table measurement) is **NOT compiled into the bundled `SQLitePCLRaw.lib.e_sqlite3`** — probed and pinned by `SqliteCapabilityTests`, so don't reach for it — the footprint is modelled from sampled row width instead (−0.8%…+12.8%, biased high). Consequence: `MaxStorageMb` no longer bounds the data volume alone; LocalDb bookkeeping is separate overhead bounded by `MaxOplogRows`/`TombstoneRetention` and is deliberately not this component's job. Decision: `docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md`. diff --git a/docker/README.md b/docker/README.md index dff7d0fd..f160cf2d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -99,7 +99,8 @@ docker/ ├── seed-live-probe.sh # Seed one deployable live-data instance on site-a (stream traffic) ├── seed-delmia-sim.sh # Seed the in-memory Delmia machine stand-in (DelmiaSim01) on site-a ├── seed-delmia-api.sh # Seed the DelmiaRecipeDownload inbound method + a scoped API key (token printed once) -├── verify-delmia-notifier.sh # DelmiaNotifier end-to-end vs the rig: standby relay, active-central crash/takeover, rejoin; step H = Flow Debugger live run + replay (CLI flow-debug) +├── seed-flow-outbound-demo.sh # Seed the Flow Debugger outbound-calls demo: RigRest external system, RigDb connection, FlowDemo list, OutboundDemo script + FlowOutboundDemo method/key +├── verify-delmia-notifier.sh # DelmiaNotifier end-to-end vs the rig: standby relay, active-central crash/takeover, rejoin; step H = Flow Debugger live run + replay (CLI flow-debug); step I = outbound API/DB/notify nodes live + replay ├── teardown.sh # Stop and remove containers ├── central-node-a/ │ ├── appsettings.Central.json # Central node A configuration diff --git a/docker/seed-flow-outbound-demo.sh b/docker/seed-flow-outbound-demo.sh new file mode 100755 index 00000000..0b4fa17f --- /dev/null +++ b/docker/seed-flow-outbound-demo.sh @@ -0,0 +1,105 @@ +#!/bin/bash +set -euo pipefail + +# Seed the Flow Debugger OUTBOUND-CALLS demo on the docker rig (plan +# docs/plans/2026-09-06-flow-debugger-outbound-calls.md, Task 10): one site Call script that makes +# all three outbound call kinds, plus an inbound method routing to it, so `flow-debug run-method` / +# the /deployment/flow-debugger page show ApiCall + DbCall + Notify nodes on the External lane. +# +# What it creates: +# - External system RigRest -> the infra REST API (scadabridge-restapi:5200, X-API-Key +# scadabridge-test-key-1) with one method Echo = POST /api/Echo. +# - DB connection RigDb -> the rig's ScadaBridgeMachineData database on scadabridge-mssql. +# - Notification list FlowDemo (email; delivery needs an SMTP configuration row — create one on +# /notifications/smtp pointing at scadabridge-smtp:1025, auth None — +# otherwise the notification node ends Parked, which is itself a +# valid terminal lifecycle for the demo). +# - Script OutboundDemo (Call) on template SimMachine (docker/seed-delmia-sim.sh): +# ExternalSystem.Call("RigRest","Echo") -> sync ApiCall, in-chain +# Database.CachedWrite("RigDb", …) -> DbCall, async (creates + inserts dbo.FlowDemo) +# Notify.To("FlowDemo").Send(…) -> Notify, async +# - Inbound method FlowOutboundDemo (Route.To(MachineCode).Call("OutboundDemo")) + an API key +# scoped to it (token printed once as FLOW_DEMO_API_KEY=…). +# - Artifacts pushed to site 1 and DelmiaSim01 redeployed so the site has the new script. +# +# Not idempotent (names must be unique). Prerequisites: docker/deploy.sh, seed-sites.sh, +# seed-delmia-sim.sh, and `infra/` up (restapi + mssql). + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +CLI="$PROJECT_ROOT/src/ZB.MOM.WW.ScadaBridge.CLI/bin/Debug/net10.0/scadabridge" +A=(--url "${SCADABRIDGE_URL:-http://localhost:9000}" --format=json --username "${SCADABRIDGE_USERNAME:-multi-role}" --password "${SCADABRIDGE_PASSWORD:-password}") +SITE_ID="${SITE_ID:-1}" +INSTANCE_NAME="${INSTANCE_NAME:-DelmiaSim01}" +TEMPLATE_NAME="${TEMPLATE_NAME:-SimMachine}" +KEY_NAME="${KEY_NAME:-flow-outbound-demo}" +RIG_DB_CS="${RIG_DB_CS:-Server=scadabridge-mssql,1433;Database=ScadaBridgeMachineData;User Id=scadabridge_app;Password=ScadaBridge_Dev1#;TrustServerCertificate=true}" + +json_id() { python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])'; } +find_id() { # $1 = json array on stdin key name, $2 = value + python3 -c 'import json,sys; k,v=sys.argv[1],sys.argv[2]; print(next(x["id"] for x in json.load(sys.stdin) if x.get(k)==v))' "$1" "$2"; } + +echo "1/7 External system RigRest -> scadabridge-restapi (method Echo = POST /api/Echo)..." +ES_ID=$("$CLI" external-system create "${A[@]}" --name RigRest --endpoint-url http://scadabridge-restapi:5200 \ + --auth-type ApiKey --auth-config '{"key":"scadabridge-test-key-1"}' --timeout-seconds 10 | json_id) +"$CLI" external-system method create "${A[@]}" --external-system-id "$ES_ID" --name Echo --http-method POST --path /api/Echo > /dev/null +echo " id=$ES_ID" + +echo "2/7 DB connection RigDb -> scadabridge-mssql/ScadaBridgeMachineData..." +DB_ID=$("$CLI" db-connection create "${A[@]}" --name RigDb --connection-string "$RIG_DB_CS" | json_id) +echo " id=$DB_ID" + +echo "3/7 Notification list FlowDemo..." +LIST_ID=$("$CLI" notification create "${A[@]}" --name FlowDemo --emails flow-demo@rig.local | json_id) +echo " id=$LIST_ID" + +echo "4/7 Script OutboundDemo (Call) on template $TEMPLATE_NAME..." +TEMPLATE_ID=$("$CLI" template list "${A[@]}" | find_id name "$TEMPLATE_NAME") +read -r -d '' DEMO_CODE <<'CS' || true +// Flow Debugger outbound-calls demo: one sync API call (in-chain), one cached DB write and one +// notification (both async branches). Each is an ordinary audited call — nothing here is +// debugger-specific; the Flow Debugger just draws the rows they leave. +System.Func P = k => Parameters.ContainsKey(k) ? Parameters[k] : null; +var wo = P("WorkOrder")?.ToString() ?? "WO-0"; + +var echo = await ExternalSystem.Call("RigRest", "Echo", + new System.Collections.Generic.Dictionary { ["workOrder"] = wo, ["source"] = "OutboundDemo" }); + +var db = await Database.CachedWrite("RigDb", + "IF OBJECT_ID('dbo.FlowDemo','U') IS NULL CREATE TABLE dbo.FlowDemo (Id INT IDENTITY(1,1) PRIMARY KEY, WorkOrder NVARCHAR(64) NOT NULL, At DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME()); INSERT INTO dbo.FlowDemo (WorkOrder) VALUES (@wo);", + new System.Collections.Generic.Dictionary { ["wo"] = wo }); + +var notification = await Notify.To("FlowDemo").Send("Flow demo " + wo, "Echo ok=" + echo.Success + "; db op=" + db.Value); + +return new { Result = echo.Success, ResultText = "echo=" + echo.Success + " db=" + db.Value + " notify=" + notification }; +CS +"$CLI" template script add "${A[@]}" --template-id "$TEMPLATE_ID" --name OutboundDemo --trigger-type Call --code "$DEMO_CODE" > /dev/null +echo " template id=$TEMPLATE_ID" + +echo "5/7 Inbound method FlowOutboundDemo + API key '$KEY_NAME'..." +read -r -d '' METHOD_CODE <<'CS' || true +System.Func P = k => Parameters.ContainsKey(k) ? Parameters[k] : null; +var code = (P("MachineCode") as string ?? "").Trim(); +if (string.IsNullOrEmpty(code)) return new { Result = false, ResultText = "MachineCode is required" }; +return await Route.To(code).Call("OutboundDemo", new { WorkOrder = P("WorkOrder") }); +CS +METHOD_ID=$("$CLI" api-method create "${A[@]}" --name FlowOutboundDemo --timeout 45 \ + --parameters '{"type":"object","properties":{"MachineCode":{"type":"string"},"WorkOrder":{"type":"string"}},"required":["MachineCode"]}' \ + --return-def '{"type":"object","properties":{"Result":{"type":"boolean"},"ResultText":{"type":"string"}},"required":["Result","ResultText"]}' \ + --script "$METHOD_CODE" | json_id) +KEY_JSON=$("$CLI" security api-key create "${A[@]}" --name "$KEY_NAME" --methods FlowOutboundDemo) +TOKEN=$(echo "$KEY_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])' 2>/dev/null || echo "$KEY_JSON" | grep -o 'sbk_[A-Za-z0-9_-]*' | head -1) +echo " method id=$METHOD_ID" + +echo "6/7 Pushing artifacts (external system + DB connection) to site $SITE_ID..." +"$CLI" site deploy-artifacts "${A[@]}" --site-id "$SITE_ID" > /dev/null + +echo "7/7 Redeploying $INSTANCE_NAME so the site has OutboundDemo..." +INSTANCE_ID=$("$CLI" instance list "${A[@]}" | find_id uniqueName "$INSTANCE_NAME") +"$CLI" instance deploy "${A[@]}" --id "$INSTANCE_ID" > /dev/null +echo " instance id=$INSTANCE_ID" + +echo +echo "FLOW_DEMO_API_KEY=$TOKEN" +echo "Try: scadabridge --url http://localhost:9000 flow-debug run-method --method-id $METHOD_ID --params '{\"MachineCode\":\"$INSTANCE_NAME\",\"WorkOrder\":\"WO-1\"}'" +echo " then open /deployment/flow-debugger?executionId=" diff --git a/docker/verify-delmia-notifier.sh b/docker/verify-delmia-notifier.sh index 6d21ec72..71b95dfd 100755 --- a/docker/verify-delmia-notifier.sh +++ b/docker/verify-delmia-notifier.sh @@ -31,6 +31,10 @@ set -euo pipefail # G reject path (DownloadPath *.bad) -> NO + "Simulated reject" # H Flow Debugger live run + replay -> YES (flow-debug run-method returns the traced # site steps inline; flow-debug replay rebuilds the same chain from the central audit store) +# I Flow Debugger OUTBOUND calls (2026-09-06) -> YES (FlowOutboundDemo — seed-flow-outbound-demo.sh — +# returns ApiOutbound + DbOutbound + Notification rows inline, the two async ones still Submitted; +# replay is retried <= 90 s until both have reached a terminal lifecycle row). Skipped with a +# note when the demo method is not seeded. : "${DELMIA_API_KEY:?export DELMIA_API_KEY=sbk_... first}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"; PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" @@ -131,4 +135,59 @@ done [ "${H_RRC:-1}" -eq 0 ] || { echo "FAIL: replay never showed the full chain after 40 s: $H_REPLAY"; exit 1; } echo " replay: $H_REPLAY" echo "H live+replay -> YES" + +# I — Flow Debugger outbound calls (plan 2026-09-06). The demo script makes a sync ExternalSystem.Call, +# a Database.CachedWrite and a Notify.Send. Live: the site returns the rows it emitted inline — the +# sync ApiCall is already Delivered, the cached write and the notification are Submitted (pending). +# Replay: the S&F forwarder (site) and the Notification Outbox (central) write the later lifecycle +# rows minutes apart, so replay is retried until both operations have a terminal row. +echo "== I Flow Debugger outbound calls (live + replay)" +I_METHOD_ID=$("$CLI" api-method list "${H_ARGS[@]}" | python3 -c 'import json,sys; ms=json.load(sys.stdin); print(next((m["id"] for m in ms if m["name"]=="FlowOutboundDemo"), ""))') +if [ -z "$I_METHOD_ID" ]; then + echo "I outbound live+replay -> SKIPPED (FlowOutboundDemo not seeded — run docker/seed-flow-outbound-demo.sh)" +else + i_check() { # $1 = live|replay; stdin = FlowDebugRunResult JSON + python3 -c ' +import json, sys +mode = sys.argv[1] +r = json.load(sys.stdin) +rows = [json.loads(e["detailsJson"]) for e in r.get("events", [])] +by = lambda ch: [d for d in rows if d.get("channel") == ch] +api, db, nt = by("ApiOutbound"), by("DbOutbound"), by("Notification") +terminal = {"Delivered", "Failed", "Parked", "Discarded"} +last = lambda ds: (ds[-1].get("status") if ds else None) +problems = [] +if r.get("completed") is not True: problems.append("incomplete") +if not any(d.get("kind") == "ApiCall" and d.get("status") == "Delivered" for d in api): problems.append("no Delivered ApiCall") +if not db: problems.append("no DbOutbound rows") +if not nt: problems.append("no Notification rows") +if mode == "live": + if last(db) != "Submitted": problems.append("db not pending inline") + if last(nt) != "Submitted": problems.append("notify not pending inline") +else: + if last(db) not in terminal: problems.append("db still " + str(last(db))) + if last(nt) not in terminal: problems.append("notify still " + str(last(nt))) + if last(db) != "Delivered": problems.append("db ended " + str(last(db))) +ok = not problems +print(("YES" if ok else "NO") + " root=" + str(r.get("rootExecutionId")) + " events=" + str(len(rows)) + + " api=" + str(last(api)) + " db=" + str(last(db)) + " notify=" + str(last(nt)) + + (" problems=" + ";".join(problems) if problems else "") + (" error=" + str(r.get("errorCode")) if r.get("errorCode") else "")) +sys.exit(0 if ok else 1)' "$1" + } + I_PARAMS=$(python3 -c 'import json,sys; print(json.dumps({"MachineCode":sys.argv[1],"WorkOrder":"WO-I"}))' "$MACHINE") + set +e; I_RUN=$("$CLI" flow-debug run-method --method-id "$I_METHOD_ID" --params "$I_PARAMS" "${H_ARGS[@]}"); I_RC=$?; set -e + [ "$I_RC" -eq 0 ] || { echo "FAIL: flow-debug run-method exited $I_RC: $I_RUN"; exit 1; } + I_LIVE=$(echo "$I_RUN" | i_check live) || { echo "FAIL: live run: $I_LIVE"; exit 1; } + echo " live: $I_LIVE" + I_ROOT=$(echo "$I_RUN" | python3 -c 'import json,sys; print(json.load(sys.stdin)["rootExecutionId"])') + I_REPLAY="" + for i in $(seq 1 45); do + set +e; I_REPLAY=$("$CLI" flow-debug replay --execution-id "$I_ROOT" "${H_ARGS[@]}" | i_check replay); I_RRC=$?; set -e + [ "$I_RRC" -eq 0 ] && break + sleep 2 + done + [ "${I_RRC:-1}" -eq 0 ] || { echo "FAIL: replay never showed both outbound operations terminal after 90 s: $I_REPLAY"; exit 1; } + echo " replay: $I_REPLAY" + echo "I outbound live+replay -> YES" +fi echo "== done. Notifier log: $RUN_DIR/delmia-notifier.log" diff --git a/docs/plans/2026-07-08-deferred-work-register.md b/docs/plans/2026-07-08-deferred-work-register.md index 58dcf9e0..d3cac413 100644 --- a/docs/plans/2026-07-08-deferred-work-register.md +++ b/docs/plans/2026-07-08-deferred-work-register.md @@ -103,6 +103,7 @@ All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round | 108 | ~~**Inbound API keys are a PER-NODE SQLite store — a central failover invalidates every inbound client's key**~~ **RESOLVED 2026-09-02 — `ZB.MOM.WW.Auth` 0.3.0 (new `ZB.MOM.WW.Auth.ApiKeys.SqlServer` package) + ScadaBridge `ScadaBridge:InboundApi:ApiKeyStore:Provider=SqlServer`.** A central pair now shares one row set: both nodes' key stores, admin stores, and audit stores point at the same `zbauth` schema in the configuration database (connection string defaults to `ScadaBridge:Database:ConfigurationDb` — never a `${secret:}` reference, to avoid a circular dependency on the SQL-backed secrets store), so create/revoke/rotate/scope changes are cluster-wide immediately instead of per-node. Each node's existing SQLite file is imported once at boot (insert-where-absent, so both nodes importing is safe and the result is the union) and renamed `*.imported`. `Provider` defaults to `Sqlite` (unchanged behavior for single-node installs, MxAccessGateway, HistorianGateway); a clustered Central left on `Sqlite` now gets a boot-time `StartupValidator` Warning naming this design instead of silently keeping the gap. Both docker rigs (`docker/`, `docker-env2/`) run `Provider=SqlServer` on both centrals; `docker/sync-inbound-api-keys.sh` is deleted, superseded by the shared store. Rig proof 2026-09-03 (`docker/`, both centrals redeployed on `Provider=SqlServer`): both nodes imported their SQLite file at the same second with no PK race — central-a `1 key / 264 audit rows inserted`, central-b `0 / 26 inserted, 1 / 264 already present` (the union), files renamed `*.imported`; a key minted via `:9001` answered `200` via `:9002`, `--enabled false` via `:9001` made both nodes answer `401` and `:9002`'s own list showed `enabled=False`, re-enable via `:9002` restored `200` on `:9001`, delete via `:9002` made `:9001` answer `401`; `verify-delmia-notifier.sh` with a freshly minted key and NO file copy: A/B/E/F `YES`, C/G `NO`, active-central `docker kill` takeover **28 s**, rejoin as standby in 2 s. `docker-env2` (second pair, `ScadaBridgeConfig2`): both centrals booted, `zbauth` v1 created, empty files imported. QA (`vq03`/`vq04`) and prod (`vd03`) still run `Provider=Sqlite` — their move is a planned deployment record (`docs/deployment/deployment-records/2026-09-XX-zb-servers-inbound-key-store.md`), not yet applied. Design: `docs/plans/2026-09-02-inbound-api-key-shared-store-design.md`. Original: | `docs/api-specs/Delmia-Recipe-Download-API.md` §8; `docker/sync-inbound-api-keys.sh` header | Fix is an architecture decision, not a patch: move the store to the shared SQL Server (as Secrets 0.5.0 did for its central store) or replicate the file; either way revocation must become cluster-wide. Needs the `ZB.MOM.WW.Auth.ApiKeys` library (scadaproj), not just this repo. | Before any 2-node central goes live with inbound API clients; before the next arch-review wave. | | 109 | **`ZB.MOM.WW.Secrets.Replicator.SqlServer`'s store migrator retries its DDL with a flat `100 * attempt` backoff and no jitter — a latent concurrent-boot flake, open in Secrets** *(Low)*. Porting that migrator's retry loop verbatim for the Auth 0.3.0 SQL Server API-key store (register row 108) reproduced an intermittent exhaustion — about once in sixty 8-way concurrent-boot trials the five flat-backoff retries ran out before a DDL/lock conflict cleared; adding jitter to the Auth port's backoff took it to 0 failures in 180 trials. Secrets' own migrator has no concurrent-migrate test, so the same defect is presumed still open there — it just hasn't been exercised at 8-way concurrency, and a ScadaBridge central pair is only ever 2-way, so this is unlikely to bite in practice. | Found 2026-09-02 while implementing `docs/plans/2026-09-02-inbound-api-key-shared-store.md` (Auth 0.3.0 `SqlServerApiKeyStoreMigrator`) | Add jitter to `SqlServerSecretsStoreMigrator`'s backoff in `~/Desktop/scadaproj/ZB.MOM.WW.Secrets` (mirror the Auth fix) and add a `Migrate_Concurrently_FromEightTasks_AllSucceed`-shaped live test to catch a regression. Lives entirely in scadaproj, not this repo. | Next time Secrets' SQL Server migrator is touched, or if a wider-than-2-way concurrent boot of a Secrets-backed cluster is ever planned. | | 110 | **Flow Debugger follow-ups: the *Vertical trace* list toggle (mockup Direction B) and console capture on `ScriptRun`** *(Low)*. v1 ships the Direction A swimlane only; the mockup's Direction B — the same steps as a vertical, filterable list with the lane as a column — was kept as an alternate for long flows and screen readers, and the detail panel's *Console* tab is absent because script `Console`/log output is not captured per run (`ScriptRun` rows carry parameters, return value, error and timing, not stdout). Both are additive: a list view over the existing `FlowGraph`, and an extra `Extra.console` field on the `ScriptRun` row's `ScriptTraceExtra` (payload cap applies). | Design `docs/plans/2026-09-05-flow-debugger-design.md` §11; plan `2026-09-05-flow-debugger.md` Task 23 | Neither was asked for by an operator; the swimlane + Timing/Writes tabs cover the acceptance flow (Delmia recipe download, rig step H), and console capture needs a per-run writer seam in `ScriptRuntimeContext` plus a redaction decision. | First operator request for a list view or for script log lines in the step detail. | +| 111 | **Flow Debugger outbound nodes: no Retry/Discard on a parked node, and no `?correlationId=` deep link from the node to the Site Calls / Notification report pages** *(Low)*. v1.1 (plan `docs/plans/2026-09-06-flow-debugger-outbound-calls.md`) draws every `ExternalSystem.*` / `Database.*` / `Notify.Send` a traced run makes as one node on the External lane and folds the cached call's / notification's later lifecycle into it (a `Parked` pill + a Lifecycle tab), but the node itself is read-only: the hint says "Retry or discard from Site Calls / Notification report" as text, without a link, and there is no per-attempt request/response diff. Both actions already exist on those pages (central→site `RetryParkedOperation` / `DiscardParkedOperation` relay; Notification Outbox Retry/Discard) and the row's `CorrelationId` (`TrackedOperationId` / `NotificationId`) is on the folded step's lifecycle rows, so a deep link is additive — the pages just don't accept a `?correlationId=` query today. | Plan 2026-09-06 "Out of scope"; `FlowStepDetail.razor` parked hint | The Site Calls page owns the retry semantics and its audit rows; duplicating the action on the debugger node would need the same Deployer gate + confirm flow. | First operator who reaches a parked node from the debugger and asks for the button. | | 68 | ~~**`deploy.sh` run from a git worktree silently rehomes every node's bind-mount data dir**~~ **RESOLVED 2026-08-15 (guard added in the same close-out).** The compose files' bind mounts are relative (`./site-a-node-a/data`), so a deploy from a worktree checkout mounts the worktree's copies — the rig keeps running and looks healthy, but its SQLite stores split from the main checkout's and everything written there is orphaned by the next deploy from main. Bit the rows-46–53 close-out live: the seeding agent's `rig-probe-1` deploy persisted into worktree-homed dirs and "vanished" after the orchestrator's redeploy (no tombstone, no product bug — LWW/tombstone forensics ruled out deletion; the row sat in `.claude/worktrees/…/docker/site-a-node-a/data/site-localdb.db` the whole time). Both `docker/deploy.sh` and `docker-env2/deploy.sh` now refuse to run from a `.claude/worktrees/` path (override: `SCADABRIDGE_ALLOW_WORKTREE_DEPLOY=1`). | Found during the rows-46–53 close-out | Closed 2026-08-15. | — | ## Resolved (verified against the code 2026-07-10) diff --git a/docs/plans/2026-09-05-flow-debugger-design.md b/docs/plans/2026-09-05-flow-debugger-design.md index 8b58c454..9a131a9f 100644 --- a/docs/plans/2026-09-05-flow-debugger-design.md +++ b/docs/plans/2026-09-05-flow-debugger-design.md @@ -1,6 +1,6 @@ # Flow Debugger — design -**Status:** implemented 2026-09-06 (branch `feat/flow-debugger`); plan `2026-09-05-flow-debugger.md` (24 tasks, all complete). Residual follow-ups: deferred-work register row 110. +**Status:** implemented 2026-09-06 (branch `feat/flow-debugger`); plan `2026-09-05-flow-debugger.md` (24 tasks, all complete). Residual follow-ups: deferred-work register row 110. **v1.1 outbound calls** (a fourth *External* lane for API / DB / notification nodes, lifecycle folding, async/spawn edges, pending refresh) — see the addendum plan `2026-09-06-flow-debugger-outbound-calls.md`; §4's "three lanes fixed" is amended by its D2. **Mockup:** Claude Design canvas *Flow Debugger Mockup* — (Direction A swimlane boards + two low-fi alternates + decision notes). **Owner components:** Central UI (#page), Inbound API, Site Runtime, Audit Log, Communication, Script Analysis, Configuration Database. diff --git a/docs/plans/2026-09-06-flow-debugger-outbound-calls.md.tasks.json b/docs/plans/2026-09-06-flow-debugger-outbound-calls.md.tasks.json index 7a57ba3e..8c5d1c02 100644 --- a/docs/plans/2026-09-06-flow-debugger-outbound-calls.md.tasks.json +++ b/docs/plans/2026-09-06-flow-debugger-outbound-calls.md.tasks.json @@ -77,7 +77,7 @@ { "id": 9, "subject": "Task 9: Flow Debugger page \u2014 pending refresh after a Live run", - "status": "in_progress", + "status": "completed", "blockedBy": [ 7, 8 @@ -86,7 +86,7 @@ { "id": 10, "subject": "Task 10: CLI projection, docs, rig demo + verification", - "status": "pending", + "status": "in_progress", "blockedBy": [ 3, 4, @@ -96,5 +96,5 @@ ] } ], - "lastUpdated": "2026-09-06T05:59:55.586055+00:00" + "lastUpdated": "2026-09-06T06:03:16.400002+00:00" } \ No newline at end of file diff --git a/docs/requirements/Component-AuditLog.md b/docs/requirements/Component-AuditLog.md index a83f29ec..b2991eb7 100644 --- a/docs/requirements/Component-AuditLog.md +++ b/docs/requirements/Component-AuditLog.md @@ -57,8 +57,9 @@ boundary: | `POST /api/{method}` (Inbound API) | External | Inbound (invokes a script) | No (gap) | | `Route.To(x).Call(y)` (Inbound API → site) | Script | Central → site | Yes — `ScriptTrace.RouteCall`, **always-on** (2026-09-05) | | Site script run / attribute write / attribute wait under a **traced** run | Script | Site-local | Yes — `ScriptTrace.ScriptRun` / `AttributeWrite` / `AttributeWait`, **gated** by the `Trace` flag (Flow Debugger Live runs, `ApiMethod.TraceAllCalls`) | +| Central method-script `Database.QuerySingle/Query/Execute` (`InboundDatabaseHelper`) | Script | Outbound | Yes — `DbOutbound.DbWrite` via `ICentralAuditWriter` (2026-09-06; was an unaudited gap) — `Target` = connection name, `Actor` = `api:{method}`, the request's `ExecutionId`, `Extra` = `{op, rowsAffected\|rowsReturned}` | -**`AuditChannel.ScriptTrace` (Flow Debugger, design `docs/plans/2026-09-05-flow-debugger-design.md` §5.1).** Trace steps ARE audit events — no parallel store. One kind is always-on (`RouteCall`, a central direct write per `Route.To().Call`, carrying the child run's id in `Extra.childExecutionId`); the three site kinds are emitted only when the run's `Trace` flag is set, ride the ordinary site `IAuditWriter` → SQLite → telemetry → central path, share the run's `ExecutionId` / `ParentExecutionId`, and are additionally returned inline on the routed reply so a Live run does not wait on the 5 s telemetry cadence. The Flow Debugger's graph (`FlowStep` / `FlowGraphBuilder`, Commons) is a central-side projection of these rows. +**`AuditChannel.ScriptTrace` (Flow Debugger, design `docs/plans/2026-09-05-flow-debugger-design.md` §5.1).** Trace steps ARE audit events — no parallel store. One kind is always-on (`RouteCall`, a central direct write per `Route.To().Call`, carrying the child run's id in `Extra.childExecutionId`); the three site kinds are emitted only when the run's `Trace` flag is set, ride the ordinary site `IAuditWriter` → SQLite → telemetry → central path, share the run's `ExecutionId` / `ParentExecutionId`, and are additionally returned inline on the routed reply so a Live run does not wait on the 5 s telemetry cadence. The Flow Debugger's graph (`FlowStep` / `FlowGraphBuilder`, Commons) is a central-side projection of these rows — **and, since 2026-09-06, of the run's `ApiOutbound` / `DbOutbound` / `Notification` rows too** (plan `docs/plans/2026-09-06-flow-debugger-outbound-calls.md`): the outbound rows already sit in the execution tree under the run's `ExecutionId`, so the replay query (`GetTraceEventsAsync`) simply widens its channel filter to the four channels plus the inbound row, and a traced site run also passes the outbound rows it emits through the same inline trace collector as its `ScriptTrace` rows (same `AuditEvent` instance the writer receives — no duplicate construction; a collector that throws can neither fail the call nor skip the durable write). Rows written after the run returned (the S&F forwarder's `Forwarded` / `Attempted` / `CachedResolve`, the Notification Outbox's `NotifyDeliver`) are not inline — the page re-reads the tree for them. Out of scope — framework traffic is not audited: diff --git a/docs/requirements/Component-CentralUI.md b/docs/requirements/Component-CentralUI.md index f20f3b1a..36b2a5f1 100644 --- a/docs/requirements/Component-CentralUI.md +++ b/docs/requirements/Component-CentralUI.md @@ -216,8 +216,9 @@ Per-leaf alarm rendering (leaf nodes are individual conditions for native alarms - **Computed alarms render unchanged** from the prior flat-table style; the enrichment is purely additive for native rows. ### Flow Debugger (Deployment Role) -- **`/deployment/flow-debugger`** (nav: Deployment → Flow Debugger; design `docs/plans/2026-09-05-flow-debugger-design.md`, plan `docs/plans/2026-09-05-flow-debugger.md`). Runs one entry point with operator-supplied inputs and renders every step the run passes through as a **three-lane swimlane** (Central / Site / Tags) — request, route call, site script run, attribute writes, attribute waits, response — each node carrying its input, output, duration and outcome; a tabbed detail panel (Input · Output · Writes · Timing, plus a *Where this ran* column) opens on the selected node with `Open in Debug View` (value-change entries, where the instance is known) and `Audit rows (n)` (→ `/audit/log?executionId=`) drill-ins. -- **Two modes.** *Live* is a real run — the method script executes on the active central node with tracing on, or a real attribute write goes through the routed-write path and the site's triggered run is polled back — audited like any other call. *Replay* executes nothing: it rebuilds the flow from an existing execution's `ScriptTrace` audit rows (`?executionId=` drill-in from the Audit Log drawer's and the Execution Chain page's **Open in Flow Debugger** actions). +- **`/deployment/flow-debugger`** (nav: Deployment → Flow Debugger; design `docs/plans/2026-09-05-flow-debugger-design.md`, plan `docs/plans/2026-09-05-flow-debugger.md`). Runs one entry point with operator-supplied inputs and renders every step the run passes through as a **four-lane swimlane** (Central / Site / Tags / **External**) — request, route call, site script run, attribute writes, attribute waits, **outbound API / database / notification calls**, response — each node carrying its input, output, duration and outcome; a tabbed detail panel (Input · Output · Writes *or* Lifecycle · Timing, plus a *Where this ran* column) opens on the selected node with `Open in Debug View` (value-change entries, where the instance is known) and `Audit rows (n)` (→ `/audit/log?executionId=`) drill-ins. +- **Two modes.** *Live* is a real run — the method script executes on the active central node with tracing on, or a real attribute write goes through the routed-write path and the site's triggered run is polled back — audited like any other call. *Replay* executes nothing: it rebuilds the flow from an existing execution's `ScriptTrace` audit rows plus its `ApiOutbound` / `DbOutbound` / `Notification` rows (`?executionId=` drill-in from the Audit Log drawer's and the Execution Chain page's **Open in Flow Debugger** actions). +- **External lane (v1.1, plan `docs/plans/2026-09-06-flow-debugger-outbound-calls.md`).** Every `ExternalSystem.Call/CachedCall`, `Database.*` and `Notify.Send` a traced run makes is a node on the fourth lane — kinds `ApiCall` / `DbCall` / `Notify` (eyebrows `API CALL` / `DATABASE` / `NOTIFY`), title = the system.method / connection / list, a status pill (`Ok` / `Pending` / `Parked` / `Failed`) plus `sync` | `cached` | `notify`. **One node per operation:** a cached call's or notification's later lifecycle rows (Forwarded → Attempted → Delivered / Parked, written minutes later by the S&F forwarder or the Notification Outbox) fold into the node (`FlowStep.Lifecycle`, a Lifecycle tab in the detail panel with *Delivered by*) instead of drawing four boxes. A sync call sits in the execution chain; a cached call or notification hangs off it on a dashed **async** connector, and a trigger cascade's spawn edge carries a bolt marker. A Live run returns only the rows the run itself emitted, so a graph left with `Pending` nodes **refreshes itself** through the Replay command every 5 s for up to 2 min (page constants), and a *Refresh outcomes* button re-reads on demand in either mode; the state chip stays *Completed* — pending outbound work is not a failed run. Retry/Discard stay on the Site Calls / Notification pages (register row 111). - **Two entry points.** An inbound API method (inputs form generated from the method's `ParameterDefinitions`, values coerced by declared type) or a value-change trigger on a named instance attribute (Site → Instance → Attribute pickers, site-scope filtered). - **Predicted graph first.** Choosing an entry point renders the statically predicted shape (`FlowShapePredictor`, Script Analysis REQ-SA-7) before anything runs — nodes whose target depends on inputs render dashed; after a run, predicted nodes the run never produced render greyed. - **Execution path.** `IFlowDebugService` dispatches `FlowDebugRunMethodCommand` / `FlowDebugRunValueChangeCommand` / `FlowDebugReplayCommand` to the **per-node `ManagementActor`** through `ManagementActorHolder` (the same seam the HTTP `/management` endpoint uses), so the server enforces the Deployer gate and the active-node refusal. A standby node answers the two run commands with `STANDBY_NODE`, which the page renders as an inline alert with the server message verbatim; Replay is served from any node. The same three commands back the CLI's `flow-debug run-method | run-value-change | replay`. diff --git a/docs/requirements/Component-InboundAPI.md b/docs/requirements/Component-InboundAPI.md index 0040a80e..13c9c235 100644 --- a/docs/requirements/Component-InboundAPI.md +++ b/docs/requirements/Component-InboundAPI.md @@ -277,6 +277,8 @@ The helper API (all asynchronous — scripts `await` them; bounded by the method - `await Database.QueryAsync("connectionName", sql, parameters)` — all rows as case-insensitive column→value dictionaries. - `await Database.ExecuteAsync("connectionName", sql, parameters)` — run a write (INSERT/UPDATE/DELETE/DDL); returns rows affected. **Writes are permitted** — the move-in integration records results, not just reads them. +**Every helper call is audited (2026-09-06).** Each `QuerySingle` / `Query` / `Execute` writes one `AuditChannel.DbOutbound` / `AuditKind.DbWrite` row through `ICentralAuditWriter` — `Status` `Delivered` or `Failed`, `Target` = the connection name, `Actor` = `api:{methodName}`, the request's `ExecutionId`, `DurationMs`, the SQL + bound parameter names/values as `RequestSummary` (capped and redacted at write time by the shared audit redactor exactly like a site `AuditingDbCommand` row), `Extra` = `{"op":"read","rowsReturned":n}` / `{"op":"write","rowsAffected":n}`, `SourceNode` from `INodeIdentityProvider`. Best-effort: an audit fault never reaches the script, and the original exception still rethrows. Under a Flow Debugger Live run the same instance is also handed to the route's trace sink so the call appears as a `DbCall` node on the External lane (Component-CentralUI.md → Flow Debugger). This closed the outbound-DB gap Component-AuditLog.md's scope table used to list for central method scripts. + Containment rules (enforced, not advisory): - **Named connections only.** `connectionName` selects one of the connections configured diff --git a/docs/requirements/Component-ScriptAnalysis.md b/docs/requirements/Component-ScriptAnalysis.md index 2aaaee10..c9e70864 100644 --- a/docs/requirements/Component-ScriptAnalysis.md +++ b/docs/requirements/Component-ScriptAnalysis.md @@ -222,13 +222,14 @@ The decorator changes **no resolution result**, only object identity: it returns ### REQ-SA-7: Flow Shape Predictor (`FlowShapePredictor`) -`FlowShapePredictor` (Flow Debugger design `docs/plans/2026-09-05-flow-debugger-design.md` §8) predicts, from script **syntax** alone, the swimlane shape a run would produce before anything executes: `PredictInboundMethod(methodName, methodScript, resolveRouteScript, maxDepth = 4)` and `PredictSiteScript(scriptName, scriptBody, maxDepth = 4)` return a `PredictedFlow` of `PredictedNode(Id, Kind, Lane, Title, InputDependent, Note)` + `PredictedEdge(From, To, CrossesBoundary)`. +`FlowShapePredictor` (Flow Debugger design `docs/plans/2026-09-05-flow-debugger-design.md` §8) predicts, from script **syntax** alone, the swimlane shape a run would produce before anything executes: `PredictInboundMethod(methodName, methodScript, resolveRouteScript, maxDepth = 4)` and `PredictSiteScript(scriptName, scriptBody, maxDepth = 4)` return a `PredictedFlow` of `PredictedNode(Id, Kind, Lane, Title, InputDependent, Note)` + `PredictedEdge(From, To, CrossesBoundary, Async = false)`. -- **Syntax walk, not semantic.** A `CSharpSyntaxWalker` over `CSharpSyntaxTree.ParseText(code, SourceCodeKind.Script)` matches the script-API member-access chains by whitespace-stripped text, exactly as `ScriptTrustValidator`'s hardening walker does: `Route.To(x).Call(y)` / `.SetAttributes` / `.WaitForAttribute`, `Scripts.Call(y)`, `…Attributes.WriteBatchAndWaitAsync` / `WaitAsync` / `WaitForAsync` / `SetAsync`, `…Attributes[x] = v`, `ExternalSystem.Call/CachedCall`, `Database.*`, `Notify.Send`. Reads (`GetAttributes`, indexer reads) produce no node because the runtime records no trace row for them. +- **Syntax walk, not semantic.** A `CSharpSyntaxWalker` over `CSharpSyntaxTree.ParseText(code, SourceCodeKind.Script)` matches the script-API member-access chains by whitespace-stripped text, exactly as `ScriptTrustValidator`'s hardening walker does: `Route.To(x).Call(y)` / `.SetAttributes` / `.WaitForAttribute`, `Scripts.Call(y)`, `…Attributes.WriteBatchAndWaitAsync` / `WaitAsync` / `WaitForAsync` / `SetAsync`, `…Attributes[x] = v`, `ExternalSystem.Call/CachedCall`, `Database.Execute/Query/QuerySingle/CachedWrite`, `Notify.Send(list)` / `Notify.To(list).Send(...)` (`Async`-suffixed spellings map identically). Reads (`GetAttributes`, indexer reads) produce no node because the runtime records no trace row for them; nor does `Database.Connection(...)` — the number of commands on a raw handle is not statically knowable. +- **Outbound calls land on the External lane (2026-09-06, plan `docs/plans/2026-09-06-flow-debugger-outbound-calls.md` D8).** `ExternalSystem.*` → `ApiCall` titled `{system}.{method}`; `Database.*` → `DbCall` titled `{connection} · {member}`; `Notify` → `Notify` titled `Notify {list}`. A `CachedCall`, `CachedWrite` or `Notify` node carries `Note = "cached"` where applicable and its incoming edge is `Async`; an async edge does **not** advance the chain — the next node links from the previous non-async node — mirroring `FlowGraphBuilder.IsAsync` so a predicted async branch renders like a real one. - **Literals resolve; everything else is `InputDependent`** and titled with the expression's source text (`Route.To(code)`). Branching is not modelled: every recognised call is emitted once in source order, and an `if (…) return/throw` guard becomes a `Note = "early return: "` on the node that **follows** it (the condition is visited first, so a guard whose condition *is* the traced call — Delmia's `if (!await …WriteBatchAndWaitAsync(…)) return …` — attributes correctly). - **DB-free by contract.** Routed and nested script bodies come from the caller-supplied `resolveRouteScript(instanceCode, scriptName)` (Central UI resolves instance → template parent chain → script; called with the literal code when constant, else with the expression text so a resolver may still answer); recursion stops at `maxDepth` with a `depth limit` note. - **Total.** Parse errors or any walker fault degrade to the request/response pair (or the root `ScriptRun`) carrying `Note = "parse error"`; the predictor never throws into the page. -- Predicted kinds are exactly the traced kinds (`FlowStepKind`), so `FlowGraphView` can overlay the real graph on the prediction by kind and grey the nodes a run never produced. There is deliberately **no separate central "method script" node** — the real graph records none, so the `InboundRequest` node stands for it. +- Predicted kinds are exactly the traced kinds (`FlowStepKind`, including `ApiCall` / `DbCall` / `Notify`), so `FlowGraphView` can overlay the real graph on the prediction by kind and grey the nodes a run never produced. There is deliberately **no separate central "method script" node** — the real graph records none, so the `InboundRequest` node stands for it. --- diff --git a/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/FlowDebugTableProjection.cs b/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/FlowDebugTableProjection.cs index fd3f2641..5f514fde 100644 --- a/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/FlowDebugTableProjection.cs +++ b/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/FlowDebugTableProjection.cs @@ -1,6 +1,8 @@ using System.Globalization; using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using ZB.MOM.WW.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging; @@ -14,11 +16,15 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands; /// error, errorCode, events: [ … ], truncated, … } whose events are canonical /// audit events (each carrying its domain fields inside a detailsJson string). Handed /// to the generic renderer unprojected, the object takes the Property/Value branch and the -/// whole event array lands in one cell as raw JSON. This unwraps it to one row per step — -/// occurred | kind | lane | target | durationMs | status | error — in the same -/// chronological order the Flow Debugger page draws them, so a terminal user can read the -/// swimlane top-to-bottom. A trailing note row reports a failed / incomplete run -/// (with the error code verbatim, e.g. STANDBY_NODE) or a truncated event list. +/// whole event array lands in one cell as raw JSON. This projects the events through the same +/// the Flow Debugger page uses and prints one row per STEP — +/// occurred | kind | lane | target | durationMs | status | lifecycle | error — in the +/// order the page draws them, so a terminal user can read the swimlane top-to-bottom. An +/// outbound operation's rows (a cached call's Submit → Forwarded → Attempted → Resolve, a +/// notification's Send → Deliver) fold into ONE row whose lifecycle column counts them +/// and whose status is the latest row's (outbound-calls plan 2026-09-06, D3). A trailing +/// note row reports a failed / incomplete run (with the error code verbatim, e.g. +/// STANDBY_NODE) or a truncated event list. /// /// /// JSON output is never touched — --format json gets the full raw result. @@ -54,14 +60,21 @@ internal static class FlowDebugTableProjection return json; } - var rows = new JsonArray(); - foreach (var evt in events.EnumerateArray()) - rows.Add(ProjectEvent(evt)); + var truncated = TryGetPropertyCI(root, "truncated", out var t) && t.ValueKind == JsonValueKind.True; + + var rows = ProjectSteps(events, truncated); + if (rows == null) + { + // The event array did not deserialize as canonical audit events (a future wire + // shape, or a proxy body) — fall back to one row per raw event so nothing is hidden. + rows = new JsonArray(); + foreach (var evt in events.EnumerateArray()) + rows.Add(ProjectEvent(evt)); + } var completed = TryGetPropertyCI(root, "completed", out var c) && c.ValueKind == JsonValueKind.True; var errorCode = StrValue(root, "errorCode"); var error = StrValue(root, "error"); - var truncated = TryGetPropertyCI(root, "truncated", out var t) && t.ValueKind == JsonValueKind.True; var notes = new List(); if (!completed || errorCode != null || error != null) @@ -81,7 +94,66 @@ internal static class FlowDebugTableProjection } } - /// Projects one canonical audit event (camelCase JSON) to its compact row. + private static readonly JsonSerializerOptions EventOptions = new(JsonSerializerDefaults.Web) + { + Converters = { new JsonStringEnumConverter() }, + }; + + /// + /// Builds the flow graph from the canonical events and projects one row per step. Returns + /// null when the array is not a list of canonical audit events. + /// + private static JsonArray? ProjectSteps(JsonElement events, bool truncated) + { + List? list; + try + { + list = JsonSerializer.Deserialize>(events.GetRawText(), EventOptions); + } + catch (JsonException) + { + return null; + } + + if (list == null || list.Any(e => e == null || e.EventId == Guid.Empty)) + return null; + + // Status per event: the graph's outcome is a coarser projection; the table keeps the + // audit status word (Delivered / Submitted / Parked …) the operator sees elsewhere. + var statusById = new Dictionary(); + foreach (var e in list) + statusById[e.EventId] = AuditDetailsCodec.Deserialize(e.DetailsJson).Status; + + var graph = FlowGraphBuilder.Build(list, truncated); + var rows = new JsonArray(); + foreach (var step in graph.Steps) + { + var lifecycle = step.Lifecycle; + var status = lifecycle is { Count: > 0 } + ? lifecycle[^1].Status + : statusById.TryGetValue(step.EventId, out var st) ? st : step.Outcome.ToString(); + + var target = step.Title; + if (step.Lane != FlowLane.External && step.Instance != null && !target.StartsWith(step.Instance, StringComparison.Ordinal)) + target = $"{step.Instance}.{target}"; + + rows.Add(new JsonObject + { + ["occurred"] = step.OccurredAtUtc.ToUniversalTime().ToString("HH:mm:ss.fff", CultureInfo.InvariantCulture), + ["kind"] = step.Kind.ToString(), + ["lane"] = step.Lane.ToString(), + ["target"] = target, + ["durationMs"] = step.DurationMs, + ["status"] = status, + ["lifecycle"] = lifecycle is { Count: > 0 } ? $"{lifecycle.Count} {(lifecycle.Count == 1 ? "row" : "rows")}" : null, + ["error"] = step.ErrorMessage, + }); + } + + return rows; + } + + /// Projects one canonical audit event (camelCase JSON) to its compact row (fallback path). private static JsonNode ProjectEvent(JsonElement evt) { if (evt.ValueKind != JsonValueKind.Object) diff --git a/src/ZB.MOM.WW.ScadaBridge.CLI/README.md b/src/ZB.MOM.WW.ScadaBridge.CLI/README.md index 29d494bb..37511025 100644 --- a/src/ZB.MOM.WW.ScadaBridge.CLI/README.md +++ b/src/ZB.MOM.WW.ScadaBridge.CLI/README.md @@ -2289,9 +2289,14 @@ Trace steps are ordinary audit rows: a run leaves them in the central `AuditLog` chain after the fact. Table output prints one row per step in chronological order — -`occurred | kind | lane | target | durationMs | status | error` — with a trailing `note` -row when the run did not complete (the error code verbatim) or the inline event list was -truncated. `--format json` returns the raw `FlowDebugRunResult` (`rootExecutionId`, +`occurred | kind | lane | target | durationMs | status | lifecycle | error` — the same +projection the Flow Debugger page draws (`FlowGraphBuilder`): an outbound `ExternalSystem.*`, +`Database.*` or `Notify` call is ONE row on the `External` lane (kinds `ApiCall` / `DbCall` / +`Notify`), and a cached call's or notification's later rows (Forwarded → Attempted → Delivered / +Parked) fold into it — `lifecycle` counts them and `status` is the latest row's (`Submitted` +while the operation is still in flight; re-run `replay` to see it settle). A trailing `note` +row reports a run that did not complete (the error code verbatim) or a truncated inline event +list. `--format json` returns the raw `FlowDebugRunResult` (`rootExecutionId`, `completed`, `error`, `errorCode`, `events`, `truncated`, `startedAtUtc`, `finishedAtUtc`). #### `flow-debug run-method` @@ -2345,7 +2350,8 @@ scadabridge --url flow-debug replay --execution-id | `--execution-id` | yes | Any execution ID in the chain — the root is resolved server-side | Only flows whose site legs were traced (a Live run, or a method with `--trace-all-calls` -set) have site steps to replay; an untraced call shows its central `RouteCall` step alone. +set) have site steps to replay; an untraced call shows its central `RouteCall` step plus the +run's outbound API / DB / notification rows, which are always audited. --- diff --git a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/FlowDebugCommandsTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/FlowDebugCommandsTests.cs index af51ab99..8e6b42ff 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/FlowDebugCommandsTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/FlowDebugCommandsTests.cs @@ -4,6 +4,7 @@ using ZB.MOM.WW.Audit; using ZB.MOM.WW.ScadaBridge.CLI.Commands; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.CLI.Tests; @@ -181,4 +182,58 @@ public class FlowDebugCommandsTests Assert.Equal("""{"error":"x"}""", FlowDebugTableProjection.ProjectEvents("""{"error":"x"}""")); Assert.Equal("[1]", FlowDebugTableProjection.ProjectEvents("[1]")); } + + [Fact] + public void Projection_FoldsAnOutboundLifecycle_IntoOneExternalLaneRow() + { + var root = Guid.NewGuid(); + var child = Guid.NewGuid(); + var corr = Guid.NewGuid(); + var t0 = new DateTime(2026, 9, 6, 9, 0, 0, 0, DateTimeKind.Utc); + var events = new List + { + ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered, + occurredAtUtc: t0, target: "DelmiaSim01.OutboundDemo", executionId: root, durationMs: 300, + extra: new ScriptTraceExtra { ChildExecutionId = child }.ToJson()), + ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered, + occurredAtUtc: t0.AddMilliseconds(10), target: "OutboundDemo", executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "DelmiaSim01", durationMs: 250), + ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiOutbound, AuditKind.ApiCall, AuditStatus.Delivered, + occurredAtUtc: t0.AddMilliseconds(20), target: "RigRest.Echo", executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "DelmiaSim01", durationMs: 40), + ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, + occurredAtUtc: t0.AddMilliseconds(70), target: "RigDb", correlationId: corr, executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "DelmiaSim01"), + ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWriteCached, AuditStatus.Attempted, + occurredAtUtc: t0.AddMilliseconds(90), target: "RigDb", correlationId: corr, executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "DelmiaSim01", durationMs: 15), + ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, + occurredAtUtc: t0.AddMilliseconds(120), target: "RigDb", correlationId: corr, executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "DelmiaSim01"), + }; + var result = new FlowDebugRunResult(root, true, null, null, events, false, new DateTimeOffset(t0), new DateTimeOffset(t0.AddSeconds(1))); + + var projected = FlowDebugTableProjection.ProjectEvents(JsonSerializer.Serialize(result, CamelCase)); + + using var doc = JsonDocument.Parse(projected); + var rows = doc.RootElement.EnumerateArray().ToList(); + Assert.Equal(4, rows.Count); // route, run, sync api call, ONE folded db write + + Assert.Equal("ApiCall", rows[2].GetProperty("kind").GetString()); + Assert.Equal("External", rows[2].GetProperty("lane").GetString()); + Assert.Equal("RigRest.Echo", rows[2].GetProperty("target").GetString()); + Assert.Equal("1 row", rows[2].GetProperty("lifecycle").GetString()); + Assert.Equal(40, rows[2].GetProperty("durationMs").GetInt32()); + + Assert.Equal("DbCall", rows[3].GetProperty("kind").GetString()); + Assert.Equal("External", rows[3].GetProperty("lane").GetString()); + Assert.Equal("RigDb", rows[3].GetProperty("target").GetString()); + Assert.Equal("3 rows", rows[3].GetProperty("lifecycle").GetString()); + Assert.Equal("Delivered", rows[3].GetProperty("status").GetString()); + Assert.Equal(50, rows[3].GetProperty("durationMs").GetInt32()); + Assert.Equal("09:00:00.070", rows[3].GetProperty("occurred").GetString()); + + Assert.Equal("DelmiaSim01.OutboundDemo", rows[1].GetProperty("target").GetString()); + Assert.Equal(JsonValueKind.Null, rows[1].GetProperty("lifecycle").ValueKind); + } }