From 7caf56f332717049c1a4369ee576f7871bc788d6 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 6 Sep 2026 11:55:52 -0400 Subject: [PATCH] fix(central-ui): Flow Debugger outcome refresh merges rows by EventId instead of replacing the graph Found in a browser Live run on the rig: the run drew API, DB and Notify nodes from its inline rows; the 5 s pending-refresh re-read the tree through Replay before the site's cached-call telemetry (12 s) had reached central, swapped the graph wholesale, and the resolved DB node vanished (7 -> 6 steps) until a manual refresh. Polling had already stopped because nothing was Pending. MergeRefreshedGraph now unions the re-read rows into the run's known rows by EventId (fresh copy wins), stores the union back on _raw and rebuilds through FlowGraphBuilder, so a refresh only ever adds nodes. The union is exact: the inline rows are the same AuditEvent objects the site writer persists (FlowTraceCollector). Tests: new PendingRefresh_KeepsStepsTheLiveRunAlreadyHad_WhenReplayLagsBehindTelemetry (fails on the old code). Two existing fixtures were internally inconsistent and only passed under the old replace semantics: ParkedThenResolved built its Resolved outcome with the Parked raw rows, and OutboundOutcome minted fresh event/correlation ids per call, so no row could ever match across live and replay. Both now share rows per run. Claude-Session: https://claude.ai/code/session_01NcMG6xXdsJbmD2wYAgkK29 --- CLAUDE.md | 2 +- docs/requirements/Component-CentralUI.md | 2 +- .../Pages/Deployment/FlowDebugger.razor.cs | 28 ++++- .../Deployment/FlowDebuggerPageTests.cs | 114 ++++++++++++++++-- 4 files changed, 126 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 40ed0aaa..1d386455 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -196,7 +196,7 @@ spec for each is `docs/requirements/Component-.md`, and `README.md` carrie - Central UI: Blazor Server (ASP.NET Core + SignalR) with Bootstrap CSS. No third-party component frameworks (no Blazorise, MudBlazor, Radzen, etc.). Build custom Blazor components for tables, grids, forms, etc. - UI design: Clean, corporate, internal-use aesthetic. Not flashy. Use the `frontend-design` skill when designing UI pages/components. - Debug view: real-time streaming via DebugStreamBridgeActor + gRPC (events via `SiteStreamGrpcClient`, snapshot via the `SiteCommandService` query surface — `QueryReply.DebugViewSnapshot`). Health dashboard: 10s polling timer. Deployment status: real-time push via SignalR. -- **Flow Debugger (2026-09-05, Deployment → `/deployment/flow-debugger` + CLI `flow-debug run-method|run-value-change|replay`).** Runs one entry point (an inbound API method, or a value change on an instance attribute) and draws every step as a Central / Site / Tags / **External** swimlane; *Live* really runs it, *Replay* rebuilds a past execution. **v1.1 (2026-09-06, `docs/plans/2026-09-06-flow-debugger-outbound-calls.md`):** every `ExternalSystem.*` / `Database.*` / `Notify.Send` a traced run makes is an `ApiCall` / `DbCall` / `Notify` node on the External lane — the site routes those already-existing `ApiOutbound` / `DbOutbound` / `Notification` rows through the same inline trace collector, the replay query widens to those channels, and `FlowGraphBuilder` folds a cached operation's rows by `(kind, CorrelationId)` into ONE node with a `Lifecycle` (sync calls in-chain, cached/notify on dashed `Async` edges, trigger cascades on `Spawn` edges, the Response linked from the inbound execution's last step). Later lifecycle rows are not inline, so after a Live run the page polls the Replay command (5 s, ≤ 2 min) while any node is `Pending`, plus a *Refresh outcomes* button. Central method-script `Database.*` calls (`InboundDatabaseHelper`) are now audited (`DbOutbound`/`DbWrite`) — they were not. Rig demo: `docker/seed-flow-outbound-demo.sh` + `verify-delmia-notifier.sh` step I. **Trace steps ARE audit events** — `AuditChannel.ScriptTrace` with kinds `RouteCall` (central, **always-on** per `Route.To().Call`, joins the two halves of a Replay) and `ScriptRun` / `AttributeWrite` / `AttributeWait` (site, **gated** by a `bool Trace` riding beside `ParentExecutionId` on `RouteToCallRequest` / `ScriptCallRequest` / `SetStaticAttributeCommand` / `AttributeValueChanged`; proto fields additive, `site_command.proto` imports `sitestream.proto`). A Live run gets the site rows **inline** on `RouteToCallResponse.TraceEvents` (+ `ChildExecutionId`) because the 5 s telemetry cadence is too slow; Replay reads them from `AuditLog` (`GetTraceEventsAsync`). A value-change on a data-sourced attribute is a REAL tag write; the Instance Actor's **trace arm** (`SiteRuntime:TraceArmWindowSeconds`, 30) stamps the device echo with the writer's execution id so the triggered run is parented and traced, and central polls the site's new `PullTraceEvents` RPC (point read, not reconciliation) for it. Execution path is three management commands on the **per-node `ManagementActor`** (`FlowDebug*`, Deployer role) — a standby refuses the two run commands with **`STANDBY_NODE`** (shown verbatim); `ApiMethod.TraceAllCalls` (migration `AddApiMethodTraceAllCalls`, CLI `--trace-all-calls`) traces every real call to a method for later Replay (~7 rows/call). Prediction before a run is a Roslyn syntax walk (`FlowShapePredictor`, REQ-SA-7, DB-free); the graph (`FlowStep`/`FlowGraphBuilder`, Commons) is a central projection, never a wire type; the renderer is custom SVG (`FlowGraphView`). Rig: `AuditLog:PerChannelRetentionDays:ScriptTrace=30` on both centrals; `verify-delmia-notifier.sh` step H proves live + replay. **Mixed-version caveat:** an older site ignores the flag → central `RouteCall` only, page says *site not traced*. **v1.2 (2026-09-06, `docs/plans/2026-09-06-flow-debugger-followups.md`):** a `Parked` External node offers **Retry / Discard in place** (same site-scope check, confirm and relay as the report pages, via `IFlowDebugService.RetryOutboundAsync`/`DiscardOutboundAsync`, then the page re-polls Replay until the node leaves Parked) and every folded node deep-links to `/site-calls/report?correlationId=` / `/notifications/report?correlationId=` (both pages accept it through additive `TrackedOperationIdFilter` / `NotificationIdFilter` query params and auto-open the row); register row 111 closed. **Trap closed the same day:** `FlowDebugService.PredictMethodAsync` used to resolve route targets with `GetAwaiter().GetResult()` inside the synchronous predictor walk — on the Blazor renderer sync context that DEADLOCKED the whole circuit (page inert, healthy websocket, no error) whenever a method routing to a site script was chosen; now a resolve-and-rewalk loop, pinned by a single-thread-pump test. Never block on a task from a Blazor event/lifecycle path. Browser checks of the page go through Playwright (`FlowDebuggerTests`, fixture login + an ephemeral method), not the login-disabled override. Design `docs/plans/2026-09-05-flow-debugger-design.md`; plan `docs/plans/2026-09-05-flow-debugger.md`. +- **Flow Debugger (2026-09-05, Deployment → `/deployment/flow-debugger` + CLI `flow-debug run-method|run-value-change|replay`).** Runs one entry point (an inbound API method, or a value change on an instance attribute) and draws every step as a Central / Site / Tags / **External** swimlane; *Live* really runs it, *Replay* rebuilds a past execution. **v1.1 (2026-09-06, `docs/plans/2026-09-06-flow-debugger-outbound-calls.md`):** every `ExternalSystem.*` / `Database.*` / `Notify.Send` a traced run makes is an `ApiCall` / `DbCall` / `Notify` node on the External lane — the site routes those already-existing `ApiOutbound` / `DbOutbound` / `Notification` rows through the same inline trace collector, the replay query widens to those channels, and `FlowGraphBuilder` folds a cached operation's rows by `(kind, CorrelationId)` into ONE node with a `Lifecycle` (sync calls in-chain, cached/notify on dashed `Async` edges, trigger cascades on `Spawn` edges, the Response linked from the inbound execution's last step). Later lifecycle rows are not inline, so after a Live run the page polls the Replay command (5 s, ≤ 2 min) while any node is `Pending`, plus a *Refresh outcomes* button. **A refresh MERGES, never replaces (2026-09-06, found live in the browser):** the page unions the re-read rows into the run's own rows by `EventId` and rebuilds the graph, because a Live run carries its site rows inline while the same rows reach central's audit store by telemetry ~5–30 s later — a Replay read taken in between lacks them, and the old wholesale swap made an already-drawn node (a resolved cached DB write) vanish until the next manual refresh. The inline rows ARE the audited rows (`FlowTraceCollector` sinks the same `AuditEvent` objects the site writer persists), so the union is exact. Central method-script `Database.*` calls (`InboundDatabaseHelper`) are now audited (`DbOutbound`/`DbWrite`) — they were not. Rig demo: `docker/seed-flow-outbound-demo.sh` + `verify-delmia-notifier.sh` step I. **Trace steps ARE audit events** — `AuditChannel.ScriptTrace` with kinds `RouteCall` (central, **always-on** per `Route.To().Call`, joins the two halves of a Replay) and `ScriptRun` / `AttributeWrite` / `AttributeWait` (site, **gated** by a `bool Trace` riding beside `ParentExecutionId` on `RouteToCallRequest` / `ScriptCallRequest` / `SetStaticAttributeCommand` / `AttributeValueChanged`; proto fields additive, `site_command.proto` imports `sitestream.proto`). A Live run gets the site rows **inline** on `RouteToCallResponse.TraceEvents` (+ `ChildExecutionId`) because the 5 s telemetry cadence is too slow; Replay reads them from `AuditLog` (`GetTraceEventsAsync`). A value-change on a data-sourced attribute is a REAL tag write; the Instance Actor's **trace arm** (`SiteRuntime:TraceArmWindowSeconds`, 30) stamps the device echo with the writer's execution id so the triggered run is parented and traced, and central polls the site's new `PullTraceEvents` RPC (point read, not reconciliation) for it. Execution path is three management commands on the **per-node `ManagementActor`** (`FlowDebug*`, Deployer role) — a standby refuses the two run commands with **`STANDBY_NODE`** (shown verbatim); `ApiMethod.TraceAllCalls` (migration `AddApiMethodTraceAllCalls`, CLI `--trace-all-calls`) traces every real call to a method for later Replay (~7 rows/call). Prediction before a run is a Roslyn syntax walk (`FlowShapePredictor`, REQ-SA-7, DB-free); the graph (`FlowStep`/`FlowGraphBuilder`, Commons) is a central projection, never a wire type; the renderer is custom SVG (`FlowGraphView`). Rig: `AuditLog:PerChannelRetentionDays:ScriptTrace=30` on both centrals; `verify-delmia-notifier.sh` step H proves live + replay. **Mixed-version caveat:** an older site ignores the flag → central `RouteCall` only, page says *site not traced*. **v1.2 (2026-09-06, `docs/plans/2026-09-06-flow-debugger-followups.md`):** a `Parked` External node offers **Retry / Discard in place** (same site-scope check, confirm and relay as the report pages, via `IFlowDebugService.RetryOutboundAsync`/`DiscardOutboundAsync`, then the page re-polls Replay until the node leaves Parked) and every folded node deep-links to `/site-calls/report?correlationId=` / `/notifications/report?correlationId=` (both pages accept it through additive `TrackedOperationIdFilter` / `NotificationIdFilter` query params and auto-open the row); register row 111 closed. **Trap closed the same day:** `FlowDebugService.PredictMethodAsync` used to resolve route targets with `GetAwaiter().GetResult()` inside the synchronous predictor walk — on the Blazor renderer sync context that DEADLOCKED the whole circuit (page inert, healthy websocket, no error) whenever a method routing to a site script was chosen; now a resolve-and-rewalk loop, pinned by a single-thread-pump test. Never block on a task from a Blazor event/lifecycle path. Browser checks of the page go through Playwright (`FlowDebuggerTests`, fixture login + an ephemeral method), not the login-disabled override. Design `docs/plans/2026-09-05-flow-debugger-design.md`; plan `docs/plans/2026-09-05-flow-debugger.md`. - Health reports: 30s interval, 60s offline threshold, monotonic sequence numbers, raw error counts per interval. - Dead letter monitoring as a health metric. - Site Event Logging: 30-day retention, 1GB storage cap, daily purge, paginated queries with keyword search. **The cap measures `site_events` + its four indexes, NOT the consolidated LocalDb file** (register row 81, 2026-08-17): whole-file measurement made the trim unwinnable on a replicated node — each batch freed event pages while adding an equal count of oplog rows + tombstones to the same file, so it stopped 2.5x over a 64 MB cap and a node with a down peer could not enforce the cap at all. `dbstat` (the exact per-table measurement) is **NOT compiled into the bundled `SQLitePCLRaw.lib.e_sqlite3`** — probed and pinned by `SqliteCapabilityTests`, so don't reach for it — the footprint is modelled from sampled row width instead (−0.8%…+12.8%, biased high). Consequence: `MaxStorageMb` no longer bounds the data volume alone; LocalDb bookkeeping is separate overhead bounded by `MaxOplogRows`/`TombstoneRetention` and is deliberately not this component's job. Decision: `docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md`. diff --git a/docs/requirements/Component-CentralUI.md b/docs/requirements/Component-CentralUI.md index 5ba639c0..bdfc2602 100644 --- a/docs/requirements/Component-CentralUI.md +++ b/docs/requirements/Component-CentralUI.md @@ -218,7 +218,7 @@ Per-leaf alarm rendering (leaf nodes are individual conditions for native alarms ### Flow Debugger (Deployment Role) - **`/deployment/flow-debugger`** (nav: Deployment → Flow Debugger; design `docs/plans/2026-09-05-flow-debugger-design.md`, plan `docs/plans/2026-09-05-flow-debugger.md`). Runs one entry point with operator-supplied inputs and renders every step the run passes through as a **four-lane swimlane** (Central / Site / Tags / **External**) — request, route call, site script run, attribute writes, attribute waits, **outbound API / database / notification calls**, response — each node carrying its input, output, duration and outcome; a tabbed detail panel (Input · Output · Writes *or* Lifecycle · Timing, plus a *Where this ran* column) opens on the selected node with `Open in Debug View` (value-change entries, where the instance is known) and `Audit rows (n)` (→ `/audit/log?executionId=`) drill-ins. - **Two modes.** *Live* is a real run — the method script executes on the active central node with tracing on, or a real attribute write goes through the routed-write path and the site's triggered run is polled back — audited like any other call. *Replay* executes nothing: it rebuilds the flow from an existing execution's `ScriptTrace` audit rows plus its `ApiOutbound` / `DbOutbound` / `Notification` rows (`?executionId=` drill-in from the Audit Log drawer's and the Execution Chain page's **Open in Flow Debugger** actions). -- **External lane (v1.1, plan `docs/plans/2026-09-06-flow-debugger-outbound-calls.md`).** Every `ExternalSystem.Call/CachedCall`, `Database.*` and `Notify.Send` a traced run makes is a node on the fourth lane — kinds `ApiCall` / `DbCall` / `Notify` (eyebrows `API CALL` / `DATABASE` / `NOTIFY`), title = the system.method / connection / list, a status pill (`Ok` / `Pending` / `Parked` / `Failed`) plus `sync` | `cached` | `notify`. **One node per operation:** a cached call's or notification's later lifecycle rows (Forwarded → Attempted → Delivered / Parked, written minutes later by the S&F forwarder or the Notification Outbox) fold into the node (`FlowStep.Lifecycle`, a Lifecycle tab in the detail panel with *Delivered by*) instead of drawing four boxes. A sync call sits in the execution chain; a cached call or notification hangs off it on a dashed **async** connector, and a trigger cascade's spawn edge carries a bolt marker. A Live run returns only the rows the run itself emitted, so a graph left with `Pending` nodes **refreshes itself** through the Replay command every 5 s for up to 2 min (page constants), and a *Refresh outcomes* button re-reads on demand in either mode; the state chip stays *Completed* — pending outbound work is not a failed run. **v1.2 (follow-ups plan `docs/plans/2026-09-06-flow-debugger-followups.md`, register row 111 closed):** a folded node carries its `CorrelationId` (`FlowStep.CorrelationId` — the `TrackedOperationId` / `NotificationId`), so the detail panel deep-links to **`/site-calls/report?correlationId=`** / **`/notifications/report?correlationId=`** (*Open in Site Calls* / *Open in Notification report*; both pages accept that query, pre-filter to the one row through an additive per-id query filter — `SiteCallQueryRequest.TrackedOperationIdFilter` / `NotificationOutboxQueryRequest.NotificationIdFilter` — show a removable *Operation* / *Notification* pill and auto-open the row's detail modal; note `NotificationId` is minted as a Guid in **`N` form** — no dashes — so the Notify link passes `N` and the report page normalises any Guid-shaped value to `N`, while `TrackedOperationId` is a `varchar(36)` `D` string and the Site Calls filter binds a `varchar` parameter for the PK seek), and a **`Parked` node offers Retry / Discard in place**: the same site-scope re-check, confirm dialog (Discard = danger) and central→site / Notification Outbox relay the report pages use (`IFlowDebugService.RetryOutboundAsync` / `DiscardOutboundAsync` → `CommunicationService`), then the page re-polls Replay (5 s / 2 min) until the node leaves `Parked`. **Prediction is async-safe:** `FlowDebugService.PredictMethodAsync` resolves route targets by a resolve-and-rewalk loop — its original blocking resolver deadlocked the whole Blazor circuit on the renderer sync context the moment a method routing to a site script was chosen (found 2026-09-06; pinned by a single-thread-pump unit test and the Playwright `FlowDebuggerTests`, which log in through the fixture and run a Live run of an ephemeral method in a real browser). +- **External lane (v1.1, plan `docs/plans/2026-09-06-flow-debugger-outbound-calls.md`).** Every `ExternalSystem.Call/CachedCall`, `Database.*` and `Notify.Send` a traced run makes is a node on the fourth lane — kinds `ApiCall` / `DbCall` / `Notify` (eyebrows `API CALL` / `DATABASE` / `NOTIFY`), title = the system.method / connection / list, a status pill (`Ok` / `Pending` / `Parked` / `Failed`) plus `sync` | `cached` | `notify`. **One node per operation:** a cached call's or notification's later lifecycle rows (Forwarded → Attempted → Delivered / Parked, written minutes later by the S&F forwarder or the Notification Outbox) fold into the node (`FlowStep.Lifecycle`, a Lifecycle tab in the detail panel with *Delivered by*) instead of drawing four boxes. A sync call sits in the execution chain; a cached call or notification hangs off it on a dashed **async** connector, and a trigger cascade's spawn edge carries a bolt marker. A Live run returns only the rows the run itself emitted, so a graph left with `Pending` nodes **refreshes itself** through the Replay command every 5 s for up to 2 min (page constants), and a *Refresh outcomes* button re-reads on demand in either mode; the state chip stays *Completed* — pending outbound work is not a failed run. A refresh **merges** the re-read rows into the run's known rows by `EventId` (fresh copy wins) and rebuilds the graph from the union, so it only ever adds nodes: the site's outbound rows reach central's audit store by telemetry seconds after the run returned them inline, and a Replay read taken in between would otherwise erase an already-drawn node (observed on the rig 2026-09-06 with a resolved cached DB write). The union is exact because the inline rows are the very `AuditEvent` objects the site writer persists. **v1.2 (follow-ups plan `docs/plans/2026-09-06-flow-debugger-followups.md`, register row 111 closed):** a folded node carries its `CorrelationId` (`FlowStep.CorrelationId` — the `TrackedOperationId` / `NotificationId`), so the detail panel deep-links to **`/site-calls/report?correlationId=`** / **`/notifications/report?correlationId=`** (*Open in Site Calls* / *Open in Notification report*; both pages accept that query, pre-filter to the one row through an additive per-id query filter — `SiteCallQueryRequest.TrackedOperationIdFilter` / `NotificationOutboxQueryRequest.NotificationIdFilter` — show a removable *Operation* / *Notification* pill and auto-open the row's detail modal; note `NotificationId` is minted as a Guid in **`N` form** — no dashes — so the Notify link passes `N` and the report page normalises any Guid-shaped value to `N`, while `TrackedOperationId` is a `varchar(36)` `D` string and the Site Calls filter binds a `varchar` parameter for the PK seek), and a **`Parked` node offers Retry / Discard in place**: the same site-scope re-check, confirm dialog (Discard = danger) and central→site / Notification Outbox relay the report pages use (`IFlowDebugService.RetryOutboundAsync` / `DiscardOutboundAsync` → `CommunicationService`), then the page re-polls Replay (5 s / 2 min) until the node leaves `Parked`. **Prediction is async-safe:** `FlowDebugService.PredictMethodAsync` resolves route targets by a resolve-and-rewalk loop — its original blocking resolver deadlocked the whole Blazor circuit on the renderer sync context the moment a method routing to a site script was chosen (found 2026-09-06; pinned by a single-thread-pump unit test and the Playwright `FlowDebuggerTests`, which log in through the fixture and run a Live run of an ephemeral method in a real browser). - **Two entry points.** An inbound API method (inputs form generated from the method's `ParameterDefinitions`, values coerced by declared type) or a value-change trigger on a named instance attribute (Site → Instance → Attribute pickers, site-scope filtered). - **Predicted graph first.** Choosing an entry point renders the statically predicted shape (`FlowShapePredictor`, Script Analysis REQ-SA-7) before anything runs — nodes whose target depends on inputs render dashed; after a run, predicted nodes the run never produced render greyed. - **Execution path.** `IFlowDebugService` dispatches `FlowDebugRunMethodCommand` / `FlowDebugRunValueChangeCommand` / `FlowDebugReplayCommand` to the **per-node `ManagementActor`** through `ManagementActorHolder` (the same seam the HTTP `/management` endpoint uses), so the server enforces the Deployer gate and the active-node refusal. A standby node answers the two run commands with `STANDBY_NODE`, which the page renders as an inline alert with the server message verbatim; Replay is served from any node. The same three commands back the CLI's `flow-debug run-method | run-value-change | replay`. diff --git a/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Deployment/FlowDebugger.razor.cs b/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Deployment/FlowDebugger.razor.cs index 71f1b25d..7e5df08e 100644 --- a/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Deployment/FlowDebugger.razor.cs +++ b/src/ZB.MOM.WW.ScadaBridge.CentralUI/Components/Pages/Deployment/FlowDebugger.razor.cs @@ -3,6 +3,7 @@ using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.AspNetCore.Components; using Microsoft.JSInterop; +using ZB.MOM.WW.Audit; using ZB.MOM.WW.ScadaBridge.CentralUI.Auth; using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared; using ZB.MOM.WW.ScadaBridge.CentralUI.Services; @@ -510,19 +511,36 @@ public partial class FlowDebugger : ComponentBase, IDisposable } /// - /// Replaces the graph with the re-read tree and keeps the selection by event id. The run's own - /// _raw (timing, root) and the state chip are deliberately left alone: pending outbound - /// work is not a failed run, and the run's Started/Total are the truthful numbers. + /// Folds the re-read tree INTO the run's known events (union by event id, re-read rows win) and + /// rebuilds the graph from the union, keeping the selection by event id. A refresh therefore only + /// ever adds rows: a Live run carries its site rows inline, while the same rows reach central's + /// audit store by telemetry seconds later — a Replay read taken in between lacks them, and + /// replacing the graph wholesale made an already-drawn node (a resolved cached DB write, on the + /// rig 2026-09-06) vanish until the next manual refresh. The run's own _raw timing, root + /// and the state chip are deliberately left alone: pending outbound work is not a failed run, and + /// the run's Started/Total are the truthful numbers. /// private void MergeRefreshedGraph(FlowDebugOutcome outcome) { - if (!outcome.Success || outcome.Graph == null || _graph == null) + if (!outcome.Success || outcome.Graph == null || _graph == null || _raw == null) return; var selectedId = _selected?.EventId; - _graph = outcome.Graph; + var merged = MergeEvents(_raw.Events, outcome.Raw?.Events); + var truncated = _raw.Truncated || outcome.Raw?.Truncated == true; + _raw = _raw with { Events = merged, Truncated = truncated }; + _graph = FlowGraphBuilder.Build(merged, truncated); _selected = selectedId is { } id ? _graph.Steps.FirstOrDefault(s => s.EventId == id) ?? _selected : _selected; } + /// Union of two audit-row lists by ; a row present in both takes the fresh copy. + internal static IReadOnlyList MergeEvents(IReadOnlyList? known, IReadOnlyList? fresh) + { + var byId = new Dictionary(); + foreach (var e in known ?? []) byId[e.EventId] = e; + foreach (var e in fresh ?? []) byId[e.EventId] = e; + return byId.Values.ToList(); + } + private void CancelPendingRefresh(bool keepWatch = false) { if (!keepWatch) diff --git a/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Deployment/FlowDebuggerPageTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Deployment/FlowDebuggerPageTests.cs index d539fe8f..841bd228 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Deployment/FlowDebuggerPageTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Deployment/FlowDebuggerPageTests.cs @@ -252,7 +252,7 @@ public class FlowDebuggerPageTests : BunitContext occurredAtUtc: t0.AddSeconds(9), target: "Delmia.Ack", correlationId: corr, executionId: child, parentExecutionId: root, sourceSiteId: "site-a", sourceInstanceId: "Sim01"), }; - var resolved = new FlowDebugOutcome(FlowGraphBuilder.Build(later), raw, null, null); + var resolved = new FlowDebugOutcome(FlowGraphBuilder.Build(later), raw with { Events = later }, null, null); return (parked, resolved, submit.EventId); } @@ -345,13 +345,35 @@ public class FlowDebuggerPageTests : BunitContext // ---- Outbound calls (2026-09-06 plan, Task 9): pending refresh after a Live run ---------- - /// Inbound → route → site run → cached API call; the call is Pending unless . + /// + /// Inbound → route → site run → cached API call; the call is Pending unless . + /// Both outcomes of one share their rows (same event/correlation ids), as the + /// real system does: the inline Live rows ARE the audit rows Replay later reads back. + /// private static FlowDebugOutcome OutboundOutcome(Guid root, bool resolved) { - var child = Guid.NewGuid(); - var corr = Guid.NewGuid(); - var t0 = DateTime.UtcNow.AddSeconds(-10); - var events = new List + var (events, child, corr, t0) = OutboundBase.GetOrAdd(root, r => + { + var child = Guid.NewGuid(); + var corr = Guid.NewGuid(); + var t0 = DateTime.UtcNow.AddSeconds(-10); + return (BuildOutboundBase(r, child, corr, t0), child, corr, t0); + }); + events = new List(events); + if (resolved) + events.Add(ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, + occurredAtUtc: t0.AddSeconds(4), target: "Delmia.Ack", correlationId: corr, executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "Sim01")); + + var raw = new FlowDebugRunResult(root, true, null, null, events, false, t0, t0.AddSeconds(1)); + return new FlowDebugOutcome(FlowGraphBuilder.Build(events), raw, null, null); + } + + private static readonly System.Collections.Concurrent.ConcurrentDictionary Events, Guid Child, Guid Corr, DateTime T0)> OutboundBase = new(); + + private static List BuildOutboundBase(Guid root, Guid child, Guid corr, DateTime t0) + { + return new List { ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered, occurredAtUtc: t0, target: "POST /api/DelmiaRecipeDownload", executionId: root, durationMs: 100, responseSummary: "{}"), @@ -365,13 +387,6 @@ public class FlowDebuggerPageTests : BunitContext occurredAtUtc: t0.AddMilliseconds(3), target: "Delmia.Ack", correlationId: corr, executionId: child, parentExecutionId: root, sourceSiteId: "site-a", sourceInstanceId: "Sim01"), }; - if (resolved) - events.Add(ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, - occurredAtUtc: t0.AddSeconds(4), target: "Delmia.Ack", correlationId: corr, executionId: child, parentExecutionId: root, - sourceSiteId: "site-a", sourceInstanceId: "Sim01")); - - var raw = new FlowDebugRunResult(root, true, null, null, events, false, t0, t0.AddSeconds(1)); - return new FlowDebugOutcome(FlowGraphBuilder.Build(events), raw, null, null); } [Fact] @@ -396,6 +411,79 @@ public class FlowDebuggerPageTests : BunitContext Assert.NotNull(cut.Find("button[data-test='flow-refresh']")); } + /// + /// Live run whose inline site rows already resolved a cached DB write, with a notification still + /// Pending; =false models the Replay read taken before that DB + /// telemetry reached central (observed live on the rig 2026-09-06: cached-call telemetry landed + /// 12 s after the run, the first poll at 5 s had none of it). + /// + private static FlowDebugOutcome DbAndNotifyOutcome(Guid root, Guid child, Guid dbCorr, Guid notifyCorr, DateTime t0, bool withDbRows, bool notifyDelivered) + { + var events = new List + { + ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered, + occurredAtUtc: t0, target: "POST /api/FlowOutboundDemo", executionId: root, durationMs: 100, responseSummary: "{}"), + ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered, + occurredAtUtc: t0.AddMilliseconds(1), target: "Sim01.OutboundDemo", executionId: root, durationMs: 90, + extra: new ScriptTraceExtra { ChildExecutionId = child }.ToJson()), + ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered, + occurredAtUtc: t0.AddMilliseconds(2), target: "OutboundDemo", executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "Sim01", durationMs: 80), + ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted, + occurredAtUtc: t0.AddMilliseconds(10), target: "FlowDemo", correlationId: notifyCorr, executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "Sim01"), + }; + if (withDbRows) + { + events.Add(ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, + occurredAtUtc: t0.AddMilliseconds(3), target: "RigDb", correlationId: dbCorr, executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "Sim01")); + events.Add(ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, + occurredAtUtc: t0.AddMilliseconds(5), target: "RigDb", correlationId: dbCorr, executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "Sim01")); + } + if (notifyDelivered) + events.Add(ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifyDeliver, AuditStatus.Delivered, + occurredAtUtc: t0.AddSeconds(1), target: "FlowDemo", correlationId: notifyCorr, executionId: child, parentExecutionId: root, + sourceSiteId: "site-a", sourceInstanceId: "Sim01")); + + var raw = new FlowDebugRunResult(root, true, null, null, events, false, t0, t0.AddSeconds(1)); + return new FlowDebugOutcome(FlowGraphBuilder.Build(events), raw, null, null); + } + + [Fact] + public void PendingRefresh_KeepsStepsTheLiveRunAlreadyHad_WhenReplayLagsBehindTelemetry() + { + RegisterServices(); + var root = Guid.NewGuid(); + var child = Guid.NewGuid(); + var dbCorr = Guid.NewGuid(); + var notifyCorr = Guid.NewGuid(); + var t0 = DateTime.UtcNow.AddSeconds(-10); + var live = DbAndNotifyOutcome(root, child, dbCorr, notifyCorr, t0, withDbRows: true, notifyDelivered: false); + var lagging = DbAndNotifyOutcome(root, child, dbCorr, notifyCorr, t0, withDbRows: false, notifyDelivered: true); + // Sanity: the two graphs disagree exactly on the DB node. + Assert.Contains(live.Graph!.Steps, s => s.Kind == FlowStepKind.DbCall); + Assert.DoesNotContain(lagging.Graph!.Steps, s => s.Kind == FlowStepKind.DbCall); + + _flow.RunMethodAsync(4, Arg.Any(), Arg.Any()).Returns(live); + _flow.ReplayAsync(root, Arg.Any()).Returns(lagging); + var cut = Render(); + cut.Instance.Delay = (_, _) => Task.CompletedTask; + cut.Find("select[data-test='flow-method-select']").Change("4"); + cut.WaitForElement("input[data-test='flow-input-MachineCode']"); + + cut.Find("button[data-test='flow-run']").Click(); + + // The poll resolves the notification (nothing Pending → polling stops) ... + cut.WaitForAssertion(() => _flow.Received(1).ReplayAsync(root, Arg.Any())); + cut.WaitForAssertion(() => Assert.Empty(cut.FindAll("[data-test='flow-pending-count']"))); + // ... but must NOT drop the DB node the live run already drew: a refresh only ever adds rows. + Assert.NotEmpty(cut.FindAll("g[data-test='flow-node'][data-kind='DbCall']")); + Assert.NotEmpty(cut.FindAll("g[data-test='flow-node'][data-kind='Notify']")); + Assert.Contains(cut.FindAll("[data-test='flow-node-pill']"), p => p.TextContent.Trim() == "Ok"); + } + [Fact] public void LiveRun_WithNoPendingStep_NeverPolls() {