Joseph Doherty
f17bae093a
Code review of the merged arch-review remediation (Tasks 1-13) found four items. 1. FlowDebugRunner returned its OWN synthesized rows un-redacted. RunMethodAsync's inbound row and RunValueChangeAsync's root row were handed to WriteCentralAsync (whose writer redacts a private copy) and then put RAW into the returned Events — so a Live run's caller saw the unfiltered row while Replay of the same execution showed the redacted one, defeating F1's "Live shows what Replay reads back". Both methods now resolve IAuditRedactor once per run and place the filtered copy into the result (including the early-return path on a failed attribute write); the raw row still reaches the writer, which redacts itself. 2. FlowGraphBuilder's spawner search degenerated to "the last step of the parent". Since executions are ordered as contiguous blocks, position[p] < childPosition holds for EVERY parent step, so a run shaped ScriptRun -> Write(A) -> Wait -> Write(B) whose Write(A) fired the cascade drew the Spawn edge from Write(B). New rule, in order: (a) the last non-async, non-Response parent step occurring at or before the child's first step — parent and child of a trigger cascade share the site clock, so this picks the write that actually fired it; (b) the position rule as fallback, which is the mixed-clock case F5 was written for (a child clock behind its parent's, where no parent step satisfies (a)); (c) parentChain[0]. The class remark's stale "Replay order is unchanged" claim is replaced with what actually happens to a central Database.* call made after Route.To().Call. 3. FlowTraceCollector's byte budget counts UTF-16 chars of DetailsJson only, so "well under the 4 MB gRPC default" overstated it. Documented honestly: ~1 MB of UTF-8 for ASCII payloads, ~3 MB worst case for non-ASCII, both under 4 MB before framing — and raise it only against a real serialized-size measurement. 4. CLI --timeout-seconds help + README row now say the management endpoint caps flow-debug run-method at ManagementService:LongRunningCommandTimeout (default 5 min), so a larger client value still 504s at the server's ceiling. Deferred-work register rows 112 (site-side inline trace truncation is invisible to the operator — needs a proto field) and 113 (the F2 method-entry site scope gates routed traffic only; ApiMethods have no site affinity to scope against) record what this review deliberately did not fix. Tests: FlowGraphBuilderTests gains the same-clock Write(A)/Wait/Write(B) cascade (red before the rule change, and the clock-behind test still passes on rule (b)); FlowDebugRunnerTests extends the redactor test to the inbound row and adds two value-change cases (settled poll, failed write). Commons 793, ManagementService 615, CentralUI 1264, CLI 462 — all green. Claude-Session: https://claude.ai/code/session_01NcMG6xXdsJbmD2wYAgkK29
592 lines
34 KiB
C#
592 lines
34 KiB
C#
using ZB.MOM.WW.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Commons.Tests.Debugging;
|
|
|
|
/// <summary>
|
|
/// Flow Debugger design 2026-09-05 §7.2: lane/edge/boundary rules over a Delmia-shaped
|
|
/// event list, truncation pass-through, and total-ness on unknown kinds.
|
|
/// </summary>
|
|
public class FlowGraphBuilderTests
|
|
{
|
|
private static readonly DateTime T0 = new(2026, 9, 5, 12, 0, 0, DateTimeKind.Utc);
|
|
private static readonly Guid Root = Guid.NewGuid();
|
|
private static readonly Guid SiteRun = Guid.NewGuid();
|
|
private static readonly Guid Cascade = Guid.NewGuid();
|
|
|
|
/// <summary>Inbound → RouteCall → site ScriptRun → write → wait → write, plus a value-change ScriptRun spawned by the second write.</summary>
|
|
private static List<AuditEvent> DelmiaShaped()
|
|
{
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, actor: "delmia-key", target: "DelmiaRecipeDownload", sourceNode: "central-a",
|
|
executionId: Root, durationMs: 900, requestSummary: "{\"MachineCode\":\"M1\"}", responseSummary: "{\"ok\":true}");
|
|
var route = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(10), actor: "inbound-api", target: "M1.ProcessRecipeDownload", sourceNode: "central-a",
|
|
executionId: Root, durationMs: 850, extra: new ScriptTraceExtra { ChildExecutionId = SiteRun }.ToJson());
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(20), actor: "ProcessRecipeDownload", target: "ProcessRecipeDownload", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 800,
|
|
extra: new ScriptTraceExtra { TriggerKind = "call" }.ToJson());
|
|
var write1 = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(30), actor: "ProcessRecipeDownload", target: "DelmiaReceiver.RecipePath", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 5,
|
|
extra: new ScriptTraceExtra { Writes = [new AttributeWriteDetail("DelmiaReceiver.RecipePath", "C:\\r.nc", true, null)] }.ToJson());
|
|
var wait = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWait, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(40), actor: "ProcessRecipeDownload", target: "DelmiaReceiver.Processed", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 700,
|
|
extra: new ScriptTraceExtra { WaitAttribute = "DelmiaReceiver.Processed", WaitOutcome = "satisfied", ObservedValue = "true" }.ToJson());
|
|
var write2 = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(750), actor: "ProcessRecipeDownload", target: "DelmiaReceiver.Trigger", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 3,
|
|
extra: new ScriptTraceExtra { Writes = [new AttributeWriteDetail("DelmiaReceiver.Trigger", "false", true, null)] }.ToJson());
|
|
var cascade = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(760), actor: "OnTriggerReset", target: "OnTriggerReset", sourceNode: "node-a",
|
|
executionId: Cascade, parentExecutionId: SiteRun, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 10,
|
|
extra: new ScriptTraceExtra { TriggerKind = "valuechange", TriggerAttribute = "DelmiaReceiver.Trigger", TriggerValue = "false" }.ToJson());
|
|
|
|
// Deliberately shuffled WITHIN each execution: the builder must order by time inside one execution.
|
|
// The executions themselves are given in emission order (root, site run, cascade) — since arch-review
|
|
// F5 that is what fixes their relative order, because they do not share a clock.
|
|
return [inbound, route, wait, run, write2, write1, cascade];
|
|
}
|
|
|
|
[Fact]
|
|
public void DelmiaShape_ProjectsLanes_Edges_AndSynthesizedResponse()
|
|
{
|
|
var graph = FlowGraphBuilder.Build(DelmiaShaped());
|
|
|
|
Assert.False(graph.Predicted);
|
|
Assert.False(graph.Truncated);
|
|
Assert.Equal(8, graph.Steps.Count); // 7 events + synthesized Response
|
|
|
|
var kinds = graph.Steps.Select(s => s.Kind).ToArray();
|
|
Assert.Equal(
|
|
new[]
|
|
{
|
|
FlowStepKind.InboundRequest, FlowStepKind.RouteCall, FlowStepKind.ScriptRun, FlowStepKind.AttributeWrite,
|
|
FlowStepKind.AttributeWait, FlowStepKind.AttributeWrite, FlowStepKind.ScriptRun, FlowStepKind.Response,
|
|
},
|
|
kinds);
|
|
|
|
Assert.Equal(FlowLane.Central, graph.Steps[0].Lane);
|
|
Assert.Equal(FlowLane.Central, graph.Steps[1].Lane);
|
|
Assert.Equal(FlowLane.Site, graph.Steps[2].Lane);
|
|
Assert.Equal(FlowLane.Tags, graph.Steps[3].Lane);
|
|
Assert.Equal(FlowLane.Site, graph.Steps[4].Lane);
|
|
Assert.Equal(FlowLane.Tags, graph.Steps[5].Lane);
|
|
Assert.Equal(FlowLane.Site, graph.Steps[6].Lane);
|
|
Assert.Equal(FlowLane.Central, graph.Steps[7].Lane);
|
|
|
|
// Exactly one boundary crossing: route → site run. The Response links from the RouteCall (D9), Central → Central.
|
|
var crossing = Assert.Single(graph.Edges, e => e.CrossesBoundary);
|
|
Assert.Equal(graph.Steps[1].EventId, crossing.FromEventId);
|
|
Assert.Equal(graph.Steps[2].EventId, crossing.ToEventId);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == graph.Steps[1].EventId && e.ToEventId == graph.Steps[7].EventId && !e.CrossesBoundary);
|
|
Assert.DoesNotContain(graph.Edges, e => e.FromEventId == graph.Steps[6].EventId && e.ToEventId == graph.Steps[7].EventId);
|
|
|
|
// One inferred echo edge: first write → satisfied wait.
|
|
var external = Assert.Single(graph.Edges, e => e.External);
|
|
Assert.Equal(graph.Steps[3].EventId, external.FromEventId);
|
|
Assert.Equal(graph.Steps[4].EventId, external.ToEventId);
|
|
|
|
// The cascade run hangs off the write that spawned it (same site, no boundary) — a Spawn edge (D9);
|
|
// the route → run edge and every in-chain edge are not.
|
|
var spawn = Assert.Single(graph.Edges, e => e.Spawn);
|
|
Assert.Equal(graph.Steps[5].EventId, spawn.FromEventId);
|
|
Assert.Equal(graph.Steps[6].EventId, spawn.ToEventId);
|
|
Assert.False(spawn.CrossesBoundary);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == graph.Steps[1].EventId && e.ToEventId == graph.Steps[2].EventId && !e.Spawn);
|
|
Assert.DoesNotContain(graph.Edges, e => e.Async);
|
|
|
|
// Inbound → route call chained inside the root execution.
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == graph.Steps[0].EventId && e.ToEventId == graph.Steps[1].EventId);
|
|
|
|
// Projection detail.
|
|
Assert.Equal("→ " + SiteRun.ToString("D")[..8], graph.Steps[1].Subtitle);
|
|
Assert.Equal("M1 · call", graph.Steps[2].Subtitle);
|
|
Assert.Equal("DelmiaReceiver.RecipePath = C:\\r.nc", graph.Steps[3].Subtitle);
|
|
Assert.Equal("satisfied · true", graph.Steps[4].Subtitle);
|
|
Assert.Equal("{\"ok\":true}", graph.Steps[7].OutputJson);
|
|
Assert.All(graph.Steps, s => Assert.Equal(FlowStepOutcome.Ok, s.Outcome));
|
|
}
|
|
|
|
[Fact]
|
|
public void Build_IsDeterministic_ResponseIdIsStable()
|
|
{
|
|
var events = DelmiaShaped();
|
|
var a = FlowGraphBuilder.Build(events);
|
|
var b = FlowGraphBuilder.Build(events);
|
|
Assert.Equal(a.Steps.Select(s => s.EventId), b.Steps.Select(s => s.EventId));
|
|
Assert.Equal(a.Edges, b.Edges);
|
|
}
|
|
|
|
[Fact]
|
|
public void Truncated_PassesThrough()
|
|
{
|
|
Assert.True(FlowGraphBuilder.Build(DelmiaShaped(), truncated: true).Truncated);
|
|
Assert.True(FlowGraphBuilder.Build(Array.Empty<AuditEvent>(), truncated: true).Truncated);
|
|
Assert.Empty(FlowGraphBuilder.Build(Array.Empty<AuditEvent>()).Steps);
|
|
}
|
|
|
|
[Fact]
|
|
public void UnknownKind_ProjectsToUnknown_NeverThrows()
|
|
{
|
|
var weird = new AuditEvent
|
|
{
|
|
EventId = Guid.NewGuid(),
|
|
OccurredAtUtc = T0,
|
|
Actor = "x",
|
|
Action = "Future.Thing",
|
|
Outcome = AuditOutcome.Success,
|
|
DetailsJson = "{\"channel\":\"Future\",\"kind\":\"Thing\",\"extra\":\"not json at all\"}",
|
|
};
|
|
var garbage = new AuditEvent
|
|
{
|
|
EventId = Guid.NewGuid(),
|
|
OccurredAtUtc = T0,
|
|
Actor = "x",
|
|
Action = "?",
|
|
Outcome = AuditOutcome.Failure,
|
|
DetailsJson = "{{{",
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build([weird, garbage]);
|
|
|
|
Assert.Equal(2, graph.Steps.Count);
|
|
Assert.All(graph.Steps, s => Assert.Equal(FlowStepKind.Unknown, s.Kind));
|
|
Assert.All(graph.Steps, s => Assert.Null(s.Extra));
|
|
Assert.Empty(graph.Edges);
|
|
}
|
|
|
|
[Fact]
|
|
public void FailedScriptRun_WithTimeoutMessage_IsTimedOut()
|
|
{
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Failed,
|
|
occurredAtUtc: T0, target: "S", executionId: SiteRun, errorMessage: "Script timed out after 30s");
|
|
var write = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Failed,
|
|
occurredAtUtc: T0.AddMilliseconds(1), target: "A", executionId: SiteRun, errorMessage: "rejected");
|
|
|
|
var graph = FlowGraphBuilder.Build([run, write]);
|
|
|
|
Assert.Equal(FlowStepOutcome.TimedOut, graph.Steps[0].Outcome);
|
|
Assert.Equal(FlowStepOutcome.Failed, graph.Steps[1].Outcome);
|
|
Assert.Single(graph.Edges);
|
|
}
|
|
|
|
[Fact]
|
|
public void ScriptTraceExtra_RoundTrips_CamelCase_AndIgnoresUnknownMembers()
|
|
{
|
|
var extra = new ScriptTraceExtra { ChildExecutionId = SiteRun, WaitTimeoutMs = 1500, Source = "flow-debugger" };
|
|
var json = extra.ToJson();
|
|
Assert.Contains("\"childExecutionId\"", json);
|
|
Assert.DoesNotContain("triggerKind", json);
|
|
|
|
var back = ScriptTraceExtra.TryParse(json.Insert(1, "\"futureField\":1,"));
|
|
Assert.Equal(extra, back);
|
|
Assert.Null(ScriptTraceExtra.TryParse(null));
|
|
Assert.Null(ScriptTraceExtra.TryParse("nope"));
|
|
}
|
|
|
|
// ---- Outbound calls (2026-09-06 plan, Task 2) --------------------------------------------
|
|
|
|
private static AuditEvent SiteRunRow(int ms, Guid executionId = default) => ScadaBridgeAuditEventFactory.Create(
|
|
AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(ms), actor: "S", target: "S", sourceNode: "node-a",
|
|
executionId: executionId == default ? SiteRun : executionId, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 800,
|
|
extra: new ScriptTraceExtra { TriggerKind = "call" }.ToJson());
|
|
|
|
private static AuditEvent WriteRow(int ms, string attr) => ScadaBridgeAuditEventFactory.Create(
|
|
AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(ms), actor: "S", target: attr, sourceNode: "node-a",
|
|
executionId: SiteRun, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 2,
|
|
extra: new ScriptTraceExtra { Writes = [new AttributeWriteDetail(attr, "1", true, null)] }.ToJson());
|
|
|
|
private static AuditEvent Outbound(AuditChannel channel, AuditKind kind, AuditStatus status, int ms, Guid? correlation, string target,
|
|
string node = "node-a", string? error = null, int? durationMs = null, string? request = null, string? response = null)
|
|
=> ScadaBridgeAuditEventFactory.Create(channel, kind, status,
|
|
occurredAtUtc: T0.AddMilliseconds(ms), actor: "S", target: target, sourceNode: node, correlationId: correlation,
|
|
executionId: SiteRun, sourceSiteId: "site-a", sourceInstanceId: "M1", errorMessage: error, durationMs: durationMs,
|
|
requestSummary: request, responseSummary: response);
|
|
|
|
[Fact]
|
|
public void CachedApiCall_FourRows_FoldIntoOneAsyncNode()
|
|
{
|
|
var corr = Guid.NewGuid();
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
WriteRow(30, "A"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, 35, corr, "Delmia.AckRecipe", request: "{\"id\":1}"),
|
|
WriteRow(40, "B"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.ApiCallCached, AuditStatus.Forwarded, 2000, corr, "Delmia.AckRecipe"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.ApiCallCached, AuditStatus.Attempted, 3000, corr, "Delmia.AckRecipe", error: "503", durationMs: 40),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, 4135, corr, "Delmia.AckRecipe", response: "{\"ok\":true}"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
Assert.Equal(4, graph.Steps.Count);
|
|
var api = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.Equal(FlowLane.External, api.Lane);
|
|
Assert.Equal(events[2].EventId, api.EventId);
|
|
Assert.Equal(T0.AddMilliseconds(35), api.OccurredAtUtc.UtcDateTime);
|
|
Assert.Equal(4, api.Lifecycle!.Count);
|
|
Assert.Equal(new[] { "CachedSubmit", "ApiCallCached", "ApiCallCached", "CachedResolve" }, api.Lifecycle.Select(r => r.Kind));
|
|
Assert.Equal(corr, api.CorrelationId);
|
|
Assert.Equal("503", api.Lifecycle[2].ErrorMessage);
|
|
Assert.Equal(FlowStepOutcome.Ok, api.Outcome);
|
|
Assert.Equal(4100, api.DurationMs);
|
|
Assert.Equal("{\"id\":1}", api.InputJson);
|
|
Assert.Equal("{\"ok\":true}", api.OutputJson);
|
|
Assert.Equal("Delivered · cached", api.Subtitle);
|
|
Assert.Null(api.ErrorMessage);
|
|
Assert.Equal(SiteRun, api.ExecutionId);
|
|
|
|
var run = graph.Steps[0]; var w1 = graph.Steps[1]; var w2 = graph.Steps[3];
|
|
Assert.Equal(new[] { FlowStepKind.ScriptRun, FlowStepKind.AttributeWrite, FlowStepKind.ApiCall, FlowStepKind.AttributeWrite },
|
|
graph.Steps.Select(s => s.Kind));
|
|
var async = Assert.Single(graph.Edges, e => e.Async);
|
|
Assert.Equal(w1.EventId, async.FromEventId);
|
|
Assert.Equal(api.EventId, async.ToEventId);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == run.EventId && e.ToEventId == w1.EventId && !e.Async);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == w1.EventId && e.ToEventId == w2.EventId && !e.Async);
|
|
Assert.DoesNotContain(graph.Edges, e => e.FromEventId == api.EventId);
|
|
Assert.Equal(3, graph.Edges.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void SyncApiCall_SitsInChain_NoAsyncEdge()
|
|
{
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
WriteRow(30, "A"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.ApiCall, AuditStatus.Delivered, 35, null, "Delmia.AckRecipe", durationMs: 212),
|
|
WriteRow(40, "B"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
var api = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.Equal(212, api.DurationMs);
|
|
Assert.Single(api.Lifecycle!);
|
|
Assert.Equal("Delivered · sync", api.Subtitle);
|
|
Assert.DoesNotContain(graph.Edges, e => e.Async);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == graph.Steps[1].EventId && e.ToEventId == api.EventId);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == api.EventId && e.ToEventId == graph.Steps[3].EventId);
|
|
Assert.DoesNotContain(graph.Edges, e => e.FromEventId == graph.Steps[1].EventId && e.ToEventId == graph.Steps[3].EventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void TwoSyncCalls_NullCorrelation_AreNotFoldedTogether()
|
|
{
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
Outbound(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered, 30, null, "MesDb"),
|
|
Outbound(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Failed, 40, null, "MesDb", error: "deadlock"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
Assert.Equal(2, graph.Steps.Count(s => s.Kind == FlowStepKind.DbCall));
|
|
Assert.Equal(FlowStepOutcome.Failed, graph.Steps[2].Outcome);
|
|
Assert.Equal(2, graph.Edges.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void NotifySend_PlusCentralDeliverParked_FoldsIntoOneParkedNotifyNode()
|
|
{
|
|
var notificationId = Guid.NewGuid();
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
WriteRow(30, "A"),
|
|
Outbound(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted, 35, notificationId, "Planners", request: "{\"subject\":\"hi\"}"),
|
|
Outbound(AuditChannel.Notification, AuditKind.NotifyDeliver, AuditStatus.Attempted, 5000, notificationId, "Planners", node: "central-a", error: "smtp down", durationMs: 30),
|
|
Outbound(AuditChannel.Notification, AuditKind.NotifyDeliver, AuditStatus.Parked, 9000, notificationId, "Planners", node: "central-a", error: "retries exhausted"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
var notify = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.Notify);
|
|
Assert.Equal(FlowStepOutcome.Parked, notify.Outcome);
|
|
Assert.Equal(3, notify.Lifecycle!.Count);
|
|
Assert.Equal("central-a", notify.Lifecycle[2].Node);
|
|
Assert.Equal("retries exhausted", notify.ErrorMessage);
|
|
Assert.Equal(8965, notify.DurationMs);
|
|
Assert.Equal("Parked", notify.Subtitle);
|
|
Assert.Equal("node-a", notify.Node);
|
|
var async = Assert.Single(graph.Edges, e => e.Async);
|
|
Assert.Equal(graph.Steps[1].EventId, async.FromEventId);
|
|
Assert.Equal(notify.EventId, async.ToEventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void CachedSubmitOnly_IsPending_WithNoDuration()
|
|
{
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
Outbound(AuditChannel.DbOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, 35, Guid.NewGuid(), "MesDb"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
var db = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.DbCall);
|
|
Assert.Equal(FlowStepOutcome.Pending, db.Outcome);
|
|
Assert.Null(db.DurationMs);
|
|
Assert.Single(db.Lifecycle!);
|
|
var async = Assert.Single(graph.Edges);
|
|
Assert.True(async.Async);
|
|
Assert.Equal(graph.Steps[0].EventId, async.FromEventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResponseLinksFromTheInboundExecutionsLastStep_NotTheLastCascadedStep()
|
|
{
|
|
var events = DelmiaShaped();
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
var response = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.Response);
|
|
var into = Assert.Single(graph.Edges, e => e.ToEventId == response.EventId);
|
|
Assert.Equal(graph.Steps[1].EventId, into.FromEventId);
|
|
Assert.Equal(FlowStepKind.RouteCall, graph.Steps[1].Kind);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResponseFallsBackToTheInboundRow_WhenNothingElseSharesItsExecution()
|
|
{
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, actor: "k", target: "Ping", sourceNode: "central-a", executionId: Root, durationMs: 5, responseSummary: "{}");
|
|
var graph = FlowGraphBuilder.Build([inbound]);
|
|
Assert.Equal(2, graph.Steps.Count);
|
|
var edge = Assert.Single(graph.Edges);
|
|
Assert.Equal(graph.Steps[0].EventId, edge.FromEventId);
|
|
Assert.Equal(graph.Steps[1].EventId, edge.ToEventId);
|
|
}
|
|
|
|
// ---- Arch-review F5: emission order between executions, timestamps within one ------------
|
|
|
|
/// <summary>
|
|
/// Central and site clocks differ. The four rows are given in EMISSION order (the Live path's
|
|
/// collector order); the graph must keep the executions in that order whichever way the skew runs.
|
|
/// </summary>
|
|
private static List<AuditEvent> SkewedRouteShape(int siteSkewMs)
|
|
{
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, actor: "k", target: "M", sourceNode: "central-a", executionId: Root, durationMs: 900,
|
|
requestSummary: "{}", responseSummary: "{\"ok\":true}");
|
|
var route = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(50), target: "M1.S", sourceNode: "central-a", executionId: Root,
|
|
extra: new ScriptTraceExtra { ChildExecutionId = SiteRun }.ToJson());
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(siteSkewMs), actor: "S", target: "S", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 20,
|
|
extra: new ScriptTraceExtra { TriggerKind = "call" }.ToJson());
|
|
var write = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(siteSkewMs + 10), actor: "S", target: "A", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 2,
|
|
extra: new ScriptTraceExtra { Writes = [new AttributeWriteDetail("A", "1", true, null)] }.ToJson());
|
|
return [inbound, route, run, write];
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteClockAhead_SiteStepsStillFollowTheirRouteCall_AndPrecedeResponse()
|
|
{
|
|
var events = SkewedRouteShape(siteSkewMs: 2000); // site 2 s ahead of central
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
Assert.Equal(
|
|
new[]
|
|
{
|
|
FlowStepKind.InboundRequest, FlowStepKind.RouteCall, FlowStepKind.ScriptRun,
|
|
FlowStepKind.AttributeWrite, FlowStepKind.Response,
|
|
},
|
|
graph.Steps.Select(s => s.Kind));
|
|
Assert.Equal(events.Select(e => e.EventId), graph.Steps.Take(4).Select(s => s.EventId));
|
|
// The Response time still comes off the inbound row's own clock (inbound + DurationMs).
|
|
Assert.Equal(T0.AddMilliseconds(900), graph.Steps[4].OccurredAtUtc.UtcDateTime);
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteClockBehind_SiteStepsDoNotSortAheadOfTheInboundRequest()
|
|
{
|
|
var events = SkewedRouteShape(siteSkewMs: -2000); // site 2 s behind central
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
Assert.Equal(
|
|
new[]
|
|
{
|
|
FlowStepKind.InboundRequest, FlowStepKind.RouteCall, FlowStepKind.ScriptRun,
|
|
FlowStepKind.AttributeWrite, FlowStepKind.Response,
|
|
},
|
|
graph.Steps.Select(s => s.Kind));
|
|
Assert.Equal(events.Select(e => e.EventId), graph.Steps.Take(4).Select(s => s.EventId));
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteClockBehind_TriggerCascade_SpawnsFromTheLastParentStep_NotTheFirst()
|
|
{
|
|
// Mixed-version shape: no RouteCall row, so the spawner is found by position, and a site
|
|
// clock behind central used to push every parent step "after" the child → parentChain[0].
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, actor: "k", target: "M", sourceNode: "central-a", executionId: Root, durationMs: 900);
|
|
var centralDb = ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(10), actor: "api:M", target: "CfgDb", sourceNode: "central-a", executionId: Root);
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(-2000), actor: "S", target: "S", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 20,
|
|
extra: new ScriptTraceExtra { TriggerKind = "call" }.ToJson());
|
|
|
|
var graph = FlowGraphBuilder.Build([inbound, centralDb, run]);
|
|
|
|
var spawn = Assert.Single(graph.Edges, e => e.Spawn);
|
|
Assert.Equal(centralDb.EventId, spawn.FromEventId); // the LAST parent step, not the inbound row
|
|
Assert.Equal(run.EventId, spawn.ToEventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void SameClock_TriggerCascade_SpawnsFromTheWriteThatFiredIt_NotTheParentsLastStep()
|
|
{
|
|
// Review follow-up: executions are contiguous blocks, so "the last parent step BEFORE the child
|
|
// in display order" is always the parent's last step. Parent and child of a cascade share the
|
|
// SITE clock, so the write that fired the trigger is the last parent step at or before the child.
|
|
var run = SiteRunRow(0);
|
|
var writeA = WriteRow(10, "DelmiaReceiver.Trigger");
|
|
var wait = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWait, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(20), actor: "S", target: "DelmiaReceiver.Processed", sourceNode: "node-a",
|
|
executionId: SiteRun, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 5,
|
|
extra: new ScriptTraceExtra { WaitAttribute = "DelmiaReceiver.Processed", WaitOutcome = "satisfied" }.ToJson());
|
|
var writeB = WriteRow(30, "DelmiaReceiver.Ack");
|
|
var cascade = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(15), actor: "OnTrigger", target: "OnTrigger", sourceNode: "node-a",
|
|
executionId: Cascade, parentExecutionId: SiteRun, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 4,
|
|
extra: new ScriptTraceExtra { TriggerKind = "valuechange", TriggerAttribute = "DelmiaReceiver.Trigger" }.ToJson());
|
|
|
|
var graph = FlowGraphBuilder.Build([run, writeA, wait, writeB, cascade]);
|
|
|
|
var spawn = Assert.Single(graph.Edges, e => e.Spawn);
|
|
Assert.Equal(writeA.EventId, spawn.FromEventId);
|
|
Assert.Equal(cascade.EventId, spawn.ToEventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void FoldedNode_TakesTheEmissionIndexOfItsEarliestEmittedRow()
|
|
{
|
|
// The folded step keeps the earliest-by-TIME row's id (the resolve here is emitted first),
|
|
// so its sequence must be the minimum input index over the rows folded into it.
|
|
var corr = Guid.NewGuid();
|
|
var resolve = Outbound(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, 4000, corr, "Delmia.Ack", response: "res");
|
|
var submit = Outbound(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, 35, corr, "Delmia.Ack", request: "req");
|
|
var run = SiteRunRow(20, Cascade);
|
|
|
|
// Emission order interleaves the executions: the folded node's earliest row is index 0, so
|
|
// its execution leads — even though the id the folded step keeps (the submit) is index 2.
|
|
var graph = FlowGraphBuilder.Build([resolve, run, submit]);
|
|
|
|
var api = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.Equal(submit.EventId, api.EventId);
|
|
Assert.Equal(new[] { FlowStepKind.ApiCall, FlowStepKind.ScriptRun }, graph.Steps.Select(s => s.Kind));
|
|
}
|
|
|
|
// ---- Review findings on Task 2 (2026-09-06) --------------------------------------------
|
|
|
|
[Fact]
|
|
public void LeadingAsyncSteps_HangOffTheSpawner_NeverOrphaned()
|
|
{
|
|
// A routed run whose ScriptTrace rows are absent (TraceAllCalls off / older site): only its outbound rows survive.
|
|
var route = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(10), target: "M1.S", sourceNode: "central-a", executionId: Root,
|
|
extra: new ScriptTraceExtra { ChildExecutionId = SiteRun }.ToJson());
|
|
var cached = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted,
|
|
occurredAtUtc: T0.AddMilliseconds(20), target: "Delmia.Ack", correlationId: Guid.NewGuid(), executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a");
|
|
var notify = ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted,
|
|
occurredAtUtc: T0.AddMilliseconds(30), target: "Planners", correlationId: Guid.NewGuid(), executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a");
|
|
var sync = ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(40), target: "MesDb", executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a");
|
|
|
|
var graph = FlowGraphBuilder.Build([route, cached, notify, sync]);
|
|
|
|
var routeStep = graph.Steps[0];
|
|
var reached = graph.Edges.Select(e => e.ToEventId).ToHashSet();
|
|
Assert.All(graph.Steps.Skip(1), s => Assert.Contains(s.EventId, reached));
|
|
Assert.Equal(2, graph.Edges.Count(e => e.Async && e.FromEventId == routeStep.EventId));
|
|
var syncEdge = Assert.Single(graph.Edges, e => e.ToEventId == graph.Steps[3].EventId);
|
|
Assert.Equal(routeStep.EventId, syncEdge.FromEventId);
|
|
Assert.False(syncEdge.Async);
|
|
Assert.False(syncEdge.Spawn);
|
|
Assert.Equal(3, graph.Edges.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void OnlyNotifyRows_InAnExecution_AllLinkFromTheSpawner()
|
|
{
|
|
var run = SiteRunRow(20);
|
|
var n1 = ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted,
|
|
occurredAtUtc: T0.AddMilliseconds(30), target: "A", correlationId: Guid.NewGuid(), executionId: Cascade, parentExecutionId: SiteRun, sourceSiteId: "site-a");
|
|
var n2 = ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted,
|
|
occurredAtUtc: T0.AddMilliseconds(31), target: "B", correlationId: Guid.NewGuid(), executionId: Cascade, parentExecutionId: SiteRun, sourceSiteId: "site-a");
|
|
|
|
var graph = FlowGraphBuilder.Build([run, n1, n2]);
|
|
|
|
Assert.Equal(2, graph.Edges.Count);
|
|
Assert.All(graph.Edges, e => { Assert.True(e.Async); Assert.True(e.Spawn); Assert.Equal(graph.Steps[0].EventId, e.FromEventId); });
|
|
}
|
|
|
|
[Fact]
|
|
public void Fold_IsOrderIndependent_AndTiesBreakByLifecycleRank()
|
|
{
|
|
var corr = Guid.NewGuid();
|
|
var submit = Outbound(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, 35, corr, "Delmia.Ack", request: "req");
|
|
var resolve = Outbound(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, 35, corr, "Delmia.Ack", response: "res");
|
|
|
|
var a = FlowGraphBuilder.Build([SiteRunRow(20), resolve, submit]);
|
|
var b = FlowGraphBuilder.Build([SiteRunRow(20), submit, resolve]);
|
|
|
|
var sa = Assert.Single(a.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
var sb = Assert.Single(b.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.Equal(submit.EventId, sa.EventId);
|
|
Assert.Equal(sa.EventId, sb.EventId);
|
|
Assert.Equal("req", sa.InputJson);
|
|
Assert.Equal("res", sa.OutputJson);
|
|
Assert.Equal(FlowStepOutcome.Ok, sa.Outcome);
|
|
Assert.Equal(0, sa.DurationMs);
|
|
Assert.True(FlowGraphBuilder.IsAsync(sa));
|
|
Assert.Equal(sa.Lifecycle!.Select(r => r.Kind), sb.Lifecycle!.Select(r => r.Kind));
|
|
}
|
|
|
|
[Fact]
|
|
public void CachedStep_WhoseSubmitRowIsMissing_IsStillAsync()
|
|
{
|
|
var corr = Guid.NewGuid();
|
|
var graph = FlowGraphBuilder.Build([
|
|
SiteRunRow(20),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.ApiCallCached, AuditStatus.Attempted, 3000, corr, "Delmia.Ack"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, 4000, corr, "Delmia.Ack"),
|
|
]);
|
|
var api = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.True(FlowGraphBuilder.IsAsync(api));
|
|
Assert.Single(graph.Edges, e => e.Async && e.ToEventId == api.EventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void ExternalLaneEdges_NeverCountAsClusterBoundaryCrossings()
|
|
{
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, target: "M", sourceNode: "central-a", executionId: Root, durationMs: 50, responseSummary: "{}");
|
|
var centralDb = ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(5), actor: "api:M", target: "CfgDb", sourceNode: "central-a", executionId: Root,
|
|
extra: "{\"op\":\"read\",\"rowsReturned\":2}");
|
|
|
|
var graph = FlowGraphBuilder.Build([inbound, centralDb]);
|
|
|
|
Assert.DoesNotContain(graph.Edges, e => e.CrossesBoundary);
|
|
Assert.Equal("read · 2 rows", graph.Steps[1].Subtitle);
|
|
// Response links from the DB call — the inbound execution's last sync step.
|
|
var into = Assert.Single(graph.Edges, e => e.ToEventId == graph.Steps[2].EventId);
|
|
Assert.Equal(graph.Steps[1].EventId, into.FromEventId);
|
|
}
|
|
}
|