Joseph Doherty
1dfb9cee20
Central rows carry central's clock, site rows the site's, so the global OccurredAtUtc sort let a 1-2 s skew float a whole site execution ahead of the inbound row / RouteCall that started it, and pushed the trigger-cascade spawner search onto its parentChain[0] fallback. FlowGraphBuilder.Build now orders executions by the input index of their earliest-given row and orders steps by timestamp only INSIDE one execution (one clock), breaking ties by kind rank then input index. A folded outbound node keeps its earliest-by-time row's identity, which need not be the row given first, so its index is the minimum over the rows folded into it. The spawner search uses position in the ordered list rather than a raw timestamp compare. FlowDebugRunner.RunMethodAsync keeps the collector's emission order instead of re-sorting; the value-change path keeps its OrderBy (all rows share one site clock). Replay input arrives timestamp-sorted from the repository, so its order is unchanged. The DelmiaShaped test fixture is still shuffled within each execution (the inside-one-execution timestamp rule is unchanged) but now lists the executions in emission order, since that is what fixes their relative order. Claude-Session: https://claude.ai/code/session_01NcMG6xXdsJbmD2wYAgkK29
567 lines
32 KiB
C#
567 lines
32 KiB
C#
using ZB.MOM.WW.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Commons.Tests.Debugging;
|
|
|
|
/// <summary>
|
|
/// Flow Debugger design 2026-09-05 §7.2: lane/edge/boundary rules over a Delmia-shaped
|
|
/// event list, truncation pass-through, and total-ness on unknown kinds.
|
|
/// </summary>
|
|
public class FlowGraphBuilderTests
|
|
{
|
|
private static readonly DateTime T0 = new(2026, 9, 5, 12, 0, 0, DateTimeKind.Utc);
|
|
private static readonly Guid Root = Guid.NewGuid();
|
|
private static readonly Guid SiteRun = Guid.NewGuid();
|
|
private static readonly Guid Cascade = Guid.NewGuid();
|
|
|
|
/// <summary>Inbound → RouteCall → site ScriptRun → write → wait → write, plus a value-change ScriptRun spawned by the second write.</summary>
|
|
private static List<AuditEvent> DelmiaShaped()
|
|
{
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, actor: "delmia-key", target: "DelmiaRecipeDownload", sourceNode: "central-a",
|
|
executionId: Root, durationMs: 900, requestSummary: "{\"MachineCode\":\"M1\"}", responseSummary: "{\"ok\":true}");
|
|
var route = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(10), actor: "inbound-api", target: "M1.ProcessRecipeDownload", sourceNode: "central-a",
|
|
executionId: Root, durationMs: 850, extra: new ScriptTraceExtra { ChildExecutionId = SiteRun }.ToJson());
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(20), actor: "ProcessRecipeDownload", target: "ProcessRecipeDownload", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 800,
|
|
extra: new ScriptTraceExtra { TriggerKind = "call" }.ToJson());
|
|
var write1 = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(30), actor: "ProcessRecipeDownload", target: "DelmiaReceiver.RecipePath", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 5,
|
|
extra: new ScriptTraceExtra { Writes = [new AttributeWriteDetail("DelmiaReceiver.RecipePath", "C:\\r.nc", true, null)] }.ToJson());
|
|
var wait = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWait, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(40), actor: "ProcessRecipeDownload", target: "DelmiaReceiver.Processed", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 700,
|
|
extra: new ScriptTraceExtra { WaitAttribute = "DelmiaReceiver.Processed", WaitOutcome = "satisfied", ObservedValue = "true" }.ToJson());
|
|
var write2 = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(750), actor: "ProcessRecipeDownload", target: "DelmiaReceiver.Trigger", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 3,
|
|
extra: new ScriptTraceExtra { Writes = [new AttributeWriteDetail("DelmiaReceiver.Trigger", "false", true, null)] }.ToJson());
|
|
var cascade = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(760), actor: "OnTriggerReset", target: "OnTriggerReset", sourceNode: "node-a",
|
|
executionId: Cascade, parentExecutionId: SiteRun, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 10,
|
|
extra: new ScriptTraceExtra { TriggerKind = "valuechange", TriggerAttribute = "DelmiaReceiver.Trigger", TriggerValue = "false" }.ToJson());
|
|
|
|
// Deliberately shuffled WITHIN each execution: the builder must order by time inside one execution.
|
|
// The executions themselves are given in emission order (root, site run, cascade) — since arch-review
|
|
// F5 that is what fixes their relative order, because they do not share a clock.
|
|
return [inbound, route, wait, run, write2, write1, cascade];
|
|
}
|
|
|
|
[Fact]
|
|
public void DelmiaShape_ProjectsLanes_Edges_AndSynthesizedResponse()
|
|
{
|
|
var graph = FlowGraphBuilder.Build(DelmiaShaped());
|
|
|
|
Assert.False(graph.Predicted);
|
|
Assert.False(graph.Truncated);
|
|
Assert.Equal(8, graph.Steps.Count); // 7 events + synthesized Response
|
|
|
|
var kinds = graph.Steps.Select(s => s.Kind).ToArray();
|
|
Assert.Equal(
|
|
new[]
|
|
{
|
|
FlowStepKind.InboundRequest, FlowStepKind.RouteCall, FlowStepKind.ScriptRun, FlowStepKind.AttributeWrite,
|
|
FlowStepKind.AttributeWait, FlowStepKind.AttributeWrite, FlowStepKind.ScriptRun, FlowStepKind.Response,
|
|
},
|
|
kinds);
|
|
|
|
Assert.Equal(FlowLane.Central, graph.Steps[0].Lane);
|
|
Assert.Equal(FlowLane.Central, graph.Steps[1].Lane);
|
|
Assert.Equal(FlowLane.Site, graph.Steps[2].Lane);
|
|
Assert.Equal(FlowLane.Tags, graph.Steps[3].Lane);
|
|
Assert.Equal(FlowLane.Site, graph.Steps[4].Lane);
|
|
Assert.Equal(FlowLane.Tags, graph.Steps[5].Lane);
|
|
Assert.Equal(FlowLane.Site, graph.Steps[6].Lane);
|
|
Assert.Equal(FlowLane.Central, graph.Steps[7].Lane);
|
|
|
|
// Exactly one boundary crossing: route → site run. The Response links from the RouteCall (D9), Central → Central.
|
|
var crossing = Assert.Single(graph.Edges, e => e.CrossesBoundary);
|
|
Assert.Equal(graph.Steps[1].EventId, crossing.FromEventId);
|
|
Assert.Equal(graph.Steps[2].EventId, crossing.ToEventId);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == graph.Steps[1].EventId && e.ToEventId == graph.Steps[7].EventId && !e.CrossesBoundary);
|
|
Assert.DoesNotContain(graph.Edges, e => e.FromEventId == graph.Steps[6].EventId && e.ToEventId == graph.Steps[7].EventId);
|
|
|
|
// One inferred echo edge: first write → satisfied wait.
|
|
var external = Assert.Single(graph.Edges, e => e.External);
|
|
Assert.Equal(graph.Steps[3].EventId, external.FromEventId);
|
|
Assert.Equal(graph.Steps[4].EventId, external.ToEventId);
|
|
|
|
// The cascade run hangs off the write that spawned it (same site, no boundary) — a Spawn edge (D9);
|
|
// the route → run edge and every in-chain edge are not.
|
|
var spawn = Assert.Single(graph.Edges, e => e.Spawn);
|
|
Assert.Equal(graph.Steps[5].EventId, spawn.FromEventId);
|
|
Assert.Equal(graph.Steps[6].EventId, spawn.ToEventId);
|
|
Assert.False(spawn.CrossesBoundary);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == graph.Steps[1].EventId && e.ToEventId == graph.Steps[2].EventId && !e.Spawn);
|
|
Assert.DoesNotContain(graph.Edges, e => e.Async);
|
|
|
|
// Inbound → route call chained inside the root execution.
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == graph.Steps[0].EventId && e.ToEventId == graph.Steps[1].EventId);
|
|
|
|
// Projection detail.
|
|
Assert.Equal("→ " + SiteRun.ToString("D")[..8], graph.Steps[1].Subtitle);
|
|
Assert.Equal("M1 · call", graph.Steps[2].Subtitle);
|
|
Assert.Equal("DelmiaReceiver.RecipePath = C:\\r.nc", graph.Steps[3].Subtitle);
|
|
Assert.Equal("satisfied · true", graph.Steps[4].Subtitle);
|
|
Assert.Equal("{\"ok\":true}", graph.Steps[7].OutputJson);
|
|
Assert.All(graph.Steps, s => Assert.Equal(FlowStepOutcome.Ok, s.Outcome));
|
|
}
|
|
|
|
[Fact]
|
|
public void Build_IsDeterministic_ResponseIdIsStable()
|
|
{
|
|
var events = DelmiaShaped();
|
|
var a = FlowGraphBuilder.Build(events);
|
|
var b = FlowGraphBuilder.Build(events);
|
|
Assert.Equal(a.Steps.Select(s => s.EventId), b.Steps.Select(s => s.EventId));
|
|
Assert.Equal(a.Edges, b.Edges);
|
|
}
|
|
|
|
[Fact]
|
|
public void Truncated_PassesThrough()
|
|
{
|
|
Assert.True(FlowGraphBuilder.Build(DelmiaShaped(), truncated: true).Truncated);
|
|
Assert.True(FlowGraphBuilder.Build(Array.Empty<AuditEvent>(), truncated: true).Truncated);
|
|
Assert.Empty(FlowGraphBuilder.Build(Array.Empty<AuditEvent>()).Steps);
|
|
}
|
|
|
|
[Fact]
|
|
public void UnknownKind_ProjectsToUnknown_NeverThrows()
|
|
{
|
|
var weird = new AuditEvent
|
|
{
|
|
EventId = Guid.NewGuid(),
|
|
OccurredAtUtc = T0,
|
|
Actor = "x",
|
|
Action = "Future.Thing",
|
|
Outcome = AuditOutcome.Success,
|
|
DetailsJson = "{\"channel\":\"Future\",\"kind\":\"Thing\",\"extra\":\"not json at all\"}",
|
|
};
|
|
var garbage = new AuditEvent
|
|
{
|
|
EventId = Guid.NewGuid(),
|
|
OccurredAtUtc = T0,
|
|
Actor = "x",
|
|
Action = "?",
|
|
Outcome = AuditOutcome.Failure,
|
|
DetailsJson = "{{{",
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build([weird, garbage]);
|
|
|
|
Assert.Equal(2, graph.Steps.Count);
|
|
Assert.All(graph.Steps, s => Assert.Equal(FlowStepKind.Unknown, s.Kind));
|
|
Assert.All(graph.Steps, s => Assert.Null(s.Extra));
|
|
Assert.Empty(graph.Edges);
|
|
}
|
|
|
|
[Fact]
|
|
public void FailedScriptRun_WithTimeoutMessage_IsTimedOut()
|
|
{
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Failed,
|
|
occurredAtUtc: T0, target: "S", executionId: SiteRun, errorMessage: "Script timed out after 30s");
|
|
var write = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Failed,
|
|
occurredAtUtc: T0.AddMilliseconds(1), target: "A", executionId: SiteRun, errorMessage: "rejected");
|
|
|
|
var graph = FlowGraphBuilder.Build([run, write]);
|
|
|
|
Assert.Equal(FlowStepOutcome.TimedOut, graph.Steps[0].Outcome);
|
|
Assert.Equal(FlowStepOutcome.Failed, graph.Steps[1].Outcome);
|
|
Assert.Single(graph.Edges);
|
|
}
|
|
|
|
[Fact]
|
|
public void ScriptTraceExtra_RoundTrips_CamelCase_AndIgnoresUnknownMembers()
|
|
{
|
|
var extra = new ScriptTraceExtra { ChildExecutionId = SiteRun, WaitTimeoutMs = 1500, Source = "flow-debugger" };
|
|
var json = extra.ToJson();
|
|
Assert.Contains("\"childExecutionId\"", json);
|
|
Assert.DoesNotContain("triggerKind", json);
|
|
|
|
var back = ScriptTraceExtra.TryParse(json.Insert(1, "\"futureField\":1,"));
|
|
Assert.Equal(extra, back);
|
|
Assert.Null(ScriptTraceExtra.TryParse(null));
|
|
Assert.Null(ScriptTraceExtra.TryParse("nope"));
|
|
}
|
|
|
|
// ---- Outbound calls (2026-09-06 plan, Task 2) --------------------------------------------
|
|
|
|
private static AuditEvent SiteRunRow(int ms, Guid executionId = default) => ScadaBridgeAuditEventFactory.Create(
|
|
AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(ms), actor: "S", target: "S", sourceNode: "node-a",
|
|
executionId: executionId == default ? SiteRun : executionId, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 800,
|
|
extra: new ScriptTraceExtra { TriggerKind = "call" }.ToJson());
|
|
|
|
private static AuditEvent WriteRow(int ms, string attr) => ScadaBridgeAuditEventFactory.Create(
|
|
AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(ms), actor: "S", target: attr, sourceNode: "node-a",
|
|
executionId: SiteRun, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 2,
|
|
extra: new ScriptTraceExtra { Writes = [new AttributeWriteDetail(attr, "1", true, null)] }.ToJson());
|
|
|
|
private static AuditEvent Outbound(AuditChannel channel, AuditKind kind, AuditStatus status, int ms, Guid? correlation, string target,
|
|
string node = "node-a", string? error = null, int? durationMs = null, string? request = null, string? response = null)
|
|
=> ScadaBridgeAuditEventFactory.Create(channel, kind, status,
|
|
occurredAtUtc: T0.AddMilliseconds(ms), actor: "S", target: target, sourceNode: node, correlationId: correlation,
|
|
executionId: SiteRun, sourceSiteId: "site-a", sourceInstanceId: "M1", errorMessage: error, durationMs: durationMs,
|
|
requestSummary: request, responseSummary: response);
|
|
|
|
[Fact]
|
|
public void CachedApiCall_FourRows_FoldIntoOneAsyncNode()
|
|
{
|
|
var corr = Guid.NewGuid();
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
WriteRow(30, "A"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, 35, corr, "Delmia.AckRecipe", request: "{\"id\":1}"),
|
|
WriteRow(40, "B"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.ApiCallCached, AuditStatus.Forwarded, 2000, corr, "Delmia.AckRecipe"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.ApiCallCached, AuditStatus.Attempted, 3000, corr, "Delmia.AckRecipe", error: "503", durationMs: 40),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, 4135, corr, "Delmia.AckRecipe", response: "{\"ok\":true}"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
Assert.Equal(4, graph.Steps.Count);
|
|
var api = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.Equal(FlowLane.External, api.Lane);
|
|
Assert.Equal(events[2].EventId, api.EventId);
|
|
Assert.Equal(T0.AddMilliseconds(35), api.OccurredAtUtc.UtcDateTime);
|
|
Assert.Equal(4, api.Lifecycle!.Count);
|
|
Assert.Equal(new[] { "CachedSubmit", "ApiCallCached", "ApiCallCached", "CachedResolve" }, api.Lifecycle.Select(r => r.Kind));
|
|
Assert.Equal(corr, api.CorrelationId);
|
|
Assert.Equal("503", api.Lifecycle[2].ErrorMessage);
|
|
Assert.Equal(FlowStepOutcome.Ok, api.Outcome);
|
|
Assert.Equal(4100, api.DurationMs);
|
|
Assert.Equal("{\"id\":1}", api.InputJson);
|
|
Assert.Equal("{\"ok\":true}", api.OutputJson);
|
|
Assert.Equal("Delivered · cached", api.Subtitle);
|
|
Assert.Null(api.ErrorMessage);
|
|
Assert.Equal(SiteRun, api.ExecutionId);
|
|
|
|
var run = graph.Steps[0]; var w1 = graph.Steps[1]; var w2 = graph.Steps[3];
|
|
Assert.Equal(new[] { FlowStepKind.ScriptRun, FlowStepKind.AttributeWrite, FlowStepKind.ApiCall, FlowStepKind.AttributeWrite },
|
|
graph.Steps.Select(s => s.Kind));
|
|
var async = Assert.Single(graph.Edges, e => e.Async);
|
|
Assert.Equal(w1.EventId, async.FromEventId);
|
|
Assert.Equal(api.EventId, async.ToEventId);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == run.EventId && e.ToEventId == w1.EventId && !e.Async);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == w1.EventId && e.ToEventId == w2.EventId && !e.Async);
|
|
Assert.DoesNotContain(graph.Edges, e => e.FromEventId == api.EventId);
|
|
Assert.Equal(3, graph.Edges.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void SyncApiCall_SitsInChain_NoAsyncEdge()
|
|
{
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
WriteRow(30, "A"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.ApiCall, AuditStatus.Delivered, 35, null, "Delmia.AckRecipe", durationMs: 212),
|
|
WriteRow(40, "B"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
var api = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.Equal(212, api.DurationMs);
|
|
Assert.Single(api.Lifecycle!);
|
|
Assert.Equal("Delivered · sync", api.Subtitle);
|
|
Assert.DoesNotContain(graph.Edges, e => e.Async);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == graph.Steps[1].EventId && e.ToEventId == api.EventId);
|
|
Assert.Contains(graph.Edges, e => e.FromEventId == api.EventId && e.ToEventId == graph.Steps[3].EventId);
|
|
Assert.DoesNotContain(graph.Edges, e => e.FromEventId == graph.Steps[1].EventId && e.ToEventId == graph.Steps[3].EventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void TwoSyncCalls_NullCorrelation_AreNotFoldedTogether()
|
|
{
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
Outbound(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered, 30, null, "MesDb"),
|
|
Outbound(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Failed, 40, null, "MesDb", error: "deadlock"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
Assert.Equal(2, graph.Steps.Count(s => s.Kind == FlowStepKind.DbCall));
|
|
Assert.Equal(FlowStepOutcome.Failed, graph.Steps[2].Outcome);
|
|
Assert.Equal(2, graph.Edges.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void NotifySend_PlusCentralDeliverParked_FoldsIntoOneParkedNotifyNode()
|
|
{
|
|
var notificationId = Guid.NewGuid();
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
WriteRow(30, "A"),
|
|
Outbound(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted, 35, notificationId, "Planners", request: "{\"subject\":\"hi\"}"),
|
|
Outbound(AuditChannel.Notification, AuditKind.NotifyDeliver, AuditStatus.Attempted, 5000, notificationId, "Planners", node: "central-a", error: "smtp down", durationMs: 30),
|
|
Outbound(AuditChannel.Notification, AuditKind.NotifyDeliver, AuditStatus.Parked, 9000, notificationId, "Planners", node: "central-a", error: "retries exhausted"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
var notify = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.Notify);
|
|
Assert.Equal(FlowStepOutcome.Parked, notify.Outcome);
|
|
Assert.Equal(3, notify.Lifecycle!.Count);
|
|
Assert.Equal("central-a", notify.Lifecycle[2].Node);
|
|
Assert.Equal("retries exhausted", notify.ErrorMessage);
|
|
Assert.Equal(8965, notify.DurationMs);
|
|
Assert.Equal("Parked", notify.Subtitle);
|
|
Assert.Equal("node-a", notify.Node);
|
|
var async = Assert.Single(graph.Edges, e => e.Async);
|
|
Assert.Equal(graph.Steps[1].EventId, async.FromEventId);
|
|
Assert.Equal(notify.EventId, async.ToEventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void CachedSubmitOnly_IsPending_WithNoDuration()
|
|
{
|
|
var events = new List<AuditEvent>
|
|
{
|
|
SiteRunRow(20),
|
|
Outbound(AuditChannel.DbOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, 35, Guid.NewGuid(), "MesDb"),
|
|
};
|
|
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
var db = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.DbCall);
|
|
Assert.Equal(FlowStepOutcome.Pending, db.Outcome);
|
|
Assert.Null(db.DurationMs);
|
|
Assert.Single(db.Lifecycle!);
|
|
var async = Assert.Single(graph.Edges);
|
|
Assert.True(async.Async);
|
|
Assert.Equal(graph.Steps[0].EventId, async.FromEventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResponseLinksFromTheInboundExecutionsLastStep_NotTheLastCascadedStep()
|
|
{
|
|
var events = DelmiaShaped();
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
var response = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.Response);
|
|
var into = Assert.Single(graph.Edges, e => e.ToEventId == response.EventId);
|
|
Assert.Equal(graph.Steps[1].EventId, into.FromEventId);
|
|
Assert.Equal(FlowStepKind.RouteCall, graph.Steps[1].Kind);
|
|
}
|
|
|
|
[Fact]
|
|
public void ResponseFallsBackToTheInboundRow_WhenNothingElseSharesItsExecution()
|
|
{
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, actor: "k", target: "Ping", sourceNode: "central-a", executionId: Root, durationMs: 5, responseSummary: "{}");
|
|
var graph = FlowGraphBuilder.Build([inbound]);
|
|
Assert.Equal(2, graph.Steps.Count);
|
|
var edge = Assert.Single(graph.Edges);
|
|
Assert.Equal(graph.Steps[0].EventId, edge.FromEventId);
|
|
Assert.Equal(graph.Steps[1].EventId, edge.ToEventId);
|
|
}
|
|
|
|
// ---- Arch-review F5: emission order between executions, timestamps within one ------------
|
|
|
|
/// <summary>
|
|
/// Central and site clocks differ. The four rows are given in EMISSION order (the Live path's
|
|
/// collector order); the graph must keep the executions in that order whichever way the skew runs.
|
|
/// </summary>
|
|
private static List<AuditEvent> SkewedRouteShape(int siteSkewMs)
|
|
{
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, actor: "k", target: "M", sourceNode: "central-a", executionId: Root, durationMs: 900,
|
|
requestSummary: "{}", responseSummary: "{\"ok\":true}");
|
|
var route = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(50), target: "M1.S", sourceNode: "central-a", executionId: Root,
|
|
extra: new ScriptTraceExtra { ChildExecutionId = SiteRun }.ToJson());
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(siteSkewMs), actor: "S", target: "S", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 20,
|
|
extra: new ScriptTraceExtra { TriggerKind = "call" }.ToJson());
|
|
var write = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(siteSkewMs + 10), actor: "S", target: "A", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 2,
|
|
extra: new ScriptTraceExtra { Writes = [new AttributeWriteDetail("A", "1", true, null)] }.ToJson());
|
|
return [inbound, route, run, write];
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteClockAhead_SiteStepsStillFollowTheirRouteCall_AndPrecedeResponse()
|
|
{
|
|
var events = SkewedRouteShape(siteSkewMs: 2000); // site 2 s ahead of central
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
Assert.Equal(
|
|
new[]
|
|
{
|
|
FlowStepKind.InboundRequest, FlowStepKind.RouteCall, FlowStepKind.ScriptRun,
|
|
FlowStepKind.AttributeWrite, FlowStepKind.Response,
|
|
},
|
|
graph.Steps.Select(s => s.Kind));
|
|
Assert.Equal(events.Select(e => e.EventId), graph.Steps.Take(4).Select(s => s.EventId));
|
|
// The Response time still comes off the inbound row's own clock (inbound + DurationMs).
|
|
Assert.Equal(T0.AddMilliseconds(900), graph.Steps[4].OccurredAtUtc.UtcDateTime);
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteClockBehind_SiteStepsDoNotSortAheadOfTheInboundRequest()
|
|
{
|
|
var events = SkewedRouteShape(siteSkewMs: -2000); // site 2 s behind central
|
|
var graph = FlowGraphBuilder.Build(events);
|
|
|
|
Assert.Equal(
|
|
new[]
|
|
{
|
|
FlowStepKind.InboundRequest, FlowStepKind.RouteCall, FlowStepKind.ScriptRun,
|
|
FlowStepKind.AttributeWrite, FlowStepKind.Response,
|
|
},
|
|
graph.Steps.Select(s => s.Kind));
|
|
Assert.Equal(events.Select(e => e.EventId), graph.Steps.Take(4).Select(s => s.EventId));
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteClockBehind_TriggerCascade_SpawnsFromTheLastParentStep_NotTheFirst()
|
|
{
|
|
// Mixed-version shape: no RouteCall row, so the spawner is found by position, and a site
|
|
// clock behind central used to push every parent step "after" the child → parentChain[0].
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, actor: "k", target: "M", sourceNode: "central-a", executionId: Root, durationMs: 900);
|
|
var centralDb = ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(10), actor: "api:M", target: "CfgDb", sourceNode: "central-a", executionId: Root);
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(-2000), actor: "S", target: "S", sourceNode: "node-a",
|
|
executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a", sourceInstanceId: "M1", durationMs: 20,
|
|
extra: new ScriptTraceExtra { TriggerKind = "call" }.ToJson());
|
|
|
|
var graph = FlowGraphBuilder.Build([inbound, centralDb, run]);
|
|
|
|
var spawn = Assert.Single(graph.Edges, e => e.Spawn);
|
|
Assert.Equal(centralDb.EventId, spawn.FromEventId); // the LAST parent step, not the inbound row
|
|
Assert.Equal(run.EventId, spawn.ToEventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void FoldedNode_TakesTheEmissionIndexOfItsEarliestEmittedRow()
|
|
{
|
|
// The folded step keeps the earliest-by-TIME row's id (the resolve here is emitted first),
|
|
// so its sequence must be the minimum input index over the rows folded into it.
|
|
var corr = Guid.NewGuid();
|
|
var resolve = Outbound(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, 4000, corr, "Delmia.Ack", response: "res");
|
|
var submit = Outbound(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, 35, corr, "Delmia.Ack", request: "req");
|
|
var run = SiteRunRow(20, Cascade);
|
|
|
|
// Emission order interleaves the executions: the folded node's earliest row is index 0, so
|
|
// its execution leads — even though the id the folded step keeps (the submit) is index 2.
|
|
var graph = FlowGraphBuilder.Build([resolve, run, submit]);
|
|
|
|
var api = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.Equal(submit.EventId, api.EventId);
|
|
Assert.Equal(new[] { FlowStepKind.ApiCall, FlowStepKind.ScriptRun }, graph.Steps.Select(s => s.Kind));
|
|
}
|
|
|
|
// ---- Review findings on Task 2 (2026-09-06) --------------------------------------------
|
|
|
|
[Fact]
|
|
public void LeadingAsyncSteps_HangOffTheSpawner_NeverOrphaned()
|
|
{
|
|
// A routed run whose ScriptTrace rows are absent (TraceAllCalls off / older site): only its outbound rows survive.
|
|
var route = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(10), target: "M1.S", sourceNode: "central-a", executionId: Root,
|
|
extra: new ScriptTraceExtra { ChildExecutionId = SiteRun }.ToJson());
|
|
var cached = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted,
|
|
occurredAtUtc: T0.AddMilliseconds(20), target: "Delmia.Ack", correlationId: Guid.NewGuid(), executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a");
|
|
var notify = ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted,
|
|
occurredAtUtc: T0.AddMilliseconds(30), target: "Planners", correlationId: Guid.NewGuid(), executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a");
|
|
var sync = ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(40), target: "MesDb", executionId: SiteRun, parentExecutionId: Root, sourceSiteId: "site-a");
|
|
|
|
var graph = FlowGraphBuilder.Build([route, cached, notify, sync]);
|
|
|
|
var routeStep = graph.Steps[0];
|
|
var reached = graph.Edges.Select(e => e.ToEventId).ToHashSet();
|
|
Assert.All(graph.Steps.Skip(1), s => Assert.Contains(s.EventId, reached));
|
|
Assert.Equal(2, graph.Edges.Count(e => e.Async && e.FromEventId == routeStep.EventId));
|
|
var syncEdge = Assert.Single(graph.Edges, e => e.ToEventId == graph.Steps[3].EventId);
|
|
Assert.Equal(routeStep.EventId, syncEdge.FromEventId);
|
|
Assert.False(syncEdge.Async);
|
|
Assert.False(syncEdge.Spawn);
|
|
Assert.Equal(3, graph.Edges.Count);
|
|
}
|
|
|
|
[Fact]
|
|
public void OnlyNotifyRows_InAnExecution_AllLinkFromTheSpawner()
|
|
{
|
|
var run = SiteRunRow(20);
|
|
var n1 = ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted,
|
|
occurredAtUtc: T0.AddMilliseconds(30), target: "A", correlationId: Guid.NewGuid(), executionId: Cascade, parentExecutionId: SiteRun, sourceSiteId: "site-a");
|
|
var n2 = ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted,
|
|
occurredAtUtc: T0.AddMilliseconds(31), target: "B", correlationId: Guid.NewGuid(), executionId: Cascade, parentExecutionId: SiteRun, sourceSiteId: "site-a");
|
|
|
|
var graph = FlowGraphBuilder.Build([run, n1, n2]);
|
|
|
|
Assert.Equal(2, graph.Edges.Count);
|
|
Assert.All(graph.Edges, e => { Assert.True(e.Async); Assert.True(e.Spawn); Assert.Equal(graph.Steps[0].EventId, e.FromEventId); });
|
|
}
|
|
|
|
[Fact]
|
|
public void Fold_IsOrderIndependent_AndTiesBreakByLifecycleRank()
|
|
{
|
|
var corr = Guid.NewGuid();
|
|
var submit = Outbound(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted, 35, corr, "Delmia.Ack", request: "req");
|
|
var resolve = Outbound(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, 35, corr, "Delmia.Ack", response: "res");
|
|
|
|
var a = FlowGraphBuilder.Build([SiteRunRow(20), resolve, submit]);
|
|
var b = FlowGraphBuilder.Build([SiteRunRow(20), submit, resolve]);
|
|
|
|
var sa = Assert.Single(a.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
var sb = Assert.Single(b.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.Equal(submit.EventId, sa.EventId);
|
|
Assert.Equal(sa.EventId, sb.EventId);
|
|
Assert.Equal("req", sa.InputJson);
|
|
Assert.Equal("res", sa.OutputJson);
|
|
Assert.Equal(FlowStepOutcome.Ok, sa.Outcome);
|
|
Assert.Equal(0, sa.DurationMs);
|
|
Assert.True(FlowGraphBuilder.IsAsync(sa));
|
|
Assert.Equal(sa.Lifecycle!.Select(r => r.Kind), sb.Lifecycle!.Select(r => r.Kind));
|
|
}
|
|
|
|
[Fact]
|
|
public void CachedStep_WhoseSubmitRowIsMissing_IsStillAsync()
|
|
{
|
|
var corr = Guid.NewGuid();
|
|
var graph = FlowGraphBuilder.Build([
|
|
SiteRunRow(20),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.ApiCallCached, AuditStatus.Attempted, 3000, corr, "Delmia.Ack"),
|
|
Outbound(AuditChannel.ApiOutbound, AuditKind.CachedResolve, AuditStatus.Delivered, 4000, corr, "Delmia.Ack"),
|
|
]);
|
|
var api = Assert.Single(graph.Steps, s => s.Kind == FlowStepKind.ApiCall);
|
|
Assert.True(FlowGraphBuilder.IsAsync(api));
|
|
Assert.Single(graph.Edges, e => e.Async && e.ToEventId == api.EventId);
|
|
}
|
|
|
|
[Fact]
|
|
public void ExternalLaneEdges_NeverCountAsClusterBoundaryCrossings()
|
|
{
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: T0, target: "M", sourceNode: "central-a", executionId: Root, durationMs: 50, responseSummary: "{}");
|
|
var centralDb = ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: T0.AddMilliseconds(5), actor: "api:M", target: "CfgDb", sourceNode: "central-a", executionId: Root,
|
|
extra: "{\"op\":\"read\",\"rowsReturned\":2}");
|
|
|
|
var graph = FlowGraphBuilder.Build([inbound, centralDb]);
|
|
|
|
Assert.DoesNotContain(graph.Edges, e => e.CrossesBoundary);
|
|
Assert.Equal("read · 2 rows", graph.Steps[1].Subtitle);
|
|
// Response links from the DB call — the inbound execution's last sync step.
|
|
var into = Assert.Single(graph.Edges, e => e.ToEventId == graph.Steps[2].EventId);
|
|
Assert.Equal(graph.Steps[1].EventId, into.FromEventId);
|
|
}
|
|
}
|