Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.InboundAPI.Tests/RouteHelperTraceTests.cs
T
Joseph Doherty 9a87d7a328 Resolve all 33 medium-severity architecture review findings
Second slice of the 2026-09-09 whole-system review remediation
(docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323.

Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor
subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush;
SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security
downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump.

Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo
removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child
process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC.

Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11
error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5
authenticated relay provenance; EI-10 body cap before audit read.

Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation;
DL-2 DataConnection protocol config encrypted at rest (migration
20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated
central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging;
DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test.

Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron
keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code;
DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs.

Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all
eight nodes with the migration applied. 46 Low findings remain open.
2026-09-09 22:05:30 -04:00

159 lines
7.6 KiB
C#

using NSubstitute;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests;
/// <summary>
/// Flow Debugger (design 2026-09-05 §5.1/§5.2/§7.1): <see cref="RouteTarget.Call"/> writes an
/// ALWAYS-ON <c>ScriptTrace.RouteCall</c> row (traced or not), propagates <c>Trace</c> on both
/// route requests, and feeds the row plus the reply's inline site events to the trace sink.
/// </summary>
public class RouteHelperTraceTests
{
private readonly IInstanceLocator _locator = Substitute.For<IInstanceLocator>();
private readonly IInstanceRouter _router = Substitute.For<IInstanceRouter>();
private readonly List<AuditEvent> _written = new();
private readonly ICentralAuditWriter _writer = Substitute.For<ICentralAuditWriter>();
private readonly INodeIdentityProvider _node = Substitute.For<INodeIdentityProvider>();
public RouteHelperTraceTests()
{
_locator.GetSiteIdForInstanceAsync("inst-1", Arg.Any<CancellationToken>()).Returns("SiteA");
_writer.WriteAsync(Arg.Do<AuditEvent>(e => _written.Add(e)), Arg.Any<CancellationToken>()).Returns(Task.CompletedTask);
_node.NodeName.Returns("central-a");
}
private RouteHelper Helper() => new(_locator, _router, _writer, _node);
private static AuditEvent SiteEvent(Guid exec) => ScadaBridgeAuditEventFactory.Create(
AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered, executionId: exec, target: "doWork");
[Fact]
public async Task EI2_RouteFailureWithBodyCaptureDisabled_DoesNotPersistEchoedSecret()
{
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, false, null, "bad payload SECRET123", DateTimeOffset.UtcNow));
await Assert.ThrowsAsync<InvalidOperationException>(() => Helper().WithBodyCapture(false).To("inst-1").Call("doWork", new { secret = "SECRET123" }));
var row = Assert.Single(_written).AsRow();
Assert.Equal(AuditStatus.Failed, row.Status);
Assert.DoesNotContain("SECRET123", System.Text.Json.JsonSerializer.Serialize(row));
}
[Fact]
public async Task Call_WritesRouteCallRow_WithParentAsExecutionId_AndChildFromReply()
{
var parent = Guid.NewGuid();
var child = Guid.NewGuid();
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 99, null,
DateTimeOffset.UtcNow, child, null));
var result = await Helper().WithParentExecutionId(parent).To("inst-1").Call("doWork", new { x = 1 });
Assert.Equal(99, result);
var evt = Assert.Single(_written);
var row = evt.AsRow();
Assert.Equal("ScriptTrace.RouteCall", evt.Action);
Assert.Equal("inbound-api", evt.Actor);
Assert.Equal("inst-1.doWork", evt.Target);
Assert.Equal("central-a", evt.SourceNode);
Assert.Equal(parent, row.ExecutionId);
Assert.Null(row.ParentExecutionId);
Assert.Equal(AuditStatus.Delivered, row.Status);
Assert.Equal(child, ScriptTraceExtra.TryParse(row.Extra)!.ChildExecutionId);
Assert.Contains("\"x\":1", row.RequestSummary);
Assert.Equal("99", row.ResponseSummary);
}
[Fact]
public async Task Call_IsAlwaysOn_EvenWhenNotTraced_AndTraceFalseOnRequest()
{
RouteToCallRequest? captured = null;
_router.RouteToCallAsync("SiteA", Arg.Do<RouteToCallRequest>(r => captured = r), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, null, null, DateTimeOffset.UtcNow));
await Helper().To("inst-1").Call("doWork");
Assert.False(captured!.Trace);
Assert.Single(_written);
}
[Fact]
public async Task WithTrace_PropagatesOnCallAndSetAttributes()
{
RouteToCallRequest? call = null;
RouteToSetAttributesRequest? set = null;
_router.RouteToCallAsync("SiteA", Arg.Do<RouteToCallRequest>(r => call = r), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, null, null, DateTimeOffset.UtcNow));
_router.RouteToSetAttributesAsync("SiteA", Arg.Do<RouteToSetAttributesRequest>(r => set = r), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToSetAttributesResponse(((RouteToSetAttributesRequest)ci[1]).CorrelationId, true, null, DateTimeOffset.UtcNow));
var traced = Helper().WithTrace(true).WithDeadline(CancellationToken.None).To("inst-1");
await traced.Call("doWork");
await traced.SetAttributes(new Dictionary<string, string> { ["A"] = "1" });
Assert.True(call!.Trace);
Assert.True(set!.Trace);
}
[Fact]
public async Task Sink_ReceivesRouteCallRow_ThenTheReplysTraceEvents_InOrder()
{
var child = Guid.NewGuid();
var siteEvents = new[] { SiteEvent(child), SiteEvent(child) };
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, null, null,
DateTimeOffset.UtcNow, child, siteEvents));
var sunk = new List<AuditEvent>();
await Helper().WithTrace(true).WithTraceSink(sunk.Add).To("inst-1").Call("doWork");
Assert.Equal(3, sunk.Count);
Assert.Equal("ScriptTrace.RouteCall", sunk[0].Action);
Assert.Same(siteEvents[0], sunk[1]);
Assert.Same(siteEvents[1], sunk[2]);
}
[Fact]
public async Task Failure_WritesFailedRow_AndStillThrows()
{
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, false, null, "script faulted", DateTimeOffset.UtcNow));
await Assert.ThrowsAsync<InvalidOperationException>(() => Helper().To("inst-1").Call("doWork"));
var row = Assert.Single(_written).AsRow();
Assert.Equal(AuditStatus.Failed, row.Status);
Assert.Equal("script faulted", row.ErrorMessage);
}
[Fact]
public async Task NoWriterAndNoSink_WritesNothing_AndNeverThrows()
{
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 1, null, DateTimeOffset.UtcNow));
var result = await new RouteHelper(_locator, _router).WithTrace(true).To("inst-1").Call("doWork");
Assert.Equal(1, result);
Assert.Empty(_written);
}
[Fact]
public async Task FaultingWriter_DoesNotFailTheCall()
{
var badWriter = Substitute.For<ICentralAuditWriter>();
badWriter.WriteAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>()).Returns(Task.FromException(new IOException("disk")));
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 5, null, DateTimeOffset.UtcNow));
var result = await new RouteHelper(_locator, _router, badWriter, null).To("inst-1").Call("doWork");
Assert.Equal(5, result);
}
}