9a87d7a328
Second slice of the 2026-09-09 whole-system review remediation
(docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323.
Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor
subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush;
SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security
downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump.
Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo
removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child
process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC.
Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11
error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5
authenticated relay provenance; EI-10 body cap before audit read.
Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation;
DL-2 DataConnection protocol config encrypted at rest (migration
20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated
central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging;
DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test.
Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron
keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code;
DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs.
Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all
eight nodes with the migration applied. 46 Low findings remain open.
159 lines
7.6 KiB
C#
159 lines
7.6 KiB
C#
using NSubstitute;
|
|
using ZB.MOM.WW.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests;
|
|
|
|
/// <summary>
|
|
/// Flow Debugger (design 2026-09-05 §5.1/§5.2/§7.1): <see cref="RouteTarget.Call"/> writes an
|
|
/// ALWAYS-ON <c>ScriptTrace.RouteCall</c> row (traced or not), propagates <c>Trace</c> on both
|
|
/// route requests, and feeds the row plus the reply's inline site events to the trace sink.
|
|
/// </summary>
|
|
public class RouteHelperTraceTests
|
|
{
|
|
private readonly IInstanceLocator _locator = Substitute.For<IInstanceLocator>();
|
|
private readonly IInstanceRouter _router = Substitute.For<IInstanceRouter>();
|
|
private readonly List<AuditEvent> _written = new();
|
|
private readonly ICentralAuditWriter _writer = Substitute.For<ICentralAuditWriter>();
|
|
private readonly INodeIdentityProvider _node = Substitute.For<INodeIdentityProvider>();
|
|
|
|
public RouteHelperTraceTests()
|
|
{
|
|
_locator.GetSiteIdForInstanceAsync("inst-1", Arg.Any<CancellationToken>()).Returns("SiteA");
|
|
_writer.WriteAsync(Arg.Do<AuditEvent>(e => _written.Add(e)), Arg.Any<CancellationToken>()).Returns(Task.CompletedTask);
|
|
_node.NodeName.Returns("central-a");
|
|
}
|
|
|
|
private RouteHelper Helper() => new(_locator, _router, _writer, _node);
|
|
|
|
private static AuditEvent SiteEvent(Guid exec) => ScadaBridgeAuditEventFactory.Create(
|
|
AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered, executionId: exec, target: "doWork");
|
|
|
|
[Fact]
|
|
public async Task EI2_RouteFailureWithBodyCaptureDisabled_DoesNotPersistEchoedSecret()
|
|
{
|
|
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, false, null, "bad payload SECRET123", DateTimeOffset.UtcNow));
|
|
await Assert.ThrowsAsync<InvalidOperationException>(() => Helper().WithBodyCapture(false).To("inst-1").Call("doWork", new { secret = "SECRET123" }));
|
|
var row = Assert.Single(_written).AsRow();
|
|
Assert.Equal(AuditStatus.Failed, row.Status);
|
|
Assert.DoesNotContain("SECRET123", System.Text.Json.JsonSerializer.Serialize(row));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Call_WritesRouteCallRow_WithParentAsExecutionId_AndChildFromReply()
|
|
{
|
|
var parent = Guid.NewGuid();
|
|
var child = Guid.NewGuid();
|
|
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 99, null,
|
|
DateTimeOffset.UtcNow, child, null));
|
|
|
|
var result = await Helper().WithParentExecutionId(parent).To("inst-1").Call("doWork", new { x = 1 });
|
|
|
|
Assert.Equal(99, result);
|
|
var evt = Assert.Single(_written);
|
|
var row = evt.AsRow();
|
|
Assert.Equal("ScriptTrace.RouteCall", evt.Action);
|
|
Assert.Equal("inbound-api", evt.Actor);
|
|
Assert.Equal("inst-1.doWork", evt.Target);
|
|
Assert.Equal("central-a", evt.SourceNode);
|
|
Assert.Equal(parent, row.ExecutionId);
|
|
Assert.Null(row.ParentExecutionId);
|
|
Assert.Equal(AuditStatus.Delivered, row.Status);
|
|
Assert.Equal(child, ScriptTraceExtra.TryParse(row.Extra)!.ChildExecutionId);
|
|
Assert.Contains("\"x\":1", row.RequestSummary);
|
|
Assert.Equal("99", row.ResponseSummary);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Call_IsAlwaysOn_EvenWhenNotTraced_AndTraceFalseOnRequest()
|
|
{
|
|
RouteToCallRequest? captured = null;
|
|
_router.RouteToCallAsync("SiteA", Arg.Do<RouteToCallRequest>(r => captured = r), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, null, null, DateTimeOffset.UtcNow));
|
|
|
|
await Helper().To("inst-1").Call("doWork");
|
|
|
|
Assert.False(captured!.Trace);
|
|
Assert.Single(_written);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task WithTrace_PropagatesOnCallAndSetAttributes()
|
|
{
|
|
RouteToCallRequest? call = null;
|
|
RouteToSetAttributesRequest? set = null;
|
|
_router.RouteToCallAsync("SiteA", Arg.Do<RouteToCallRequest>(r => call = r), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, null, null, DateTimeOffset.UtcNow));
|
|
_router.RouteToSetAttributesAsync("SiteA", Arg.Do<RouteToSetAttributesRequest>(r => set = r), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToSetAttributesResponse(((RouteToSetAttributesRequest)ci[1]).CorrelationId, true, null, DateTimeOffset.UtcNow));
|
|
|
|
var traced = Helper().WithTrace(true).WithDeadline(CancellationToken.None).To("inst-1");
|
|
await traced.Call("doWork");
|
|
await traced.SetAttributes(new Dictionary<string, string> { ["A"] = "1" });
|
|
|
|
Assert.True(call!.Trace);
|
|
Assert.True(set!.Trace);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Sink_ReceivesRouteCallRow_ThenTheReplysTraceEvents_InOrder()
|
|
{
|
|
var child = Guid.NewGuid();
|
|
var siteEvents = new[] { SiteEvent(child), SiteEvent(child) };
|
|
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, null, null,
|
|
DateTimeOffset.UtcNow, child, siteEvents));
|
|
var sunk = new List<AuditEvent>();
|
|
|
|
await Helper().WithTrace(true).WithTraceSink(sunk.Add).To("inst-1").Call("doWork");
|
|
|
|
Assert.Equal(3, sunk.Count);
|
|
Assert.Equal("ScriptTrace.RouteCall", sunk[0].Action);
|
|
Assert.Same(siteEvents[0], sunk[1]);
|
|
Assert.Same(siteEvents[1], sunk[2]);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Failure_WritesFailedRow_AndStillThrows()
|
|
{
|
|
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, false, null, "script faulted", DateTimeOffset.UtcNow));
|
|
|
|
await Assert.ThrowsAsync<InvalidOperationException>(() => Helper().To("inst-1").Call("doWork"));
|
|
|
|
var row = Assert.Single(_written).AsRow();
|
|
Assert.Equal(AuditStatus.Failed, row.Status);
|
|
Assert.Equal("script faulted", row.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task NoWriterAndNoSink_WritesNothing_AndNeverThrows()
|
|
{
|
|
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 1, null, DateTimeOffset.UtcNow));
|
|
|
|
var result = await new RouteHelper(_locator, _router).WithTrace(true).To("inst-1").Call("doWork");
|
|
|
|
Assert.Equal(1, result);
|
|
Assert.Empty(_written);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task FaultingWriter_DoesNotFailTheCall()
|
|
{
|
|
var badWriter = Substitute.For<ICentralAuditWriter>();
|
|
badWriter.WriteAsync(Arg.Any<AuditEvent>(), Arg.Any<CancellationToken>()).Returns(Task.FromException(new IOException("disk")));
|
|
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
|
|
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, true, 5, null, DateTimeOffset.UtcNow));
|
|
|
|
var result = await new RouteHelper(_locator, _router, badWriter, null).To("inst-1").Call("doWork");
|
|
Assert.Equal(5, result);
|
|
}
|
|
}
|