Joseph Doherty
9a87d7a328
Second slice of the 2026-09-09 whole-system review remediation
(docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323.
Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor
subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush;
SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security
downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump.
Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo
removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child
process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC.
Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11
error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5
authenticated relay provenance; EI-10 body cap before audit read.
Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation;
DL-2 DataConnection protocol config encrypted at rest (migration
20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated
central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging;
DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test.
Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron
keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code;
DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs.
Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all
eight nodes with the migration applied. 46 Low findings remain open.
183 lines
12 KiB
C#
183 lines
12 KiB
C#
using Microsoft.EntityFrameworkCore;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase;
|
|
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
|
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
|
using Xunit;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Repositories;
|
|
|
|
/// <summary>
|
|
/// Flow Debugger Replay (design 2026-09-05 §7.1): <see cref="AuditLogRepository.GetTraceEventsAsync"/>
|
|
/// returns the whole chain's ScriptTrace rows plus the inbound row, from ANY entry id, ordered,
|
|
/// and excludes unrelated and non-trace rows. Runs against the shared MSSQL fixture.
|
|
/// </summary>
|
|
public class AuditLogRepositoryTraceEventsTests : IClassFixture<MsSqlMigrationFixture>
|
|
{
|
|
private readonly MsSqlMigrationFixture _fixture;
|
|
public AuditLogRepositoryTraceEventsTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
|
|
|
|
private ScadaBridgeDbContext CreateContext() => new(
|
|
new DbContextOptionsBuilder<ScadaBridgeDbContext>().UseSqlServer(_fixture.ConnectionString).Options);
|
|
|
|
[SkippableFact]
|
|
public async Task ScopedTreeAndReplay_ExcludeOtherSite_AndRetainCentralRows()
|
|
{
|
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
|
var root = Guid.NewGuid(); var childA = Guid.NewGuid(); var childB = Guid.NewGuid();
|
|
var central = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered, executionId: root);
|
|
var allowed = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered, executionId: childA, parentExecutionId: root, sourceSiteId: "site-a");
|
|
var denied = ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered, executionId: childB, parentExecutionId: root, sourceSiteId: "site-b", responseSummary: "sensitive-sql-values");
|
|
await using var context = CreateContext(); var repo = new AuditLogRepository(context);
|
|
foreach (var row in new[] { central, allowed, denied }) await repo.InsertIfNotExistsAsync(row);
|
|
var tree = await repo.GetExecutionTreeAsync(childA, permittedSiteIds: new[] { "site-a" });
|
|
Assert.Contains(tree, node => node.ExecutionId == root);
|
|
Assert.DoesNotContain(tree, node => node.ExecutionId == childB);
|
|
var replay = await repo.GetTraceEventsAsync(childA, permittedSiteIds: new[] { "site-a" });
|
|
Assert.Equal(new[] { central.EventId, allowed.EventId }.Order(), replay.Select(e => e.EventId).Order());
|
|
Assert.DoesNotContain(replay, row => row.DetailsJson?.Contains("sensitive-sql-values") == true);
|
|
}
|
|
|
|
[SkippableTheory]
|
|
[InlineData(600, 0)]
|
|
[InlineData(200, 10000)]
|
|
[InlineData(1, 1100000)]
|
|
public async Task Replay_BoundsRowsAndUtf8Bytes_AndKeepsRoot(int count, int payloadLength)
|
|
{
|
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
|
var root = Guid.NewGuid(); var start = DateTime.UtcNow;
|
|
await using var context = CreateContext(); var repo = new AuditLogRepository(context);
|
|
for (var i = 0; i < count; i++)
|
|
await repo.InsertIfNotExistsAsync(ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace,
|
|
AuditKind.ScriptRun, AuditStatus.Delivered, executionId: root, occurredAtUtc: payloadLength == 0 ? start : start.AddMilliseconds(i),
|
|
responseSummary: new string('界', payloadLength)));
|
|
var rows = await repo.GetTraceEventsAsync(root);
|
|
var batch = Assert.IsType<TraceEventBatch>(rows);
|
|
Assert.True(batch.Truncated);
|
|
Assert.Equal(root, batch.RootExecutionId);
|
|
Assert.True(batch.Count <= 500);
|
|
if (payloadLength == 0) Assert.Equal(500, batch.Count);
|
|
if (payloadLength > 1024 * 1024) Assert.Empty(batch);
|
|
Assert.Equal(batch.Count, batch.Select(e => e.EventId).Distinct().Count());
|
|
Assert.True(batch.Sum(e => System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(e).Length) <= 1024 * 1024);
|
|
}
|
|
|
|
[SkippableFact]
|
|
public async Task ReturnsInboundPlusChainTraceRows_Ordered_FromAnyEntry()
|
|
{
|
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
|
var siteId = "test-trace-" + Guid.NewGuid().ToString("N")[..8];
|
|
var t0 = new DateTime(2026, 9, 5, 12, 0, 0, DateTimeKind.Utc);
|
|
var root = Guid.NewGuid();
|
|
var siteRun = Guid.NewGuid();
|
|
var cascade = Guid.NewGuid();
|
|
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: t0, actor: "key", target: "M", executionId: root, sourceSiteId: siteId, responseSummary: "{}");
|
|
var route = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(10), target: "I.S", executionId: root, sourceSiteId: siteId,
|
|
extra: new ScriptTraceExtra { ChildExecutionId = siteRun }.ToJson());
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(20), target: "S", executionId: siteRun, parentExecutionId: root, sourceSiteId: siteId);
|
|
var write = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.AttributeWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(30), target: "A", executionId: siteRun, parentExecutionId: root, sourceSiteId: siteId);
|
|
var cascadeRun = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(40), target: "C", executionId: cascade, parentExecutionId: siteRun, sourceSiteId: siteId);
|
|
// Same chain but outside the replay channel set: a secured-write row — excluded
|
|
// (outbound-call plan 2026-09-06 D6 widened the set; SecuredWrite stays out).
|
|
var outbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.SecuredWrite, AuditKind.SecuredWriteSubmit, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(25), target: "ERP.x", executionId: siteRun, parentExecutionId: root, sourceSiteId: siteId);
|
|
// Unrelated chain.
|
|
var other = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(5), target: "Z", executionId: Guid.NewGuid(), sourceSiteId: siteId);
|
|
|
|
await using (var ctx = CreateContext())
|
|
{
|
|
var repo = new AuditLogRepository(ctx);
|
|
foreach (var e in new[] { cascadeRun, other, write, outbound, inbound, run, route })
|
|
await repo.InsertIfNotExistsAsync(e);
|
|
}
|
|
|
|
await using (var ctx = CreateContext())
|
|
{
|
|
var repo = new AuditLogRepository(ctx);
|
|
foreach (var entry in new[] { root, siteRun, cascade })
|
|
{
|
|
var events = await repo.GetTraceEventsAsync(entry);
|
|
Assert.Equal(
|
|
new[] { inbound.EventId, route.EventId, run.EventId, write.EventId, cascadeRun.EventId },
|
|
events.Select(e => e.EventId).ToArray());
|
|
Assert.Equal(inbound, events[0]);
|
|
Assert.Equal(cascadeRun, events[4]);
|
|
}
|
|
|
|
Assert.Empty(await repo.GetTraceEventsAsync(Guid.NewGuid()));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Outbound-calls plan (2026-09-06, D6): replay returns the chain's ScriptTrace, ApiOutbound,
|
|
/// DbOutbound and Notification rows plus the InboundRequest row, in (OccurredAtUtc, EventId)
|
|
/// order; SecuredWrite, Cluster and InboundAuthFailure rows in the same tree stay excluded.
|
|
/// </summary>
|
|
[SkippableFact]
|
|
public async Task IncludesOutboundApiDbAndNotificationRows_ExcludesOtherChannels()
|
|
{
|
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
|
var siteId = "test-trace-" + Guid.NewGuid().ToString("N")[..8];
|
|
var t0 = new DateTime(2026, 9, 6, 12, 0, 0, DateTimeKind.Utc);
|
|
var root = Guid.NewGuid();
|
|
var siteRun = Guid.NewGuid();
|
|
|
|
var inbound = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundRequest, AuditStatus.Delivered,
|
|
occurredAtUtc: t0, actor: "key", target: "M", executionId: root, sourceSiteId: siteId, responseSummary: "{}");
|
|
var route = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.RouteCall, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(10), target: "I.S", executionId: root, sourceSiteId: siteId,
|
|
extra: new ScriptTraceExtra { ChildExecutionId = siteRun }.ToJson());
|
|
var run = ScadaBridgeAuditEventFactory.Create(AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(20), target: "S", executionId: siteRun, parentExecutionId: root, sourceSiteId: siteId);
|
|
var apiSubmit = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiOutbound, AuditKind.CachedSubmit, AuditStatus.Submitted,
|
|
occurredAtUtc: t0.AddMilliseconds(30), target: "Delmia.AckRecipe", executionId: siteRun, parentExecutionId: root,
|
|
sourceSiteId: siteId, correlationId: Guid.NewGuid());
|
|
var dbWrite = ScadaBridgeAuditEventFactory.Create(AuditChannel.DbOutbound, AuditKind.DbWrite, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(40), target: "MesDb", executionId: siteRun, parentExecutionId: root, sourceSiteId: siteId);
|
|
var notify = ScadaBridgeAuditEventFactory.Create(AuditChannel.Notification, AuditKind.NotifySend, AuditStatus.Submitted,
|
|
occurredAtUtc: t0.AddMilliseconds(50), target: "Planners", executionId: siteRun, parentExecutionId: root,
|
|
sourceSiteId: siteId, correlationId: Guid.NewGuid());
|
|
// Same execution ids, but channels outside the replay set — all excluded.
|
|
var secured = ScadaBridgeAuditEventFactory.Create(AuditChannel.SecuredWrite, AuditKind.SecuredWriteSubmit, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(35), target: "Tag", executionId: siteRun, parentExecutionId: root, sourceSiteId: siteId);
|
|
var authFailure = ScadaBridgeAuditEventFactory.Create(AuditChannel.ApiInbound, AuditKind.InboundAuthFailure, AuditStatus.Failed,
|
|
occurredAtUtc: t0.AddMilliseconds(1), actor: "key", target: "M", executionId: root, sourceSiteId: siteId);
|
|
var cluster = ScadaBridgeAuditEventFactory.Create(AuditChannel.Cluster, AuditKind.ManualFailover, AuditStatus.Delivered,
|
|
occurredAtUtc: t0.AddMilliseconds(45), actor: "admin", target: "central-a", executionId: root, sourceSiteId: siteId);
|
|
|
|
await using (var ctx = CreateContext())
|
|
{
|
|
var repo = new AuditLogRepository(ctx);
|
|
foreach (var e in new[] { notify, secured, dbWrite, cluster, run, authFailure, apiSubmit, inbound, route })
|
|
await repo.InsertIfNotExistsAsync(e);
|
|
}
|
|
|
|
await using (var ctx = CreateContext())
|
|
{
|
|
var repo = new AuditLogRepository(ctx);
|
|
foreach (var entry in new[] { root, siteRun })
|
|
{
|
|
var events = await repo.GetTraceEventsAsync(entry);
|
|
Assert.Equal(
|
|
new[] { inbound.EventId, route.EventId, run.EventId, apiSubmit.EventId, dbWrite.EventId, notify.EventId },
|
|
events.Select(e => e.EventId).ToArray());
|
|
Assert.Equal(apiSubmit, events[3]);
|
|
Assert.Equal(dbWrite, events[4]);
|
|
Assert.Equal(notify, events[5]);
|
|
var decomposed = events.Select(AuditRowProjection.Decompose).ToList();
|
|
Assert.DoesNotContain(decomposed, v => v.Channel is AuditChannel.SecuredWrite or AuditChannel.Cluster);
|
|
Assert.DoesNotContain(decomposed, v => v.Kind == AuditKind.InboundAuthFailure);
|
|
}
|
|
}
|
|
}
|
|
}
|