dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Resolve all 33 medium-severity architecture review findings

Browse Source 
Second slice of the 2026-09-09 whole-system review remediation
(docs/plans/2026-09-09-arch-review.md §10), on top of the High slice 146a1323.

Runtime/devices: SR-2 alarm on-trigger integration scope; SR-3 correlated NativeAlarmActor
subscribe retry; SR-4 immutable audited DB target; SR-5 ordered InstanceActor stop/flush;
SR-12 rehydration fence; SR-13 S&F gate recheck per delivery; DC-4 no OPC UA security
downgrade; DC-5 alarm snapshot for every new subscriber; ORCH-1 sync-dispose joins the pump.

Management/UI: MG-2 site scope on audit tree + Flow Debugger Replay; MG-3 stale-probe memo
removed; MG-4 server-owned bulk-deploy deadline; UI-5 sandbox runs in a killable child
process (new CentralUI.SandboxWorker project); UI-14 bounded sandbox output/IPC.

Edge/audit: EI-1 relay audit ownership; EI-2 SkipBodyCapture suppresses error text; EI-3/EI-11
error-field redaction + caps; EI-4 single relay deadline, pre-execution-only retry; EI-5
authenticated relay provenance; EI-10 body cap before audit read.

Persistence/contracts: DL-1 global Instance.UniqueName unique index + conflict translation;
DL-2 DataConnection protocol config encrypted at rest (migration
20260909203046_EnforceGlobalInstanceNamesAndProtectProtocolConfiguration — coordinated
central upgrade, old binaries cannot read protected rows); DL-3 bounded Replay paging;
DL-6 cross-site Overwrite of a deployed instance rejected; XC-4 proto descriptor contract test.

Config/hygiene: XC-1 central listener isolation by local port; XC-2/XC-3 inert SMTP + cron
keys removed from rig settings; CM-2 env2 bootstrap guard enabled; DC-6 deploy site exit code;
DC-7 notifier retries only explicit standby rejection; XC-5 ClusterClient purged from docs.

Gates recorded in the report: build 0/0, 9 397 passed / 0 failed, rig redeployed on all
eight nodes with the migration applied. 46 Low findings remain open.
This commit is contained in:
Joseph Doherty
2026-09-09 22:05:30 -04:00
parent 146a13238f
commit 9a87d7a328
136 changed files with 5441 additions and 648 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.InboundAPI.Tests/Middleware/AuditWriteMiddlewareTests.cs
+69
View File
@@ -1,4 +1,5 @@
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Http;
@@ -24,6 +25,74 @@ namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests.Middleware;
/// </summary>
public class AuditWriteMiddlewareTests
{
[Theory]
[InlineData(0, 1)]
[InlineData(1, 1)]
[InlineData(2, 0)]
[InlineData(3, 0)]
public async Task EI1_RelayAuditOwnership_OnlyServingPeerSuppressesLocalFailure(int scenario, int localRows)
{
IResult forwarded = scenario switch
{
0 => Results.Json(new { code = "ACTIVE_NODE_UNREACHABLE" }, statusCode: 503),
1 => Results.Json(new { code = "STANDBY_NODE" }, statusCode: 503),
_ => new HttpStandbyRequestForwarder.RelayedResult(scenario == 2 ? 200 : 500, "application/json", "{}"u8.ToArray())
};
var services = new ServiceCollection().AddLogging();
services.AddSingleton<IActiveNodeGate>(new StandbyGate());
services.AddSingleton<IStandbyRequestForwarder>(new ResultForwarder(forwarded));
using var provider = services.BuildServiceProvider();
var writer = new RecordingAuditWriter();
var middleware = CreateMiddleware(async ctx =>
{
var filter = new InboundApiEndpointFilter(NullLogger<InboundApiEndpointFilter>.Instance,
new InboundOptions(new InboundApiOptions()));
var result = await filter.InvokeAsync(EndpointFilterInvocationContext.Create(ctx), _ => throw new InvalidOperationException("Must not execute locally"));
await ((IResult)result!).ExecuteAsync(ctx);
}, writer);
var context = BuildContext("echo", "{}");
context.RequestServices = provider;
context.Response.Body = new MemoryStream();
await middleware.InvokeAsync(context);
Assert.Equal(localRows, writer.Events.Count);
if (localRows > 0) Assert.NotNull(writer.Events[0].ExecutionId);
}
private sealed class StandbyGate : IActiveNodeGate { public bool IsActiveNode => false; }
private sealed class ResultForwarder(IResult result) : IStandbyRequestForwarder
{
public bool IsConfigured => true;
public Task<IResult> ForwardAsync(HttpContext context, CancellationToken cancellationToken) => Task.FromResult(result);
}
private sealed class InboundOptions(InboundApiOptions value) : IOptionsMonitor<InboundApiOptions>
{
public InboundApiOptions CurrentValue => value;
public InboundApiOptions Get(string? name) => value;
public IDisposable? OnChange(Action<InboundApiOptions, string?> listener) => null;
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task EI2_SkipBodyCapture_SuppressesExceptionText(bool escaped)
{
var writer = new RecordingAuditWriter();
var opts = new AuditLogOptions();
opts.PerTargetOverrides["secret"] = new PerTargetRedactionOverride { SkipBodyCapture = true };
var middleware = CreateMiddleware(ctx =>
{
ctx.Items[AuditWriteMiddleware.ErrorDetailItemKey] = "payload SECRET123";
if (escaped) throw new ArgumentException("payload SECRET123");
ctx.Response.StatusCode = 500;
return Task.CompletedTask;
}, writer, opts);
var context = BuildContext("secret", "SECRET123");
if (escaped) await Assert.ThrowsAsync<ArgumentException>(() => middleware.InvokeAsync(context));
else await middleware.InvokeAsync(context);
var row = Assert.Single(writer.Events);
Assert.DoesNotContain("SECRET123", System.Text.Json.JsonSerializer.Serialize(row));
}
/// <summary>
/// Test-only recording <see cref="ICentralAuditWriter"/>. Captures every
/// <see cref="AuditEvent"/> the middleware emits so each test can assert on
tests/ZB.MOM.WW.ScadaBridge.InboundAPI.Tests/RelayProvenanceTests.cs
+152
View File
@@ -0,0 +1,152 @@
using System.Net;
using System.Text;
using System.Text.Json;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.AuditLog.Configuration;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.InboundAPI.Middleware;
namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests;
public class RelayProvenanceTests
{
[Theory]
[InlineData("none")]
[InlineData("body")]
[InlineData("path")]
[InlineData("credential")]
[InlineData("expired")]
[InlineData("wrongkey")]
public async Task EI5_GenuineRelayPreservesCallerIp_OnlyForIntactSignedRequest(string tamper)
{
var options = new Monitor<InboundApiOptions>(new() { ApiKeyPepper = new string('p', 32), StandbyForwardPeerEndpoints = ["http://peer"] });
var writer = new Writer();
var handler = new Handler(async request =>
{
var active = Context(tamper == "body" ? "changed" : await request.Content!.ReadAsStringAsync(), "10.0.0.2");
foreach (var header in request.Headers) active.Request.Headers[header.Key] = header.Value.ToArray();
active.Request.ContentType = request.Content!.Headers.ContentType?.ToString();
if (tamper == "path") active.Request.Path = "/api/another";
if (tamper == "credential") active.Request.Headers["X-API-Key"] = "changed-key";
if (tamper is "expired" or "wrongkey")
{
var envelope = active.Request.Headers["X-ScadaBridge-Relay"].ToString().Split('.');
var node = System.Text.Json.Nodes.JsonNode.Parse(Convert.FromBase64String(envelope[0]))!;
if (tamper == "expired") node["Time"] = DateTimeOffset.UtcNow.AddHours(-1).ToUnixTimeSeconds();
var bytes = JsonSerializer.SerializeToUtf8Bytes(node);
var pepper = Encoding.UTF8.GetBytes(new string(tamper == "wrongkey" ? 'x' : 'p', 32));
var key = System.Security.Cryptography.HMACSHA256.HashData(pepper, "ScadaBridge.InboundRelay.v1"u8);
var mac = System.Security.Cryptography.HMACSHA256.HashData(key, bytes);
active.Request.Headers["X-ScadaBridge-Relay"] = Convert.ToBase64String(bytes) + "." + Convert.ToHexString(mac);
}
await AuditAsync(active, writer, options);
return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") };
});
var forwarder = new HttpStandbyRequestForwarder(new Factory(handler), options, NullLogger<HttpStandbyRequestForwarder>.Instance);
var original = Context("{}", "10.4.5.6");
original.Items[AuditWriteMiddleware.InboundExecutionIdItemKey] = Guid.NewGuid();
original.Request.Headers["X-ScadaBridge-Relay"] = "caller-forged-proof";
original.Request.Headers["X-Forwarded-For"] = "6.6.6.6";
await forwarder.ForwardAsync(original, CancellationToken.None);
var row = Assert.Single(writer.Events).AsRow();
using var extra = JsonDocument.Parse(row.Extra!);
Assert.Equal(tamper != "none" ? "10.0.0.2" : "10.4.5.6", extra.RootElement.GetProperty("remoteIp").GetString());
if (tamper == "none") Assert.Equal(original.Items[AuditWriteMiddleware.InboundExecutionIdItemKey], row.ExecutionId);
}
[Fact]
public async Task EI5_DirectForgedForwardingHeadersCannotChangeAuditedCaller()
{
var context = Context("{}", "10.4.5.6");
context.Request.Headers[HttpStandbyRequestForwarder.ForwardedRequestHeader] = "trusted-peer";
context.Request.Headers["X-ScadaBridge-Relay"] = "forged";
context.Request.Headers["X-Forwarded-For"] = "6.6.6.6";
var writer = new Writer();
await AuditAsync(context, writer, new Monitor<InboundApiOptions>(new() { ApiKeyPepper = new string('p', 32) }));
using var extra = JsonDocument.Parse(Assert.Single(writer.Events).AsRow().Extra!);
Assert.Equal("10.4.5.6", extra.RootElement.GetProperty("remoteIp").GetString());
}
[Fact]
public async Task EI1_AllAuthenticatedPeersStandby_OnlyOriginEmitsFailureAudit()
{
var options = new Monitor<InboundApiOptions>(new() { ApiKeyPepper = new string('p', 32), StandbyForwardPeerEndpoints = ["http://peer-a", "http://peer-b"] });
var writer = new Writer();
var attempts = 0;
var handler = new Handler(async request =>
{
attempts++;
var peer = Context(await request.Content!.ReadAsStringAsync(), "10.0.0.2");
foreach (var header in request.Headers) peer.Request.Headers[header.Key] = header.Value.ToArray();
peer.Request.ContentType = request.Content!.Headers.ContentType?.ToString();
await AuditAsync(peer, writer, options, new Standby());
var response = new HttpResponseMessage((HttpStatusCode)peer.Response.StatusCode) { Content = new StringContent("{}") };
response.Headers.Add(HttpStandbyRequestForwarder.StandbyResponseHeader, "1");
return response;
});
var forwarder = new HttpStandbyRequestForwarder(new Factory(handler), options, NullLogger<HttpStandbyRequestForwarder>.Instance);
var original = Context("{}", "10.4.5.6");
await AuditAsync(original, writer, options, new Standby(), forwarder);
Assert.Equal(2, attempts);
Assert.Equal(503, original.Response.StatusCode);
var row = Assert.Single(writer.Events).AsRow();
Assert.Equal(original.Items[AuditWriteMiddleware.InboundExecutionIdItemKey], row.ExecutionId);
}
private sealed class Standby : IActiveNodeGate { public bool IsActiveNode => false; }
private static async Task AuditAsync(HttpContext context, Writer writer, IOptionsMonitor<InboundApiOptions> options, IActiveNodeGate? gate = null, IStandbyRequestForwarder? forwarder = null)
{
var services = new ServiceCollection().AddLogging();
services.AddSingleton<ICentralAuditWriter>(writer);
services.AddSingleton(options);
if (gate is not null) services.AddSingleton(gate);
if (forwarder is not null) services.AddSingleton(forwarder);
services.AddSingleton<IOptionsMonitor<AuditLogOptions>>(new Monitor<AuditLogOptions>(new()));
using var provider = services.BuildServiceProvider();
context.RequestServices = provider;
var middleware = ActivatorUtilities.CreateInstance<AuditWriteMiddleware>(provider, (RequestDelegate)(async ctx =>
{
if (gate is null) { ctx.Response.StatusCode = 200; return; }
var filter = new InboundApiEndpointFilter(NullLogger<InboundApiEndpointFilter>.Instance, options);
var result = await filter.InvokeAsync(EndpointFilterInvocationContext.Create(ctx), _ => ValueTask.FromResult<object?>(Results.Ok()));
await ((IResult)result!).ExecuteAsync(ctx);
}));
await middleware.InvokeAsync(context);
}
private static DefaultHttpContext Context(string body, string ip)
{
var context = new DefaultHttpContext();
context.Request.Method = "POST"; context.Request.Path = "/api/echo";
context.Request.RouteValues["methodName"] = "echo";
context.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(body));
context.Request.ContentType = "application/json";
context.Response.Body = new MemoryStream();
context.Connection.RemoteIpAddress = IPAddress.Parse(ip);
return context;
}
private sealed class Writer : ICentralAuditWriter
{
public List<AuditEvent> Events { get; } = [];
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default) { Events.Add(evt); return Task.CompletedTask; }
}
private sealed class Monitor<T>(T value) : IOptionsMonitor<T>
{
public T CurrentValue => value;
public T Get(string? name) => value;
public IDisposable? OnChange(Action<T, string?> listener) => null;
}
private sealed class Handler(Func<HttpRequestMessage, Task<HttpResponseMessage>> send) : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => send(request);
}
private sealed class Factory(HttpMessageHandler handler) : IHttpClientFactory
{
public HttpClient CreateClient(string name) => new(handler, false);
}
}
tests/ZB.MOM.WW.ScadaBridge.InboundAPI.Tests/RouteHelperTraceTests.cs
+11
View File
@@ -33,6 +33,17 @@ public class RouteHelperTraceTests
private static AuditEvent SiteEvent(Guid exec) => ScadaBridgeAuditEventFactory.Create(
AuditChannel.ScriptTrace, AuditKind.ScriptRun, AuditStatus.Delivered, executionId: exec, target: "doWork");
[Fact]
public async Task EI2_RouteFailureWithBodyCaptureDisabled_DoesNotPersistEchoedSecret()
{
_router.RouteToCallAsync("SiteA", Arg.Any<RouteToCallRequest>(), Arg.Any<CancellationToken>())
.Returns(ci => new RouteToCallResponse(((RouteToCallRequest)ci[1]).CorrelationId, false, null, "bad payload SECRET123", DateTimeOffset.UtcNow));
await Assert.ThrowsAsync<InvalidOperationException>(() => Helper().WithBodyCapture(false).To("inst-1").Call("doWork", new { secret = "SECRET123" }));
var row = Assert.Single(_written).AsRow();
Assert.Equal(AuditStatus.Failed, row.Status);
Assert.DoesNotContain("SECRET123", System.Text.Json.JsonSerializer.Serialize(row));
}
[Fact]
public async Task Call_WritesRouteCallRow_WithParentAsExecutionId_AndChildFromReply()
{
tests/ZB.MOM.WW.ScadaBridge.InboundAPI.Tests/StandbyRequestForwardingTests.cs
+28 -6
View File
@@ -20,10 +20,32 @@ public class StandbyRequestForwardingTests
// Filter: when does a standby relay, and when does it refuse?
// ------------------------------------------------------------------
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task EI4_AppliedThenResponseLost_IsNotReplayed(bool timeout)
{
var effects = 0;
var handler = new FakeHandler(_ =>
{
effects++;
if (timeout) throw new TaskCanceledException("response lost after apply");
throw new HttpRequestException("response lost after apply");
});
var result = await CreateForwarder(handler, "http://a", "http://b")
.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
Assert.Equal(1, effects);
var ctx = new DefaultHttpContext { RequestServices = new ServiceCollection().AddLogging().BuildServiceProvider() };
ctx.Response.Body = new MemoryStream();
await result.ExecuteAsync(ctx);
ctx.Response.Body.Position = 0;
Assert.Contains("OUTCOME_UNKNOWN", await new StreamReader(ctx.Response.Body).ReadToEndAsync());
}
[Fact]
public async Task Standby_WithForwarderConfigured_RelaysAndDoesNotRunHandler()
{
var forwarder = new StubForwarder(configured: true, result: Results.Text("relayed"));
var forwarder = new StubForwarder(configured: true, result: new HttpStandbyRequestForwarder.RelayedResult(200, "text/plain", "relayed"u8.ToArray()));
var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder);
var next = NextSentinel(out var handlerRan);
@@ -163,7 +185,7 @@ public class StandbyRequestForwardingTests
{
bodies.Add(await req.Content!.ReadAsStringAsync());
if (req.RequestUri!.Host == "dead")
throw new HttpRequestException("connection refused");
throw new HttpRequestException("connection refused", new System.Net.Sockets.SocketException((int)System.Net.Sockets.SocketError.ConnectionRefused));
return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") };
});
var forwarder = CreateForwarder(handler, "http://dead:5000", "http://alive:5000");
@@ -224,7 +246,7 @@ public class StandbyRequestForwardingTests
[Fact]
public async Task Forwarder_AllPeersFail_Returns503ActiveNodeUnreachable()
{
var handler = new FakeHandler(_ => throw new HttpRequestException("down"));
var handler = new FakeHandler(_ => throw new HttpRequestException(HttpRequestError.NameResolutionError, "down"));
var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000");
var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
@@ -234,7 +256,7 @@ public class StandbyRequestForwardingTests
}
[Fact]
public async Task Forwarder_PeerTimesOut_TriesNextPeer()
public async Task Forwarder_PeerTimesOut_ReturnsUncertainWithoutTryingNextPeer()
{
var handler = new FakeHandler(async (req, ct) =>
{
@@ -248,8 +270,8 @@ public class StandbyRequestForwardingTests
var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
Assert.Equal(200, relayed.StatusCode);
Assert.IsNotType<HttpStandbyRequestForwarder.RelayedResult>(result);
AssertStatus(result, StatusCodes.Status502BadGateway);
}
[Fact]