Resolve all high-severity architecture review findings

This commit is contained in:
Joseph Doherty
2026-09-09 16:13:58 -04:00
parent 8bfe65512e
commit 146a13238f
192 changed files with 3160 additions and 753 deletions
@@ -83,6 +83,51 @@ public class CentralControlAuthInterceptorTests
=> interceptor.UnaryServerHandler<string, string>(
"request", context, (_, _) => Task.FromResult("ok"));
[Theory]
[InlineData("notification")]
[InlineData("reconcile")]
[InlineData("health")]
[InlineData("heartbeat")]
[InlineData("audit")]
[InlineData("cached-audit")]
[InlineData("cached-operational")]
public async Task VerifiedSiteA_CannotSubmitSiteBOwnership(string kind)
{
var service = new CentralControlGrpcService(
NullLogger<CentralControlGrpcService>.Instance,
Microsoft.Extensions.Options.Options.Create(new ZB.MOM.WW.ScadaBridge.Communication.CommunicationOptions()));
object request = kind switch
{
"notification" => new NotificationSubmitDto { SourceSiteId = SiteB },
"reconcile" => new ReconcileSiteRequestDto { SiteIdentifier = SiteB },
"health" => new SiteHealthReportDto { SiteId = SiteB },
"heartbeat" => new HeartbeatDto { SiteId = SiteB },
"audit" => new AuditEventBatch { Events = { new AuditEventDto { SourceSiteId = SiteA }, new AuditEventDto { SourceSiteId = SiteB } } },
_ => new CachedTelemetryBatch { Packets = { new CachedTelemetryPacket {
AuditEvent = new AuditEventDto { SourceSiteId = kind == "cached-audit" ? SiteB : SiteA },
Operational = new SiteCallOperationalDto { SourceSite = kind == "cached-operational" ? SiteB : SiteA }
} } }
};
var continued = false;
var ex = await Assert.ThrowsAsync<RpcException>(() => CreateInterceptor().UnaryServerHandler<object, object>(
request, CreateContext(ControlMethod, SiteA, $"Bearer {SiteAKey}"), async (r, context) =>
{
continued = true;
return r switch
{
NotificationSubmitDto n => await service.SubmitNotification(n, context),
ReconcileSiteRequestDto n => await service.ReconcileSite(n, context),
SiteHealthReportDto n => await service.ReportSiteHealth(n, context),
HeartbeatDto n => await service.Heartbeat(n, context),
AuditEventBatch n => await service.IngestAuditEvents(n, context),
CachedTelemetryBatch n => await service.IngestCachedTelemetry(n, context),
_ => throw new InvalidOperationException()
};
}));
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
Assert.False(continued); // no actor dispatch, token mint, or partial batch ingest
}
[Fact]
public async Task NonGatedMethod_PassesThrough()
{
@@ -47,6 +47,7 @@ public class CentralControlEndToEndTests : IAsyncLifetime
// bridge can be asserted without extracting ids out of a decoded AuditEvent.
private static readonly Guid AcceptedId = Guid.Parse("11111111-1111-1111-1111-111111111111");
private int _dispatches;
private IHost _host = null!;
private TestServer _server = null!;
private ActorSystem _actorSystem = null!;
@@ -56,7 +57,7 @@ public class CentralControlEndToEndTests : IAsyncLifetime
public async Task InitializeAsync()
{
_actorSystem = ActorSystem.Create("centralcontrol-e2e-test");
var stub = _actorSystem.ActorOf(Props.Create(() => new StubCentralActor(AcceptedId)), "central-stub");
var stub = _actorSystem.ActorOf(Props.Create(() => new StubCentralActor(AcceptedId, () => Interlocked.Increment(ref _dispatches))), "central-stub");
_service = new CentralControlGrpcService(
NullLogger<CentralControlGrpcService>.Instance,
@@ -117,6 +118,35 @@ public class CentralControlEndToEndTests : IAsyncLifetime
// ---- Auth positives / negatives, all through the real pipeline ----
[Theory]
[InlineData("reconcile")]
[InlineData("notification")]
[InlineData("audit")]
[InlineData("cached")]
[InlineData("health")]
[InlineData("heartbeat")]
public async Task SiteAKey_WithForeignOwner_IsDeniedBeforeAnyActorDispatch(string rpc)
{
using var channel = Channel(SiteAKey, SiteA);
var client = new CentralControlService.CentralControlServiceClient(channel);
var deadline = DateTime.UtcNow.AddSeconds(2);
var error = await Assert.ThrowsAsync<RpcException>(async () =>
{
switch (rpc)
{
case "reconcile": await client.ReconcileSiteAsync(new ReconcileSiteRequestDto { SiteIdentifier = SiteB }, deadline: deadline); break;
case "notification": await client.SubmitNotificationAsync(new NotificationSubmitDto { SourceSiteId = SiteB }, deadline: deadline); break;
case "audit": await client.IngestAuditEventsAsync(new AuditEventBatch { Events = { new AuditEventDto { SourceSiteId = SiteA }, new AuditEventDto { SourceSiteId = SiteB } } }, deadline: deadline); break;
case "cached": await client.IngestCachedTelemetryAsync(new CachedTelemetryBatch { Packets = { new CachedTelemetryPacket {
AuditEvent = new AuditEventDto { SourceSiteId = SiteA }, Operational = new SiteCallOperationalDto { SourceSite = SiteB } } } }, deadline: deadline); break;
case "health": await client.ReportSiteHealthAsync(new SiteHealthReportDto { SiteId = SiteB }, deadline: deadline); break;
default: await client.HeartbeatAsync(new HeartbeatDto { SiteId = SiteB }, deadline: deadline); break;
}
});
Assert.Equal(StatusCode.PermissionDenied, error.StatusCode);
Assert.Equal(0, Volatile.Read(ref _dispatches));
}
[Fact]
public async Task CorrectSiteAndKey_ReachesTheService_OnAUnaryCall()
{
@@ -211,6 +241,7 @@ public class CentralControlEndToEndTests : IAsyncLifetime
{
NotificationId = Guid.NewGuid().ToString(),
ListName = "ops",
SourceSiteId = SiteA,
SiteEnqueuedAt = Timestamp.FromDateTime(
DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 0, 0), DateTimeKind.Utc)),
};
@@ -249,13 +280,14 @@ public class CentralControlEndToEndTests : IAsyncLifetime
/// </summary>
private sealed class StubCentralActor : ReceiveActor
{
public StubCentralActor(Guid acceptedId)
public StubCentralActor(Guid acceptedId, Action dispatched)
{
Receive<NotificationSubmit>(msg =>
Sender.Tell(new NotificationSubmitAck(msg.NotificationId, Accepted: true, Error: null)));
Receive<NotificationSubmit>(msg => { dispatched();
Sender.Tell(new NotificationSubmitAck(msg.NotificationId, Accepted: true, Error: null)); });
Receive<IngestAuditEventsCommand>(_ =>
Sender.Tell(new IngestAuditEventsReply(new[] { acceptedId })));
Receive<IngestAuditEventsCommand>(_ => { dispatched();
Sender.Tell(new IngestAuditEventsReply(new[] { acceptedId })); });
ReceiveAny(_ => dispatched());
}
}
}
@@ -83,7 +83,7 @@ public class ControlPlaneAuthEndToEndTests : IAsyncLifetime
/// </summary>
private GrpcChannel Channel(string? key, string siteId = "site-a")
{
var options = new GrpcChannelOptions { HttpHandler = _server.CreateHandler() };
var options = new GrpcChannelOptions { HttpHandler = _server.CreateHandler() }.WithMessageLimits();
if (key is not null)
{
options.WithSiteCredentials(new FixedPskProvider(key), siteId);
@@ -116,10 +116,25 @@ public class ControlPlaneAuthEndToEndTests : IAsyncLifetime
IServerStreamWriter<SiteStreamEvent> responseStream,
ServerCallContext context)
{
await responseStream.WriteAsync(new SiteStreamEvent { CorrelationId = request.CorrelationId });
await responseStream.WriteAsync(new SiteStreamEvent
{
CorrelationId = request.CorrelationId == "large-audit-window"
? new string('x', 17 * 1024 * 1024)
: request.CorrelationId,
});
}
}
[Fact]
public async Task CM1_ControlPlaneChannelReceivesWorstCaseAuditSizedMessage()
{
using var channel = Channel(SiteKey);
var client = new SiteStreamService.SiteStreamServiceClient(channel);
using var call = client.SubscribeInstance(new InstanceStreamRequest { CorrelationId = "large-audit-window" });
Assert.True(await call.ResponseStream.MoveNext(CancellationToken.None));
Assert.Equal(17 * 1024 * 1024, call.ResponseStream.Current.CorrelationId.Length);
}
[Fact]
public async Task CorrectKey_IsAccepted_OnAUnaryCall()
{
@@ -1,3 +1,4 @@
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using System.Collections.Concurrent;
using Akka.Actor;
using Microsoft.AspNetCore.Builder;
@@ -69,16 +70,32 @@ public class GrpcCentralTransportTests : IAsyncLifetime
/// instead of reaching its TestServer. See <see cref="GrpcStatusHandler"/>.
/// </summary>
private Grpc.Core.StatusCode? _nodeAForcedStatus;
private bool _loseReconcileResponse;
private HttpMessageHandler HandlerFor(string endpoint) => endpoint == EndpointA
? new GrpcStatusHandler(
new ToggleHandler(_nodeA.Server.CreateHandler(), () => _nodeA.IsUp),
new LostReconcileResponseHandler(new ToggleHandler(_nodeA.Server.CreateHandler(), () => _nodeA.IsUp), () => _loseReconcileResponse),
() => _nodeAForcedStatus)
: new ToggleHandler(_nodeB.Server.CreateHandler(), () => _nodeB.IsUp);
private GrpcCentralTransport NewTransport(CentralChannelProvider provider, CommunicationOptions? options = null)
=> new(provider, options ?? new CommunicationOptions(), NullLogger<GrpcCentralTransport>.Instance);
[Fact]
public void ReconcileAppliedThenResponseLost_DoesNotMintAgainOnPeer()
{
using var provider = NewProvider();
_loseReconcileResponse = true;
var inbox = new Capture(_system);
NewTransport(provider).ReconcileSite(new ReconcileSiteRequest(SiteA, "node-a", new Dictionary<string, string>()), inbox.Ref);
var failure = Assert.IsType<Akka.Actor.Status.Failure>(inbox.Receive(TimeSpan.FromSeconds(5)));
var rpc = Assert.IsType<Grpc.Core.RpcException>(failure.Cause);
Assert.Equal(Grpc.Core.StatusCode.Aborted, rpc.StatusCode);
Assert.Contains("OUTCOME_UNKNOWN", rpc.Status.Detail);
Assert.Equal(1, _nodeA.ReconcileCount);
Assert.Equal(0, _nodeB.ReconcileCount);
}
[Fact]
public async Task HappyPath_ReachesThePreferredNode_AndRoutesTheAckBack()
{
@@ -343,6 +360,21 @@ public class GrpcCentralTransportTests : IAsyncLifetime
}
/// <summary>A gRPC channel handler that throws (a refused connection) while its node is "down".</summary>
private sealed class LostReconcileResponseHandler(HttpMessageHandler inner, Func<bool> lose) : DelegatingHandler(inner)
{
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct)
{
var response = await base.SendAsync(request, ct);
if (!lose() || !request.RequestUri!.AbsolutePath.EndsWith("/ReconcileSite", StringComparison.Ordinal)) return response;
await response.Content.ReadAsByteArrayAsync(ct); // application has replied; only its response is lost
response.Dispose();
var failure = new HttpResponseMessage(System.Net.HttpStatusCode.OK) { Version = new Version(2, 0), Content = new ByteArrayContent([]), RequestMessage = request };
failure.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/grpc");
failure.Headers.TryAddWithoutValidation("grpc-status", "14");
return failure;
}
}
private sealed class ToggleHandler : DelegatingHandler
{
private readonly Func<bool> _isUp;
@@ -382,6 +414,7 @@ public class GrpcCentralTransportTests : IAsyncLifetime
public TestServer Server { get; private set; } = null!;
public volatile bool IsUp = true;
public int SubmitCount => _counters.Submits;
public int ReconcileCount => Volatile.Read(ref _counters.Reconciles);
public static async Task<CentralNode> StartAsync(
ActorSystem system, string label, string site, string key, bool repliesToSubmit)
@@ -429,6 +462,7 @@ public class GrpcCentralTransportTests : IAsyncLifetime
private sealed class StubCounters
{
private int _submits;
public int Reconciles;
public int Submits => Volatile.Read(ref _submits);
public void IncrementSubmits() => Interlocked.Increment(ref _submits);
public volatile bool BlackHole;
@@ -447,6 +481,12 @@ public class GrpcCentralTransportTests : IAsyncLifetime
}
});
Receive<ReconcileSiteRequest>(_ =>
{
Interlocked.Increment(ref counters.Reconciles);
Sender.Tell(new ReconcileSiteResponse([], [], "http://central/"));
});
// Heartbeat lands here as a Tell (the failback probe); ignore it, no reply expected.
ReceiveAny(_ => { });
}
@@ -114,9 +114,9 @@ public sealed class GrpcSiteTransportFailoverTests : IAsyncLifetime
}
[Fact]
public async Task Unavailable_OnNodeA_FailsOverToNodeB_AndStaysSticky()
public async Task NotDispatched_OnNodeA_FailsOverToNodeB_AndStaysSticky()
{
_stubA.ThrowUnavailable = true;
_stubA.RejectBeforeDispatch = true;
var reply = await CallAsync();
@@ -135,12 +135,12 @@ public sealed class GrpcSiteTransportFailoverTests : IAsyncLifetime
[Fact]
public async Task Failback_ReturnsToPreferredNodeA_OncePreferredIsReachableAgain()
{
_stubA.ThrowUnavailable = true;
_stubA.RejectBeforeDispatch = true;
await CallAsync(); // flips to B
Assert.False(_provider.IsOnPreferredNode(SiteId));
// NodeA recovers; the failback probe reports it reachable.
_stubA.ThrowUnavailable = false;
_stubA.RejectBeforeDispatch = false;
_preferredReachable = true;
var back = await _provider.TryFailbackAsync(SiteId, CancellationToken.None);
@@ -156,7 +156,7 @@ public sealed class GrpcSiteTransportFailoverTests : IAsyncLifetime
public async Task DeadlineExceeded_OnNodeA_IsNotRetriedOnNodeB()
{
// A DeadlineExceeded is ambiguous (a WriteTag/Deploy/Failover may already have executed), so
// — unlike Unavailable — it must NOT fail over to B. Modelled by A returning the status
// it must NOT fail over to B. Modelled by A returning the status
// directly, isolating the retry-decision from TestServer's own timeout mechanics.
_stubA.StatusToThrow = StatusCode.DeadlineExceeded;
@@ -168,6 +168,19 @@ public sealed class GrpcSiteTransportFailoverTests : IAsyncLifetime
Assert.True(_provider.IsOnPreferredNode(SiteId), "a deadline must not flip stickiness");
}
[Fact]
public async Task AmbiguousUnavailable_DoesNotReplayLifecycleOnPeer()
{
_stubA.StatusToThrow = StatusCode.Unavailable;
var error = await Assert.ThrowsAsync<RpcException>(() => CallAsync());
Assert.Equal(StatusCode.Aborted, error.StatusCode);
Assert.Contains("OUTCOME_UNKNOWN", error.Status.Detail);
Assert.Equal(1, _stubA.LifecycleCalls);
Assert.Equal(0, _stubB.LifecycleCalls);
}
[Fact]
public async Task UnknownSite_Throws_SiteChannelUnavailable()
{
@@ -190,7 +203,7 @@ public sealed class GrpcSiteTransportFailoverTests : IAsyncLifetime
public string? LastAuthHeader { get; private set; }
public string? LastSiteHeader { get; private set; }
public bool DeadlineWasSet { get; private set; }
public volatile bool ThrowUnavailable;
public volatile bool RejectBeforeDispatch;
public StatusCode? StatusToThrow;
public override Task<LifecycleReply> ExecuteLifecycle(LifecycleRequest request, ServerCallContext context)
@@ -202,9 +215,9 @@ public sealed class GrpcSiteTransportFailoverTests : IAsyncLifetime
.FirstOrDefault(h => h.Key == ControlPlaneCredentials.SiteHeader)?.Value;
DeadlineWasSet = context.Deadline != DateTime.MaxValue;
if (ThrowUnavailable)
if (RejectBeforeDispatch)
{
throw new RpcException(new Status(StatusCode.Unavailable, "node down"));
throw GrpcRetrySafety.NotDispatched("node not ready before dispatch");
}
if (StatusToThrow is { } status)
{
@@ -0,0 +1,136 @@
using System.Net;
using System.Security.Claims;
using Microsoft.AspNetCore.Antiforgery;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication.Cookies;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.DependencyInjection;
using ZB.MOM.WW.Auth.Abstractions.Roles;
using ZB.MOM.WW.ScadaBridge.CentralUI.Auth;
using ZB.MOM.WW.ScadaBridge.Security;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
public class SessionActivityHttpIntegrationTests
{
[Fact]
public async Task PassivePing_RoleReplacementDoesNotRenewCookie_AndIdleExpires()
{
await using var fixture = await Fixture.CreateAsync();
await fixture.LoginAsync();
fixture.Clock.Advance(16);
using var alive = await fixture.SendAsync(HttpMethod.Get, "/auth/ping");
Assert.Equal(HttpStatusCode.OK, alive.StatusCode);
Assert.False(alive.Headers.Contains("Set-Cookie"));
fixture.Clock.Advance(15);
using var expired = await fixture.SendAsync(HttpMethod.Get, "/auth/ping");
Assert.Equal(HttpStatusCode.Unauthorized, expired.StatusCode);
Assert.False((await fixture.Circuit!.GetAuthoritativeStateAsync()).User.Identity!.IsAuthenticated);
}
[Fact]
public async Task AntiforgeryActivity_RenewsHttpAndExistingCircuit_ThenBothExpire()
{
await using var fixture = await Fixture.CreateAsync();
await fixture.LoginAsync();
var token = await fixture.TokenAsync();
fixture.Clock.Advance(20);
using var activity = await fixture.SendAsync(HttpMethod.Post, "/auth/activity", token);
Assert.Equal(HttpStatusCode.NoContent, activity.StatusCode);
Assert.True(activity.Headers.Contains("Set-Cookie"));
fixture.Clock.Advance(15);
using var alive = await fixture.SendAsync(HttpMethod.Get, "/auth/ping");
Assert.Equal(HttpStatusCode.OK, alive.StatusCode);
Assert.True((await fixture.Circuit!.GetAuthoritativeStateAsync()).User.Identity!.IsAuthenticated);
using var again = await fixture.SendAsync(HttpMethod.Post, "/auth/activity", token);
Assert.Equal(HttpStatusCode.NoContent, again.StatusCode);
fixture.Clock.Advance(31);
using var expired = await fixture.SendAsync(HttpMethod.Get, "/auth/ping");
Assert.Equal(HttpStatusCode.Unauthorized, expired.StatusCode);
Assert.False((await fixture.Circuit.GetAuthoritativeStateAsync()).User.Identity!.IsAuthenticated);
}
[Fact]
public async Task MissingAntiforgeryDoesNotRenew_AndExpiredActivityCannotRevive()
{
await using var fixture = await Fixture.CreateAsync();
await fixture.LoginAsync();
var token = await fixture.TokenAsync();
fixture.Clock.Advance(20);
using var rejected = await fixture.SendAsync(HttpMethod.Post, "/auth/activity");
Assert.Equal(HttpStatusCode.BadRequest, rejected.StatusCode);
Assert.False(rejected.Headers.Contains("Set-Cookie"));
fixture.Clock.Advance(11);
using var expired = await fixture.SendAsync(HttpMethod.Post, "/auth/activity", token);
Assert.Equal(HttpStatusCode.Unauthorized, expired.StatusCode);
Assert.False((await fixture.Circuit!.GetAuthoritativeStateAsync()).User.Identity!.IsAuthenticated);
}
private sealed class Fixture : IAsyncDisposable
{
private readonly WebApplication _app;
private HttpClient _client = null!;
private readonly Dictionary<string, string> _cookies = [];
public Clock Clock { get; }
public CookieAuthenticationStateProvider? Circuit { get; private set; }
private Fixture(WebApplication app, Clock clock) { _app = app; Clock = clock; }
public static async Task<Fixture> CreateAsync()
{
var builder = WebApplication.CreateBuilder();
builder.WebHost.UseTestServer();
var clock = new Clock();
builder.Services.AddSingleton<TimeProvider>(clock);
builder.Services.AddSecurity();
builder.Services.Configure<SecurityOptions>(o => { o.RequireHttpsCookie = false; o.JwtSigningKey = new string('x', 64); });
builder.Services.AddSingleton<IGroupRoleMapper<string>, Mapper>();
builder.Services.AddOptions<CookieAuthenticationOptions>("Cookies").PostConfigure(o => o.TimeProvider = clock);
builder.Services.AddAntiforgery(o => o.HeaderName = "X-CSRF-TOKEN");
var app = builder.Build();
var fixture = new Fixture(app, clock);
app.UseAuthentication();
app.UseAuthorization();
app.MapGet("/test/login", async (HttpContext context) =>
{
var principal = SessionClaimBuilder.Build("alice", "Alice", ["group"],
new RoleMappingResult([Roles.Deployer], ["1"], false), clock.GetUtcNow());
fixture.Circuit = new CookieAuthenticationStateProvider(new HttpContextAccessor
{ HttpContext = new DefaultHttpContext { User = principal } }, app.Services.GetRequiredService<IServiceScopeFactory>());
await context.SignInAsync("Cookies", principal);
});
app.MapGet("/test/token", (HttpContext context, IAntiforgery antiforgery) =>
antiforgery.GetAndStoreTokens(context).RequestToken!);
app.MapAuthEndpoints();
await app.StartAsync();
fixture._client = app.GetTestClient();
return fixture;
}
public async Task LoginAsync() { using var response = await SendAsync(HttpMethod.Get, "/test/login"); response.EnsureSuccessStatusCode(); }
public async Task<string> TokenAsync() { using var response = await SendAsync(HttpMethod.Get, "/test/token"); response.EnsureSuccessStatusCode(); return await response.Content.ReadAsStringAsync(); }
public async Task<HttpResponseMessage> SendAsync(HttpMethod method, string path, string? token = null)
{
using var request = new HttpRequestMessage(method, path);
if (_cookies.Count > 0) request.Headers.Add("Cookie", string.Join("; ", _cookies.Select(c => $"{c.Key}={c.Value}")));
if (token is not null) request.Headers.Add("X-CSRF-TOKEN", token);
var response = await _client.SendAsync(request);
if (response.Headers.TryGetValues("Set-Cookie", out var values))
foreach (var value in values) { var pair = value.Split(';')[0].Split('=', 2); _cookies[pair[0]] = pair[1]; }
return response;
}
public async ValueTask DisposeAsync() { _client.Dispose(); await _app.DisposeAsync(); }
}
private sealed class Clock : TimeProvider
{
public DateTimeOffset Now { get; set; } = new(2026, 9, 9, 12, 0, 0, TimeSpan.Zero);
public override DateTimeOffset GetUtcNow() => Now;
public void Advance(int minutes) => Now = Now.AddMinutes(minutes);
}
private sealed class Mapper : IGroupRoleMapper<string>
{
public Task<GroupRoleMapping<string>> MapAsync(IReadOnlyList<string> groups, CancellationToken ct)
=> Task.FromResult(new GroupRoleMapping<string>([Roles.Deployer], new RoleMappingResult([Roles.Deployer], ["1"], false)));
}
}