feat(grpc): host CentralControlService on the central node (T1A.2)
Central now ALSO listens for the seven site→central control messages over gRPC, alongside the existing ClusterClient path. Nothing flips to gRPC yet — sites keep CentralTransport=Akka (T1A.3's job); central simply starts also accepting. - CentralControlGrpcService (Communication.Grpc): decodes each RPC onto the SAME in-process message the ClusterClient path carries, Asks the existing CentralCommunicationActor (zero handler-logic changes), encodes the reply via the T1A.1 mapper. Readiness-gated like SiteStreamGrpcServer.SetReady — Unavailable until AkkaHostedService hands the actor over. Heartbeat stays fire-and-forget (Tell, always-OK, never gated on readiness). Ingest reuses the shared SiteStreamGrpcServer.AuditIngestAskTimeout constant. Fault→status mapping is retry-aware: Unavailable (never dispatched, safe to cross-node retry) vs DeadlineExceeded/Internal (it ran, do not re-send elsewhere). - CentralControlAuthInterceptor (Host): a SEPARATE interceptor class, not a variant constructor on ControlPlaneAuthInterceptor. Central's model is per-site (verify the Bearer token against the key for the site in the required x-scadabridge-site header, via ISitePskProvider) where a site verifies its one own-key — a genuinely different model. Fail-closed on every branch: missing or blank header, unresolvable key, and mismatched token all → PermissionDenied, never pass-through. One public constructor only (the explicit-prefix ctor is internal), pinned by a reflection test — a second public ctor makes Grpc.AspNetCore's GetFactory() throw per-call and silently disables the gate. - Explicit Kestrel h2c listener on new option ScadaBridge:Node:CentralGrpcPort (default 8083, symmetric with sites), mirroring the Site branch. Additive to central's :5000 HTTP/1 surface, which is untouched — gRPC does NOT go through Traefik (HTTP/1 only). Registered by type on AddGrpc; service mapped with MapGrpcService. Port range-validated by NodeOptionsValidator. - Rig: publish the central gRPC port 9013:8083 / 9014:8083 on both central nodes so a later task can exercise it. Tests: CentralControlEndToEndTests (Host.Tests, TestServer + real interceptor + real service over a stub actor) proves auth positives/negatives are distinguishable and covers unary + the ingest bridge shapes; the interceptor is registered BY TYPE, never in DI. CentralControlAuthInterceptorTests pins the per-site gate + one-public-ctor invariant. CentralControlGrpcServiceTests (Communication.Tests, TestKit) covers the readiness gate, fire-and-forget heartbeat, and the DeadlineExceeded-vs-Unavailable status mapping. No active <Protobuf> item. Communication.Tests (356) + Host.Tests (384) green.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,256 @@
|
||||
using Akka.Actor;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Grpc.Net.Client;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.TestHost;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end proof of the central-hosted gRPC control plane (T1A.2): the real
|
||||
/// <see cref="CentralControlAuthInterceptor"/> AND the real
|
||||
/// <see cref="CentralControlGrpcService"/> over a real gRPC stack, the service Asking a real
|
||||
/// (stub) <c>CentralCommunicationActor</c> stand-in.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The interceptor is registered <b>BY TYPE on <c>AddGrpc</c></b>, exactly as <c>Program.cs</c>
|
||||
/// does, and is deliberately NOT placed in DI as a singleton — pre-registering it lets DI hand
|
||||
/// the instance back and bypasses <c>Grpc.AspNetCore</c>'s own activation path, which is how the
|
||||
/// two-public-constructor defect escaped a green suite once before. This harness copies the
|
||||
/// shape of <see cref="ControlPlaneAuthEndToEndTests"/> for the same reason.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Runs in-process over <see cref="TestServer"/>: no ports, no containers. A tiny Akka actor
|
||||
/// stands in for <c>CentralCommunicationActor</c> so the test never touches MSSQL or a cluster;
|
||||
/// the method paths, message types and mapper are the real generated/production ones.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class CentralControlEndToEndTests : IAsyncLifetime
|
||||
{
|
||||
private const string SiteA = "site-a";
|
||||
private const string SiteAKey = "site-a-preshared-key";
|
||||
private const string SiteB = "site-b";
|
||||
private const string SiteBKey = "site-b-preshared-key";
|
||||
|
||||
// The deterministic id the stub actor "accepts" for every ingest batch, so the ingest
|
||||
// bridge can be asserted without extracting ids out of a decoded AuditEvent.
|
||||
private static readonly Guid AcceptedId = Guid.Parse("11111111-1111-1111-1111-111111111111");
|
||||
|
||||
private IHost _host = null!;
|
||||
private TestServer _server = null!;
|
||||
private ActorSystem _actorSystem = null!;
|
||||
private CentralControlGrpcService _service = null!;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task InitializeAsync()
|
||||
{
|
||||
_actorSystem = ActorSystem.Create("centralcontrol-e2e-test");
|
||||
var stub = _actorSystem.ActorOf(Props.Create(() => new StubCentralActor(AcceptedId)), "central-stub");
|
||||
|
||||
_service = new CentralControlGrpcService(
|
||||
NullLogger<CentralControlGrpcService>.Instance,
|
||||
Options.Create(new CommunicationOptions()));
|
||||
_service.SetReady(stub);
|
||||
|
||||
var pskProvider = new MapPskProvider(new Dictionary<string, string>
|
||||
{
|
||||
[SiteA] = SiteAKey,
|
||||
[SiteB] = SiteBKey,
|
||||
});
|
||||
|
||||
_host = await new HostBuilder()
|
||||
.ConfigureWebHost(web => web
|
||||
.UseTestServer()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
// BY TYPE, and NOT also in DI — see the class remarks.
|
||||
services.AddGrpc(o => o.Interceptors.Add<CentralControlAuthInterceptor>());
|
||||
services.AddSingleton<ISitePskProvider>(pskProvider);
|
||||
services.AddSingleton(_service);
|
||||
})
|
||||
.Configure(app =>
|
||||
{
|
||||
app.UseRouting();
|
||||
app.UseEndpoints(e => e.MapGrpcService<CentralControlGrpcService>());
|
||||
}))
|
||||
.StartAsync();
|
||||
|
||||
_server = _host.GetTestServer();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
await _host.StopAsync();
|
||||
_host.Dispose();
|
||||
await _actorSystem.Terminate();
|
||||
}
|
||||
|
||||
/// <summary>Builds a channel credentialed for <paramref name="siteId"/> with <paramref name="key"/>.</summary>
|
||||
private GrpcChannel Channel(string? key, string siteId)
|
||||
{
|
||||
var options = new GrpcChannelOptions { HttpHandler = _server.CreateHandler() };
|
||||
if (key is not null)
|
||||
{
|
||||
options.WithSiteCredentials(new FixedPskProvider(key), siteId);
|
||||
}
|
||||
return GrpcChannel.ForAddress(_server.BaseAddress, options);
|
||||
}
|
||||
|
||||
private CentralControlService.CentralControlServiceClient Client(string? key, string siteId)
|
||||
=> new(Channel(key, siteId));
|
||||
|
||||
// ---- Auth positives / negatives, all through the real pipeline ----
|
||||
|
||||
[Fact]
|
||||
public async Task CorrectSiteAndKey_ReachesTheService_OnAUnaryCall()
|
||||
{
|
||||
var client = Client(SiteAKey, SiteA);
|
||||
|
||||
var ack = await client.SubmitNotificationAsync(NewNotificationDto());
|
||||
|
||||
Assert.True(ack.Accepted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NoCredentialsAtAll_IsRejected_WithPermissionDenied()
|
||||
{
|
||||
var client = Client(key: null, SiteA);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await client.SubmitNotificationAsync(new NotificationSubmitDto()));
|
||||
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WrongKeyForTheSite_IsRejected_WithPermissionDenied()
|
||||
{
|
||||
// site-a presents site-b's (valid, but wrong-for-this-site) key.
|
||||
var client = Client(SiteBKey, SiteA);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await client.SubmitNotificationAsync(new NotificationSubmitDto()));
|
||||
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnknownSite_IsRejected_WithPermissionDenied()
|
||||
{
|
||||
var client = Client("any-key", "site-nonexistent");
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await client.SubmitNotificationAsync(new NotificationSubmitDto()));
|
||||
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TheAcceptAndRejectPathsAreDistinguishable()
|
||||
{
|
||||
// A gate whose accept and reject paths produce the same observable result is not a gate.
|
||||
// Correct key → the service answers (Accepted:true); wrong key → PermissionDenied,
|
||||
// never reaching the service. These are two different outcomes, which is the whole point.
|
||||
var ok = await Client(SiteAKey, SiteA).SubmitNotificationAsync(NewNotificationDto());
|
||||
Assert.True(ok.Accepted);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(
|
||||
async () => await Client("wrong", SiteA).SubmitNotificationAsync(new NotificationSubmitDto()));
|
||||
Assert.Equal(StatusCode.PermissionDenied, ex.StatusCode);
|
||||
}
|
||||
|
||||
// ---- One RPC per shape: unary (above) + the ingest bridge ----
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAuditEvents_DecodesTheBatch_AsksTheActor_EncodesTheAck()
|
||||
{
|
||||
var client = Client(SiteAKey, SiteA);
|
||||
|
||||
var batch = new AuditEventBatch();
|
||||
batch.Events.Add(NewAuditDto());
|
||||
batch.Events.Add(NewAuditDto());
|
||||
|
||||
var ack = await client.IngestAuditEventsAsync(batch);
|
||||
|
||||
// The stub actor accepts one deterministic id per non-empty batch; its presence proves
|
||||
// the full DTO→command→Ask→reply→ack bridge ran, gated call and all.
|
||||
Assert.Contains(AcceptedId.ToString(), ack.AcceptedEventIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAuditEvents_EmptyBatch_ShortCircuits_WithoutAskingTheActor()
|
||||
{
|
||||
// Even the empty-batch fast path is behind the gate — it still needs a valid key.
|
||||
var client = Client(SiteAKey, SiteA);
|
||||
|
||||
var ack = await client.IngestAuditEventsAsync(new AuditEventBatch());
|
||||
|
||||
Assert.Empty(ack.AcceptedEventIds);
|
||||
}
|
||||
|
||||
// The mapper reads SiteEnqueuedAt unconditionally; only DTOs that clear the gate reach it,
|
||||
// so the accept-path tests carry a timestamp while the negative tests can pass a bare DTO.
|
||||
private static NotificationSubmitDto NewNotificationDto() => new()
|
||||
{
|
||||
NotificationId = Guid.NewGuid().ToString(),
|
||||
ListName = "ops",
|
||||
SiteEnqueuedAt = Timestamp.FromDateTime(
|
||||
DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 0, 0), DateTimeKind.Utc)),
|
||||
};
|
||||
|
||||
private static AuditEventDto NewAuditDto() => new()
|
||||
{
|
||||
EventId = Guid.NewGuid().ToString(),
|
||||
OccurredAtUtc = Timestamp.FromDateTime(
|
||||
DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 0, 0), DateTimeKind.Utc)),
|
||||
Channel = "ApiOutbound",
|
||||
Kind = "ApiCall",
|
||||
Status = "Delivered",
|
||||
SourceSiteId = SiteA,
|
||||
};
|
||||
|
||||
private sealed class FixedPskProvider(string key) : ISitePskProvider
|
||||
{
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct) => new(key);
|
||||
public void Invalidate(string siteId) { }
|
||||
}
|
||||
|
||||
private sealed class MapPskProvider(IReadOnlyDictionary<string, string> keys) : ISitePskProvider
|
||||
{
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct)
|
||||
=> keys.TryGetValue(siteId, out var key)
|
||||
? new ValueTask<string>(key)
|
||||
: throw new InvalidOperationException($"no key for '{siteId}'");
|
||||
|
||||
public void Invalidate(string siteId) { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal stand-in for <c>CentralCommunicationActor</c>: answers the two RPC shapes this
|
||||
/// test exercises. Replies straight to the Ask's temp sender, exactly as the real actor's
|
||||
/// Forward/PipeTo paths do.
|
||||
/// </summary>
|
||||
private sealed class StubCentralActor : ReceiveActor
|
||||
{
|
||||
public StubCentralActor(Guid acceptedId)
|
||||
{
|
||||
Receive<NotificationSubmit>(msg =>
|
||||
Sender.Tell(new NotificationSubmitAck(msg.NotificationId, Accepted: true, Error: null)));
|
||||
|
||||
Receive<IngestAuditEventsCommand>(_ =>
|
||||
Sender.Tell(new IngestAuditEventsReply(new[] { acceptedId })));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user