perf(comms+audit): close phase-2 residuals — direct ingest path, monotonic timeouts, synthetic probe, not-reporting set, cursor-exact audit pull

2026-08-14 21:38:23 -04:00
parent 4cd1441984
commit a5882753dd
38 changed files with 1254 additions and 443 deletions
@@ -0,0 +1,201 @@
+using System.Buffers.Binary;
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Google.Protobuf.WellKnownTypes;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
+using ZB.MOM.WW.ScadaBridge.Communication.Actors;
+using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
+using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
+
+namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
+
+/// <summary>
+/// Arch-review phase-2 residual #3: <see cref="CentralChannelProvider"/>'s failback probe reuses
+/// the <c>Heartbeat</c> RPC to ask "does the preferred central endpoint answer again?". It is
+/// emitted by a site's TRANSPORT layer, not by any node's heartbeat timer, so central must not
+/// count it as liveness — otherwise a site whose real heartbeats had stopped keeps looking alive
+/// on the health dashboard for as long as its transport keeps probing.
+/// <para>
+/// The contract is the explicit additive <c>Synthetic</c> flag (<c>HeartbeatDto.synthetic</c>,
+/// proto field 5), NOT the <c>failback-probe</c> hostname, which is a log label only.
+/// </para>
+/// </summary>
+public class SyntheticHeartbeatTests : TestKit
+{
+    private static readonly DateTimeOffset T0 =
+        new(2026, 8, 14, 9, 0, 0, TimeSpan.Zero);
+
+    // ── The consumer: central skips liveness bookkeeping for a synthetic heartbeat ───
+
+    [Fact]
+    public void SyntheticHeartbeat_DoesNotMarkTheHealthAggregator()
+    {
+        var (actor, aggregator) = CreateCentralActor();
+
+        actor.Tell(new HeartbeatMessage("site-1", CentralChannelProvider.SyntheticProbeHostname,
+            IsActive: false, Timestamp: T0, Synthetic: true));
+
+        // Give the actor a real chance to (wrongly) mark before asserting the negative.
+        ExpectNoMsg(TimeSpan.FromMilliseconds(300));
+        aggregator.DidNotReceiveWithAnyArgs().MarkHeartbeat(default!, default);
+    }
+
+    [Fact]
+    public void RealHeartbeat_StillMarksTheHealthAggregator()
+    {
+        // The guard must be keyed on the flag alone — an ordinary heartbeat is unaffected,
+        // including one from a node that predates the field (proto3 defaults it to false).
+        var (actor, aggregator) = CreateCentralActor();
+
+        actor.Tell(new HeartbeatMessage("site-1", "node-a", IsActive: true, Timestamp: T0));
+
+        AwaitAssert(() => aggregator.Received(1).MarkHeartbeat("site-1", T0));
+    }
+
+    [Fact]
+    public void SyntheticHeartbeatReplica_IsAlsoSkipped()
+    {
+        // Belt-and-braces on the last hop before the aggregator: a peer central node that
+        // predates the flag could still replicate one.
+        var (actor, aggregator) = CreateCentralActor();
+
+        actor.Tell(new SiteHeartbeatReplica(new HeartbeatMessage(
+            "site-1", CentralChannelProvider.SyntheticProbeHostname,
+            IsActive: false, Timestamp: T0, Synthetic: true)));
+
+        ExpectNoMsg(TimeSpan.FromMilliseconds(300));
+        aggregator.DidNotReceiveWithAnyArgs().MarkHeartbeat(default!, default);
+    }
+
+    // ── The wire contract: the flag survives the round trip ─────────────────────────
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public void TheSyntheticFlag_RoundTripsThroughTheDto(bool synthetic)
+    {
+        var msg = new HeartbeatMessage("site-1", "node-a", IsActive: true, Timestamp: T0,
+            Synthetic: synthetic);
+
+        var back = CentralControlDtoMapper.FromDto(CentralControlDtoMapper.ToDto(msg));
+
+        Assert.Equal(synthetic, back.Synthetic);
+        Assert.Equal(msg with { Synthetic = synthetic }, back);
+    }
+
+    [Fact]
+    public void ADtoFromAnOlderSite_DefaultsToNotSynthetic()
+    {
+        // proto3 default: a peer that never sets field 5 sends a REAL heartbeat, as before.
+        var dto = new HeartbeatDto
+        {
+            SiteId = "site-1",
+            NodeHostname = "node-a",
+            IsActive = true,
+            Timestamp = Timestamp.FromDateTimeOffset(T0),
+        };
+
+        Assert.False(CentralControlDtoMapper.FromDto(dto).Synthetic);
+    }
+
+    // ── The producer: the failback probe marks itself synthetic ─────────────────────
+
+    [Fact]
+    public async Task TheFailbackProbe_MarksItsHeartbeatSynthetic()
+    {
+        // Two endpoints so a flip is possible; the capture handler answers nothing useful, so
+        // the probe faults and re-arms — we only care about the request it put on the wire.
+        var capture = new HeartbeatCapturingHandler();
+        using var provider = new CentralChannelProvider(
+            new[] { "http://central-a:8083", "http://central-b:8083" },
+            new FixedPskProvider("k"),
+            "site-1",
+            new CommunicationOptions(),
+            NullLogger.Instance,
+            handlerFactory: _ => capture,
+            probeDeadline: TimeSpan.FromSeconds(2),
+            backoffBase: TimeSpan.FromMilliseconds(20),
+            backoffCap: TimeSpan.FromMilliseconds(50));
+
+        // Off the preferred endpoint → the background failback probe arms.
+        provider.ReportUnavailable(0);
+
+        var probe = await capture.WaitForHeartbeatAsync(TimeSpan.FromSeconds(10));
+
+        Assert.True(probe.Synthetic);
+        Assert.Equal("site-1", probe.SiteId);
+        // The hostname is a human-readable label that rides ALONGSIDE the flag; central keys
+        // its skip on the flag, never on this string.
+        Assert.Equal(CentralChannelProvider.SyntheticProbeHostname, probe.NodeHostname);
+        Assert.False(probe.IsActive);
+    }
+
+    private (IActorRef Actor, ICentralHealthAggregator Aggregator) CreateCentralActor()
+    {
+        var siteRepo = Substitute.For<ISiteRepository>();
+        siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<Site>());
+
+        var aggregator = Substitute.For<ICentralHealthAggregator>();
+
+        var services = new ServiceCollection();
+        services.AddScoped(_ => siteRepo);
+        services.AddSingleton(aggregator);
+        var sp = services.BuildServiceProvider();
+
+        var actor = Sys.ActorOf(Props.Create(() =>
+            new CentralCommunicationActor(sp, Substitute.For<ISiteCommandTransport>())));
+        return (actor, aggregator);
+    }
+
+    private sealed class FixedPskProvider(string key) : ISitePskProvider
+    {
+        public ValueTask<string> GetAsync(string siteId, CancellationToken ct) => new(key);
+        public void Invalidate(string siteId) { }
+    }
+
+    /// <summary>
+    /// Captures the first <c>Heartbeat</c> request body and decodes the length-prefixed gRPC
+    /// frame back into a <see cref="HeartbeatDto"/>. The response is deliberately a bare 500 so
+    /// the probe treats the endpoint as still down; the provider swallows that and re-arms.
+    /// </summary>
+    private sealed class HeartbeatCapturingHandler : HttpMessageHandler
+    {
+        private readonly TaskCompletionSource<HeartbeatDto> _captured =
+            new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+        public async Task<HeartbeatDto> WaitForHeartbeatAsync(TimeSpan timeout)
+        {
+            var completed = await Task.WhenAny(_captured.Task, Task.Delay(timeout));
+            Assert.Same(_captured.Task, completed);
+            return await _captured.Task;
+        }
+
+        protected override async Task<HttpResponseMessage> SendAsync(
+            HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            if (request.Content is not null &&
+                request.RequestUri?.AbsolutePath.EndsWith("/Heartbeat", StringComparison.Ordinal) == true)
+            {
+                var body = await request.Content.ReadAsByteArrayAsync(cancellationToken);
+                // gRPC frame: 1 compression byte + 4-byte big-endian length + payload.
+                if (body.Length >= 5)
+                {
+                    var length = BinaryPrimitives.ReadInt32BigEndian(body.AsSpan(1, 4));
+                    _captured.TrySetResult(
+                        HeartbeatDto.Parser.ParseFrom(body.AsSpan(5, length).ToArray()));
+                }
+            }
+
+            return new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError)
+            {
+                Version = request.Version,
+            };
+        }
+    }
+}