using System.Buffers.Binary; using Akka.Actor; using Akka.TestKit.Xunit2; using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health; using ZB.MOM.WW.ScadaBridge.Communication.Actors; using ZB.MOM.WW.ScadaBridge.Communication.Grpc; using ZB.MOM.WW.ScadaBridge.HealthMonitoring; namespace ZB.MOM.WW.ScadaBridge.Communication.Tests; /// /// Arch-review phase-2 residual #3: 's failback probe reuses /// the Heartbeat RPC to ask "does the preferred central endpoint answer again?". It is /// emitted by a site's TRANSPORT layer, not by any node's heartbeat timer, so central must not /// count it as liveness — otherwise a site whose real heartbeats had stopped keeps looking alive /// on the health dashboard for as long as its transport keeps probing. /// /// The contract is the explicit additive Synthetic flag (HeartbeatDto.synthetic, /// proto field 5), NOT the failback-probe hostname, which is a log label only. /// /// public class SyntheticHeartbeatTests : TestKit { private static readonly DateTimeOffset T0 = new(2026, 8, 14, 9, 0, 0, TimeSpan.Zero); // ── The consumer: central skips liveness bookkeeping for a synthetic heartbeat ─── [Fact] public void SyntheticHeartbeat_DoesNotMarkTheHealthAggregator() { var (actor, aggregator) = CreateCentralActor(); actor.Tell(new HeartbeatMessage("site-1", CentralChannelProvider.SyntheticProbeHostname, IsActive: false, Timestamp: T0, Synthetic: true)); // Give the actor a real chance to (wrongly) mark before asserting the negative. ExpectNoMsg(TimeSpan.FromMilliseconds(300)); aggregator.DidNotReceiveWithAnyArgs().MarkHeartbeat(default!, default); } [Fact] public void RealHeartbeat_StillMarksTheHealthAggregator() { // The guard must be keyed on the flag alone — an ordinary heartbeat is unaffected, // including one from a node that predates the field (proto3 defaults it to false). var (actor, aggregator) = CreateCentralActor(); actor.Tell(new HeartbeatMessage("site-1", "node-a", IsActive: true, Timestamp: T0)); AwaitAssert(() => aggregator.Received(1).MarkHeartbeat("site-1", T0)); } [Fact] public void SyntheticHeartbeatReplica_IsAlsoSkipped() { // Belt-and-braces on the last hop before the aggregator: a peer central node that // predates the flag could still replicate one. var (actor, aggregator) = CreateCentralActor(); actor.Tell(new SiteHeartbeatReplica(new HeartbeatMessage( "site-1", CentralChannelProvider.SyntheticProbeHostname, IsActive: false, Timestamp: T0, Synthetic: true))); ExpectNoMsg(TimeSpan.FromMilliseconds(300)); aggregator.DidNotReceiveWithAnyArgs().MarkHeartbeat(default!, default); } // ── The wire contract: the flag survives the round trip ───────────────────────── [Theory] [InlineData(true)] [InlineData(false)] public void TheSyntheticFlag_RoundTripsThroughTheDto(bool synthetic) { var msg = new HeartbeatMessage("site-1", "node-a", IsActive: true, Timestamp: T0, Synthetic: synthetic); var back = CentralControlDtoMapper.FromDto(CentralControlDtoMapper.ToDto(msg)); Assert.Equal(synthetic, back.Synthetic); Assert.Equal(msg with { Synthetic = synthetic }, back); } [Fact] public void ADtoFromAnOlderSite_DefaultsToNotSynthetic() { // proto3 default: a peer that never sets field 5 sends a REAL heartbeat, as before. var dto = new HeartbeatDto { SiteId = "site-1", NodeHostname = "node-a", IsActive = true, Timestamp = Timestamp.FromDateTimeOffset(T0), }; Assert.False(CentralControlDtoMapper.FromDto(dto).Synthetic); } // ── The producer: the failback probe marks itself synthetic ───────────────────── [Fact] public async Task TheFailbackProbe_MarksItsHeartbeatSynthetic() { // Two endpoints so a flip is possible; the capture handler answers nothing useful, so // the probe faults and re-arms — we only care about the request it put on the wire. var capture = new HeartbeatCapturingHandler(); using var provider = new CentralChannelProvider( new[] { "http://central-a:8083", "http://central-b:8083" }, new FixedPskProvider("k"), "site-1", new CommunicationOptions(), NullLogger.Instance, handlerFactory: _ => capture, probeDeadline: TimeSpan.FromSeconds(2), backoffBase: TimeSpan.FromMilliseconds(20), backoffCap: TimeSpan.FromMilliseconds(50)); // Off the preferred endpoint → the background failback probe arms. provider.ReportUnavailable(0); var probe = await capture.WaitForHeartbeatAsync(TimeSpan.FromSeconds(10)); Assert.True(probe.Synthetic); Assert.Equal("site-1", probe.SiteId); // The hostname is a human-readable label that rides ALONGSIDE the flag; central keys // its skip on the flag, never on this string. Assert.Equal(CentralChannelProvider.SyntheticProbeHostname, probe.NodeHostname); Assert.False(probe.IsActive); } private (IActorRef Actor, ICentralHealthAggregator Aggregator) CreateCentralActor() { var siteRepo = Substitute.For(); siteRepo.GetAllSitesAsync(Arg.Any()).Returns(new List()); var aggregator = Substitute.For(); var services = new ServiceCollection(); services.AddScoped(_ => siteRepo); services.AddSingleton(aggregator); var sp = services.BuildServiceProvider(); var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, Substitute.For()))); return (actor, aggregator); } private sealed class FixedPskProvider(string key) : ISitePskProvider { public ValueTask GetAsync(string siteId, CancellationToken ct) => new(key); public void Invalidate(string siteId) { } } /// /// Captures the first Heartbeat request body and decodes the length-prefixed gRPC /// frame back into a . The response is deliberately a bare 500 so /// the probe treats the endpoint as still down; the provider swallows that and re-arms. /// private sealed class HeartbeatCapturingHandler : HttpMessageHandler { private readonly TaskCompletionSource _captured = new(TaskCreationOptions.RunContinuationsAsynchronously); public async Task WaitForHeartbeatAsync(TimeSpan timeout) { var completed = await Task.WhenAny(_captured.Task, Task.Delay(timeout)); Assert.Same(_captured.Task, completed); return await _captured.Task; } protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { if (request.Content is not null && request.RequestUri?.AbsolutePath.EndsWith("/Heartbeat", StringComparison.Ordinal) == true) { var body = await request.Content.ReadAsByteArrayAsync(cancellationToken); // gRPC frame: 1 compression byte + 4-byte big-endian length + payload. if (body.Length >= 5) { var length = BinaryPrimitives.ReadInt32BigEndian(body.AsSpan(1, 4)); _captured.TrySetResult( HeartbeatDto.Parser.ParseFrom(body.AsSpan(5, length).ToArray())); } } return new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError) { Version = request.Version, }; } } }