feat(saf): resync ack confirmation + completed/ack-missing telemetry; doc resync rewrite (plan R2-02 T7)

tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/SiteReplicationActorTests.cs

@@ -1,4 +1,5 @@
 using System.Collections.Concurrent;
+using System.Diagnostics.Metrics;
 using Akka.Actor;
 using Akka.TestKit.Xunit2;
 using Microsoft.Extensions.Logging;
@@ -9,6 +10,7 @@ using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
 using ZB.MOM.WW.ScadaBridge.StoreAndForward;
+using ZB.MOM.WW.ScadaBridge.Commons.Observability;
 using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
 
 namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
@@ -386,6 +388,69 @@ akka {
         ExpectNoMsg(TimeSpan.FromMilliseconds(300));
     }
 
+    // ── R2 T7: active-side resync ack confirmation + telemetry ──
+    //
+    // NOTE (deviation from plan): the actor logs via Microsoft ILogger (NullLogger in
+    // tests), NOT Akka's EventStream, so the plan's EventFilter.Warning assertions can
+    // never observe these warnings. We observe the two OTel counters via a MeterListener
+    // instead — the equivalent, and stronger, observable signal.
+
+    [Fact]
+    public async Task ActiveNode_ReceivingAck_CountsResyncCompleted()
+    {
+        long completed = 0;
+        using var listener = ListenCounter("scadabridge.store_and_forward.resync.completed",
+            m => Interlocked.Add(ref completed, m));
+
+        await _sfStorage.EnqueueAsync(NewMessage("m1"));
+        var actor = CreateResyncActor(isActive: () => true, ackTimeout: TimeSpan.FromSeconds(30));
+        actor.Tell(new RequestSfBufferResync(), TestActor);
+        var chunk = ExpectMsg<SfBufferSnapshotChunk>(TimeSpan.FromSeconds(5));
+
+        actor.Tell(new SfBufferResyncAck(chunk.ResyncId, 1), TestActor);
+
+        await AwaitAssertAsync(() =>
+        {
+            Assert.True(Interlocked.Read(ref completed) >= 1); // ack recorded the completion
+            return Task.CompletedTask;
+        }, TimeSpan.FromSeconds(5));
+    }
+
+    [Fact]
+    public async Task ActiveNode_MissingAck_WarnsAfterAckTimeout()
+    {
+        long ackMissing = 0;
+        using var listener = ListenCounter("scadabridge.store_and_forward.resync.ack_missing",
+            m => Interlocked.Add(ref ackMissing, m));
+
+        await _sfStorage.EnqueueAsync(NewMessage("m1"));
+        var actor = CreateResyncActor(isActive: () => true, ackTimeout: TimeSpan.FromMilliseconds(200));
+        actor.Tell(new RequestSfBufferResync(), TestActor);
+        ExpectMsg<SfBufferSnapshotChunk>(TimeSpan.FromSeconds(5));
+        // No ack is sent → the ack window expires and the resync is counted unacknowledged.
+
+        await AwaitAssertAsync(() =>
+        {
+            Assert.True(Interlocked.Read(ref ackMissing) >= 1);
+            return Task.CompletedTask;
+        }, TimeSpan.FromSeconds(5));
+    }
+
+    /// <summary>Attaches a <see cref="MeterListener"/> to a single ScadaBridge counter by name,
+    /// forwarding each recorded increment to <paramref name="onMeasurement"/>.</summary>
+    private static MeterListener ListenCounter(string instrumentName, Action<long> onMeasurement)
+    {
+        var listener = new MeterListener();
+        listener.InstrumentPublished = (inst, l) =>
+        {
+            if (inst.Meter.Name == ScadaBridgeTelemetry.MeterName && inst.Name == instrumentName)
+                l.EnableMeasurementEvents(inst);
+        };
+        listener.SetMeasurementEventCallback<long>((_, m, _, _) => onMeasurement(m));
+        listener.Start();
+        return listener;
+    }
+
     private static StoreAndForwardMessage NewSfMessage(string id) => new()
     {
         Id = id,
@@ -417,11 +482,12 @@ akka {
     };
 
     /// <summary>Constructs a <see cref="ResyncTestActor"/> with the given active-node check
-    /// (the resync chunk/ack tests Tell to and expect from <see cref="TestKit.TestActor"/>).</summary>
-    private IActorRef CreateResyncActor(Func<bool> isActive) =>
+    /// (the resync chunk/ack tests Tell to and expect from <see cref="TestKit.TestActor"/>).
+    /// <paramref name="ackTimeout"/> is the active-side ack window seam (T7).</summary>
+    private IActorRef CreateResyncActor(Func<bool> isActive, TimeSpan? ackTimeout = null) =>
         ActorOf(Props.Create(() => new ResyncTestActor(
             _storage, _sfStorage, _replicationService, SiteRole,
-            NullLogger<SiteReplicationActor>.Instance, CreateTestProbe().Ref, isActive)));
+            NullLogger<SiteReplicationActor>.Instance, CreateTestProbe().Ref, isActive, ackTimeout)));
 
     /// <summary>Test message: drives <see cref="SiteReplicationActor.OnPeerTracked"/> directly,
     /// standing in for the MemberUp→TryTrackPeer path (a single-node TestKit cannot form a real peer).</summary>
@@ -438,9 +504,10 @@ akka {
         public ResyncTestActor(
             SiteStorageService storage, StoreAndForwardStorage sfStorage,
             ReplicationService replicationService, string siteRole,
-            ILogger<SiteReplicationActor> logger, IActorRef peerProbe, Func<bool> isActive)
+            ILogger<SiteReplicationActor> logger, IActorRef peerProbe, Func<bool> isActive,
+            TimeSpan? ackTimeout = null)
             : base(storage, sfStorage, replicationService, siteRole, logger,
-                   configFetcher: null, isActiveOverride: isActive)
+                   configFetcher: null, isActiveOverride: isActive, resyncAckTimeout: ackTimeout)
         {
             _peerProbe = peerProbe;
             Receive<TriggerPeerTracked>(_ => OnPeerTracked());