feat(saf): resync ack confirmation + completed/ack-missing telemetry; doc resync rewrite (plan R2-02 T7)

src/ZB.MOM.WW.ScadaBridge.Commons/Observability/ScadaBridgeTelemetry.cs

@@ -47,6 +47,14 @@ public static class ScadaBridgeTelemetry
         Meter.CreateCounter<long>("scadabridge.store_and_forward.replication.failures", unit: "1",
             description: "S&F buffer replication operations that failed to dispatch/deliver to the peer node");
 
+    private static readonly Counter<long> _sfResyncCompleted =
+        Meter.CreateCounter<long>("scadabridge.store_and_forward.resync.completed", unit: "1",
+            description: "S&F anti-entropy resyncs the standby acknowledged as applied");
+
+    private static readonly Counter<long> _sfResyncAckMissing =
+        Meter.CreateCounter<long>("scadabridge.store_and_forward.resync.ack_missing", unit: "1",
+            description: "S&F resyncs answered by the active node but never acknowledged by the standby within the ack window (lost chunks / dead peer)");
+
     /// <summary>
     /// Incremented each time a per-site live-alarm aggregator re-establishes its site-wide
     /// gRPC stream — a NodeA↔NodeB failover flip or a reconcile-driven reopen after the
@@ -107,6 +115,12 @@ public static class ScadaBridgeTelemetry
     /// <summary>Records one failed S&amp;F replication dispatch.</summary>
     public static void RecordReplicationFailure() => _replicationFailures.Add(1);
 
+    /// <summary>Records one acknowledged (applied) S&amp;F buffer resync.</summary>
+    public static void RecordSfResyncCompleted() => _sfResyncCompleted.Add(1);
+
+    /// <summary>Records one resync whose ack window expired.</summary>
+    public static void RecordSfResyncAckMissing() => _sfResyncAckMissing.Add(1);
+
     /// <summary>Records that a site connection opened (increments the up-count gauge).</summary>
     public static void SiteConnectionOpened() => Interlocked.Increment(ref _siteConnectionsUp);
 

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/SiteReplicationActor.cs

@@ -47,6 +47,15 @@ public class SiteReplicationActor : ReceiveActor, IWithTimers
     /// test seam; production default 30 s.</summary>
     private readonly TimeSpan _resyncAssemblyTimeout;
 
+    // ── Resync delivery confirmation (active side; actor-thread only) ──
+    private string? _pendingAckResyncId;
+    private const string ResyncAckTimerKey = "sf-resync-ack-timeout";
+
+    /// <summary>How long the active node waits for the standby's <see cref="SfBufferResyncAck"/>
+    /// before warning + counting the resync as unacknowledged (lost chunks / dead peer). Ctor
+    /// test seam; production default 60 s.</summary>
+    private readonly TimeSpan _resyncAckTimeout;
+
     /// <summary>
     /// Maximum rows an active node returns in a single anti-entropy resync snapshot.
     /// A standby whose buffer exceeded this (Truncated snapshot) resyncs the oldest
@@ -129,7 +138,8 @@ public class SiteReplicationActor : ReceiveActor, IWithTimers
         Func<bool>? isActiveOverride = null,
         SiteRuntimeOptions? options = null,
         TimeSpan? configFetchRetryDelay = null,
-        TimeSpan? resyncAssemblyTimeout = null)
+        TimeSpan? resyncAssemblyTimeout = null,
+        TimeSpan? resyncAckTimeout = null)
     {
         _storage = storage;
         _sfStorage = sfStorage;
@@ -140,6 +150,7 @@ public class SiteReplicationActor : ReceiveActor, IWithTimers
         _cluster = Cluster.Get(Context.System);
         _isActive = isActiveOverride ?? DefaultIsActive;
         _resyncAssemblyTimeout = resyncAssemblyTimeout ?? TimeSpan.FromSeconds(30);
+        _resyncAckTimeout = resyncAckTimeout ?? TimeSpan.FromSeconds(60);
         // UA2: bound the standby's replicated-config fetch retries. At least one
         // attempt always runs; the fixed inter-attempt delay is a test seam
         // (production default 2 s).
@@ -173,6 +184,26 @@ public class SiteReplicationActor : ReceiveActor, IWithTimers
         Receive<SfResyncSnapshotLoaded>(HandleSfResyncSnapshotLoaded);
         Receive<SfBufferSnapshotChunk>(HandleSfBufferSnapshotChunk);
         Receive<ResyncAssemblyTimedOut>(HandleResyncAssemblyTimedOut);
+        Receive<SfBufferResyncAck>(msg =>
+        {
+            if (msg.ResyncId == _pendingAckResyncId)
+            {
+                _pendingAckResyncId = null;
+                Timers.Cancel(ResyncAckTimerKey);
+            }
+            ScadaBridgeTelemetry.RecordSfResyncCompleted();
+            _logger.LogInformation("S&F resync {ResyncId} acknowledged by standby: {Rows} row(s) applied",
+                msg.ResyncId, msg.RowCount);
+        });
+        Receive<ResyncAckTimedOut>(msg =>
+        {
+            if (msg.ResyncId != _pendingAckResyncId) return;
+            _pendingAckResyncId = null;
+            ScadaBridgeTelemetry.RecordSfResyncAckMissing();
+            _logger.LogWarning(
+                "S&F resync {ResyncId} was never acknowledged within {Window} — snapshot chunks may have been lost (frame drop / dead peer); the next peer-track retries",
+                msg.ResyncId, _resyncAckTimeout);
+        });
         Receive<SfBufferSnapshot>(HandleSfBufferSnapshot); // legacy monolithic handler — retained for rolling compat
     }
 
@@ -491,7 +522,11 @@ public class SiteReplicationActor : ReceiveActor, IWithTimers
         _logger.LogInformation(
             "Answered S&F resync request with {Rows} row(s) in {Chunks} chunk(s), resyncId={ResyncId}",
             msg.Messages.Count, chunks.Count, resyncId);
-        // Task 7 registers the pending-ack entry here.
+        // Arm the ack window: absence of an SfBufferResyncAck within it surfaces as a
+        // Warning + counter (the silent-loss mode N2 flagged). Single-outstanding-resync
+        // bookkeeping: a new request supersedes by overwriting the id and restarting the timer.
+        _pendingAckResyncId = resyncId;
+        Timers.StartSingleTimer(ResyncAckTimerKey, new ResyncAckTimedOut(resyncId), _resyncAckTimeout);
     }
 
     /// <summary>
@@ -630,6 +665,9 @@ public class SiteReplicationActor : ReceiveActor, IWithTimers
 
     /// <summary>Internal: a partial chunk assembly for <paramref name="ResyncId"/> exceeded its window.</summary>
     internal sealed record ResyncAssemblyTimedOut(string ResyncId);
+
+    /// <summary>Internal: the active node's ack window for <paramref name="ResyncId"/> expired.</summary>
+    internal sealed record ResyncAckTimedOut(string ResyncId);
 }
 
 /// <summary>