fix(saf): chunk the resync snapshot to fit Akka remoting frames (additive protocol, byte-budgeted) (plan R2-02 T5)

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/SiteReplicationActor.cs

@@ -39,6 +39,48 @@ public class SiteReplicationActor : ReceiveActor
     /// </summary>
     private const int MaxResyncRows = 10_000;
 
+    /// <summary>
+    /// Estimated per-chunk payload budget for a resync snapshot. Akka remoting's default
+    /// <c>maximum-frame-size</c> is 128 000 bytes and <c>BuildHocon</c> sets no override,
+    /// so the monolithic <see cref="SfBufferSnapshot"/> is silently undeliverable for any
+    /// realistic backlog (review 02 round 2, N2). 64 000 bytes leaves ≈50% headroom for
+    /// the JSON envelope, CLR type manifests, and the non-payload columns.
+    /// </summary>
+    internal const int MaxResyncChunkBytes = 64_000;
+
+    /// <summary>Row cap per resync chunk (bounds a chunk even when every row is tiny).</summary>
+    internal const int MaxResyncChunkRows = 200;
+
+    /// <summary>
+    /// Splits a resync snapshot into chunks that fit Akka remoting's default
+    /// 128 000-byte frame (review 02 round 2, N2): rows accumulate until the estimated
+    /// payload budget or the row cap is hit. Estimation is payload-dominated
+    /// (payload_json length + 512 bytes fixed overhead per row); a single row whose
+    /// payload exceeds the budget ships alone (Warning at the call site). Order is
+    /// preserved (oldest-first, matching GetAllMessagesAsync).
+    /// </summary>
+    internal static List<List<StoreAndForwardMessage>> ChunkForRemoting(
+        IReadOnlyList<StoreAndForwardMessage> rows, int maxChunkBytes, int maxChunkRows)
+    {
+        var chunks = new List<List<StoreAndForwardMessage>>();
+        var current = new List<StoreAndForwardMessage>();
+        var currentBytes = 0;
+        foreach (var row in rows)
+        {
+            var estimate = (row.PayloadJson?.Length ?? 0) + 512;
+            if (current.Count > 0 && (currentBytes + estimate > maxChunkBytes || current.Count >= maxChunkRows))
+            {
+                chunks.Add(current);
+                current = new List<StoreAndForwardMessage>();
+                currentBytes = 0;
+            }
+            current.Add(row);
+            currentBytes += estimate;
+        }
+        if (current.Count > 0) chunks.Add(current);
+        return chunks;
+    }
+
     /// <summary>
     /// Initializes a new <see cref="SiteReplicationActor"/> and registers Akka message handlers.
     /// </summary>
@@ -111,7 +153,8 @@ public class SiteReplicationActor : ReceiveActor
 
         // Anti-entropy — full S&F buffer resync on peer (re)join
         Receive<RequestSfBufferResync>(HandleRequestSfBufferResync);
-        Receive<SfBufferSnapshot>(HandleSfBufferSnapshot);
+        Receive<SfResyncSnapshotLoaded>(HandleSfResyncSnapshotLoaded);
+        Receive<SfBufferSnapshot>(HandleSfBufferSnapshot); // legacy monolithic handler — retained for rolling compat
     }
 
     /// <inheritdoc />
@@ -388,9 +431,10 @@ public class SiteReplicationActor : ReceiveActor
 
     /// <summary>
     /// Active-node side of the anti-entropy resync: answers a standby's
-    /// <see cref="RequestSfBufferResync"/> with a full-buffer <see cref="SfBufferSnapshot"/>
-    /// (up to <see cref="MaxResyncRows"/> oldest rows). A non-active node ignores the
-    /// request — only the authoritative node may answer.
+    /// <see cref="RequestSfBufferResync"/> with a sequence of byte-budgeted
+    /// <see cref="SfBufferSnapshotChunk"/>s (up to <see cref="MaxResyncRows"/> oldest rows).
+    /// A non-active node ignores the request — only the authoritative node may answer.
+    /// The snapshot is piped back to Self so chunking + ack bookkeeping stays actor-safe.
     /// </summary>
     private void HandleRequestSfBufferResync(RequestSfBufferResync msg)
     {
@@ -402,9 +446,33 @@ public class SiteReplicationActor : ReceiveActor
 
         var replyTo = Sender;
         _sfStorage.GetAllMessagesAsync(MaxResyncRows).PipeTo(
-            replyTo,
             Self,
-            success: result => new SfBufferSnapshot(result.Messages, result.Truncated));
+            failure: ex => new Status.Failure(ex),
+            success: result => new SfResyncSnapshotLoaded(replyTo, result.Messages, result.Truncated));
+    }
+
+    /// <summary>
+    /// Active-node continuation: the resync snapshot finished loading; chunk it to fit the
+    /// remoting frame and send the sequenced chunks to the requester (all sharing one
+    /// resyncId). Task 7 arms the ack-timeout here.
+    /// </summary>
+    private void HandleSfResyncSnapshotLoaded(SfResyncSnapshotLoaded msg)
+    {
+        var chunks = ChunkForRemoting(msg.Messages, MaxResyncChunkBytes, MaxResyncChunkRows);
+        if (chunks.Count == 0) chunks.Add(new List<StoreAndForwardMessage>()); // empty buffer still resyncs (clears the standby)
+        var resyncId = Guid.NewGuid().ToString("N");
+        for (var i = 0; i < chunks.Count; i++)
+        {
+            if (chunks[i].Count == 1 && (chunks[i][0].PayloadJson?.Length ?? 0) + 512 > MaxResyncChunkBytes)
+                _logger.LogWarning(
+                    "Resync row {Id} alone exceeds the chunk budget ({Bytes}B payload); sending solo — it may exceed the remoting frame",
+                    chunks[i][0].Id, chunks[i][0].PayloadJson?.Length ?? 0);
+            msg.ReplyTo.Tell(new SfBufferSnapshotChunk(resyncId, i + 1, chunks.Count, chunks[i], msg.Truncated), Self);
+        }
+        _logger.LogInformation(
+            "Answered S&F resync request with {Rows} row(s) in {Chunks} chunk(s), resyncId={ResyncId}",
+            msg.Messages.Count, chunks.Count, resyncId);
+        // Task 7 registers the pending-ack entry here.
     }
 
     /// <summary>
@@ -451,6 +519,10 @@ public class SiteReplicationActor : ReceiveActor
                     _logger.LogError(t.Exception, "Failed to apply S&F buffer resync snapshot");
             });
     }
+
+    /// <summary>Internal: the resync snapshot finished loading; chunk and send to the requester.</summary>
+    internal sealed record SfResyncSnapshotLoaded(
+        IActorRef ReplyTo, List<StoreAndForwardMessage> Messages, bool Truncated);
 }
 
 /// <summary>
@@ -467,3 +539,24 @@ public sealed record RequestSfBufferResync;
 /// Akka remoting; the message list rides the default serializer.
 /// </summary>
 public sealed record SfBufferSnapshot(List<StoreAndForwardMessage> Messages, bool Truncated);
+
+/// <summary>
+/// Active→standby: one sequenced chunk of a full-buffer anti-entropy snapshot
+/// (review 02 round 2, N2 — the monolithic <see cref="SfBufferSnapshot"/> exceeds Akka
+/// remoting's default 128 000-byte frame for any realistic backlog). All chunks of one
+/// resync share <paramref name="ResyncId"/>; <paramref name="Sequence"/> is 1-based up to
+/// <paramref name="TotalChunks"/>. Additive message — the legacy monolithic snapshot
+/// handler is retained for rolling upgrades. Crosses intra-site Akka remoting (NOT
+/// ClusterClient — ClusterClientContractLockTests is intentionally not involved).
+/// </summary>
+public sealed record SfBufferSnapshotChunk(
+    string ResyncId, int Sequence, int TotalChunks,
+    List<StoreAndForwardMessage> Messages, bool Truncated);
+
+/// <summary>
+/// Standby→active: delivery confirmation — the standby assembled all chunks of
+/// <paramref name="ResyncId"/> and applied them atomically (<paramref name="RowCount"/>
+/// rows installed). Absence within the ack window is surfaced by the active node
+/// (Warning + counter) — the silent-loss mode N2 flagged.
+/// </summary>
+public sealed record SfBufferResyncAck(string ResyncId, int RowCount);