using Microsoft.Extensions.Logging; using ZB.MOM.WW.ScadaBridge.Commons.Observability; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.StoreAndForward; /// /// Async replication of buffer operations to standby node. /// /// - Forwards add/remove/park operations to standby via a replication handler. /// - No ack wait (fire-and-forget per design). /// - Standby applies operations to its own SQLite. /// - On failover, standby resumes delivery from its replicated state. /// public class ReplicationService { private readonly StoreAndForwardOptions _options; private readonly ILogger _logger; private Func? _replicationHandler; /// Initializes a new instance of . /// Store-and-forward configuration options. /// Logger instance. public ReplicationService( StoreAndForwardOptions options, ILogger logger) { _options = options; _logger = logger; } /// /// Sets the handler for forwarding replication operations to the standby node. /// Typically wraps Akka Tell to the standby's replication actor. /// /// The async delegate that forwards each replication operation to the standby. public void SetReplicationHandler(Func handler) { _replicationHandler = handler; } /// /// Replicates an enqueue operation to standby (fire-and-forget). /// /// The message that was enqueued on the active node. public void ReplicateEnqueue(StoreAndForwardMessage message) { if (!_options.ReplicationEnabled || _replicationHandler == null) return; FireAndForget(new ReplicationOperation( ReplicationOperationType.Add, message.Id, message)); } /// /// Replicates a remove operation to standby (fire-and-forget). /// /// The identifier of the message to remove from the standby buffer. public void ReplicateRemove(string messageId) { if (!_options.ReplicationEnabled || _replicationHandler == null) return; FireAndForget(new ReplicationOperation( ReplicationOperationType.Remove, messageId, null)); } /// /// Replicates a park operation to standby (fire-and-forget). /// /// The message that was parked on the active node. public void ReplicatePark(StoreAndForwardMessage message) { if (!_options.ReplicationEnabled || _replicationHandler == null) return; FireAndForget(new ReplicationOperation( ReplicationOperationType.Park, message.Id, message)); } /// /// Replicates an operator-initiated requeue (a parked /// message moved back to the pending queue) to standby (fire-and-forget). The /// carried message reflects the active node's post-requeue state (Pending, /// retry_count = 0) so the standby's copy can be brought into sync. /// /// The message in its post-requeue (Pending, retry_count=0) state. public void ReplicateRequeue(StoreAndForwardMessage message) { if (!_options.ReplicationEnabled || _replicationHandler == null) return; FireAndForget(new ReplicationOperation( ReplicationOperationType.Requeue, message.Id, message)); } /// /// Applies a replicated operation received from the active node. /// Used by the standby node to keep its SQLite in sync. /// /// Add/Park/Requeue are applied as upserts (), /// not blind INSERT/UPDATE: the full message rides in every one of those operations, /// so a Park/Requeue whose original Add was lost (fire-and-forget replication is /// best-effort) self-heals by materialising the row, and a duplicate Add (e.g. /// re-issued by an anti-entropy resync) applies newest-wins instead of throwing a /// primary-key violation. Remove is a plain delete. /// /// The replication operation to apply. /// The standby node's store-and-forward storage to update. /// A task representing the asynchronous apply operation. public async Task ApplyReplicatedOperationAsync( ReplicationOperation operation, StoreAndForwardStorage storage) { switch (operation.OperationType) { case ReplicationOperationType.Add when operation.Message != null: await storage.UpsertMessageAsync(operation.Message); break; case ReplicationOperationType.Remove: await storage.RemoveMessageAsync(operation.MessageId); break; case ReplicationOperationType.Park when operation.Message != null: operation.Message.Status = StoreAndForwardMessageStatus.Parked; await storage.UpsertMessageAsync(operation.Message); break; case ReplicationOperationType.Requeue when operation.Message != null: operation.Message.Status = StoreAndForwardMessageStatus.Pending; operation.Message.RetryCount = 0; await storage.UpsertMessageAsync(operation.Message); break; } } private void FireAndForget(ReplicationOperation operation) { // Invoked inline, NOT via Task.Run: the handler is a non-blocking Akka // Tell, and thread-pool hand-off destroyed Add/Remove ordering for the // same message id (arch review 02). Inline invocation preserves issue // order; Akka's per-sender/receiver guarantee preserves it on the wire. try { var task = _replicationHandler!.Invoke(operation); if (!task.IsCompletedSuccessfully) { task.ContinueWith(t => { ScadaBridgeTelemetry.RecordReplicationFailure(); _logger.LogWarning(t.Exception, "Replication of {OpType} for message {MessageId} failed (best-effort); standby buffer may be diverging", operation.OperationType, operation.MessageId); }, TaskContinuationOptions.OnlyOnFaulted); } } catch (Exception ex) { ScadaBridgeTelemetry.RecordReplicationFailure(); _logger.LogWarning(ex, "Replication of {OpType} for message {MessageId} failed (best-effort); standby buffer may be diverging", operation.OperationType, operation.MessageId); } } } /// /// Represents a buffer operation to be replicated to standby. /// public record ReplicationOperation( ReplicationOperationType OperationType, string MessageId, StoreAndForwardMessage? Message); /// /// Types of buffer operations that are replicated. /// public enum ReplicationOperationType { Add, Remove, Park, /// /// An operator moved a parked message back to the pending /// queue. The standby resets its matching row to Pending with retry_count = 0. /// Requeue }