feat(localdb)!: replicate site config + sf_messages via CDC, delete the bespoke replicators
Tasks 14, 15 and 16, landed as ONE commit. PLAN DEFECT: these three tasks cannot compile separately. SiteReplicationActor takes a ReplicationService and calls ReplaceAllAsync (Task 14 deletes both); DeploymentManagerActor Tells message types declared in ReplicationMessages.cs (Task 15 deletes it); AkkaHostedService constructs the actor (Task 16). Any ordering leaves a broken intermediate. Combining them also strengthens the invariant Task 14 already stated for itself — the two mechanisms never both run, and never neither. Registered 8 tables in SiteLocalDbSetup.OnReady: sf_messages plus the 7 site config tables. notification_lists and smtp_configurations are deliberately NOT registered — permanently empty by design, so registering them would open a standing replication channel whose only historical payload was plaintext SMTP passwords. Migrate stays the LAST call in OnReady, after all registrations, so migrated rows enter the oplog through live capture triggers. Deleted: SiteReplicationActor, ReplicationMessages.cs, ReplicationService, StoreAndForwardStorage.ReplaceAllAsync, and 6 test files. ReplaceAllAsync is not merely unused but unsafe to keep: a mass DELETE on a now-replicated table would be captured and shipped to the peer. Kept ActiveNodeEvaluator (delivery gate + heartbeat still need it) with its doc corrected, and activeNodeCheck in AkkaHostedService (SiteCommunicationActor). The positional-argument hazard the plan flagged was real: removing DeploymentManagerActor's optional IActorRef? replicationActor shifted 6 trailing optionals, and 4 test call sites bound the wrong arguments with no compile error at some positions. Converted them to named arguments where possible — Props.Create builds an expression tree, which rejects out-of-position named args, so the rest are padded positionally with a comment saying why. The Task 7 'not yet registered' test was INVERTED rather than deleted, and is exact in both directions: too few means a table silently stops replicating, too many means the SMTP tables leak. Added a separate security-named test for those two, and a composite-PK test (LWW keys on the full PK, so a truncated key set would collapse distinct rows). The convergence suites now get their registrations from the real OnReady — their temporary harness registration is deleted, so they prove the cutover rather than agreeing with themselves. Verified: build 0 warnings; SiteRuntime 512, StoreAndForward 130, Host 330, AuditLog 355, ExternalSystemGateway 142, HealthMonitoring 97, LocalDb integration 16 — all pass, 0 failures. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
@@ -52,7 +52,6 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
/// </summary>
|
||||
private readonly ILoggerFactory _loggerFactory;
|
||||
private readonly IActorRef? _dclManager;
|
||||
private readonly IActorRef? _replicationActor;
|
||||
private readonly ISiteHealthCollector? _healthCollector;
|
||||
private readonly IServiceProvider? _serviceProvider;
|
||||
/// <summary>
|
||||
@@ -166,7 +165,6 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
SiteRuntimeOptions options,
|
||||
ILogger<DeploymentManagerActor> logger,
|
||||
IActorRef? dclManager = null,
|
||||
IActorRef? replicationActor = null,
|
||||
ISiteHealthCollector? healthCollector = null,
|
||||
IServiceProvider? serviceProvider = null,
|
||||
ILoggerFactory? loggerFactory = null,
|
||||
@@ -181,7 +179,6 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
_streamManager = streamManager;
|
||||
_options = options;
|
||||
_dclManager = dclManager;
|
||||
_replicationActor = replicationActor;
|
||||
_healthCollector = healthCollector;
|
||||
_serviceProvider = serviceProvider;
|
||||
_configFetcher = configFetcher;
|
||||
@@ -785,15 +782,14 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
await _storage.ClearStaticOverridesAsync(instanceName);
|
||||
await _storage.ClearNativeAlarmsForInstanceAsync(instanceName);
|
||||
|
||||
// Replicate to standby node — notify-and-fetch: send only the deployment id +
|
||||
// central fetch coordinates (NOT the config JSON). The standby fetches the
|
||||
// config over HTTP itself, so a large config never crosses the intra-site Akka
|
||||
// hop (which would silently drop on the 128 KB frame trap). When the coords are
|
||||
// absent (deploy paths other than RefreshDeployment), the standby fetch is a
|
||||
// no-op miss and reconciliation is the durable backstop.
|
||||
_replicationActor?.Tell(new ReplicateConfigDeploy(
|
||||
instanceName, command.DeploymentId, command.RevisionHash, true,
|
||||
command.CentralFetchBaseUrl ?? "", command.FetchToken ?? ""));
|
||||
// No explicit replication step: deployed_configurations is a replicated table,
|
||||
// so the write above is captured and shipped to the peer like any other row.
|
||||
// This used to be a notify-and-fetch Tell — the standby was sent the deployment
|
||||
// id plus central fetch coordinates and pulled the config over HTTP itself,
|
||||
// because a large config would silently drop on the intra-site Akka hop's
|
||||
// 128 KB frame trap. Replication carries the row directly and has no such limit,
|
||||
// so the standby no longer fetches on deploy. (SiteReconciliationActor still
|
||||
// fetches at node startup when central reports gaps — a different path.)
|
||||
|
||||
return new DeployPersistenceResult(
|
||||
command.DeploymentId, instanceName, true, null, sender, isRedeploy);
|
||||
@@ -967,7 +963,6 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
if (t.IsCompletedSuccessfully)
|
||||
{
|
||||
_replicationActor?.Tell(new ReplicateConfigSetEnabled(instanceName, false));
|
||||
// Operational `deployment` event — disable succeeded.
|
||||
LogDeploymentEvent("Info", instanceName, $"Instance {instanceName} disabled");
|
||||
}
|
||||
@@ -1001,7 +996,6 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
Task.Run(async () =>
|
||||
{
|
||||
await _storage.SetInstanceEnabledAsync(instanceName, true);
|
||||
_replicationActor?.Tell(new ReplicateConfigSetEnabled(instanceName, true));
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var config = configs.FirstOrDefault(c => c.InstanceUniqueName == instanceName);
|
||||
return new EnableResult(command, config, null, sender);
|
||||
@@ -1099,7 +1093,6 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
if (t.IsCompletedSuccessfully)
|
||||
{
|
||||
_replicationActor?.Tell(new ReplicateConfigRemove(instanceName));
|
||||
// Operational `deployment` event — delete succeeded.
|
||||
LogDeploymentEvent("Info", instanceName, $"Instance {instanceName} deleted");
|
||||
}
|
||||
@@ -1949,7 +1942,6 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
// central-only and is never stored on a site (see the purge above).
|
||||
|
||||
// Replicate artifacts to standby node
|
||||
_replicationActor?.Tell(new ReplicateArtifacts(command));
|
||||
|
||||
return new ArtifactDeploymentResponse(
|
||||
command.DeploymentId, "", true, null, DateTimeOffset.UtcNow);
|
||||
|
||||
@@ -1,707 +0,0 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Event;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Runs on every site node (not a singleton). Handles both config and S&F replication
|
||||
/// between site cluster peers.
|
||||
///
|
||||
/// Outbound: receives local replication requests and forwards to peer via ActorSelection.
|
||||
/// Inbound: receives replicated operations from peer and applies to local SQLite.
|
||||
/// Uses fire-and-forget (Tell) — no ack wait per design.
|
||||
/// </summary>
|
||||
public class SiteReplicationActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly StoreAndForwardStorage _sfStorage;
|
||||
private readonly ReplicationService _replicationService;
|
||||
private readonly IDeploymentConfigFetcher? _configFetcher;
|
||||
private readonly string _siteRole;
|
||||
private readonly ILogger<SiteReplicationActor> _logger;
|
||||
private readonly Cluster _cluster;
|
||||
private readonly Func<bool> _isActive;
|
||||
private readonly int _configFetchRetryCount;
|
||||
private readonly TimeSpan _configFetchRetryDelay;
|
||||
private Address? _peerAddress;
|
||||
|
||||
/// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
// ── Chunked-resync assembly (standby side; actor-thread only) ──
|
||||
private string? _assemblingResyncId;
|
||||
private int _assemblingTotalChunks;
|
||||
private bool _assemblingTruncated;
|
||||
private readonly Dictionary<int, List<StoreAndForwardMessage>> _assemblingChunks = new();
|
||||
private const string ResyncAssemblyTimerKey = "sf-resync-assembly-timeout";
|
||||
|
||||
/// <summary>How long a partial chunk assembly may wait for its missing chunks before
|
||||
/// being discarded (a lost chunk = lost resync; the next peer-track retries). Ctor
|
||||
/// test seam; production default 30 s.</summary>
|
||||
private readonly TimeSpan _resyncAssemblyTimeout;
|
||||
|
||||
// ── Resync delivery confirmation (active side; actor-thread only) ──
|
||||
private string? _pendingAckResyncId;
|
||||
private const string ResyncAckTimerKey = "sf-resync-ack-timeout";
|
||||
|
||||
/// <summary>How long the active node waits for the standby's <see cref="SfBufferResyncAck"/>
|
||||
/// before warning + counting the resync as unacknowledged (lost chunks / dead peer). Ctor
|
||||
/// test seam; production default 60 s.</summary>
|
||||
private readonly TimeSpan _resyncAckTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum rows an active node returns in a single anti-entropy resync snapshot.
|
||||
/// A standby whose buffer exceeded this (Truncated snapshot) resyncs the oldest
|
||||
/// 10 000 rows; further divergence drains naturally as the active node delivers.
|
||||
/// </summary>
|
||||
private const int MaxResyncRows = 10_000;
|
||||
|
||||
/// <summary>
|
||||
/// Estimated per-chunk payload budget for a resync snapshot. Akka remoting's default
|
||||
/// <c>maximum-frame-size</c> is 128 000 bytes and <c>BuildHocon</c> sets no override,
|
||||
/// so the monolithic <see cref="SfBufferSnapshot"/> is silently undeliverable for any
|
||||
/// realistic backlog (review 02 round 2, N2). 64 000 bytes leaves ≈50% headroom for
|
||||
/// the JSON envelope, CLR type manifests, and the non-payload columns.
|
||||
/// </summary>
|
||||
internal const int MaxResyncChunkBytes = 64_000;
|
||||
|
||||
/// <summary>Row cap per resync chunk (bounds a chunk even when every row is tiny).</summary>
|
||||
internal const int MaxResyncChunkRows = 200;
|
||||
|
||||
/// <summary>
|
||||
/// Splits a resync snapshot into chunks that fit Akka remoting's default
|
||||
/// 128 000-byte frame (review 02 round 2, N2): rows accumulate until the estimated
|
||||
/// payload budget or the row cap is hit. Estimation is payload-dominated
|
||||
/// (payload_json length + 512 bytes fixed overhead per row); a single row whose
|
||||
/// payload exceeds the budget ships alone (Warning at the call site). Order is
|
||||
/// preserved (oldest-first, matching GetAllMessagesAsync).
|
||||
/// </summary>
|
||||
internal static List<List<StoreAndForwardMessage>> ChunkForRemoting(
|
||||
IReadOnlyList<StoreAndForwardMessage> rows, int maxChunkBytes, int maxChunkRows)
|
||||
{
|
||||
var chunks = new List<List<StoreAndForwardMessage>>();
|
||||
var current = new List<StoreAndForwardMessage>();
|
||||
var currentBytes = 0;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
var estimate = (row.PayloadJson?.Length ?? 0) + 512;
|
||||
if (current.Count > 0 && (currentBytes + estimate > maxChunkBytes || current.Count >= maxChunkRows))
|
||||
{
|
||||
chunks.Add(current);
|
||||
current = new List<StoreAndForwardMessage>();
|
||||
currentBytes = 0;
|
||||
}
|
||||
current.Add(row);
|
||||
currentBytes += estimate;
|
||||
}
|
||||
if (current.Count > 0) chunks.Add(current);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new <see cref="SiteReplicationActor"/> and registers Akka message handlers.
|
||||
/// </summary>
|
||||
/// <param name="storage">Service for accessing local site storage.</param>
|
||||
/// <param name="sfStorage">Store-and-forward SQLite storage for replication of buffered messages.</param>
|
||||
/// <param name="replicationService">Service providing replication transport logic.</param>
|
||||
/// <param name="siteRole">Akka cluster role used to identify peer nodes to replicate to.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="configFetcher">
|
||||
/// Fetches a deployed instance's config JSON from central over HTTP. Used by the
|
||||
/// notify-and-fetch standby apply path (<see cref="HandleApplyConfigDeploy"/>): the peer
|
||||
/// replicates only the deployment id, and the standby fetches the config itself so a large
|
||||
/// config never crosses the intra-site Akka hop. Null on nodes/tests without a fetcher.
|
||||
/// </param>
|
||||
/// <param name="isActiveOverride">
|
||||
/// Active-node check that gates the buffer-resync roles (a standby requests a
|
||||
/// resync, the active node answers). Production wiring passes the Host's
|
||||
/// <c>IClusterNodeProvider.SelfIsPrimary</c> delegate (the same instance gating the
|
||||
/// S&F delivery sweep); null falls back to the shared oldest-Up evaluator
|
||||
/// (<see cref="Communication.ClusterState.ActiveNodeEvaluator"/>).
|
||||
/// </param>
|
||||
/// <param name="options">Site runtime options, including the config-fetch retry count; production defaults apply when null.</param>
|
||||
/// <param name="configFetchRetryDelay">Delay between config-fetch retry attempts; defaults to 2 seconds when null.</param>
|
||||
public SiteReplicationActor(
|
||||
SiteStorageService storage,
|
||||
StoreAndForwardStorage sfStorage,
|
||||
ReplicationService replicationService,
|
||||
string siteRole,
|
||||
ILogger<SiteReplicationActor> logger,
|
||||
IDeploymentConfigFetcher? configFetcher = null,
|
||||
Func<bool>? isActiveOverride = null,
|
||||
SiteRuntimeOptions? options = null,
|
||||
TimeSpan? configFetchRetryDelay = null,
|
||||
TimeSpan? resyncAssemblyTimeout = null,
|
||||
TimeSpan? resyncAckTimeout = null)
|
||||
{
|
||||
_storage = storage;
|
||||
_sfStorage = sfStorage;
|
||||
_replicationService = replicationService;
|
||||
_configFetcher = configFetcher;
|
||||
_siteRole = siteRole;
|
||||
_logger = logger;
|
||||
_cluster = Cluster.Get(Context.System);
|
||||
_isActive = isActiveOverride ?? DefaultIsActive;
|
||||
_resyncAssemblyTimeout = resyncAssemblyTimeout ?? TimeSpan.FromSeconds(30);
|
||||
_resyncAckTimeout = resyncAckTimeout ?? TimeSpan.FromSeconds(60);
|
||||
// UA2: bound the standby's replicated-config fetch retries. At least one
|
||||
// attempt always runs; the fixed inter-attempt delay is a test seam
|
||||
// (production default 2 s).
|
||||
_configFetchRetryCount = Math.Max(1, options?.ConfigFetchRetryCount ?? 1);
|
||||
_configFetchRetryDelay = configFetchRetryDelay ?? TimeSpan.FromSeconds(2);
|
||||
|
||||
// Cluster member events
|
||||
Receive<ClusterEvent.MemberUp>(HandleMemberUp);
|
||||
Receive<ClusterEvent.MemberRemoved>(HandleMemberRemoved);
|
||||
Receive<ClusterEvent.CurrentClusterState>(HandleCurrentClusterState);
|
||||
|
||||
// Outbound — forward to peer
|
||||
Receive<ReplicateConfigDeploy>(msg => SendToPeer(new ApplyConfigDeploy(
|
||||
msg.InstanceName, msg.DeploymentId, msg.RevisionHash, msg.IsEnabled,
|
||||
msg.CentralFetchBaseUrl, msg.FetchToken)));
|
||||
Receive<ReplicateConfigRemove>(msg => SendToPeer(new ApplyConfigRemove(msg.InstanceName)));
|
||||
Receive<ReplicateConfigSetEnabled>(msg => SendToPeer(new ApplyConfigSetEnabled(
|
||||
msg.InstanceName, msg.IsEnabled)));
|
||||
Receive<ReplicateArtifacts>(msg => SendToPeer(new ApplyArtifacts(msg.Command)));
|
||||
Receive<ReplicateStoreAndForward>(msg => SendToPeer(new ApplyStoreAndForward(msg.Operation)));
|
||||
|
||||
// Inbound — apply from peer
|
||||
Receive<ApplyConfigDeploy>(HandleApplyConfigDeploy);
|
||||
Receive<ApplyConfigRemove>(HandleApplyConfigRemove);
|
||||
Receive<ApplyConfigSetEnabled>(HandleApplyConfigSetEnabled);
|
||||
Receive<ApplyArtifacts>(HandleApplyArtifacts);
|
||||
Receive<ApplyStoreAndForward>(HandleApplyStoreAndForward);
|
||||
|
||||
// Anti-entropy — full S&F buffer resync on peer (re)join
|
||||
Receive<RequestSfBufferResync>(HandleRequestSfBufferResync);
|
||||
Receive<SfResyncSnapshotLoaded>(HandleSfResyncSnapshotLoaded);
|
||||
Receive<SfBufferSnapshotChunk>(HandleSfBufferSnapshotChunk);
|
||||
Receive<ResyncAssemblyTimedOut>(HandleResyncAssemblyTimedOut);
|
||||
Receive<SfBufferResyncAck>(msg =>
|
||||
{
|
||||
if (msg.ResyncId == _pendingAckResyncId)
|
||||
{
|
||||
_pendingAckResyncId = null;
|
||||
Timers.Cancel(ResyncAckTimerKey);
|
||||
}
|
||||
ScadaBridgeTelemetry.RecordSfResyncCompleted();
|
||||
_logger.LogInformation("S&F resync {ResyncId} acknowledged by standby: {Rows} row(s) applied",
|
||||
msg.ResyncId, msg.RowCount);
|
||||
});
|
||||
Receive<ResyncAckTimedOut>(msg =>
|
||||
{
|
||||
if (msg.ResyncId != _pendingAckResyncId) return;
|
||||
_pendingAckResyncId = null;
|
||||
ScadaBridgeTelemetry.RecordSfResyncAckMissing();
|
||||
_logger.LogWarning(
|
||||
"S&F resync {ResyncId} was never acknowledged within {Window} — snapshot chunks may have been lost (frame drop / dead peer); the next peer-track retries",
|
||||
msg.ResyncId, _resyncAckTimeout);
|
||||
});
|
||||
Receive<SfBufferSnapshot>(HandleSfBufferSnapshot); // legacy monolithic handler — retained for rolling compat
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
base.PreStart();
|
||||
_cluster.Subscribe(Self, ClusterEvent.SubscriptionInitialStateMode.InitialStateAsSnapshot,
|
||||
typeof(ClusterEvent.MemberUp),
|
||||
typeof(ClusterEvent.MemberRemoved));
|
||||
_logger.LogInformation("SiteReplicationActor started, subscribing to cluster events for role {Role}", _siteRole);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
_cluster.Unsubscribe(Self);
|
||||
base.PostStop();
|
||||
}
|
||||
|
||||
private void HandleCurrentClusterState(ClusterEvent.CurrentClusterState state)
|
||||
{
|
||||
foreach (var member in state.Members)
|
||||
{
|
||||
if (member.Status == MemberStatus.Up)
|
||||
TryTrackPeer(member);
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleMemberUp(ClusterEvent.MemberUp evt)
|
||||
{
|
||||
TryTrackPeer(evt.Member);
|
||||
}
|
||||
|
||||
private void HandleMemberRemoved(ClusterEvent.MemberRemoved evt)
|
||||
{
|
||||
if (evt.Member.Address.Equals(_peerAddress))
|
||||
{
|
||||
_logger.LogInformation("Peer node removed: {Address}", _peerAddress);
|
||||
_peerAddress = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void TryTrackPeer(Member member)
|
||||
{
|
||||
// Must have our site role, and must not be self
|
||||
if (member.HasRole(_siteRole) && !member.Address.Equals(_cluster.SelfAddress))
|
||||
{
|
||||
_peerAddress = member.Address;
|
||||
_logger.LogInformation("Peer node tracked: {Address}", _peerAddress);
|
||||
OnPeerTracked();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Side-effect run whenever a peer is (re)tracked. A <b>standby</b> requests a
|
||||
/// full S&F buffer snapshot for anti-entropy resync — this closes the "a
|
||||
/// standby down for an hour rejoins and diverges forever" gap: it may have missed
|
||||
/// replicated Add/Remove/Park ops while it was gone. The active node never
|
||||
/// requests. <see langword="protected virtual"/> so tests can drive it without a
|
||||
/// real two-node cluster.
|
||||
/// </summary>
|
||||
protected virtual void OnPeerTracked()
|
||||
{
|
||||
if (!SafeIsActive())
|
||||
{
|
||||
SendToPeer(new RequestSfBufferResync());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Repo-standard active-node check: this node is active when it is the OLDEST Up
|
||||
/// member carrying the site role — the same oldest-Up semantics as the S&F delivery
|
||||
/// gate (IClusterNodeProvider.SelfIsPrimary → ClusterActivityEvaluator → shared
|
||||
/// ActiveNodeEvaluator). NEVER the cluster leader: leadership is lowest-address and
|
||||
/// diverges from singleton/delivery placement permanently after the lower-address
|
||||
/// node restarts — the divergence that made the delivering node wipe its own live
|
||||
/// buffer via a wrong-direction resync (review 02 round 2, N1 Critical). Any other
|
||||
/// state reports standby — safe-by-default.
|
||||
/// </summary>
|
||||
private bool DefaultIsActive() =>
|
||||
Communication.ClusterState.ActiveNodeEvaluator.SelfIsOldestUp(_cluster, _siteRole);
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates the active-node check, treating a throwing check as standby
|
||||
/// (safe-by-default: a standby never delivers or answers resyncs).
|
||||
/// </summary>
|
||||
private bool SafeIsActive()
|
||||
{
|
||||
try
|
||||
{
|
||||
return _isActive();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Active-node check threw; treating node as standby");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Forwards a replication message to the tracked peer node's <c>site-replication</c> actor
|
||||
/// (fire-and-forget, dropped when no peer is tracked). <see langword="protected virtual"/>
|
||||
/// so tests can intercept the peer send without standing up a real two-node cluster.
|
||||
/// </summary>
|
||||
/// <param name="message">The replication message to forward to the peer.</param>
|
||||
protected virtual void SendToPeer(object message)
|
||||
{
|
||||
if (_peerAddress == null)
|
||||
{
|
||||
// A dropped op is a lost delta — surface it at Warning + a metric so a
|
||||
// never-tracked peer (a dead standby) is visible, not silent (arch
|
||||
// review 02). In single-node dev the peer is legitimately absent; the
|
||||
// per-op warning is rate-tolerable (accepted per review).
|
||||
ScadaBridgeTelemetry.RecordReplicationFailure();
|
||||
_logger.LogWarning("No peer available, dropping replication message {Type}", message.GetType().Name);
|
||||
return;
|
||||
}
|
||||
|
||||
var path = new RootActorPath(_peerAddress) / "user" / "site-replication";
|
||||
Context.ActorSelection(path).Tell(message);
|
||||
}
|
||||
|
||||
// ── Inbound handlers ──
|
||||
|
||||
private void HandleApplyConfigDeploy(ApplyConfigDeploy msg)
|
||||
{
|
||||
if (string.IsNullOrEmpty(msg.CentralFetchBaseUrl))
|
||||
{
|
||||
// The direct DeployInstanceCommand cross-cluster wire path was retired.
|
||||
// This guard is a defensive fallback: skip quietly rather than calling FetchAsync("")
|
||||
// and logging a spurious error. Reconciliation backstops any missed writes.
|
||||
_logger.LogDebug(
|
||||
"No fetch coords for {Instance} (deployment {DeploymentId}) — skipping replicated fetch; T18 reconciliation is the backstop",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (_configFetcher is null)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"No config fetcher available; cannot apply replicated config for {Instance} (deployment {DeploymentId}) — reconciliation will backstop",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Replicating config for {Instance} (deployment {DeploymentId}) — fetching from central",
|
||||
msg.InstanceName, msg.DeploymentId);
|
||||
|
||||
// Notify-and-fetch: the peer sent only the id, so the standby fetches the config
|
||||
// itself (off-thread; best-effort fire-and-forget, matching the no-ack replication
|
||||
// model). The guarded write only overwrites a strictly-older local row. The fetch
|
||||
// is retried up to ConfigFetchRetryCount times with a fixed delay (UA2) — a transient
|
||||
// central hiccup no longer defers to the slower reconciliation backstop, which still
|
||||
// covers a total failure after the last attempt.
|
||||
_ = FetchWithRetryAsync();
|
||||
return;
|
||||
|
||||
async Task FetchWithRetryAsync()
|
||||
{
|
||||
for (var attempt = 1; attempt <= _configFetchRetryCount; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Non-null: the outer method returns early when _configFetcher is null.
|
||||
var json = await _configFetcher!.FetchAsync(
|
||||
msg.CentralFetchBaseUrl, msg.DeploymentId, msg.FetchToken, CancellationToken.None);
|
||||
await _storage.StoreDeployedConfigIfNewerAsync(
|
||||
msg.InstanceName, json, msg.DeploymentId, msg.RevisionHash, msg.IsEnabled);
|
||||
return;
|
||||
}
|
||||
catch (DeploymentConfigFetchException fex) when (fex.IsSuperseded)
|
||||
{
|
||||
// A superseded/expired fetch never heals by retrying — a newer deploy
|
||||
// will replicate its own id.
|
||||
_logger.LogInformation(
|
||||
"Skip replicated config for {Instance}: superseded/expired (a newer deploy will replicate)",
|
||||
msg.InstanceName);
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (attempt < _configFetchRetryCount)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Replicated config fetch attempt {Attempt}/{Max} failed for {Instance} (deployment {DeploymentId}) — retrying in {Delay}",
|
||||
attempt, _configFetchRetryCount, msg.InstanceName, msg.DeploymentId, _configFetchRetryDelay);
|
||||
await Task.Delay(_configFetchRetryDelay);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"Replicated config fetch failed after {Attempts} attempt(s) for {Instance} (deployment {DeploymentId}) — reconciliation will backstop",
|
||||
_configFetchRetryCount, msg.InstanceName, msg.DeploymentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void HandleApplyConfigRemove(ApplyConfigRemove msg)
|
||||
{
|
||||
_logger.LogInformation("Applying replicated config remove for {Instance}", msg.InstanceName);
|
||||
_storage.RemoveDeployedConfigAsync(msg.InstanceName)
|
||||
.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
_logger.LogError(t.Exception, "Failed to apply replicated remove for {Instance}", msg.InstanceName);
|
||||
});
|
||||
}
|
||||
|
||||
private void HandleApplyConfigSetEnabled(ApplyConfigSetEnabled msg)
|
||||
{
|
||||
_logger.LogInformation("Applying replicated set-enabled={Enabled} for {Instance}", msg.IsEnabled, msg.InstanceName);
|
||||
_storage.SetInstanceEnabledAsync(msg.InstanceName, msg.IsEnabled)
|
||||
.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
_logger.LogError(t.Exception, "Failed to apply replicated set-enabled for {Instance}", msg.InstanceName);
|
||||
});
|
||||
}
|
||||
|
||||
private void HandleApplyArtifacts(ApplyArtifacts msg)
|
||||
{
|
||||
var command = msg.Command;
|
||||
_logger.LogInformation("Applying replicated artifacts, deploymentId={DeploymentId}", command.DeploymentId);
|
||||
|
||||
Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
if (command.SharedScripts != null)
|
||||
foreach (var s in command.SharedScripts)
|
||||
await _storage.StoreSharedScriptAsync(s.Name, s.Code, s.ParameterDefinitions, s.ReturnDefinition);
|
||||
|
||||
if (command.ExternalSystems != null)
|
||||
foreach (var es in command.ExternalSystems)
|
||||
await _storage.StoreExternalSystemAsync(es.Name, es.EndpointUrl, es.AuthType, es.AuthConfiguration, es.MethodDefinitionsJson, es.TimeoutSeconds);
|
||||
|
||||
if (command.DatabaseConnections != null)
|
||||
foreach (var db in command.DatabaseConnections)
|
||||
await _storage.StoreDatabaseConnectionAsync(db.Name, db.ConnectionString, db.MaxRetries, db.RetryDelay);
|
||||
|
||||
// Notification lists and SMTP
|
||||
// configuration are central-only and are never persisted on a site.
|
||||
// Mirror the primary apply path: purge any pre-fix rows (including the
|
||||
// plaintext SMTP password) instead of writing the command's
|
||||
// (now-always-null) NotificationLists/SmtpConfigurations.
|
||||
await _storage.PurgeCentralOnlyNotificationConfigAsync();
|
||||
|
||||
if (command.DataConnections != null)
|
||||
foreach (var dc in command.DataConnections)
|
||||
await _storage.StoreDataConnectionDefinitionAsync(dc.Name, dc.Protocol, dc.PrimaryConfigurationJson, dc.BackupConfigurationJson, dc.FailoverRetryCount);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to apply replicated artifacts");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void HandleApplyStoreAndForward(ApplyStoreAndForward msg)
|
||||
{
|
||||
_logger.LogDebug("Applying replicated S&F operation {OpType} for message {Id}",
|
||||
msg.Operation.OperationType, msg.Operation.MessageId);
|
||||
|
||||
_replicationService.ApplyReplicatedOperationAsync(msg.Operation, _sfStorage)
|
||||
.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
_logger.LogError(t.Exception, "Failed to apply replicated S&F operation {Id}", msg.Operation.MessageId);
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Active-node side of the anti-entropy resync: answers a standby's
|
||||
/// <see cref="RequestSfBufferResync"/> with a sequence of byte-budgeted
|
||||
/// <see cref="SfBufferSnapshotChunk"/>s (up to <see cref="MaxResyncRows"/> oldest rows).
|
||||
/// A non-active node ignores the request — only the authoritative node may answer.
|
||||
/// The snapshot is piped back to Self so chunking + ack bookkeeping stays actor-safe.
|
||||
/// </summary>
|
||||
private void HandleRequestSfBufferResync(RequestSfBufferResync msg)
|
||||
{
|
||||
if (!SafeIsActive())
|
||||
{
|
||||
_logger.LogDebug("Ignoring S&F buffer resync request — this node is not active");
|
||||
return;
|
||||
}
|
||||
|
||||
var replyTo = Sender;
|
||||
_sfStorage.GetAllMessagesAsync(MaxResyncRows).PipeTo(
|
||||
Self,
|
||||
failure: ex => new Status.Failure(ex),
|
||||
success: result => new SfResyncSnapshotLoaded(replyTo, result.Messages, result.Truncated));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Active-node continuation: the resync snapshot finished loading; chunk it to fit the
|
||||
/// remoting frame and send the sequenced chunks to the requester (all sharing one
|
||||
/// resyncId). Task 7 arms the ack-timeout here.
|
||||
/// </summary>
|
||||
private void HandleSfResyncSnapshotLoaded(SfResyncSnapshotLoaded msg)
|
||||
{
|
||||
var chunks = ChunkForRemoting(msg.Messages, MaxResyncChunkBytes, MaxResyncChunkRows);
|
||||
if (chunks.Count == 0) chunks.Add(new List<StoreAndForwardMessage>()); // empty buffer still resyncs (clears the standby)
|
||||
var resyncId = Guid.NewGuid().ToString("N");
|
||||
for (var i = 0; i < chunks.Count; i++)
|
||||
{
|
||||
if (chunks[i].Count == 1 && (chunks[i][0].PayloadJson?.Length ?? 0) + 512 > MaxResyncChunkBytes)
|
||||
_logger.LogWarning(
|
||||
"Resync row {Id} alone exceeds the chunk budget ({Bytes}B payload); sending solo — it may exceed the remoting frame",
|
||||
chunks[i][0].Id, chunks[i][0].PayloadJson?.Length ?? 0);
|
||||
msg.ReplyTo.Tell(new SfBufferSnapshotChunk(resyncId, i + 1, chunks.Count, chunks[i], msg.Truncated), Self);
|
||||
}
|
||||
_logger.LogInformation(
|
||||
"Answered S&F resync request with {Rows} row(s) in {Chunks} chunk(s), resyncId={ResyncId}",
|
||||
msg.Messages.Count, chunks.Count, resyncId);
|
||||
// Arm the ack window: absence of an SfBufferResyncAck within it surfaces as a
|
||||
// Warning + counter (the silent-loss mode N2 flagged). Single-outstanding-resync
|
||||
// bookkeeping: a new request supersedes by overwriting the id and restarting the timer.
|
||||
_pendingAckResyncId = resyncId;
|
||||
Timers.StartSingleTimer(ResyncAckTimerKey, new ResyncAckTimedOut(resyncId), _resyncAckTimeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Standby-node side of the anti-entropy resync: replaces the local buffer
|
||||
/// wholesale with the active node's snapshot. Combined with the upsert-based
|
||||
/// replicated applies (arch review 02), any replicated op that lands
|
||||
/// after this resync merges cleanly onto the resynced state. An active node
|
||||
/// ignores a snapshot — it is the source of truth, never a resync target.
|
||||
/// </summary>
|
||||
private void HandleSfBufferSnapshot(SfBufferSnapshot msg)
|
||||
{
|
||||
if (SafeIsActive())
|
||||
{
|
||||
_logger.LogDebug("Ignoring S&F buffer snapshot — this node is active");
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.Truncated)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"S&F buffer resync snapshot truncated at {Cap} rows; divergence beyond the cap drains naturally",
|
||||
MaxResyncRows);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Applying S&F buffer resync snapshot ({Count} rows), replacing local buffer", msg.Messages.Count);
|
||||
|
||||
Task.Run(async () =>
|
||||
{
|
||||
// Belt-and-braces (N1): re-check at apply time. ReplaceAllAsync discards
|
||||
// every in-flight row (StoreAndForwardStorage.cs "Never call on an active
|
||||
// node"); a flip between message receipt and this point must abort.
|
||||
if (SafeIsActive())
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Discarding S&F buffer resync snapshot: this node became active before apply");
|
||||
return;
|
||||
}
|
||||
await _sfStorage.ReplaceAllAsync(msg.Messages);
|
||||
})
|
||||
.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
_logger.LogError(t.Exception, "Failed to apply S&F buffer resync snapshot");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Standby-node side of the chunked anti-entropy resync: accumulates the chunks of one
|
||||
/// <c>ResyncId</c>, and once all have arrived, assembles them in sequence order and
|
||||
/// replaces the local buffer atomically, then acks. A new <c>ResyncId</c> discards any
|
||||
/// stale partial assembly (review 02 round 2, N2). An active node ignores chunks.
|
||||
/// </summary>
|
||||
private void HandleSfBufferSnapshotChunk(SfBufferSnapshotChunk msg)
|
||||
{
|
||||
if (SafeIsActive())
|
||||
{
|
||||
_logger.LogDebug("Ignoring S&F resync chunk — this node is active");
|
||||
return;
|
||||
}
|
||||
|
||||
if (_assemblingResyncId != msg.ResyncId)
|
||||
{
|
||||
if (_assemblingResyncId != null)
|
||||
_logger.LogWarning(
|
||||
"Discarding partial S&F resync assembly {Old} ({Have}/{Want} chunks): a new resync {New} superseded it",
|
||||
_assemblingResyncId, _assemblingChunks.Count, _assemblingTotalChunks, msg.ResyncId);
|
||||
_assemblingResyncId = msg.ResyncId;
|
||||
_assemblingTotalChunks = msg.TotalChunks;
|
||||
_assemblingTruncated = msg.Truncated;
|
||||
_assemblingChunks.Clear();
|
||||
}
|
||||
|
||||
_assemblingChunks[msg.Sequence] = msg.Messages;
|
||||
Timers.StartSingleTimer(ResyncAssemblyTimerKey, new ResyncAssemblyTimedOut(msg.ResyncId), _resyncAssemblyTimeout);
|
||||
|
||||
if (_assemblingChunks.Count < _assemblingTotalChunks)
|
||||
return;
|
||||
|
||||
// Complete: assemble in sequence order and apply atomically.
|
||||
var assembled = Enumerable.Range(1, _assemblingTotalChunks)
|
||||
.SelectMany(seq => _assemblingChunks[seq])
|
||||
.ToList();
|
||||
var resyncId = _assemblingResyncId!;
|
||||
var truncated = _assemblingTruncated;
|
||||
_assemblingResyncId = null;
|
||||
_assemblingChunks.Clear();
|
||||
Timers.Cancel(ResyncAssemblyTimerKey);
|
||||
|
||||
if (truncated)
|
||||
_logger.LogWarning(
|
||||
"S&F buffer resync snapshot truncated at {Cap} rows; divergence beyond the cap drains naturally",
|
||||
MaxResyncRows);
|
||||
_logger.LogInformation(
|
||||
"Applying chunked S&F resync {ResyncId} ({Count} rows), replacing local buffer", resyncId, assembled.Count);
|
||||
|
||||
// KNOWN, ACCEPTED race (review 02 round 2, N5 — do NOT "fix" this into something
|
||||
// worse): a replicated Remove sent after the active node read its snapshot but
|
||||
// before the snapshot's chunks is ordered BEFORE them on the wire (same
|
||||
// sender/receiver pair), so this apply can re-add the removed row → an orphan
|
||||
// Pending row that a later failover re-delivers ONCE. Bounded, self-correcting at
|
||||
// the next resync, and inherent to no-ack replication; a delivered-side dedup or
|
||||
// op-sequencing scheme would cost far more than one rare duplicate.
|
||||
var replyTo = Sender;
|
||||
Task.Run(async () =>
|
||||
{
|
||||
if (SafeIsActive()) // belt-and-braces, mirrors the monolithic path (T3)
|
||||
{
|
||||
_logger.LogWarning("Discarding chunked S&F resync {ResyncId}: node became active before apply", resyncId);
|
||||
return;
|
||||
}
|
||||
await _sfStorage.ReplaceAllAsync(assembled);
|
||||
replyTo.Tell(new SfBufferResyncAck(resyncId, assembled.Count));
|
||||
})
|
||||
.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
_logger.LogError(t.Exception, "Failed to apply chunked S&F resync {ResyncId}", resyncId);
|
||||
});
|
||||
}
|
||||
|
||||
private void HandleResyncAssemblyTimedOut(ResyncAssemblyTimedOut msg)
|
||||
{
|
||||
if (_assemblingResyncId != msg.ResyncId) return; // superseded already
|
||||
_logger.LogWarning(
|
||||
"S&F resync assembly {ResyncId} timed out with {Have}/{Want} chunks — discarding partial (a lost chunk; the next peer-track retries)",
|
||||
msg.ResyncId, _assemblingChunks.Count, _assemblingTotalChunks);
|
||||
ScadaBridgeTelemetry.RecordReplicationFailure();
|
||||
_assemblingResyncId = null;
|
||||
_assemblingChunks.Clear();
|
||||
}
|
||||
|
||||
/// <summary>Internal: the resync snapshot finished loading; chunk and send to the requester.</summary>
|
||||
internal sealed record SfResyncSnapshotLoaded(
|
||||
IActorRef ReplyTo, List<StoreAndForwardMessage> Messages, bool Truncated);
|
||||
|
||||
/// <summary>Internal: a partial chunk assembly for <paramref name="ResyncId"/> exceeded its window.</summary>
|
||||
internal sealed record ResyncAssemblyTimedOut(string ResyncId);
|
||||
|
||||
/// <summary>Internal: the active node's ack window for <paramref name="ResyncId"/> expired.</summary>
|
||||
internal sealed record ResyncAckTimedOut(string ResyncId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Standby→active: request a full S&F buffer snapshot for anti-entropy resync
|
||||
/// (sent when a standby (re)tracks a peer). Crosses Akka remoting between the two
|
||||
/// site nodes; the POCO rides the default serializer.
|
||||
/// </summary>
|
||||
public sealed record RequestSfBufferResync;
|
||||
|
||||
/// <summary>
|
||||
/// Active→standby: full-buffer snapshot. <paramref name="Truncated"/> is true when
|
||||
/// the active node's buffer exceeded <c>MaxResyncRows</c> (the standby logs a Warning —
|
||||
/// divergence beyond the cap drains naturally as the active node delivers). Crosses
|
||||
/// Akka remoting; the message list rides the default serializer.
|
||||
/// </summary>
|
||||
public sealed record SfBufferSnapshot(List<StoreAndForwardMessage> Messages, bool Truncated);
|
||||
|
||||
/// <summary>
|
||||
/// Active→standby: one sequenced chunk of a full-buffer anti-entropy snapshot
|
||||
/// (review 02 round 2, N2 — the monolithic <see cref="SfBufferSnapshot"/> exceeds Akka
|
||||
/// remoting's default 128 000-byte frame for any realistic backlog). All chunks of one
|
||||
/// resync share <paramref name="ResyncId"/>; <paramref name="Sequence"/> is 1-based up to
|
||||
/// <paramref name="TotalChunks"/>. Additive message — the legacy monolithic snapshot
|
||||
/// handler is retained for rolling upgrades. Crosses intra-site Akka remoting (NOT
|
||||
/// ClusterClient — ClusterClientContractLockTests is intentionally not involved).
|
||||
/// </summary>
|
||||
public sealed record SfBufferSnapshotChunk(
|
||||
string ResyncId, int Sequence, int TotalChunks,
|
||||
List<StoreAndForwardMessage> Messages, bool Truncated);
|
||||
|
||||
/// <summary>
|
||||
/// Standby→active: delivery confirmation — the standby assembled all chunks of
|
||||
/// <paramref name="ResyncId"/> and applied them atomically (<paramref name="RowCount"/>
|
||||
/// rows installed). Absence within the ack window is surfaced by the active node
|
||||
/// (Warning + counter) — the silent-loss mode N2 flagged.
|
||||
/// </summary>
|
||||
public sealed record SfBufferResyncAck(string ResyncId, int RowCount);
|
||||
@@ -1,44 +0,0 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
|
||||
|
||||
// Outbound messages — sent by local DeploymentManagerActor/S&F service
|
||||
// to the local SiteReplicationActor for forwarding to the peer node.
|
||||
|
||||
/// <summary>Outbound: tell the peer to fetch+apply a deployed instance config by id (notify-and-fetch; no inline config).</summary>
|
||||
public record ReplicateConfigDeploy(
|
||||
string InstanceName, string DeploymentId, string RevisionHash, bool IsEnabled,
|
||||
string CentralFetchBaseUrl, string FetchToken);
|
||||
|
||||
/// <summary>Outbound: replicate removal of a deployed instance config to the peer node.</summary>
|
||||
public record ReplicateConfigRemove(string InstanceName);
|
||||
|
||||
/// <summary>Outbound: replicate an instance enabled/disabled flag change to the peer node.</summary>
|
||||
public record ReplicateConfigSetEnabled(string InstanceName, bool IsEnabled);
|
||||
|
||||
/// <summary>Outbound: replicate a system-wide artifact deployment (shared scripts, external systems, etc.) to the peer node.</summary>
|
||||
public record ReplicateArtifacts(DeployArtifactsCommand Command);
|
||||
|
||||
/// <summary>Outbound: replicate a store-and-forward buffer mutation (enqueue/dequeue/park/etc.) to the peer node.</summary>
|
||||
public record ReplicateStoreAndForward(ReplicationOperation Operation);
|
||||
|
||||
// Inbound messages — received from the peer's SiteReplicationActor
|
||||
// and applied to local SQLite storage.
|
||||
|
||||
/// <summary>Inbound: peer-replicated config deploy — the standby fetches the config by id and writes it (guarded).</summary>
|
||||
public record ApplyConfigDeploy(
|
||||
string InstanceName, string DeploymentId, string RevisionHash, bool IsEnabled,
|
||||
string CentralFetchBaseUrl, string FetchToken);
|
||||
|
||||
/// <summary>Inbound: apply peer-replicated removal of a deployed instance config to local SQLite.</summary>
|
||||
public record ApplyConfigRemove(string InstanceName);
|
||||
|
||||
/// <summary>Inbound: apply a peer-replicated instance enabled/disabled flag change to local SQLite.</summary>
|
||||
public record ApplyConfigSetEnabled(string InstanceName, bool IsEnabled);
|
||||
|
||||
/// <summary>Inbound: apply a peer-replicated system-wide artifact deployment to local SQLite.</summary>
|
||||
public record ApplyArtifacts(DeployArtifactsCommand Command);
|
||||
|
||||
/// <summary>Inbound: apply a peer-replicated store-and-forward buffer mutation to the local buffer.</summary>
|
||||
public record ApplyStoreAndForward(ReplicationOperation Operation);
|
||||
Reference in New Issue
Block a user