feat(localdb)!: replicate site config + sf_messages via CDC, delete the bespoke replicators
Tasks 14, 15 and 16, landed as ONE commit. PLAN DEFECT: these three tasks cannot compile separately. SiteReplicationActor takes a ReplicationService and calls ReplaceAllAsync (Task 14 deletes both); DeploymentManagerActor Tells message types declared in ReplicationMessages.cs (Task 15 deletes it); AkkaHostedService constructs the actor (Task 16). Any ordering leaves a broken intermediate. Combining them also strengthens the invariant Task 14 already stated for itself — the two mechanisms never both run, and never neither. Registered 8 tables in SiteLocalDbSetup.OnReady: sf_messages plus the 7 site config tables. notification_lists and smtp_configurations are deliberately NOT registered — permanently empty by design, so registering them would open a standing replication channel whose only historical payload was plaintext SMTP passwords. Migrate stays the LAST call in OnReady, after all registrations, so migrated rows enter the oplog through live capture triggers. Deleted: SiteReplicationActor, ReplicationMessages.cs, ReplicationService, StoreAndForwardStorage.ReplaceAllAsync, and 6 test files. ReplaceAllAsync is not merely unused but unsafe to keep: a mass DELETE on a now-replicated table would be captured and shipped to the peer. Kept ActiveNodeEvaluator (delivery gate + heartbeat still need it) with its doc corrected, and activeNodeCheck in AkkaHostedService (SiteCommunicationActor). The positional-argument hazard the plan flagged was real: removing DeploymentManagerActor's optional IActorRef? replicationActor shifted 6 trailing optionals, and 4 test call sites bound the wrong arguments with no compile error at some positions. Converted them to named arguments where possible — Props.Create builds an expression tree, which rejects out-of-position named args, so the rest are padded positionally with a comment saying why. The Task 7 'not yet registered' test was INVERTED rather than deleted, and is exact in both directions: too few means a table silently stops replicating, too many means the SMTP tables leak. Added a separate security-named test for those two, and a composite-PK test (LWW keys on the full PK, so a truncated key set would collapse distinct rows). The convergence suites now get their registrations from the real OnReady — their temporary harness registration is deleted, so they prove the cutover rather than agreeing with themselves. Verified: build 0 warnings; SiteRuntime 512, StoreAndForward 130, Host 330, AuditLog 355, ExternalSystemGateway 142, HealthMonitoring 97, LocalDb integration 16 — all pass, 0 failures. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
+20
-14
@@ -67,12 +67,14 @@ public class DeploymentManagerActorTests : TestKit, IDisposable
|
||||
null, // no stream manager in tests
|
||||
options,
|
||||
NullLogger<DeploymentManagerActor>.Instance,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
serviceProvider,
|
||||
null,
|
||||
configFetcher)));
|
||||
// Named from here on. These trailing parameters are all optional and several
|
||||
// share a type, so a positional list silently binds the wrong argument when the
|
||||
// signature changes — which is exactly what removing replicationActor did.
|
||||
dclManager: null,
|
||||
healthCollector: null,
|
||||
serviceProvider: serviceProvider,
|
||||
loggerFactory: null,
|
||||
configFetcher: configFetcher)));
|
||||
}
|
||||
|
||||
private static string MakeConfigJson(string instanceName)
|
||||
@@ -301,8 +303,11 @@ public class DeploymentManagerActorTests : TestKit, IDisposable
|
||||
var dm = ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage, _compilationService, _sharedScriptLibrary, null,
|
||||
new SiteRuntimeOptions(), NullLogger<DeploymentManagerActor>.Instance,
|
||||
null, null, null, null, null, null,
|
||||
TimeSpan.FromMilliseconds(200), loader)));
|
||||
// dclManager, healthCollector, serviceProvider, loggerFactory, configFetcher.
|
||||
// Props.Create builds an expression tree, which rejects named arguments that
|
||||
// are out of position, so the optional tail has to be padded positionally.
|
||||
null, null, null, null, null,
|
||||
startupLoadRetryInterval: TimeSpan.FromMilliseconds(200), configLoader: loader)));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
@@ -892,13 +897,14 @@ public class DeploymentManagerActorTests : TestKit, IDisposable
|
||||
// Security cleanup. notification_lists and smtp_configurations can hold plaintext
|
||||
// SMTP passwords written by a pre-2026-07-10 build, and the ACTIVE node's artifact
|
||||
// apply is what clears them (DeploymentManagerActor.HandleDeployArtifacts). The
|
||||
// standby's copy of this call lives in SiteReplicationActor and dies with it at
|
||||
// Task 15, which makes this call site the ONLY remaining one.
|
||||
// standby used to hold a second copy of this call in SiteReplicationActor; LocalDb
|
||||
// Phase 2 deleted that actor, so this is now the ONLY call site that keeps the
|
||||
// tables empty.
|
||||
//
|
||||
// Nothing pins it today: ArtifactStorageTests covers the storage method, not the
|
||||
// actor's call to it, so Task 16's edits to this actor could drop the call and every
|
||||
// suite would stay green. That is precisely the kind of silent security regression
|
||||
// this test exists to prevent — verified red-first by commenting out the call.
|
||||
// ArtifactStorageTests covers the storage method, not the actor's call to it, so
|
||||
// without this test the call could be dropped and every suite would stay green.
|
||||
// That is precisely the kind of silent security regression it exists to prevent —
|
||||
// verified red-first by commenting out the call.
|
||||
await SeedCentralOnlyRowsAsync();
|
||||
Assert.Equal(1, await RowCountAsync("notification_lists"));
|
||||
Assert.Equal(1, await RowCountAsync("smtp_configurations"));
|
||||
|
||||
+1
-2
@@ -80,8 +80,7 @@ akka {
|
||||
private IActorRef CreateDeploymentManager() =>
|
||||
ActorOf(Props.Create(() => new DeploymentManagerActor(
|
||||
_storage, _compilationService, _sharedScriptLibrary,
|
||||
null, new SiteRuntimeOptions(), NullLogger<DeploymentManagerActor>.Instance,
|
||||
null, null, null, null, null, null, null, null)));
|
||||
null, new SiteRuntimeOptions(), NullLogger<DeploymentManagerActor>.Instance)));
|
||||
|
||||
[Fact]
|
||||
public void SiteNodeJoined_PushesLocalTrustedCertsToJoinedNode()
|
||||
|
||||
+4
-5
@@ -103,11 +103,10 @@ public class DeploymentManagerLoggerFactoryTests : TestKit, IDisposable
|
||||
null,
|
||||
new SiteRuntimeOptions { StartupBatchSize = 100, StartupBatchDelayMs = 5 },
|
||||
NullLogger<DeploymentManagerActor>.Instance,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
loggerFactory)));
|
||||
// dclManager, healthCollector, serviceProvider — padded positionally because
|
||||
// Props.Create is an expression tree and rejects out-of-position named args.
|
||||
null, null, null,
|
||||
loggerFactory: loggerFactory)));
|
||||
|
||||
// Allow async startup (load configs + staggered creation).
|
||||
await Task.Delay(2000);
|
||||
|
||||
+4
-3
@@ -61,10 +61,11 @@ public class DeploymentManagerRedeployTests : TestKit, IDisposable
|
||||
null,
|
||||
new SiteRuntimeOptions(),
|
||||
NullLogger<DeploymentManagerActor>.Instance,
|
||||
// dclManager — padded positionally because Props.Create is an expression tree
|
||||
// and rejects out-of-position named args.
|
||||
null,
|
||||
null,
|
||||
healthCollector,
|
||||
serviceProvider)));
|
||||
healthCollector: healthCollector,
|
||||
serviceProvider: serviceProvider)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -1,568 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.Metrics;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteReplicationActor"/>'s notify-and-fetch config replication:
|
||||
/// the active node now replicates an id-only <see cref="ReplicateConfigDeploy"/> (no inline
|
||||
/// config JSON — killing the intra-site 128 KB frame trap), and the standby fetches the
|
||||
/// config from central over HTTP and writes it with the older-write guard.
|
||||
/// </summary>
|
||||
public class SiteReplicationActorTests : TestKit, IDisposable
|
||||
{
|
||||
// Cluster provider is required because SiteReplicationActor calls Cluster.Get in its ctor
|
||||
// and subscribes to cluster events in PreStart. We use the in-memory TestTransport (not
|
||||
// dot-netty) so no real socket is bound and no DNS lookup happens — the actor only needs
|
||||
// the cluster extension to load; these tests never form a real two-node cluster.
|
||||
private const string ClusterConfig = @"
|
||||
akka {
|
||||
actor { provider = cluster }
|
||||
remote {
|
||||
enabled-transports = [""akka.remote.test""]
|
||||
test {
|
||||
transport-class = ""Akka.Remote.Transport.TestTransport, Akka.Remote""
|
||||
applied-adapters = []
|
||||
registry-key = site-repl-test
|
||||
local-address = ""test://site-repl@localhost:1""
|
||||
maximum-payload-bytes = 128000b
|
||||
scheme-identifier = test
|
||||
}
|
||||
}
|
||||
cluster { roles = [""site-test""] }
|
||||
loglevel = WARNING
|
||||
}";
|
||||
|
||||
private const string SiteRole = "site-test";
|
||||
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly TestLocalDb _siteLocalDb;
|
||||
private readonly TestLocalDb _sfLocalDb;
|
||||
private readonly StoreAndForwardStorage _sfStorage;
|
||||
private readonly ReplicationService _replicationService;
|
||||
private readonly string _sfDbFile;
|
||||
|
||||
public SiteReplicationActorTests() : base(ClusterConfig, "site-repl")
|
||||
{
|
||||
_sfDbFile = Path.Combine(Path.GetTempPath(), $"site-repl-sf-{Guid.NewGuid():N}.db");
|
||||
|
||||
// SiteStorageService takes an ILocalDb now; LocalDb has no in-memory mode, so the
|
||||
// site store gets its own temp-file database alongside the S&F one.
|
||||
_siteLocalDb = TestLocalDb.CreateTemp("site-repl-test");
|
||||
_storage = new SiteStorageService(
|
||||
_siteLocalDb.Db, NullLogger<SiteStorageService>.Instance);
|
||||
_storage.InitializeAsync().GetAwaiter().GetResult();
|
||||
|
||||
_sfLocalDb = TestLocalDb.Create(_sfDbFile);
|
||||
_sfStorage = new StoreAndForwardStorage(
|
||||
_sfLocalDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
|
||||
_sfStorage.InitializeAsync().GetAwaiter().GetResult();
|
||||
|
||||
_replicationService = new ReplicationService(
|
||||
new StoreAndForwardOptions(), NullLogger<ReplicationService>.Instance);
|
||||
}
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
Shutdown();
|
||||
// The master connection anchors the WAL — dispose before deleting.
|
||||
var siteDbPath = _siteLocalDb.Path;
|
||||
_siteLocalDb.Dispose();
|
||||
_sfLocalDb.Dispose();
|
||||
TestLocalDb.DeleteFiles(siteDbPath);
|
||||
TestLocalDb.DeleteFiles(_sfDbFile);
|
||||
}
|
||||
|
||||
private IActorRef CreateReplicationActor(IDeploymentConfigFetcher fetcher) =>
|
||||
ActorOf(Props.Create(() => new SiteReplicationActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, fetcher)));
|
||||
|
||||
private IActorRef CreateReplicationActor(
|
||||
IDeploymentConfigFetcher fetcher, SiteRuntimeOptions options, TimeSpan retryDelay) =>
|
||||
ActorOf(Props.Create(() => new SiteReplicationActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, fetcher, null, options, retryDelay)));
|
||||
|
||||
[Fact]
|
||||
public async Task ReplicatedFetch_RetriesUpToConfigFetchRetryCount()
|
||||
{
|
||||
// The first two fetches fail transiently; the third succeeds. With
|
||||
// ConfigFetchRetryCount = 3 the standby must retry to the third attempt and
|
||||
// then guarded-write the fetched config (a short retry delay keeps the test fast).
|
||||
var attempts = 0;
|
||||
var fetcher = new FakeConfigFetcher(_ =>
|
||||
Interlocked.Increment(ref attempts) < 3
|
||||
? Task.FromException<string>(new InvalidOperationException("central hiccup"))
|
||||
: Task.FromResult("{\"instanceUniqueName\":\"RetryPump\"}"));
|
||||
var actor = CreateReplicationActor(
|
||||
fetcher, new SiteRuntimeOptions { ConfigFetchRetryCount = 3 },
|
||||
TimeSpan.FromMilliseconds(50));
|
||||
|
||||
actor.Tell(new ApplyConfigDeploy(
|
||||
"RetryPump", "dep-r1", "sha256:r1", true,
|
||||
"http://central:9000", "tok-r1"));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
Assert.Equal(3, Volatile.Read(ref attempts));
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.Single(configs, c => c.InstanceUniqueName == "RetryPump");
|
||||
}, TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyConfigDeploy_StandbyFetchesConfigAndGuardedWrites()
|
||||
{
|
||||
// The standby receives an id-only ApplyConfigDeploy; it fetches the config from
|
||||
// central using the message's coords, then guarded-writes the fetched config.
|
||||
const string configJson = "{\"instanceUniqueName\":\"Pump1\"}";
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult(configJson));
|
||||
var actor = CreateReplicationActor(fetcher);
|
||||
|
||||
actor.Tell(new ApplyConfigDeploy(
|
||||
"Pump1", "dep-100", "sha256:abc", true,
|
||||
"http://central:9000", "tok-xyz"));
|
||||
|
||||
// The continuation runs off-thread; await the guarded write landing.
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
var row = Assert.Single(configs, c => c.InstanceUniqueName == "Pump1");
|
||||
Assert.Equal(configJson, row.ConfigJson);
|
||||
Assert.Equal("dep-100", row.DeploymentId);
|
||||
Assert.Equal("sha256:abc", row.RevisionHash);
|
||||
Assert.True(row.IsEnabled);
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// The fetcher was called with the message's coords.
|
||||
var call = Assert.Single(fetcher.Calls);
|
||||
Assert.Equal("http://central:9000", call.BaseUrl);
|
||||
Assert.Equal("dep-100", call.DeploymentId);
|
||||
Assert.Equal("tok-xyz", call.Token);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyConfigDeploy_Superseded404_SkipsWriteAndActorSurvives()
|
||||
{
|
||||
// A 404 (superseded/expired) surfaces as DeploymentConfigFetchException{IsSuperseded}.
|
||||
// The standby must skip the write, observe the exception (no crash), and stay alive.
|
||||
var fetcher = new FakeConfigFetcher(_ =>
|
||||
Task.FromException<string>(
|
||||
new DeploymentConfigFetchException("expired", isSuperseded: true)));
|
||||
var actor = CreateReplicationActor(fetcher);
|
||||
|
||||
actor.Tell(new ApplyConfigDeploy(
|
||||
"GonePump", "dep-stale", "sha256:gone", true,
|
||||
"http://central:9000", "tok-stale"));
|
||||
|
||||
// The fetch was attempted...
|
||||
await AwaitAssertAsync(() =>
|
||||
{
|
||||
Assert.Single(fetcher.Calls);
|
||||
return Task.CompletedTask;
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
|
||||
// ...the actor did not crash (no Terminated to its watcher within the window)...
|
||||
Watch(actor);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
// ...and nothing was written for the superseded instance.
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "GonePump");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ApplyConfigDeploy_EmptyFetchCoords_SkipsFetchAndWrite()
|
||||
{
|
||||
// The direct DeployInstanceCommand cross-cluster wire path was retired in Task 14.
|
||||
// This tests the defensive guard: if empty coords arrive, the actor must skip quietly
|
||||
// — no FetchAsync("") call, no write — rather than erroring.
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("never"));
|
||||
var actor = CreateReplicationActor(fetcher);
|
||||
|
||||
actor.Tell(new ApplyConfigDeploy(
|
||||
"NoCoordsPump", "dep-direct", "sha256:nc", true,
|
||||
CentralFetchBaseUrl: "", FetchToken: ""));
|
||||
|
||||
// Give any (erroneous) async continuation time to run, then prove neither happened.
|
||||
Watch(actor);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
Assert.Empty(fetcher.Calls);
|
||||
var configs = await _storage.GetAllDeployedConfigsAsync();
|
||||
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "NoCoordsPump");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReplicateConfigDeploy_MapsToIdOnlyApplyConfigDeploy_ForPeer()
|
||||
{
|
||||
// The outbound mapping must forward an id-only ApplyConfigDeploy carrying the fetch
|
||||
// coords (and NO inline config) to the peer.
|
||||
var probe = CreateTestProbe();
|
||||
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("unused"));
|
||||
var actor = ActorOf(Props.Create(() => new ProbeForwardingReplicationActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, fetcher, probe.Ref)));
|
||||
|
||||
actor.Tell(new ReplicateConfigDeploy(
|
||||
"Pump2", "dep-200", "sha256:def", false,
|
||||
"http://central:9000", "tok-abc"));
|
||||
|
||||
var applied = probe.ExpectMsg<ApplyConfigDeploy>(TimeSpan.FromSeconds(3));
|
||||
Assert.Equal("Pump2", applied.InstanceName);
|
||||
Assert.Equal("dep-200", applied.DeploymentId);
|
||||
Assert.Equal("sha256:def", applied.RevisionHash);
|
||||
Assert.False(applied.IsEnabled);
|
||||
Assert.Equal("http://central:9000", applied.CentralFetchBaseUrl);
|
||||
Assert.Equal("tok-abc", applied.FetchToken);
|
||||
}
|
||||
|
||||
// ── Task 21: peer-join S&F buffer resync (anti-entropy) ──
|
||||
|
||||
[Fact]
|
||||
public void StandbyTrackingPeer_SendsResyncRequest()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var actor = ActorOf(Props.Create(() => new ResyncTestActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, probe.Ref, () => false)));
|
||||
|
||||
actor.Tell(new TriggerPeerTracked()); // stands in for TryTrackPeer's MemberUp path
|
||||
|
||||
probe.ExpectMsg<RequestSfBufferResync>(TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ActiveTrackingPeer_DoesNotRequestResync()
|
||||
{
|
||||
var probe = CreateTestProbe();
|
||||
var actor = ActorOf(Props.Create(() => new ResyncTestActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, probe.Ref, () => true)));
|
||||
|
||||
actor.Tell(new TriggerPeerTracked());
|
||||
|
||||
probe.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); // active node never requests a resync
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ActiveNode_AnswersResyncRequest_WithChunkedSnapshot()
|
||||
{
|
||||
// Post-R2-T5 the active node answers with byte-budgeted SfBufferSnapshotChunk(s)
|
||||
// (a single small row rides one chunk) rather than the monolithic SfBufferSnapshot.
|
||||
await _sfStorage.EnqueueAsync(NewSfMessage("m1"));
|
||||
var probe = CreateTestProbe();
|
||||
var actor = ActorOf(Props.Create(() => new ResyncTestActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, probe.Ref, () => true)));
|
||||
|
||||
actor.Tell(new RequestSfBufferResync(), TestActor);
|
||||
|
||||
var chunk = ExpectMsg<SfBufferSnapshotChunk>(TimeSpan.FromSeconds(3));
|
||||
Assert.Equal(1, chunk.TotalChunks);
|
||||
Assert.Equal(1, chunk.Sequence);
|
||||
Assert.Single(chunk.Messages);
|
||||
Assert.False(chunk.Truncated);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StandbyNode_AppliesSnapshot_ReplacingItsBuffer()
|
||||
{
|
||||
await _sfStorage.EnqueueAsync(NewSfMessage("stale"));
|
||||
var probe = CreateTestProbe();
|
||||
var actor = ActorOf(Props.Create(() => new ResyncTestActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, probe.Ref, () => false)));
|
||||
|
||||
actor.Tell(new SfBufferSnapshot(new List<StoreAndForwardMessage> { NewSfMessage("fresh") }, false));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
Assert.Null(await _sfStorage.GetMessageByIdAsync("stale"));
|
||||
Assert.NotNull(await _sfStorage.GetMessageByIdAsync("fresh"));
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ── R2 T5: chunked resync answer ──
|
||||
|
||||
[Fact]
|
||||
public void ChunkForRemoting_SplitsByByteBudget_PreservingOrderAndSequence()
|
||||
{
|
||||
var rows = Enumerable.Range(0, 10)
|
||||
.Select(i => NewMessage($"m{i}", payloadJson: new string('x', 20_000)))
|
||||
.ToList();
|
||||
|
||||
var chunks = SiteReplicationActor.ChunkForRemoting(rows, maxChunkBytes: 64_000, maxChunkRows: 200);
|
||||
|
||||
Assert.True(chunks.Count > 1); // 10 × 20 KB cannot ride one 64 KB chunk
|
||||
Assert.Equal(rows.Select(r => r.Id), chunks.SelectMany(c => c).Select(r => r.Id)); // order preserved
|
||||
Assert.All(chunks, c => Assert.True(
|
||||
c.Sum(r => r.PayloadJson.Length) <= 64_000 || c.Count == 1)); // budget honored (oversized row isolated)
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ChunkForRemoting_RowCapHonored_AndSingleOversizedRowIsolated()
|
||||
{
|
||||
var many = Enumerable.Range(0, 500).Select(i => NewMessage($"s{i}", payloadJson: "{}")).ToList();
|
||||
Assert.All(SiteReplicationActor.ChunkForRemoting(many, 64_000, 200), c => Assert.True(c.Count <= 200));
|
||||
|
||||
var oversized = new List<StoreAndForwardMessage>
|
||||
{ NewMessage("big", payloadJson: new string('y', 100_000)), NewMessage("small", payloadJson: "{}") };
|
||||
var chunks = SiteReplicationActor.ChunkForRemoting(oversized, 64_000, 200);
|
||||
Assert.Equal(2, chunks.Count); // the oversized row rides alone
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ActiveNode_AnswersResyncRequest_WithSequencedChunks_SharingOneResyncId()
|
||||
{
|
||||
for (var i = 0; i < 3; i++)
|
||||
await _sfStorage.EnqueueAsync(NewMessage($"c{i}", payloadJson: new string('z', 30_000)));
|
||||
var actor = CreateResyncActor(isActive: () => true);
|
||||
|
||||
actor.Tell(new RequestSfBufferResync(), TestActor);
|
||||
|
||||
var first = ExpectMsg<SfBufferSnapshotChunk>(TimeSpan.FromSeconds(5));
|
||||
var rest = Enumerable.Range(1, first.TotalChunks - 1)
|
||||
.Select(_ => ExpectMsg<SfBufferSnapshotChunk>(TimeSpan.FromSeconds(5)))
|
||||
.Prepend(first)
|
||||
.ToList();
|
||||
|
||||
Assert.True(first.TotalChunks > 1);
|
||||
Assert.All(rest, c => Assert.Equal(first.ResyncId, c.ResyncId));
|
||||
Assert.Equal(Enumerable.Range(1, first.TotalChunks), rest.Select(c => c.Sequence));
|
||||
Assert.Equal(3, rest.Sum(c => c.Messages.Count));
|
||||
}
|
||||
|
||||
// ── R2 T6: standby chunk assembly + atomic apply + ack ──
|
||||
|
||||
[Fact]
|
||||
public async Task StandbyNode_AssemblesChunks_AppliesOnce_AndAcks()
|
||||
{
|
||||
await _sfStorage.EnqueueAsync(NewMessage("stale"));
|
||||
var actor = CreateResyncActor(isActive: () => false);
|
||||
var resyncId = "r1";
|
||||
|
||||
actor.Tell(new SfBufferSnapshotChunk(resyncId, 1, 2,
|
||||
new List<StoreAndForwardMessage> { NewMessage("f1") }, false), TestActor);
|
||||
actor.Tell(new SfBufferSnapshotChunk(resyncId, 2, 2,
|
||||
new List<StoreAndForwardMessage> { NewMessage("f2") }, false), TestActor);
|
||||
|
||||
var ack = ExpectMsg<SfBufferResyncAck>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(resyncId, ack.ResyncId);
|
||||
Assert.Equal(2, ack.RowCount);
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
Assert.Null(await _sfStorage.GetMessageByIdAsync("stale")); // replaced wholesale
|
||||
Assert.NotNull(await _sfStorage.GetMessageByIdAsync("f1"));
|
||||
Assert.NotNull(await _sfStorage.GetMessageByIdAsync("f2"));
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task StandbyNode_NewResyncId_DiscardsStalePartialAssembly()
|
||||
{
|
||||
var actor = CreateResyncActor(isActive: () => false);
|
||||
actor.Tell(new SfBufferSnapshotChunk("old", 1, 2,
|
||||
new List<StoreAndForwardMessage> { NewMessage("orphan") }, false), TestActor);
|
||||
actor.Tell(new SfBufferSnapshotChunk("new", 1, 1,
|
||||
new List<StoreAndForwardMessage> { NewMessage("fresh") }, false), TestActor);
|
||||
|
||||
ExpectMsg<SfBufferResyncAck>(TimeSpan.FromSeconds(5)); // "new" completed
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
Assert.NotNull(await _sfStorage.GetMessageByIdAsync("fresh"));
|
||||
Assert.Null(await _sfStorage.GetMessageByIdAsync("orphan")); // stale partial never applied
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ActiveNode_IgnoresChunks_NeverAcks()
|
||||
{
|
||||
var actor = CreateResyncActor(isActive: () => true);
|
||||
actor.Tell(new SfBufferSnapshotChunk("r", 1, 1,
|
||||
new List<StoreAndForwardMessage> { NewMessage("x") }, false), TestActor);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
// ── R2 T7: active-side resync ack confirmation + telemetry ──
|
||||
//
|
||||
// NOTE (deviation from plan): the actor logs via Microsoft ILogger (NullLogger in
|
||||
// tests), NOT Akka's EventStream, so the plan's EventFilter.Warning assertions can
|
||||
// never observe these warnings. We observe the two OTel counters via a MeterListener
|
||||
// instead — the equivalent, and stronger, observable signal.
|
||||
|
||||
[Fact]
|
||||
public async Task ActiveNode_ReceivingAck_CountsResyncCompleted()
|
||||
{
|
||||
long completed = 0;
|
||||
using var listener = ListenCounter("scadabridge.store_and_forward.resync.completed",
|
||||
m => Interlocked.Add(ref completed, m));
|
||||
|
||||
await _sfStorage.EnqueueAsync(NewMessage("m1"));
|
||||
var actor = CreateResyncActor(isActive: () => true, ackTimeout: TimeSpan.FromSeconds(30));
|
||||
actor.Tell(new RequestSfBufferResync(), TestActor);
|
||||
var chunk = ExpectMsg<SfBufferSnapshotChunk>(TimeSpan.FromSeconds(5));
|
||||
|
||||
actor.Tell(new SfBufferResyncAck(chunk.ResyncId, 1), TestActor);
|
||||
|
||||
await AwaitAssertAsync(() =>
|
||||
{
|
||||
Assert.True(Interlocked.Read(ref completed) >= 1); // ack recorded the completion
|
||||
return Task.CompletedTask;
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ActiveNode_MissingAck_WarnsAfterAckTimeout()
|
||||
{
|
||||
long ackMissing = 0;
|
||||
using var listener = ListenCounter("scadabridge.store_and_forward.resync.ack_missing",
|
||||
m => Interlocked.Add(ref ackMissing, m));
|
||||
|
||||
await _sfStorage.EnqueueAsync(NewMessage("m1"));
|
||||
var actor = CreateResyncActor(isActive: () => true, ackTimeout: TimeSpan.FromMilliseconds(200));
|
||||
actor.Tell(new RequestSfBufferResync(), TestActor);
|
||||
ExpectMsg<SfBufferSnapshotChunk>(TimeSpan.FromSeconds(5));
|
||||
// No ack is sent → the ack window expires and the resync is counted unacknowledged.
|
||||
|
||||
await AwaitAssertAsync(() =>
|
||||
{
|
||||
Assert.True(Interlocked.Read(ref ackMissing) >= 1);
|
||||
return Task.CompletedTask;
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
/// <summary>Attaches a <see cref="MeterListener"/> to a single ScadaBridge counter by name,
|
||||
/// forwarding each recorded increment to <paramref name="onMeasurement"/>.</summary>
|
||||
private static MeterListener ListenCounter(string instrumentName, Action<long> onMeasurement)
|
||||
{
|
||||
var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (inst, l) =>
|
||||
{
|
||||
if (inst.Meter.Name == ScadaBridgeTelemetry.MeterName && inst.Name == instrumentName)
|
||||
l.EnableMeasurementEvents(inst);
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((_, m, _, _) => onMeasurement(m));
|
||||
listener.Start();
|
||||
return listener;
|
||||
}
|
||||
|
||||
private static StoreAndForwardMessage NewSfMessage(string id) => new()
|
||||
{
|
||||
Id = id,
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "t",
|
||||
PayloadJson = "{}",
|
||||
RetryCount = 0,
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Builds a resync-test message with a settable payload (additive to
|
||||
/// <see cref="NewSfMessage"/> — the chunker sizes on <c>PayloadJson</c> length).
|
||||
/// </summary>
|
||||
private static StoreAndForwardMessage NewMessage(string id, string payloadJson = "{}") => new()
|
||||
{
|
||||
Id = id,
|
||||
Category = StoreAndForwardCategory.ExternalSystem,
|
||||
Target = "t",
|
||||
PayloadJson = payloadJson,
|
||||
RetryCount = 0,
|
||||
MaxRetries = 50,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Pending,
|
||||
};
|
||||
|
||||
/// <summary>Constructs a <see cref="ResyncTestActor"/> with the given active-node check
|
||||
/// (the resync chunk/ack tests Tell to and expect from <see cref="TestKit.TestActor"/>).
|
||||
/// <paramref name="ackTimeout"/> is the active-side ack window seam (T7).</summary>
|
||||
private IActorRef CreateResyncActor(Func<bool> isActive, TimeSpan? ackTimeout = null) =>
|
||||
ActorOf(Props.Create(() => new ResyncTestActor(
|
||||
_storage, _sfStorage, _replicationService, SiteRole,
|
||||
NullLogger<SiteReplicationActor>.Instance, CreateTestProbe().Ref, isActive, ackTimeout)));
|
||||
|
||||
/// <summary>Test message: drives <see cref="SiteReplicationActor.OnPeerTracked"/> directly,
|
||||
/// standing in for the MemberUp→TryTrackPeer path (a single-node TestKit cannot form a real peer).</summary>
|
||||
private sealed record TriggerPeerTracked;
|
||||
|
||||
/// <summary>
|
||||
/// Test subclass for the resync tests: captures peer sends to a probe, injects the
|
||||
/// active-node check, and exposes <see cref="OnPeerTracked"/> via a test message.
|
||||
/// </summary>
|
||||
private sealed class ResyncTestActor : SiteReplicationActor
|
||||
{
|
||||
private readonly IActorRef _peerProbe;
|
||||
|
||||
public ResyncTestActor(
|
||||
SiteStorageService storage, StoreAndForwardStorage sfStorage,
|
||||
ReplicationService replicationService, string siteRole,
|
||||
ILogger<SiteReplicationActor> logger, IActorRef peerProbe, Func<bool> isActive,
|
||||
TimeSpan? ackTimeout = null)
|
||||
: base(storage, sfStorage, replicationService, siteRole, logger,
|
||||
configFetcher: null, isActiveOverride: isActive, resyncAckTimeout: ackTimeout)
|
||||
{
|
||||
_peerProbe = peerProbe;
|
||||
Receive<TriggerPeerTracked>(_ => OnPeerTracked());
|
||||
}
|
||||
|
||||
protected override void SendToPeer(object message) => _peerProbe.Tell(message, Self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Test subclass exposing the peer send: <see cref="SiteReplicationActor.SendToPeer"/> is
|
||||
/// overridden to forward to a probe so the outbound mapping can be asserted without a real
|
||||
/// two-node cluster (a single-node TestKit has no peer address, so the real send is dropped).
|
||||
/// </summary>
|
||||
private sealed class ProbeForwardingReplicationActor : SiteReplicationActor
|
||||
{
|
||||
private readonly IActorRef _peerProbe;
|
||||
|
||||
public ProbeForwardingReplicationActor(
|
||||
SiteStorageService storage, StoreAndForwardStorage sfStorage,
|
||||
ReplicationService replicationService, string siteRole,
|
||||
ILogger<SiteReplicationActor> logger, IDeploymentConfigFetcher configFetcher,
|
||||
IActorRef peerProbe)
|
||||
: base(storage, sfStorage, replicationService, siteRole, logger, configFetcher)
|
||||
=> _peerProbe = peerProbe;
|
||||
|
||||
protected override void SendToPeer(object message) => _peerProbe.Tell(message, Self);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In-test fake <see cref="IDeploymentConfigFetcher"/>: runs a per-deploymentId behavior
|
||||
/// (return config JSON or throw, as a Task — mirroring the real async HTTP fetcher) and
|
||||
/// records every call's coords thread-safely (the continuation runs on a pool thread).
|
||||
/// </summary>
|
||||
private sealed class FakeConfigFetcher : IDeploymentConfigFetcher
|
||||
{
|
||||
private readonly Func<string, Task<string>> _behavior;
|
||||
public ConcurrentQueue<(string BaseUrl, string DeploymentId, string Token)> Calls { get; } = new();
|
||||
|
||||
public FakeConfigFetcher(Func<string, Task<string>> behavior) => _behavior = behavior;
|
||||
|
||||
public async Task<string> FetchAsync(
|
||||
string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
|
||||
{
|
||||
Calls.Enqueue((centralFetchBaseUrl, deploymentId, token));
|
||||
await Task.Yield();
|
||||
return await _behavior(deploymentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
using Akka.TestKit.Xunit2;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Characterization pin for the chunked anti-entropy resync contract (review 02 round 2,
|
||||
/// N2). <see cref="SfBufferSnapshotChunk"/> (active→standby) and <see cref="SfBufferResyncAck"/>
|
||||
/// (standby→active) ride intra-site Akka remoting on the default reflective-JSON wire
|
||||
/// format. A rename/move, dropped setter, or non-default-constructible message would
|
||||
/// silently break resync across a rolling upgrade and only surface as a divergent buffer
|
||||
/// after a failover. These pin round-trip fidelity and type identity. (These messages are
|
||||
/// NOT ClusterClient traffic, so they are intentionally absent from ClusterClientContractLockTests.)
|
||||
/// </summary>
|
||||
public class ResyncWireSerializationPinTests : TestKit
|
||||
{
|
||||
private T RoundTrip<T>(T message)
|
||||
{
|
||||
var serialization = Sys.Serialization;
|
||||
var serializer = serialization.FindSerializerFor(message);
|
||||
var bytes = serializer.ToBinary(message);
|
||||
return (T)serialization.Deserialize(bytes, serializer.Identifier, message!.GetType());
|
||||
}
|
||||
|
||||
private static StoreAndForwardMessage FullMessage() => new()
|
||||
{
|
||||
Id = Guid.NewGuid().ToString("N"),
|
||||
Category = StoreAndForwardCategory.Notification,
|
||||
Target = "Operators",
|
||||
PayloadJson = "{\"notificationId\":\"abc\"}",
|
||||
RetryCount = 4,
|
||||
MaxRetries = 0,
|
||||
RetryIntervalMs = 30000,
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
LastAttemptAt = DateTimeOffset.UtcNow,
|
||||
Status = StoreAndForwardMessageStatus.Parked,
|
||||
LastError = "central rejected",
|
||||
OriginInstanceName = "Plant.Pump3",
|
||||
ExecutionId = Guid.NewGuid(),
|
||||
SourceScript = "ScriptActor:MonitorSpeed",
|
||||
ParentExecutionId = Guid.NewGuid(),
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void SfBufferSnapshotChunk_WithFullMessage_RoundTripsOnTheWire()
|
||||
{
|
||||
var message = FullMessage();
|
||||
var original = new SfBufferSnapshotChunk(
|
||||
"resync-1", 2, 5, new List<StoreAndForwardMessage> { message }, Truncated: true);
|
||||
|
||||
var back = RoundTrip(original);
|
||||
|
||||
Assert.Equal(original.ResyncId, back.ResyncId);
|
||||
Assert.Equal(original.Sequence, back.Sequence);
|
||||
Assert.Equal(original.TotalChunks, back.TotalChunks);
|
||||
Assert.Equal(original.Truncated, back.Truncated);
|
||||
var m = Assert.Single(back.Messages);
|
||||
Assert.Equal(message.Id, m.Id);
|
||||
Assert.Equal(message.Category, m.Category);
|
||||
Assert.Equal(message.Target, m.Target);
|
||||
Assert.Equal(message.PayloadJson, m.PayloadJson);
|
||||
Assert.Equal(message.RetryCount, m.RetryCount);
|
||||
Assert.Equal(message.MaxRetries, m.MaxRetries);
|
||||
Assert.Equal(message.RetryIntervalMs, m.RetryIntervalMs);
|
||||
Assert.Equal(message.CreatedAt, m.CreatedAt);
|
||||
Assert.Equal(message.LastAttemptAt, m.LastAttemptAt);
|
||||
Assert.Equal(message.Status, m.Status);
|
||||
Assert.Equal(message.LastError, m.LastError);
|
||||
Assert.Equal(message.OriginInstanceName, m.OriginInstanceName);
|
||||
Assert.Equal(message.ExecutionId, m.ExecutionId);
|
||||
Assert.Equal(message.SourceScript, m.SourceScript);
|
||||
Assert.Equal(message.ParentExecutionId, m.ParentExecutionId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SfBufferResyncAck_RoundTripsOnTheWire()
|
||||
{
|
||||
var original = new SfBufferResyncAck("resync-1", 42);
|
||||
|
||||
var back = RoundTrip(original);
|
||||
|
||||
Assert.Equal(original.ResyncId, back.ResyncId);
|
||||
Assert.Equal(original.RowCount, back.RowCount);
|
||||
}
|
||||
|
||||
// Type-identity pins: the reflective-JSON wire embeds CLR type manifests, so a
|
||||
// rename/move of either type silently breaks resync across a rolling upgrade.
|
||||
[Theory]
|
||||
[InlineData(typeof(SfBufferSnapshotChunk), "ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors.SfBufferSnapshotChunk")]
|
||||
[InlineData(typeof(SfBufferResyncAck), "ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors.SfBufferResyncAck")]
|
||||
public void ResyncContract_TypeIdentity_IsPinned(Type type, string expectedFullName) =>
|
||||
Assert.Equal(expectedFullName, type.FullName);
|
||||
}
|
||||
Reference in New Issue
Block a user