using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Messages;
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Actors;
///
/// Tests for 's notify-and-fetch config replication:
/// the active node now replicates an id-only (no inline
/// config JSON — killing the intra-site 128 KB frame trap), and the standby fetches the
/// config from central over HTTP and writes it with the older-write guard.
///
public class SiteReplicationActorTests : TestKit, IDisposable
{
// Cluster provider is required because SiteReplicationActor calls Cluster.Get in its ctor
// and subscribes to cluster events in PreStart. We use the in-memory TestTransport (not
// dot-netty) so no real socket is bound and no DNS lookup happens — the actor only needs
// the cluster extension to load; these tests never form a real two-node cluster.
private const string ClusterConfig = @"
akka {
actor { provider = cluster }
remote {
enabled-transports = [""akka.remote.test""]
test {
transport-class = ""Akka.Remote.Transport.TestTransport, Akka.Remote""
applied-adapters = []
registry-key = site-repl-test
local-address = ""test://site-repl@localhost:1""
maximum-payload-bytes = 128000b
scheme-identifier = test
}
}
cluster { roles = [""site-test""] }
loglevel = WARNING
}";
private const string SiteRole = "site-test";
private readonly SiteStorageService _storage;
private readonly StoreAndForwardStorage _sfStorage;
private readonly ReplicationService _replicationService;
private readonly string _dbFile;
private readonly string _sfDbFile;
public SiteReplicationActorTests() : base(ClusterConfig, "site-repl")
{
_dbFile = Path.Combine(Path.GetTempPath(), $"site-repl-test-{Guid.NewGuid():N}.db");
_sfDbFile = Path.Combine(Path.GetTempPath(), $"site-repl-sf-{Guid.NewGuid():N}.db");
_storage = new SiteStorageService(
$"Data Source={_dbFile}", NullLogger.Instance);
_storage.InitializeAsync().GetAwaiter().GetResult();
_sfStorage = new StoreAndForwardStorage(
$"Data Source={_sfDbFile}", NullLogger.Instance);
_sfStorage.InitializeAsync().GetAwaiter().GetResult();
_replicationService = new ReplicationService(
new StoreAndForwardOptions(), NullLogger.Instance);
}
void IDisposable.Dispose()
{
Shutdown();
try { File.Delete(_dbFile); } catch { /* cleanup */ }
try { File.Delete(_sfDbFile); } catch { /* cleanup */ }
}
private IActorRef CreateReplicationActor(IDeploymentConfigFetcher fetcher) =>
ActorOf(Props.Create(() => new SiteReplicationActor(
_storage, _sfStorage, _replicationService, SiteRole,
NullLogger.Instance, fetcher)));
private IActorRef CreateReplicationActor(
IDeploymentConfigFetcher fetcher, SiteRuntimeOptions options, TimeSpan retryDelay) =>
ActorOf(Props.Create(() => new SiteReplicationActor(
_storage, _sfStorage, _replicationService, SiteRole,
NullLogger.Instance, fetcher, null, options, retryDelay)));
[Fact]
public async Task ReplicatedFetch_RetriesUpToConfigFetchRetryCount()
{
// The first two fetches fail transiently; the third succeeds. With
// ConfigFetchRetryCount = 3 the standby must retry to the third attempt and
// then guarded-write the fetched config (a short retry delay keeps the test fast).
var attempts = 0;
var fetcher = new FakeConfigFetcher(_ =>
Interlocked.Increment(ref attempts) < 3
? Task.FromException(new InvalidOperationException("central hiccup"))
: Task.FromResult("{\"instanceUniqueName\":\"RetryPump\"}"));
var actor = CreateReplicationActor(
fetcher, new SiteRuntimeOptions { ConfigFetchRetryCount = 3 },
TimeSpan.FromMilliseconds(50));
actor.Tell(new ApplyConfigDeploy(
"RetryPump", "dep-r1", "sha256:r1", true,
"http://central:9000", "tok-r1"));
await AwaitAssertAsync(async () =>
{
Assert.Equal(3, Volatile.Read(ref attempts));
var configs = await _storage.GetAllDeployedConfigsAsync();
Assert.Single(configs, c => c.InstanceUniqueName == "RetryPump");
}, TimeSpan.FromSeconds(10));
}
[Fact]
public async Task ApplyConfigDeploy_StandbyFetchesConfigAndGuardedWrites()
{
// The standby receives an id-only ApplyConfigDeploy; it fetches the config from
// central using the message's coords, then guarded-writes the fetched config.
const string configJson = "{\"instanceUniqueName\":\"Pump1\"}";
var fetcher = new FakeConfigFetcher(_ => Task.FromResult(configJson));
var actor = CreateReplicationActor(fetcher);
actor.Tell(new ApplyConfigDeploy(
"Pump1", "dep-100", "sha256:abc", true,
"http://central:9000", "tok-xyz"));
// The continuation runs off-thread; await the guarded write landing.
await AwaitAssertAsync(async () =>
{
var configs = await _storage.GetAllDeployedConfigsAsync();
var row = Assert.Single(configs, c => c.InstanceUniqueName == "Pump1");
Assert.Equal(configJson, row.ConfigJson);
Assert.Equal("dep-100", row.DeploymentId);
Assert.Equal("sha256:abc", row.RevisionHash);
Assert.True(row.IsEnabled);
}, TimeSpan.FromSeconds(5));
// The fetcher was called with the message's coords.
var call = Assert.Single(fetcher.Calls);
Assert.Equal("http://central:9000", call.BaseUrl);
Assert.Equal("dep-100", call.DeploymentId);
Assert.Equal("tok-xyz", call.Token);
}
[Fact]
public async Task ApplyConfigDeploy_Superseded404_SkipsWriteAndActorSurvives()
{
// A 404 (superseded/expired) surfaces as DeploymentConfigFetchException{IsSuperseded}.
// The standby must skip the write, observe the exception (no crash), and stay alive.
var fetcher = new FakeConfigFetcher(_ =>
Task.FromException(
new DeploymentConfigFetchException("expired", isSuperseded: true)));
var actor = CreateReplicationActor(fetcher);
actor.Tell(new ApplyConfigDeploy(
"GonePump", "dep-stale", "sha256:gone", true,
"http://central:9000", "tok-stale"));
// The fetch was attempted...
await AwaitAssertAsync(() =>
{
Assert.Single(fetcher.Calls);
return Task.CompletedTask;
}, TimeSpan.FromSeconds(5));
// ...the actor did not crash (no Terminated to its watcher within the window)...
Watch(actor);
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
// ...and nothing was written for the superseded instance.
var configs = await _storage.GetAllDeployedConfigsAsync();
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "GonePump");
}
[Fact]
public async Task ApplyConfigDeploy_EmptyFetchCoords_SkipsFetchAndWrite()
{
// The direct DeployInstanceCommand cross-cluster wire path was retired in Task 14.
// This tests the defensive guard: if empty coords arrive, the actor must skip quietly
// — no FetchAsync("") call, no write — rather than erroring.
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("never"));
var actor = CreateReplicationActor(fetcher);
actor.Tell(new ApplyConfigDeploy(
"NoCoordsPump", "dep-direct", "sha256:nc", true,
CentralFetchBaseUrl: "", FetchToken: ""));
// Give any (erroneous) async continuation time to run, then prove neither happened.
Watch(actor);
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
Assert.Empty(fetcher.Calls);
var configs = await _storage.GetAllDeployedConfigsAsync();
Assert.DoesNotContain(configs, c => c.InstanceUniqueName == "NoCoordsPump");
}
[Fact]
public void ReplicateConfigDeploy_MapsToIdOnlyApplyConfigDeploy_ForPeer()
{
// The outbound mapping must forward an id-only ApplyConfigDeploy carrying the fetch
// coords (and NO inline config) to the peer.
var probe = CreateTestProbe();
var fetcher = new FakeConfigFetcher(_ => Task.FromResult("unused"));
var actor = ActorOf(Props.Create(() => new ProbeForwardingReplicationActor(
_storage, _sfStorage, _replicationService, SiteRole,
NullLogger.Instance, fetcher, probe.Ref)));
actor.Tell(new ReplicateConfigDeploy(
"Pump2", "dep-200", "sha256:def", false,
"http://central:9000", "tok-abc"));
var applied = probe.ExpectMsg(TimeSpan.FromSeconds(3));
Assert.Equal("Pump2", applied.InstanceName);
Assert.Equal("dep-200", applied.DeploymentId);
Assert.Equal("sha256:def", applied.RevisionHash);
Assert.False(applied.IsEnabled);
Assert.Equal("http://central:9000", applied.CentralFetchBaseUrl);
Assert.Equal("tok-abc", applied.FetchToken);
}
// ── Task 21: peer-join S&F buffer resync (anti-entropy) ──
[Fact]
public void StandbyTrackingPeer_SendsResyncRequest()
{
var probe = CreateTestProbe();
var actor = ActorOf(Props.Create(() => new ResyncTestActor(
_storage, _sfStorage, _replicationService, SiteRole,
NullLogger.Instance, probe.Ref, () => false)));
actor.Tell(new TriggerPeerTracked()); // stands in for TryTrackPeer's MemberUp path
probe.ExpectMsg(TimeSpan.FromSeconds(3));
}
[Fact]
public void ActiveTrackingPeer_DoesNotRequestResync()
{
var probe = CreateTestProbe();
var actor = ActorOf(Props.Create(() => new ResyncTestActor(
_storage, _sfStorage, _replicationService, SiteRole,
NullLogger.Instance, probe.Ref, () => true)));
actor.Tell(new TriggerPeerTracked());
probe.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); // active node never requests a resync
}
[Fact]
public async Task ActiveNode_AnswersResyncRequest_WithChunkedSnapshot()
{
// Post-R2-T5 the active node answers with byte-budgeted SfBufferSnapshotChunk(s)
// (a single small row rides one chunk) rather than the monolithic SfBufferSnapshot.
await _sfStorage.EnqueueAsync(NewSfMessage("m1"));
var probe = CreateTestProbe();
var actor = ActorOf(Props.Create(() => new ResyncTestActor(
_storage, _sfStorage, _replicationService, SiteRole,
NullLogger.Instance, probe.Ref, () => true)));
actor.Tell(new RequestSfBufferResync(), TestActor);
var chunk = ExpectMsg(TimeSpan.FromSeconds(3));
Assert.Equal(1, chunk.TotalChunks);
Assert.Equal(1, chunk.Sequence);
Assert.Single(chunk.Messages);
Assert.False(chunk.Truncated);
}
[Fact]
public async Task StandbyNode_AppliesSnapshot_ReplacingItsBuffer()
{
await _sfStorage.EnqueueAsync(NewSfMessage("stale"));
var probe = CreateTestProbe();
var actor = ActorOf(Props.Create(() => new ResyncTestActor(
_storage, _sfStorage, _replicationService, SiteRole,
NullLogger.Instance, probe.Ref, () => false)));
actor.Tell(new SfBufferSnapshot(new List { NewSfMessage("fresh") }, false));
await AwaitAssertAsync(async () =>
{
Assert.Null(await _sfStorage.GetMessageByIdAsync("stale"));
Assert.NotNull(await _sfStorage.GetMessageByIdAsync("fresh"));
}, TimeSpan.FromSeconds(5));
}
// ── R2 T5: chunked resync answer ──
[Fact]
public void ChunkForRemoting_SplitsByByteBudget_PreservingOrderAndSequence()
{
var rows = Enumerable.Range(0, 10)
.Select(i => NewMessage($"m{i}", payloadJson: new string('x', 20_000)))
.ToList();
var chunks = SiteReplicationActor.ChunkForRemoting(rows, maxChunkBytes: 64_000, maxChunkRows: 200);
Assert.True(chunks.Count > 1); // 10 × 20 KB cannot ride one 64 KB chunk
Assert.Equal(rows.Select(r => r.Id), chunks.SelectMany(c => c).Select(r => r.Id)); // order preserved
Assert.All(chunks, c => Assert.True(
c.Sum(r => r.PayloadJson.Length) <= 64_000 || c.Count == 1)); // budget honored (oversized row isolated)
}
[Fact]
public void ChunkForRemoting_RowCapHonored_AndSingleOversizedRowIsolated()
{
var many = Enumerable.Range(0, 500).Select(i => NewMessage($"s{i}", payloadJson: "{}")).ToList();
Assert.All(SiteReplicationActor.ChunkForRemoting(many, 64_000, 200), c => Assert.True(c.Count <= 200));
var oversized = new List
{ NewMessage("big", payloadJson: new string('y', 100_000)), NewMessage("small", payloadJson: "{}") };
var chunks = SiteReplicationActor.ChunkForRemoting(oversized, 64_000, 200);
Assert.Equal(2, chunks.Count); // the oversized row rides alone
}
[Fact]
public async Task ActiveNode_AnswersResyncRequest_WithSequencedChunks_SharingOneResyncId()
{
for (var i = 0; i < 3; i++)
await _sfStorage.EnqueueAsync(NewMessage($"c{i}", payloadJson: new string('z', 30_000)));
var actor = CreateResyncActor(isActive: () => true);
actor.Tell(new RequestSfBufferResync(), TestActor);
var first = ExpectMsg(TimeSpan.FromSeconds(5));
var rest = Enumerable.Range(1, first.TotalChunks - 1)
.Select(_ => ExpectMsg(TimeSpan.FromSeconds(5)))
.Prepend(first)
.ToList();
Assert.True(first.TotalChunks > 1);
Assert.All(rest, c => Assert.Equal(first.ResyncId, c.ResyncId));
Assert.Equal(Enumerable.Range(1, first.TotalChunks), rest.Select(c => c.Sequence));
Assert.Equal(3, rest.Sum(c => c.Messages.Count));
}
// ── R2 T6: standby chunk assembly + atomic apply + ack ──
[Fact]
public async Task StandbyNode_AssemblesChunks_AppliesOnce_AndAcks()
{
await _sfStorage.EnqueueAsync(NewMessage("stale"));
var actor = CreateResyncActor(isActive: () => false);
var resyncId = "r1";
actor.Tell(new SfBufferSnapshotChunk(resyncId, 1, 2,
new List { NewMessage("f1") }, false), TestActor);
actor.Tell(new SfBufferSnapshotChunk(resyncId, 2, 2,
new List { NewMessage("f2") }, false), TestActor);
var ack = ExpectMsg(TimeSpan.FromSeconds(5));
Assert.Equal(resyncId, ack.ResyncId);
Assert.Equal(2, ack.RowCount);
await AwaitAssertAsync(async () =>
{
Assert.Null(await _sfStorage.GetMessageByIdAsync("stale")); // replaced wholesale
Assert.NotNull(await _sfStorage.GetMessageByIdAsync("f1"));
Assert.NotNull(await _sfStorage.GetMessageByIdAsync("f2"));
});
}
[Fact]
public async Task StandbyNode_NewResyncId_DiscardsStalePartialAssembly()
{
var actor = CreateResyncActor(isActive: () => false);
actor.Tell(new SfBufferSnapshotChunk("old", 1, 2,
new List { NewMessage("orphan") }, false), TestActor);
actor.Tell(new SfBufferSnapshotChunk("new", 1, 1,
new List { NewMessage("fresh") }, false), TestActor);
ExpectMsg(TimeSpan.FromSeconds(5)); // "new" completed
await AwaitAssertAsync(async () =>
{
Assert.NotNull(await _sfStorage.GetMessageByIdAsync("fresh"));
Assert.Null(await _sfStorage.GetMessageByIdAsync("orphan")); // stale partial never applied
});
}
[Fact]
public void ActiveNode_IgnoresChunks_NeverAcks()
{
var actor = CreateResyncActor(isActive: () => true);
actor.Tell(new SfBufferSnapshotChunk("r", 1, 1,
new List { NewMessage("x") }, false), TestActor);
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
}
// ── R2 T7: active-side resync ack confirmation + telemetry ──
//
// NOTE (deviation from plan): the actor logs via Microsoft ILogger (NullLogger in
// tests), NOT Akka's EventStream, so the plan's EventFilter.Warning assertions can
// never observe these warnings. We observe the two OTel counters via a MeterListener
// instead — the equivalent, and stronger, observable signal.
[Fact]
public async Task ActiveNode_ReceivingAck_CountsResyncCompleted()
{
long completed = 0;
using var listener = ListenCounter("scadabridge.store_and_forward.resync.completed",
m => Interlocked.Add(ref completed, m));
await _sfStorage.EnqueueAsync(NewMessage("m1"));
var actor = CreateResyncActor(isActive: () => true, ackTimeout: TimeSpan.FromSeconds(30));
actor.Tell(new RequestSfBufferResync(), TestActor);
var chunk = ExpectMsg(TimeSpan.FromSeconds(5));
actor.Tell(new SfBufferResyncAck(chunk.ResyncId, 1), TestActor);
await AwaitAssertAsync(() =>
{
Assert.True(Interlocked.Read(ref completed) >= 1); // ack recorded the completion
return Task.CompletedTask;
}, TimeSpan.FromSeconds(5));
}
[Fact]
public async Task ActiveNode_MissingAck_WarnsAfterAckTimeout()
{
long ackMissing = 0;
using var listener = ListenCounter("scadabridge.store_and_forward.resync.ack_missing",
m => Interlocked.Add(ref ackMissing, m));
await _sfStorage.EnqueueAsync(NewMessage("m1"));
var actor = CreateResyncActor(isActive: () => true, ackTimeout: TimeSpan.FromMilliseconds(200));
actor.Tell(new RequestSfBufferResync(), TestActor);
ExpectMsg(TimeSpan.FromSeconds(5));
// No ack is sent → the ack window expires and the resync is counted unacknowledged.
await AwaitAssertAsync(() =>
{
Assert.True(Interlocked.Read(ref ackMissing) >= 1);
return Task.CompletedTask;
}, TimeSpan.FromSeconds(5));
}
/// Attaches a to a single ScadaBridge counter by name,
/// forwarding each recorded increment to .
private static MeterListener ListenCounter(string instrumentName, Action onMeasurement)
{
var listener = new MeterListener();
listener.InstrumentPublished = (inst, l) =>
{
if (inst.Meter.Name == ScadaBridgeTelemetry.MeterName && inst.Name == instrumentName)
l.EnableMeasurementEvents(inst);
};
listener.SetMeasurementEventCallback((_, m, _, _) => onMeasurement(m));
listener.Start();
return listener;
}
private static StoreAndForwardMessage NewSfMessage(string id) => new()
{
Id = id,
Category = StoreAndForwardCategory.ExternalSystem,
Target = "t",
PayloadJson = "{}",
RetryCount = 0,
MaxRetries = 50,
RetryIntervalMs = 30000,
CreatedAt = DateTimeOffset.UtcNow,
Status = StoreAndForwardMessageStatus.Pending,
};
///
/// Builds a resync-test message with a settable payload (additive to
/// — the chunker sizes on PayloadJson length).
///
private static StoreAndForwardMessage NewMessage(string id, string payloadJson = "{}") => new()
{
Id = id,
Category = StoreAndForwardCategory.ExternalSystem,
Target = "t",
PayloadJson = payloadJson,
RetryCount = 0,
MaxRetries = 50,
RetryIntervalMs = 30000,
CreatedAt = DateTimeOffset.UtcNow,
Status = StoreAndForwardMessageStatus.Pending,
};
/// Constructs a with the given active-node check
/// (the resync chunk/ack tests Tell to and expect from ).
/// is the active-side ack window seam (T7).
private IActorRef CreateResyncActor(Func isActive, TimeSpan? ackTimeout = null) =>
ActorOf(Props.Create(() => new ResyncTestActor(
_storage, _sfStorage, _replicationService, SiteRole,
NullLogger.Instance, CreateTestProbe().Ref, isActive, ackTimeout)));
/// Test message: drives directly,
/// standing in for the MemberUp→TryTrackPeer path (a single-node TestKit cannot form a real peer).
private sealed record TriggerPeerTracked;
///
/// Test subclass for the resync tests: captures peer sends to a probe, injects the
/// active-node check, and exposes via a test message.
///
private sealed class ResyncTestActor : SiteReplicationActor
{
private readonly IActorRef _peerProbe;
public ResyncTestActor(
SiteStorageService storage, StoreAndForwardStorage sfStorage,
ReplicationService replicationService, string siteRole,
ILogger logger, IActorRef peerProbe, Func isActive,
TimeSpan? ackTimeout = null)
: base(storage, sfStorage, replicationService, siteRole, logger,
configFetcher: null, isActiveOverride: isActive, resyncAckTimeout: ackTimeout)
{
_peerProbe = peerProbe;
Receive(_ => OnPeerTracked());
}
protected override void SendToPeer(object message) => _peerProbe.Tell(message, Self);
}
///
/// Test subclass exposing the peer send: is
/// overridden to forward to a probe so the outbound mapping can be asserted without a real
/// two-node cluster (a single-node TestKit has no peer address, so the real send is dropped).
///
private sealed class ProbeForwardingReplicationActor : SiteReplicationActor
{
private readonly IActorRef _peerProbe;
public ProbeForwardingReplicationActor(
SiteStorageService storage, StoreAndForwardStorage sfStorage,
ReplicationService replicationService, string siteRole,
ILogger logger, IDeploymentConfigFetcher configFetcher,
IActorRef peerProbe)
: base(storage, sfStorage, replicationService, siteRole, logger, configFetcher)
=> _peerProbe = peerProbe;
protected override void SendToPeer(object message) => _peerProbe.Tell(message, Self);
}
///
/// In-test fake : runs a per-deploymentId behavior
/// (return config JSON or throw, as a Task — mirroring the real async HTTP fetcher) and
/// records every call's coords thread-safely (the continuation runs on a pool thread).
///
private sealed class FakeConfigFetcher : IDeploymentConfigFetcher
{
private readonly Func> _behavior;
public ConcurrentQueue<(string BaseUrl, string DeploymentId, string Token)> Calls { get; } = new();
public FakeConfigFetcher(Func> behavior) => _behavior = behavior;
public async Task FetchAsync(
string centralFetchBaseUrl, string deploymentId, string token, CancellationToken ct)
{
Calls.Enqueue((centralFetchBaseUrl, deploymentId, token));
await Task.Yield();
return await _behavior(deploymentId);
}
}
}