using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb;
///
/// LocalDb Phase 2 — two driver nodes replicating the alarm store-and-forward buffer over a real
/// loopback h2c transport, through the real fail-closed interceptor.
///
///
///
/// This is the test the phase exists to pass. Everything upstream — the schema, the
/// registration, the sink rewire, the drain gate — can be individually green while a
/// redundant pair still fails to share the undelivered alarm history that is the whole
/// point of moving the queue here.
///
///
/// Rows are written through the production rather
/// than by hand-rolled INSERTs. The id derivation, the column set and the delete-on-ack
/// semantics under test are then the ones production actually uses.
///
///
[Collection("LocalDbPairConvergence")]
public sealed class AlarmSfConvergenceTests
{
private static readonly Serilog.ILogger Log = new LoggerConfiguration().CreateLogger();
/// A writer that acknowledges everything — the drain's happy path.
private sealed class AckWriter : IAlarmHistorianWriter
{
public Task> WriteBatchAsync(
IReadOnlyList batch, CancellationToken cancellationToken) =>
Task.FromResult>(
batch.Select(_ => HistorianWriteOutcome.Ack).ToList());
}
/// A writer that never acknowledges — a historian outage, which is when the buffer matters.
private sealed class OutageWriter : IAlarmHistorianWriter
{
public Task> WriteBatchAsync(
IReadOnlyList batch, CancellationToken cancellationToken) =>
Task.FromResult>(
batch.Select(_ => HistorianWriteOutcome.RetryPlease).ToList());
}
private static AlarmHistorianEvent Event(string alarmId, int second) => new(
AlarmId: alarmId,
EquipmentPath: "line-1/eq-1",
AlarmName: "temp-high",
AlarmTypeName: "LimitAlarm",
Severity: AlarmSeverity.High,
EventKind: "Activated",
Message: "temperature high",
User: "system",
Comment: null,
TimestampUtc: new DateTime(2026, 7, 21, 0, 0, second, DateTimeKind.Utc));
/// The set of row ids on a node, ordered — equal sets on both nodes means converged.
private static async Task IdsAsync(ILocalDb db)
{
var rows = await db.QueryAsync(
"SELECT id FROM alarm_sf_events ORDER BY id",
static r => r.GetString(0));
return string.Join(",", rows);
}
private static async Task CountAsync(ILocalDb db)
{
var rows = await db.QueryAsync(
"SELECT COUNT(*) FROM alarm_sf_events", static r => r.GetInt64(0));
return rows[0];
}
[Fact]
public async Task AlarmBurstOnA_ConvergesToB_AndOplogDrains()
{
await using var pair = new LocalDbPairHarness();
await pair.StartAsync();
// The Primary's sink. Its drain is left unstarted so the burst stays queued and observable;
// an outage is exactly the state in which this buffer has to survive a node loss.
using var sink = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log);
for (var i = 0; i < 25; i++)
await sink.EnqueueAsync(Event($"eq/alarm-{i}", i), TestContext.Current.CancellationToken);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CountAsync(pair.B) == 25,
"node B to hold all 25 buffered alarm events");
(await IdsAsync(pair.B)).ShouldBe(await IdsAsync(pair.A));
// Oplog drains to empty once B acks: the steady state of a converged pair, and the proof
// the rows were genuinely acknowledged rather than merely sent.
await LocalDbPairHarness.WaitUntilAsync(
async () => await LocalDbPairHarness.OplogDepthAsync(pair.A) == 0,
"node A's oplog to drain after node B acknowledges the burst");
}
[Fact]
public async Task DeliveredRowsAreRemovedFromBothNodes()
{
// The no-redeliver-after-failover property, asserted at the data layer. An acknowledged row
// is DELETEd, and the delete has to reach the peer as a tombstone — otherwise a promoted
// standby would find the Primary's already-delivered events still queued and send them again.
await using var pair = new LocalDbPairHarness();
await pair.StartAsync();
using var buffering = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log);
for (var i = 0; i < 5; i++)
await buffering.EnqueueAsync(Event($"eq/alarm-{i}", i), TestContext.Current.CancellationToken);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CountAsync(pair.B) == 5,
"node B to receive the buffered events");
// The historian comes back and the Primary drains.
using var draining = new LocalDbStoreAndForwardSink(pair.A, new AckWriter(), Log);
await draining.DrainOnceAsync(TestContext.Current.CancellationToken);
(await CountAsync(pair.A)).ShouldBe(0);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CountAsync(pair.B) == 0,
"node B to drop its copies once node A delivered them");
(await LocalDbPairHarness.TombstoneCountAsync(pair.B, "alarm_sf_events"))
.ShouldBe(5, "the deletes must arrive as tombstones, not vanish silently");
}
[Fact]
public async Task WritesWhileTransportDown_SurviveRejoin()
{
// The realistic outage shape: alarms keep arriving on the Primary while the pair link is
// down. Nothing may be lost when it comes back.
await using var pair = new LocalDbPairHarness();
await pair.StartAsync();
using var sink = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log);
await sink.EnqueueAsync(Event("eq/before", 1), TestContext.Current.CancellationToken);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CountAsync(pair.B) == 1,
"the pre-outage event to reach node B");
await pair.StopPassiveAsync();
for (var i = 0; i < 10; i++)
await sink.EnqueueAsync(Event($"eq/during-{i}", 10 + i), TestContext.Current.CancellationToken);
await pair.RestartPairAsync();
await LocalDbPairHarness.WaitUntilAsync(
async () => await CountAsync(pair.B) == 11,
"every event buffered during the outage to reach node B after the rejoin");
(await IdsAsync(pair.B)).ShouldBe(await IdsAsync(pair.A));
}
[Fact]
public async Task TheSameEventOnBothNodes_ConvergesToOneRow()
{
// Both adapters default-write while their redundancy role is unknown, so in the boot window
// the pair genuinely double-accepts the same fanned transition. Payload-derived ids are what
// turn that into one row instead of two; random GUIDs would leave the duplicate in the
// buffer for the eventual Primary to deliver twice.
await using var pair = new LocalDbPairHarness();
await pair.StartAsync();
var shared = Event("eq/shared", 1);
using var sinkA = new LocalDbStoreAndForwardSink(pair.A, new OutageWriter(), Log);
using var sinkB = new LocalDbStoreAndForwardSink(pair.B, new OutageWriter(), Log);
await sinkA.EnqueueAsync(shared, TestContext.Current.CancellationToken);
await sinkB.EnqueueAsync(shared, TestContext.Current.CancellationToken);
// A second, DISTINCT event on B. Without it the scenario would pass with replication
// switched off entirely — each node would hold its own single copy and "count == 1 on
// both" would be satisfied by two unconverged databases.
await sinkB.EnqueueAsync(Event("eq/only-on-b", 2), TestContext.Current.CancellationToken);
await LocalDbPairHarness.WaitUntilAsync(
async () => await CountAsync(pair.A) == 2 && await CountAsync(pair.B) == 2,
"both nodes to hold two rows: the doubly-accepted event collapsed, plus B's own");
(await IdsAsync(pair.A)).ShouldBe(await IdsAsync(pair.B));
// Held, not merely reached: a late-arriving replica of the peer's copy of the shared event
// must not add a third row a moment later.
(await LocalDbPairHarness.HeldWithinAsync(
async () => await CountAsync(pair.A) > 2 || await CountAsync(pair.B) > 2,
TimeSpan.FromSeconds(3)))
.ShouldBeFalse("the duplicate must stay collapsed");
}
}