using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Serilog.Core;
using Serilog.Events;
using Shouldly;
using Xunit;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Registration;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
///
/// LocalDb Phase 1 (Task 2) — pins the deployment-cache schema and the load-bearing
/// DDL → RegisterReplicated → writes ordering inside .
///
///
///
/// The ordering assertion is the point of this file. RegisterReplicated installs the
/// capture triggers; rows written before it are never captured into the oplog and
/// therefore never reach the peer — silently, and forever. A reordered OnReady passes
/// every schema-shape test while shipping a replication path that quietly moves nothing.
///
///
/// BEHAVIOUR CHANGE (2026-08-18): registration is CONDITIONAL on this node actually
/// being configured to replicate. Until now OnReady registered all four tables
/// unconditionally, so every test here could build a database from LocalDb:Path
/// alone and still find them registered. They now pass a replication section, because that
/// is what makes registration happen at all — see
///
/// for the other branch. The registered-table-set pin is therefore now a pin on
/// configured nodes; the unconfigured node's pin is the deregistration self-heal.
///
///
/// Built against a real temp-file rather than a hand-written schema:
/// the library has no in-memory mode, and asserting against a schema the test itself wrote
/// would only prove the test agrees with itself. That is doubly true here — the baseline and
/// deregistration behaviours live entirely inside the library, so only a real file can show
/// them.
///
///
public sealed class LocalDbSetupTests : IDisposable
{
///
/// The initiating half of a replicating pair: it dials, so it carries PeerAddress.
///
private static readonly KeyValuePair[] InitiatorReplication =
[
new("LocalDb:Replication:PeerAddress", "http://peer:9001"),
new("LocalDb:Replication:ApiKey", "test-sync-key"),
];
///
/// The passive half of the same pair. It never dials, so it has no PeerAddress
/// — only the shared key (plus a listen port, which is Program.cs's concern). Keying the
/// predicate on PeerAddress alone would strip capture from this node and leave the
/// pair converging in one direction only.
///
private static readonly KeyValuePair[] PassiveReplication =
[
new("LocalDb:Replication:ApiKey", "test-sync-key"),
];
private readonly string _dbPath =
Path.Combine(Path.GetTempPath(), $"otopcua-localdb-setup-{Guid.NewGuid():N}.db");
private readonly List _providers = [];
///
/// Boots over the test's database file.
///
///
///
/// Successive calls reopen the same file, which is how the self-heal and
/// late-opt-in cases are expressed: boot once under one replication configuration, write
/// rows, then boot again under another. That is exactly the sequence an operator
/// performs by editing config and restarting the node.
///
///
/// A null lets OnReady fall back to Serilog's static
/// Log.Logger, which in this assembly is never assigned and is therefore the
/// library's silent default. That is a load-bearing assumption for the two cases that
/// DO pass a logger: were anything here to install a static logger, they would still
/// pass (they assert on their own sink) but the silent cases would start writing to it.
/// Pass a logger whenever the assertion is about logging.
///
///
private ILocalDb Boot(
KeyValuePair[]? replication = null,
Serilog.ILogger? logger = null)
{
// A previous boot's provider still holds the file open; the library has no second-handle
// mode. Closing it (and the pool) is what makes a reopen behave like a process restart.
DisposeProviders();
var settings = new List>
{
new("LocalDb:Path", _dbPath),
};
settings.AddRange(replication ?? []);
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(settings)
.Build();
var provider = new ServiceCollection()
.AddZbLocalDb(configuration, db => LocalDbSetup.OnReady(db, configuration, logger))
.BuildServiceProvider();
_providers.Add(provider);
return provider.GetRequiredService();
}
private static async Task ScalarAsync(ILocalDb db, string sql) =>
(await db.QueryAsync(sql, r => r.GetInt32(0), parameters: null, TestContext.Current.CancellationToken))[0];
private static Task InsertPointerAsync(ILocalDb db, string clusterId) =>
db.ExecuteAsync(
"""
INSERT INTO deployment_pointer
(cluster_id, deployment_id, revision_hash, artifact_sha256, applied_at_utc)
VALUES (@ClusterId, @DeploymentId, @RevisionHash, @Sha, @AppliedAtUtc)
""",
new
{
ClusterId = clusterId,
DeploymentId = "0123456789abcdef0123456789abcdef",
RevisionHash = new string('a', 64),
Sha = new string('b', 64),
AppliedAtUtc = "2026-07-20T00:00:00.0000000Z",
},
TestContext.Current.CancellationToken);
[Fact]
public void OnReady_WhenReplicationIsConfigured_RegistersExactlyTheFourReplicatedTables()
{
// Both directions are load-bearing. "No fewer" catches a dropped RegisterReplicated call
// (that table then never replicates). "No more" catches an accidental registration —
// every replicated table costs three triggers and oplog volume on every write.
var db = Boot(InitiatorReplication);
db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal)
.ShouldBe(["alarm_condition_state", "alarm_sf_events", "deployment_artifacts", "deployment_pointer"]);
}
[Fact]
public void OnReady_WhenOnlyTheSyncApiKeyIsSet_StillRegistersTheFourTables()
{
// The passive half of a pair carries the key and nothing else. If the predicate were
// "PeerAddress is set", this node would install no capture triggers and its OWN writes
// would never reach the initiator — a pair that replicates in one direction only, with
// nothing logged anywhere. This test is the falsifier for that narrower predicate.
var db = Boot(PassiveReplication);
db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal)
.ShouldBe(["alarm_condition_state", "alarm_sf_events", "deployment_artifacts", "deployment_pointer"]);
}
[Fact]
public async Task OnReady_WhenReplicationIsNotConfigured_RegistersNothingAndInstallsNoTriggers()
{
// The default-OFF node. It has no peer, so capture buys it nothing and costs it two INSERTs
// plus a json_object of the whole row inside every write transaction — for an oplog that is
// never drained. Asserting the trigger count as well as ReplicatedTables matters: the
// dictionary is in-process bookkeeping, the triggers are what actually run.
var db = Boot();
db.ReplicatedTables.ShouldBeEmpty();
// None of the three application schemas defines a trigger of its own, so every trigger in
// this file would be a capture trigger.
(await ScalarAsync(db, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger'")).ShouldBe(0);
}
[Fact]
public void DeploymentArtifacts_PkIsDeploymentIdPlusChunkIndex()
{
// The composite PK is what makes an artifact chunkable. A single-column PK here would
// make every chunk of a deployment collide onto one row under LWW.
var db = Boot(InitiatorReplication);
db.ReplicatedTables["deployment_artifacts"].PkColumns
.ShouldBe(["deployment_id", "chunk_index"]);
}
[Fact]
public void DeploymentPointer_PkIsClusterId()
{
// One current-deployment pointer per cluster: the pair's two nodes converge on the same
// row rather than each keeping its own.
var db = Boot(InitiatorReplication);
db.ReplicatedTables["deployment_pointer"].PkColumns.ShouldBe(["cluster_id"]);
}
[Fact]
public void AlarmSfEvents_PkIsTheAppMintedId()
{
// A single TEXT PK the application mints. The legacy queue this table replaces keyed on
// `RowId INTEGER PRIMARY KEY AUTOINCREMENT`, which LWW cannot replicate: two nodes would
// independently allocate rowid 7 for different alarms and silently overwrite each other.
var db = Boot(InitiatorReplication);
db.ReplicatedTables["alarm_sf_events"].PkColumns.ShouldBe(["id"]);
}
[Fact]
public void AlarmConditionState_PkIsTheScriptedAlarmId()
{
// One condition-state row per alarm identity. The pair's two nodes converge on the same row
// under LWW rather than each keeping its own, and Save is an unconditional upsert onto it.
var db = Boot(InitiatorReplication);
db.ReplicatedTables["alarm_condition_state"].PkColumns.ShouldBe(["scripted_alarm_id"]);
}
[Fact]
public async Task AlarmSfEventRows_EnterTheOplog()
{
// Same ordering assertion as the deployment-pointer test below, for the table Phase 2 adds.
// Worth its own case because the alarm table is registered by a different call site, and a
// registration added ahead of its DDL would fail loudly while one added after the migrator
// would fail silently.
var db = Boot(InitiatorReplication);
await db.ExecuteAsync(
"""
INSERT INTO alarm_sf_events
(id, alarm_id, enqueued_at_utc, payload_json, attempt_count)
VALUES (@Id, @AlarmId, @EnqueuedAtUtc, @PayloadJson, 0)
""",
new
{
Id = new string('c', 64),
AlarmId = "equip/alarm-1",
EnqueuedAtUtc = "2026-07-21T00:00:00.0000000Z",
PayloadJson = """{"AlarmId":"equip/alarm-1"}""",
},
TestContext.Current.CancellationToken);
(await ScalarAsync(db, "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'alarm_sf_events'"))
.ShouldBe(1);
}
[Fact]
public async Task RowsWrittenAfterOnReady_EnterTheOplog()
{
// THE ordering assertion. If DDL and RegisterReplicated were swapped — or a write were
// added to OnReady ahead of registration — the triggers would not exist at write time and
// this count would stay 0 while every other test in this file still passed.
var db = Boot(InitiatorReplication);
await InsertPointerAsync(db, "SITE-A");
(await ScalarAsync(db, "SELECT COUNT(*) FROM __localdb_oplog")).ShouldBeGreaterThanOrEqualTo(1);
}
[Fact]
public async Task EnablingReplicationLater_BaselinesTheRowsAlreadyInTheFile()
{
// The defect this whole change exists for. Capture is change-data-capture and the snapshot
// streamer pages from __localdb_row_version, so rows written while replication was off are
// in neither — a late opt-in used to converge on subsequent writes only, silently and
// forever. RegisterReplicated(table, baselineExistingRows: true) seeds the ledger at the LWW
// floor and flags a resync so they actually travel.
//
// The assertion is on the ledger entry's HLC, not merely on its existence. A row captured by
// a trigger carries a genuine (large, non-zero) stamp; a BASELINED row is seeded at the LWW
// floor of HLC 0, which is what makes it lose to any real remote write of the same key and
// win only where the peer holds no version at all. Asserting existence alone would pass
// against the old unconditional registration, where the trigger simply captured the write —
// i.e. it would prove nothing about the flag.
var unreplicated = Boot();
await InsertPointerAsync(unreplicated, "SITE-A");
var replicated = Boot(InitiatorReplication);
(await ScalarAsync(
replicated,
"""
SELECT COUNT(*) FROM __localdb_row_version
WHERE table_name = 'deployment_pointer' AND hlc = 0
"""))
.ShouldBe(1);
}
[Fact]
public async Task DisablingReplication_DropsTheCaptureTriggersAnEarlierBootInstalled()
{
// The self-heal half. Merely declining to register would leave a file registered by an
// earlier build capturing forever — on precisely the node that has no peer to capture for.
// DeregisterReplicated drops the triggers and prunes that table's oplog and row-version
// rows.
var replicated = Boot(InitiatorReplication);
await InsertPointerAsync(replicated, "SITE-A");
(await ScalarAsync(replicated, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger'"))
.ShouldBe(12);
var unreplicated = Boot();
unreplicated.ReplicatedTables.ShouldBeEmpty();
(await ScalarAsync(unreplicated, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger'"))
.ShouldBe(0);
}
[Fact]
public async Task DisablingReplication_PrunesBookkeepingButNeverTouchesTheCachedDataItself()
{
// Deregistration prunes the table's oplog and row-version rows — BOOKKEEPING. It must not
// touch the base table, because deployment_pointer/deployment_artifacts ARE the
// boot-from-cache backstop: a node whose central SQL is unreachable serves its address space
// from exactly these rows. A library regression that cleaned the base table along with the
// ledger would take that safety net out silently, on the very nodes that never replicate
// (central, site-b) and so would never notice a replication fault either.
var replicated = Boot(InitiatorReplication);
await InsertPointerAsync(replicated, "SITE-A");
(await ScalarAsync(replicated, "SELECT COUNT(*) FROM __localdb_row_version")).ShouldBeGreaterThan(0);
var unreplicated = Boot();
// The row survives, and with its content intact — a count alone would pass against a
// cleanup that blanked the columns it does not own.
var pointers = await unreplicated.QueryAsync(
"SELECT cluster_id || '|' || deployment_id || '|' || revision_hash FROM deployment_pointer",
static r => r.GetString(0),
parameters: null,
TestContext.Current.CancellationToken);
pointers.ShouldBe([$"SITE-A|0123456789abcdef0123456789abcdef|{new string('a', 64)}"]);
// …while the bookkeeping it DOES own is gone.
(await ScalarAsync(unreplicated, "SELECT COUNT(*) FROM __localdb_row_version")).ShouldBe(0);
(await ScalarAsync(unreplicated, "SELECT COUNT(*) FROM __localdb_oplog")).ShouldBe(0);
}
[Fact]
public async Task DisablingReplication_LogsOnceWhenItActuallyCleanedSomething()
{
// An operator-visible record that a node just stopped capturing, and how much it dropped.
// Silent trigger removal on a node someone MEANT to replicate is the failure this line
// exists to make diagnosable.
var replicated = Boot(InitiatorReplication);
await InsertPointerAsync(replicated, "SITE-A");
var sink = new CollectingSink();
Boot(replication: null, logger: BuildLogger(sink));
sink.Events.Count(e => e.Level == LogEventLevel.Information).ShouldBe(1);
}
[Fact]
public void DisablingReplication_SaysNothingWhenThereWasNothingToClean()
{
// The steady state of every default-OFF node in the fleet: it boots unreplicated every day
// and has nothing to deregister. A log line here would be a daily false alarm, which is how
// the real one gets ignored.
var sink = new CollectingSink();
Boot(replication: null, logger: BuildLogger(sink));
sink.Events.ShouldBeEmpty();
}
[Fact]
public void AFailureMidwayThroughRegistration_FailsBootLoudly_AndTheRetriedBootConverges()
{
// OnReady flips four tables in a loop with no try/catch, so a failure on the third leaves
// two registered and two not. That is the RIGHT behaviour and this pins both halves of why:
// the throw propagates out of the AddZbLocalDb singleton factory and fails host start
// (a node half-capturing is a node whose replication is silently partial), and because both
// library APIs are idempotent the retried boot completes the flip rather than compounding it.
//
// Swallowing the throw would leave the node up, serving, and replicating two of four tables
// with nothing in the log to say so.
var real = Boot(InitiatorReplication);
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection([new KeyValuePair("LocalDb:Path", _dbPath), .. InitiatorReplication])
.Build();
var faulty = new FailsOnNthRegistration(real, failOnCall: 3);
Should.Throw(() => LocalDbSetup.OnReady(faulty, configuration));
// The wrapper really did reach the third call — otherwise the assertion above would be
// pinning some unrelated throw, and the "partial flip" it claims to model never happened.
faulty.Calls.ShouldBe(3);
// The retry: same file, no wrapper. Idempotent re-registration converges on all four.
LocalDbSetup.OnReady(real, configuration);
real.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal)
.ShouldBe(["alarm_condition_state", "alarm_sf_events", "deployment_artifacts", "deployment_pointer"]);
}
///
/// Passes everything through to a real database except the failOnCall-th
/// RegisterReplicated, which throws — the mid-loop partial-flip shape.
///
private sealed class FailsOnNthRegistration(ILocalDb inner, int failOnCall) : ILocalDb
{
/// How many registrations were attempted, successful or not.
public int Calls { get; private set; }
public IReadOnlyDictionary ReplicatedTables => inner.ReplicatedTables;
public SqliteConnection CreateConnection() => inner.CreateConnection();
public Task ExecuteAsync(string sql, object? parameters, CancellationToken cancellationToken) =>
inner.ExecuteAsync(sql, parameters, cancellationToken);
public Task> QueryAsync(
string sql, Func map, object? parameters, CancellationToken cancellationToken) =>
inner.QueryAsync(sql, map, parameters, cancellationToken);
public Task BeginTransactionAsync(CancellationToken cancellationToken) =>
inner.BeginTransactionAsync(cancellationToken);
public ReplicatedTable RegisterReplicated(string tableName) =>
RegisterReplicated(tableName, baselineExistingRows: false);
public ReplicatedTable RegisterReplicated(string tableName, bool baselineExistingRows)
{
Calls++;
if (Calls == failOnCall)
throw new InvalidOperationException($"simulated registration failure on '{tableName}'");
return inner.RegisterReplicated(tableName, baselineExistingRows);
}
public bool DeregisterReplicated(string tableName) => inner.DeregisterReplicated(tableName);
}
private static Serilog.ILogger BuildLogger(ILogEventSink sink) =>
new Serilog.LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sink).CreateLogger();
/// Captures whatever logs.
private sealed class CollectingSink : ILogEventSink
{
public List Events { get; } = [];
public void Emit(LogEvent logEvent) => Events.Add(logEvent);
}
private void DisposeProviders()
{
foreach (var provider in _providers)
provider.Dispose();
_providers.Clear();
// No in-memory mode, so these are real files. Pooled connections keep a handle open past
// dispose; clearing the pools first is what lets a reopen (or the delete below) succeed.
SqliteConnection.ClearAllPools();
}
public void Dispose()
{
DisposeProviders();
foreach (var path in new[] { _dbPath, $"{_dbPath}-wal", $"{_dbPath}-shm" })
{
if (File.Exists(path))
File.Delete(path);
}
}
}