using Microsoft.Data.Sqlite; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Serilog.Core; using Serilog.Events; using Shouldly; using Xunit; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.LocalDb.Registration; using ZB.MOM.WW.OtOpcUa.Host.Configuration; namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; /// /// LocalDb Phase 1 (Task 2) — pins the deployment-cache schema and the load-bearing /// DDL → RegisterReplicated → writes ordering inside . /// /// /// /// The ordering assertion is the point of this file. RegisterReplicated installs the /// capture triggers; rows written before it are never captured into the oplog and /// therefore never reach the peer — silently, and forever. A reordered OnReady passes /// every schema-shape test while shipping a replication path that quietly moves nothing. /// /// /// BEHAVIOUR CHANGE (2026-08-18): registration is CONDITIONAL on this node actually /// being configured to replicate. Until now OnReady registered all four tables /// unconditionally, so every test here could build a database from LocalDb:Path /// alone and still find them registered. They now pass a replication section, because that /// is what makes registration happen at all — see /// /// for the other branch. The registered-table-set pin is therefore now a pin on /// configured nodes; the unconfigured node's pin is the deregistration self-heal. /// /// /// Built against a real temp-file rather than a hand-written schema: /// the library has no in-memory mode, and asserting against a schema the test itself wrote /// would only prove the test agrees with itself. That is doubly true here — the baseline and /// deregistration behaviours live entirely inside the library, so only a real file can show /// them. /// /// public sealed class LocalDbSetupTests : IDisposable { /// /// The initiating half of a replicating pair: it dials, so it carries PeerAddress. /// private static readonly KeyValuePair[] InitiatorReplication = [ new("LocalDb:Replication:PeerAddress", "http://peer:9001"), new("LocalDb:Replication:ApiKey", "test-sync-key"), ]; /// /// The passive half of the same pair. It never dials, so it has no PeerAddress /// — only the shared key (plus a listen port, which is Program.cs's concern). Keying the /// predicate on PeerAddress alone would strip capture from this node and leave the /// pair converging in one direction only. /// private static readonly KeyValuePair[] PassiveReplication = [ new("LocalDb:Replication:ApiKey", "test-sync-key"), ]; private readonly string _dbPath = Path.Combine(Path.GetTempPath(), $"otopcua-localdb-setup-{Guid.NewGuid():N}.db"); private readonly List _providers = []; /// /// Boots over the test's database file. /// /// /// /// Successive calls reopen the same file, which is how the self-heal and /// late-opt-in cases are expressed: boot once under one replication configuration, write /// rows, then boot again under another. That is exactly the sequence an operator /// performs by editing config and restarting the node. /// /// /// A null lets OnReady fall back to Serilog's static /// Log.Logger, which in this assembly is never assigned and is therefore the /// library's silent default. That is a load-bearing assumption for the two cases that /// DO pass a logger: were anything here to install a static logger, they would still /// pass (they assert on their own sink) but the silent cases would start writing to it. /// Pass a logger whenever the assertion is about logging. /// /// private ILocalDb Boot( KeyValuePair[]? replication = null, Serilog.ILogger? logger = null) { // A previous boot's provider still holds the file open; the library has no second-handle // mode. Closing it (and the pool) is what makes a reopen behave like a process restart. DisposeProviders(); var settings = new List> { new("LocalDb:Path", _dbPath), }; settings.AddRange(replication ?? []); var configuration = new ConfigurationBuilder() .AddInMemoryCollection(settings) .Build(); var provider = new ServiceCollection() .AddZbLocalDb(configuration, db => LocalDbSetup.OnReady(db, configuration, logger)) .BuildServiceProvider(); _providers.Add(provider); return provider.GetRequiredService(); } private static async Task ScalarAsync(ILocalDb db, string sql) => (await db.QueryAsync(sql, r => r.GetInt32(0), parameters: null, TestContext.Current.CancellationToken))[0]; private static Task InsertPointerAsync(ILocalDb db, string clusterId) => db.ExecuteAsync( """ INSERT INTO deployment_pointer (cluster_id, deployment_id, revision_hash, artifact_sha256, applied_at_utc) VALUES (@ClusterId, @DeploymentId, @RevisionHash, @Sha, @AppliedAtUtc) """, new { ClusterId = clusterId, DeploymentId = "0123456789abcdef0123456789abcdef", RevisionHash = new string('a', 64), Sha = new string('b', 64), AppliedAtUtc = "2026-07-20T00:00:00.0000000Z", }, TestContext.Current.CancellationToken); [Fact] public void OnReady_WhenReplicationIsConfigured_RegistersExactlyTheFourReplicatedTables() { // Both directions are load-bearing. "No fewer" catches a dropped RegisterReplicated call // (that table then never replicates). "No more" catches an accidental registration — // every replicated table costs three triggers and oplog volume on every write. var db = Boot(InitiatorReplication); db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal) .ShouldBe(["alarm_condition_state", "alarm_sf_events", "deployment_artifacts", "deployment_pointer"]); } [Fact] public void OnReady_WhenOnlyTheSyncApiKeyIsSet_StillRegistersTheFourTables() { // The passive half of a pair carries the key and nothing else. If the predicate were // "PeerAddress is set", this node would install no capture triggers and its OWN writes // would never reach the initiator — a pair that replicates in one direction only, with // nothing logged anywhere. This test is the falsifier for that narrower predicate. var db = Boot(PassiveReplication); db.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal) .ShouldBe(["alarm_condition_state", "alarm_sf_events", "deployment_artifacts", "deployment_pointer"]); } [Fact] public async Task OnReady_WhenReplicationIsNotConfigured_RegistersNothingAndInstallsNoTriggers() { // The default-OFF node. It has no peer, so capture buys it nothing and costs it two INSERTs // plus a json_object of the whole row inside every write transaction — for an oplog that is // never drained. Asserting the trigger count as well as ReplicatedTables matters: the // dictionary is in-process bookkeeping, the triggers are what actually run. var db = Boot(); db.ReplicatedTables.ShouldBeEmpty(); // None of the three application schemas defines a trigger of its own, so every trigger in // this file would be a capture trigger. (await ScalarAsync(db, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger'")).ShouldBe(0); } [Fact] public void DeploymentArtifacts_PkIsDeploymentIdPlusChunkIndex() { // The composite PK is what makes an artifact chunkable. A single-column PK here would // make every chunk of a deployment collide onto one row under LWW. var db = Boot(InitiatorReplication); db.ReplicatedTables["deployment_artifacts"].PkColumns .ShouldBe(["deployment_id", "chunk_index"]); } [Fact] public void DeploymentPointer_PkIsClusterId() { // One current-deployment pointer per cluster: the pair's two nodes converge on the same // row rather than each keeping its own. var db = Boot(InitiatorReplication); db.ReplicatedTables["deployment_pointer"].PkColumns.ShouldBe(["cluster_id"]); } [Fact] public void AlarmSfEvents_PkIsTheAppMintedId() { // A single TEXT PK the application mints. The legacy queue this table replaces keyed on // `RowId INTEGER PRIMARY KEY AUTOINCREMENT`, which LWW cannot replicate: two nodes would // independently allocate rowid 7 for different alarms and silently overwrite each other. var db = Boot(InitiatorReplication); db.ReplicatedTables["alarm_sf_events"].PkColumns.ShouldBe(["id"]); } [Fact] public void AlarmConditionState_PkIsTheScriptedAlarmId() { // One condition-state row per alarm identity. The pair's two nodes converge on the same row // under LWW rather than each keeping its own, and Save is an unconditional upsert onto it. var db = Boot(InitiatorReplication); db.ReplicatedTables["alarm_condition_state"].PkColumns.ShouldBe(["scripted_alarm_id"]); } [Fact] public async Task AlarmSfEventRows_EnterTheOplog() { // Same ordering assertion as the deployment-pointer test below, for the table Phase 2 adds. // Worth its own case because the alarm table is registered by a different call site, and a // registration added ahead of its DDL would fail loudly while one added after the migrator // would fail silently. var db = Boot(InitiatorReplication); await db.ExecuteAsync( """ INSERT INTO alarm_sf_events (id, alarm_id, enqueued_at_utc, payload_json, attempt_count) VALUES (@Id, @AlarmId, @EnqueuedAtUtc, @PayloadJson, 0) """, new { Id = new string('c', 64), AlarmId = "equip/alarm-1", EnqueuedAtUtc = "2026-07-21T00:00:00.0000000Z", PayloadJson = """{"AlarmId":"equip/alarm-1"}""", }, TestContext.Current.CancellationToken); (await ScalarAsync(db, "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'alarm_sf_events'")) .ShouldBe(1); } [Fact] public async Task RowsWrittenAfterOnReady_EnterTheOplog() { // THE ordering assertion. If DDL and RegisterReplicated were swapped — or a write were // added to OnReady ahead of registration — the triggers would not exist at write time and // this count would stay 0 while every other test in this file still passed. var db = Boot(InitiatorReplication); await InsertPointerAsync(db, "SITE-A"); (await ScalarAsync(db, "SELECT COUNT(*) FROM __localdb_oplog")).ShouldBeGreaterThanOrEqualTo(1); } [Fact] public async Task EnablingReplicationLater_BaselinesTheRowsAlreadyInTheFile() { // The defect this whole change exists for. Capture is change-data-capture and the snapshot // streamer pages from __localdb_row_version, so rows written while replication was off are // in neither — a late opt-in used to converge on subsequent writes only, silently and // forever. RegisterReplicated(table, baselineExistingRows: true) seeds the ledger at the LWW // floor and flags a resync so they actually travel. // // The assertion is on the ledger entry's HLC, not merely on its existence. A row captured by // a trigger carries a genuine (large, non-zero) stamp; a BASELINED row is seeded at the LWW // floor of HLC 0, which is what makes it lose to any real remote write of the same key and // win only where the peer holds no version at all. Asserting existence alone would pass // against the old unconditional registration, where the trigger simply captured the write — // i.e. it would prove nothing about the flag. var unreplicated = Boot(); await InsertPointerAsync(unreplicated, "SITE-A"); var replicated = Boot(InitiatorReplication); (await ScalarAsync( replicated, """ SELECT COUNT(*) FROM __localdb_row_version WHERE table_name = 'deployment_pointer' AND hlc = 0 """)) .ShouldBe(1); } [Fact] public async Task DisablingReplication_DropsTheCaptureTriggersAnEarlierBootInstalled() { // The self-heal half. Merely declining to register would leave a file registered by an // earlier build capturing forever — on precisely the node that has no peer to capture for. // DeregisterReplicated drops the triggers and prunes that table's oplog and row-version // rows. var replicated = Boot(InitiatorReplication); await InsertPointerAsync(replicated, "SITE-A"); (await ScalarAsync(replicated, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger'")) .ShouldBe(12); var unreplicated = Boot(); unreplicated.ReplicatedTables.ShouldBeEmpty(); (await ScalarAsync(unreplicated, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger'")) .ShouldBe(0); } [Fact] public async Task DisablingReplication_PrunesBookkeepingButNeverTouchesTheCachedDataItself() { // Deregistration prunes the table's oplog and row-version rows — BOOKKEEPING. It must not // touch the base table, because deployment_pointer/deployment_artifacts ARE the // boot-from-cache backstop: a node whose central SQL is unreachable serves its address space // from exactly these rows. A library regression that cleaned the base table along with the // ledger would take that safety net out silently, on the very nodes that never replicate // (central, site-b) and so would never notice a replication fault either. var replicated = Boot(InitiatorReplication); await InsertPointerAsync(replicated, "SITE-A"); (await ScalarAsync(replicated, "SELECT COUNT(*) FROM __localdb_row_version")).ShouldBeGreaterThan(0); var unreplicated = Boot(); // The row survives, and with its content intact — a count alone would pass against a // cleanup that blanked the columns it does not own. var pointers = await unreplicated.QueryAsync( "SELECT cluster_id || '|' || deployment_id || '|' || revision_hash FROM deployment_pointer", static r => r.GetString(0), parameters: null, TestContext.Current.CancellationToken); pointers.ShouldBe([$"SITE-A|0123456789abcdef0123456789abcdef|{new string('a', 64)}"]); // …while the bookkeeping it DOES own is gone. (await ScalarAsync(unreplicated, "SELECT COUNT(*) FROM __localdb_row_version")).ShouldBe(0); (await ScalarAsync(unreplicated, "SELECT COUNT(*) FROM __localdb_oplog")).ShouldBe(0); } [Fact] public async Task DisablingReplication_LogsOnceWhenItActuallyCleanedSomething() { // An operator-visible record that a node just stopped capturing, and how much it dropped. // Silent trigger removal on a node someone MEANT to replicate is the failure this line // exists to make diagnosable. var replicated = Boot(InitiatorReplication); await InsertPointerAsync(replicated, "SITE-A"); var sink = new CollectingSink(); Boot(replication: null, logger: BuildLogger(sink)); sink.Events.Count(e => e.Level == LogEventLevel.Information).ShouldBe(1); } [Fact] public void DisablingReplication_SaysNothingWhenThereWasNothingToClean() { // The steady state of every default-OFF node in the fleet: it boots unreplicated every day // and has nothing to deregister. A log line here would be a daily false alarm, which is how // the real one gets ignored. var sink = new CollectingSink(); Boot(replication: null, logger: BuildLogger(sink)); sink.Events.ShouldBeEmpty(); } [Fact] public void AFailureMidwayThroughRegistration_FailsBootLoudly_AndTheRetriedBootConverges() { // OnReady flips four tables in a loop with no try/catch, so a failure on the third leaves // two registered and two not. That is the RIGHT behaviour and this pins both halves of why: // the throw propagates out of the AddZbLocalDb singleton factory and fails host start // (a node half-capturing is a node whose replication is silently partial), and because both // library APIs are idempotent the retried boot completes the flip rather than compounding it. // // Swallowing the throw would leave the node up, serving, and replicating two of four tables // with nothing in the log to say so. var real = Boot(InitiatorReplication); var configuration = new ConfigurationBuilder() .AddInMemoryCollection([new KeyValuePair("LocalDb:Path", _dbPath), .. InitiatorReplication]) .Build(); var faulty = new FailsOnNthRegistration(real, failOnCall: 3); Should.Throw(() => LocalDbSetup.OnReady(faulty, configuration)); // The wrapper really did reach the third call — otherwise the assertion above would be // pinning some unrelated throw, and the "partial flip" it claims to model never happened. faulty.Calls.ShouldBe(3); // The retry: same file, no wrapper. Idempotent re-registration converges on all four. LocalDbSetup.OnReady(real, configuration); real.ReplicatedTables.Keys.OrderBy(k => k, StringComparer.Ordinal) .ShouldBe(["alarm_condition_state", "alarm_sf_events", "deployment_artifacts", "deployment_pointer"]); } /// /// Passes everything through to a real database except the failOnCall-th /// RegisterReplicated, which throws — the mid-loop partial-flip shape. /// private sealed class FailsOnNthRegistration(ILocalDb inner, int failOnCall) : ILocalDb { /// How many registrations were attempted, successful or not. public int Calls { get; private set; } public IReadOnlyDictionary ReplicatedTables => inner.ReplicatedTables; public SqliteConnection CreateConnection() => inner.CreateConnection(); public Task ExecuteAsync(string sql, object? parameters, CancellationToken cancellationToken) => inner.ExecuteAsync(sql, parameters, cancellationToken); public Task> QueryAsync( string sql, Func map, object? parameters, CancellationToken cancellationToken) => inner.QueryAsync(sql, map, parameters, cancellationToken); public Task BeginTransactionAsync(CancellationToken cancellationToken) => inner.BeginTransactionAsync(cancellationToken); public ReplicatedTable RegisterReplicated(string tableName) => RegisterReplicated(tableName, baselineExistingRows: false); public ReplicatedTable RegisterReplicated(string tableName, bool baselineExistingRows) { Calls++; if (Calls == failOnCall) throw new InvalidOperationException($"simulated registration failure on '{tableName}'"); return inner.RegisterReplicated(tableName, baselineExistingRows); } public bool DeregisterReplicated(string tableName) => inner.DeregisterReplicated(tableName); } private static Serilog.ILogger BuildLogger(ILogEventSink sink) => new Serilog.LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sink).CreateLogger(); /// Captures whatever logs. private sealed class CollectingSink : ILogEventSink { public List Events { get; } = []; public void Emit(LogEvent logEvent) => Events.Add(logEvent); } private void DisposeProviders() { foreach (var provider in _providers) provider.Dispose(); _providers.Clear(); // No in-memory mode, so these are real files. Pooled connections keep a handle open past // dispose; clearing the pools first is what lets a reopen (or the delete below) succeed. SqliteConnection.ClearAllPools(); } public void Dispose() { DisposeProviders(); foreach (var path in new[] { _dbPath, $"{_dbPath}-wal", $"{_dbPath}-shm" }) { if (File.Exists(path)) File.Delete(path); } } }