using System.Net;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Xunit;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb;
///
/// Serializes the two-node convergence tests against one another: each stands up a real Kestrel
/// h2c listener plus two SQLite files, and running them concurrently under CI contention is a
/// flakiness risk. The rest of the assembly parallelizes normally.
///
[CollectionDefinition("LocalDbPairConvergence")]
public sealed class LocalDbPairConvergenceCollection;
///
/// Two driver-role OtOpcUa nodes replicating the deployment-artifact cache over a REAL loopback
/// gRPC transport (Kestrel h2c on 127.0.0.1), through the REAL fail-closed
/// . Node A is the initiator (it dials the peer); node B
/// is passive (it hosts MapZbLocalDbSync).
///
///
///
/// Both databases are initialised through the production —
/// a hand-written schema here would prove only that the test agrees with itself. The tables,
/// their primary keys, and the DDL→RegisterReplicated ordering under test are the ones
/// the host actually runs.
///
///
/// The two instances are owned by the harness and registered into the
/// hosts as pre-constructed singletons. MS.DI does not dispose instances it did not create,
/// so tearing a host down (the transport-loss scenario) leaves the databases intact and
/// writable — which is exactly what lets node A accumulate writes while B is down.
///
///
/// A real loopback socket (not an in-memory TestServer) is deliberate: killing the passive
/// host disposes the server and closes the socket, faulting the initiator's active stream
/// promptly. An in-memory TestServer does not model a connection drop and leaves the client
/// hanging.
///
///
/// The API keys are per-node so the wrong-key scenario can hand A and B different keys and
/// assert non-convergence — with a matching-key positive control proving the same harness
/// does converge when the keys agree (an absence assertion without a positive control passed
/// vacuously in ScadaBridge).
///
/// Offline: no docker, no external services.
///
public sealed class LocalDbPairHarness : IAsyncDisposable
{
/// The key both nodes share unless a test overrides one of them.
public const string DefaultApiKey = "otopcua-localdb-pair-convergence-key";
/// How long a scenario waits for the pair to agree (or to stay diverged) before deciding.
public static readonly TimeSpan ConvergeTimeout = TimeSpan.FromSeconds(30);
private readonly string _apiKeyA;
private readonly string _apiKeyB;
private readonly string _pathA = Path.Combine(Path.GetTempPath(), $"otopcua-pairA-{Guid.NewGuid():N}.db");
// Not readonly: WipePassiveAsync replaces node B's database wholesale, which is what a rebuilt
// node with a lost volume actually is — a new file, and with it a new node id.
private string _pathB = Path.Combine(Path.GetTempPath(), $"otopcua-pairB-{Guid.NewGuid():N}.db");
private readonly ServiceProvider _dbProviderA;
private ServiceProvider _dbProviderB;
private IHost? _serverHost; // node B — passive
private IHost? _initiatorHost; // node A — dials the peer
static LocalDbPairHarness() =>
// Grpc.Net.Client dials the loopback server over HTTP/2 cleartext (h2c).
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
/// Node A's replication key. Defaults to .
/// Node B's replication key. Defaults to .
public LocalDbPairHarness(string? apiKeyA = null, string? apiKeyB = null)
{
_apiKeyA = apiKeyA ?? DefaultApiKey;
_apiKeyB = apiKeyB ?? DefaultApiKey;
_dbProviderA = BuildDatabaseProvider(_pathA);
_dbProviderB = BuildDatabaseProvider(_pathB);
}
/// Node A — the initiator, which dials the peer.
public ILocalDb A => _dbProviderA.GetRequiredService();
/// Node B — the passive node, which listens.
public ILocalDb B => _dbProviderB.GetRequiredService();
/// The production artifact cache over node A's database.
public IDeploymentArtifactCache CacheA =>
new LocalDbDeploymentArtifactCache(A, NullLogger.Instance);
/// The production artifact cache over node B's database.
public IDeploymentArtifactCache CacheB =>
new LocalDbDeploymentArtifactCache(B, NullLogger.Instance);
// ---- lifecycle --------------------------------------------------------------------------
/// Forces both databases to construct (running OnReady) then brings the pair online.
public async Task StartAsync()
{
_ = A;
_ = B;
await StartPassiveAsync();
await StartInitiatorAsync();
}
/// Takes node B's listener down, leaving its database intact and writable.
public async Task StopPassiveAsync()
{
await StopHostAsync(_serverHost);
_serverHost = null;
}
///
/// Destroys node B's database and replaces it with an empty one, leaving the pairing config
/// untouched — the rebuilt-node case: a lost volume, a re-imaged host, a fresh container.
///
///
/// A new file, not a truncated one, because that is what the real thing is: the node id lives
/// in the database, so a rebuilt node comes back with a new identity and a zero watermark. The
/// caller follows with to bring the wiped node back online.
///
public async Task WipePassiveAsync()
{
await StopPassiveAsync();
await _dbProviderB.DisposeAsync();
// Pooled connections outlive the provider; without this the delete silently no-ops on
// Windows and the "wiped" node would still hold its rows.
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
DeleteDatabaseFiles(_pathB);
_pathB = Path.Combine(Path.GetTempPath(), $"otopcua-pairB-{Guid.NewGuid():N}.db");
_dbProviderB = BuildDatabaseProvider(_pathB);
_ = B;
}
///
/// Brings node B back on a NEW loopback port and re-dials from A. The initiator re-reads the
/// peer address on each reconnect, so this is a genuine rejoin over the same databases.
///
public async Task RestartPairAsync()
{
await StartPassiveAsync();
await StopHostAsync(_initiatorHost);
_initiatorHost = null;
await StartInitiatorAsync();
}
// ---- convergence helpers ----------------------------------------------------------------
/// Polls until true or the deadline passes; fails otherwise.
public static async Task WaitUntilAsync(Func> condition, string because)
{
var deadline = DateTime.UtcNow + ConvergeTimeout;
while (DateTime.UtcNow < deadline)
{
if (await condition())
return;
await Task.Delay(50);
}
Assert.Fail($"Timed out after {ConvergeTimeout.TotalSeconds:0}s waiting for: {because}");
}
/// Waits out a bounded window and returns whether ever held.
///
/// For the negative half of the wrong-key scenario: it must NOT converge. A short, fixed
/// window keeps the test quick while still giving a matching-key control ample time to
/// converge (the control uses with the full timeout).
///
public static async Task HeldWithinAsync(Func> condition, TimeSpan window)
{
var deadline = DateTime.UtcNow + window;
while (DateTime.UtcNow < deadline)
{
if (await condition())
return true;
await Task.Delay(50);
}
return false;
}
///
/// Dumps the __localdb_row_version rows for one table, ordered by pk, as
/// pk_json|hlc|node_id|is_tombstone. Equal dumps on both nodes prove B holds A's
/// origin-stamped row (same HLC + node id), not a locally re-derived one.
///
public static async Task DumpRowVersionAsync(ILocalDb db, string table)
{
var rows = await db.QueryAsync(
"""
SELECT pk_json, hlc, node_id, is_tombstone
FROM __localdb_row_version
WHERE table_name = @Table
ORDER BY pk_json
""",
static r => $"{r.GetString(0)}|{r.GetInt64(1)}|{r.GetString(2)}|{r.GetInt64(3)}",
new { Table = table });
return string.Join("\n", rows);
}
/// Counts deployment_artifacts chunk rows for one deployment on a node.
public static async Task ChunkCountAsync(ILocalDb db, string deploymentId)
{
var rows = await db.QueryAsync(
"SELECT COUNT(*) FROM deployment_artifacts WHERE deployment_id = @DeploymentId",
static r => r.GetInt64(0),
new { DeploymentId = deploymentId });
return rows[0];
}
///
/// Counts oplog rows on a node. Zero means every local change has been acked by the peer and
/// pruned — the steady state of a converged pair, and the state from which a wiped peer can
/// only be healed by a snapshot.
///
public static async Task OplogDepthAsync(ILocalDb db)
{
var rows = await db.QueryAsync("SELECT COUNT(*) FROM __localdb_oplog", static r => r.GetInt64(0));
return rows[0];
}
/// Counts tombstone rows in __localdb_row_version for one table on a node.
public static async Task TombstoneCountAsync(ILocalDb db, string table)
{
var rows = await db.QueryAsync(
"""
SELECT COUNT(*) FROM __localdb_row_version
WHERE table_name = @Table AND is_tombstone = 1
""",
static r => r.GetInt64(0),
new { Table = table });
return rows[0];
}
// ---- fixture internals ------------------------------------------------------------------
///
/// A provider owning one deployment-cache database, initialised through the host's own
/// — same schema, same registration order.
///
private static ServiceProvider BuildDatabaseProvider(string path)
{
var config = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = path })
.Build();
return new ServiceCollection()
.AddZbLocalDb(config, db => LocalDbSetup.OnReady(db, config))
.BuildServiceProvider();
}
private IConfiguration ReplicationConfig(string apiKey, string? peerAddress)
{
var values = new Dictionary
{
// Tight flush + bounded reconnect backoff so convergence is observable well inside the
// poll deadline. The 60 s production default would let the doubling backoff overrun it
// after a peer outage.
["LocalDb:Replication:FlushInterval"] = "00:00:00.050",
["LocalDb:Replication:ReconnectBackoffMax"] = "00:00:02",
["LocalDb:Replication:ApiKey"] = apiKey,
};
if (peerAddress is not null)
values["LocalDb:Replication:PeerAddress"] = peerAddress;
return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
}
/// Starts node B, the passive listener, behind the real auth interceptor.
private async Task StartPassiveAsync()
{
var config = ReplicationConfig(_apiKeyB, peerAddress: null);
_serverHost = await new HostBuilder()
.ConfigureWebHost(web =>
{
web.UseKestrel(o =>
o.Listen(IPAddress.Loopback, 0, listen => listen.Protocols = HttpProtocols.Http2));
web.ConfigureServices(services =>
{
services.AddLogging();
services.AddRouting();
// The REAL interceptor, not a stand-in. If it rejected legitimate peer traffic,
// every matching-key scenario would fail — which is the point.
services.AddGrpc(o => o.Interceptors.Add());
services.AddSingleton(B);
services.AddZbLocalDbReplication(config);
})
.Configure(app =>
{
app.UseRouting();
app.UseEndpoints(e => e.MapZbLocalDbSync());
});
})
.StartAsync();
}
/// Starts node A, which dials the passive node.
private async Task StartInitiatorAsync()
{
var config = ReplicationConfig(_apiKeyA, PassiveAddress());
_initiatorHost = await new HostBuilder()
.ConfigureServices(services =>
{
services.AddLogging();
services.AddSingleton(A);
services.AddZbLocalDbReplication(config);
})
.StartAsync();
}
private string PassiveAddress()
=> _serverHost!.Services.GetRequiredService()
.Features.Get()!.Addresses.Single();
private static async Task StopHostAsync(IHost? host)
{
if (host is null)
return;
try { await host.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ }
host.Dispose();
}
public async ValueTask DisposeAsync()
{
await StopHostAsync(_initiatorHost);
await StopHostAsync(_serverHost);
await _dbProviderA.DisposeAsync();
await _dbProviderB.DisposeAsync();
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
DeleteDatabaseFiles(_pathA);
DeleteDatabaseFiles(_pathB);
}
/// Deletes a SQLite database and its WAL/shm sidecars, best effort.
private static void DeleteDatabaseFiles(string path)
{
foreach (var suffix in new[] { "", "-wal", "-shm" })
{
try { File.Delete(path + suffix); } catch { /* best effort */ }
}
}
}