using System.Net;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.ScadaBridge.Host;
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
///
/// Serializes the site-pair convergence tests against each other: each one stands up a real
/// Kestrel listener plus two SQLite files, and running them concurrently under CI
/// contention is a flakiness risk.
///
[CollectionDefinition("LocalDbSitePairConvergence")]
public sealed class LocalDbSitePairConvergenceCollection;
///
/// Two ScadaBridge site nodes replicating the consolidated site database over a REAL
/// loopback gRPC transport, through the REAL fail-closed auth interceptor.
///
///
///
/// Extracted from the Phase 1 convergence tests once Phase 2 needed the same pair for the
/// store-and-forward buffer and the configuration tables. Everything here is fixture; the
/// derived classes hold only their own data helpers and scenarios.
///
///
/// It uses , not a hand-written schema, so the tables,
/// their primary keys, and the registration ORDER under test are the ones the host actually
/// runs. A separate schema here would prove only that the test agrees with itself.
///
///
/// Offline: no docker, no external services. Loopback Kestrel with h2c.
///
///
public abstract class LocalDbSitePairHarness : IAsyncLifetime
{
private const string SharedApiKey = "site-pair-convergence-key";
/// How long a scenario waits for the pair to agree before failing.
protected static readonly TimeSpan ConvergeTimeout = TimeSpan.FromSeconds(30);
private readonly string _pathA = Path.Combine(Path.GetTempPath(), $"sitepairA-{Guid.NewGuid():N}.db");
private readonly string _pathB = Path.Combine(Path.GetTempPath(), $"sitepairB-{Guid.NewGuid():N}.db");
// The databases are owned by the fixture, in their own providers, and registered into the
// hosts as pre-constructed instances. MS.DI does not dispose instances it did not create,
// so tearing a host down (the offline-peer scenario) leaves the databases intact and
// writable — which is exactly what lets node A accumulate writes while B is down.
private ServiceProvider _dbProviderA = null!;
private ServiceProvider _dbProviderB = null!;
private IHost? _serverHost; // node B — passive
private IHost? _initiatorHost; // node A — dials the peer
static LocalDbSitePairHarness() =>
// Grpc.Net.Client dials the loopback server over HTTP/2 cleartext.
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
/// Node A — the initiator, which dials the peer.
protected ILocalDb A => _dbProviderA.GetRequiredService();
/// Node B — the passive node, which listens.
protected ILocalDb B => _dbProviderB.GetRequiredService();
public async Task InitializeAsync()
{
_dbProviderA = BuildDatabaseProvider(_pathA, "node-a");
_dbProviderB = BuildDatabaseProvider(_pathB, "node-b");
// Force construction (and therefore OnReady) before anything replicates.
_ = A;
_ = B;
await StartPassiveAsync();
await StartInitiatorAsync();
}
public async Task DisposeAsync()
{
await StopHostAsync(_initiatorHost);
await StopHostAsync(_serverHost);
await _dbProviderA.DisposeAsync();
await _dbProviderB.DisposeAsync();
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (var path in new[] { _pathA, _pathB })
{
foreach (var suffix in new[] { "", "-wal", "-shm" })
{
try { File.Delete(path + suffix); } catch { /* best effort */ }
}
}
}
// ---- fixture internals ------------------------------------------------------------
///
/// A provider owning one consolidated site database, initialized through the host's own
/// — same schema, same registration order.
///
private static ServiceProvider BuildDatabaseProvider(string path, string nodeName)
{
var config = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary
{
["LocalDb:Path"] = path,
["ScadaBridge:Node:NodeName"] = nodeName,
// Point the legacy migrators at paths that do not exist, so they no-op rather
// than picking up stray files from the test working directory. The two Phase 2
// defaults matter most: unlike the Phase 1 pair they resolve inside ./data/,
// and a migration would also RENAME whatever it found.
["ScadaBridge:OperationTracking:ConnectionString"] =
$"Data Source={Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db")}",
["ScadaBridge:SiteEventLog:DatabasePath"] =
Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"),
["ScadaBridge:StoreAndForward:SqliteDbPath"] =
Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"),
["ScadaBridge:Database:SiteDbPath"] =
Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"),
})
.Build();
return new ServiceCollection()
.AddZbLocalDb(config, db =>
{
SiteLocalDbSetup.OnReady(db, config);
RegisterPhase2TablesUntilCutover(db);
})
.BuildServiceProvider();
}
///
/// Registers the Phase 2 tables for capture, which production OnReady does not do
/// until the Task 14 cutover.
///
///
/// Delete this method at Task 14, along with its call above. Until the cutover the
/// bespoke SiteReplicationActor and ReplicationService still own these
/// tables, so OnReady deliberately leaves them unregistered — but the convergence
/// specifications the cutover has to satisfy need capture triggers to mean anything. The
/// tables are empty at this point, so registering here captures nothing retroactively;
/// the ordering guarantee under test is unaffected.
///
/// After Task 14 these registrations come from OnReady itself, and leaving this
/// method behind would mask a cutover that forgot one — the whole point of these tests.
///
///
private static void RegisterPhase2TablesUntilCutover(ILocalDb db)
{
foreach (var table in Phase2ReplicatedTables)
db.RegisterReplicated(table);
}
///
/// The eight tables Task 14 registers. Listed literally rather than derived from
/// production code, so that a cutover which registers the wrong set fails these tests
/// instead of agreeing with itself.
///
///
/// notification_lists and smtp_configurations are absent by design. They
/// are permanently empty (no site writer since 2026-07-10, the migrator skips them, the
/// active-node purge keeps them empty), so registering them would open a standing
/// replication channel whose only historical payload was plaintext SMTP passwords.
///
protected static readonly string[] Phase2ReplicatedTables =
[
"sf_messages",
"deployed_configurations", "static_attribute_overrides", "shared_scripts",
"external_systems", "database_connections", "data_connection_definitions",
"native_alarm_state",
];
private static IConfiguration ReplicationConfig(string? peerAddress)
{
var values = new Dictionary
{
// Tight flush + bounded reconnect backoff so convergence is observable well
// inside the poll deadline. The 60 s production default would let the doubling
// backoff overrun it after a peer outage.
["LocalDb:Replication:FlushInterval"] = "00:00:00.050",
["LocalDb:Replication:ReconnectBackoffMax"] = "00:00:02",
// Both nodes share one key — the interceptor is fail-closed, so a mismatch here
// turns every scenario below red (verified by deliberately breaking it).
["LocalDb:Replication:ApiKey"] = SharedApiKey,
};
if (peerAddress is not null)
values["LocalDb:Replication:PeerAddress"] = peerAddress;
return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
}
/// Starts node B, the passive listener.
protected async Task StartPassiveAsync()
{
var config = ReplicationConfig(peerAddress: null);
_serverHost = await new HostBuilder()
.ConfigureWebHost(web =>
{
web.UseKestrel(o =>
o.Listen(IPAddress.Loopback, 0, listen => listen.Protocols = HttpProtocols.Http2));
web.ConfigureServices(services =>
{
services.AddLogging();
services.AddRouting();
// The REAL interceptor, not a stand-in. If it rejected legitimate peer
// traffic, every scenario below would fail — which is the point.
services.AddGrpc(o => o.Interceptors.Add());
services.AddSingleton(B);
services.AddZbLocalDbReplication(config);
});
web.Configure(app =>
{
app.UseRouting();
app.UseEndpoints(e => e.MapZbLocalDbSync());
});
})
.StartAsync();
}
/// Starts node A, which dials the passive node.
protected async Task StartInitiatorAsync()
{
var config = ReplicationConfig(PassiveAddress());
_initiatorHost = await new HostBuilder()
.ConfigureServices(services =>
{
services.AddLogging();
services.AddSingleton(A);
services.AddZbLocalDbReplication(config);
})
.StartAsync();
}
/// Takes node B's listener down, leaving its database intact and writable.
protected async Task StopPassiveAsync()
{
await StopHostAsync(_serverHost);
_serverHost = null;
}
///
/// Brings node B back on a NEW loopback port and re-dials from A. The initiator's channel
/// factory re-reads the peer address on each reconnect, so this is a genuine rejoin.
///
protected async Task RestartPairAsync()
{
await StartPassiveAsync();
await StopHostAsync(_initiatorHost);
await StartInitiatorAsync();
}
private string PassiveAddress()
=> _serverHost!.Services.GetRequiredService()
.Features.Get()!.Addresses.Single();
private static async Task StopHostAsync(IHost? host)
{
if (host is null) return;
try { await host.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ }
host.Dispose();
}
/// Polls until true or the deadline passes.
protected static async Task WaitUntilAsync(Func> condition, string because)
{
var deadline = DateTime.UtcNow + ConvergeTimeout;
while (DateTime.UtcNow < deadline)
{
if (await condition()) return;
await Task.Delay(50);
}
Assert.Fail($"Timed out after {ConvergeTimeout.TotalSeconds:0}s waiting for: {because}");
}
}