using System.Net; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.LocalDb.Replication; using ZB.MOM.WW.ScadaBridge.Host; namespace ZB.MOM.WW.ScadaBridge.IntegrationTests; /// /// Serializes the site-pair convergence tests against each other: each one stands up a real /// Kestrel listener plus two SQLite files, and running them concurrently under CI /// contention is a flakiness risk. /// [CollectionDefinition("LocalDbSitePairConvergence")] public sealed class LocalDbSitePairConvergenceCollection; /// /// Two ScadaBridge site nodes replicating the consolidated site database over a REAL /// loopback gRPC transport, through the REAL fail-closed auth interceptor. /// /// /// /// Extracted from the Phase 1 convergence tests once Phase 2 needed the same pair for the /// store-and-forward buffer and the configuration tables. Everything here is fixture; the /// derived classes hold only their own data helpers and scenarios. /// /// /// It uses , not a hand-written schema, so the tables, /// their primary keys, and the registration ORDER under test are the ones the host actually /// runs. A separate schema here would prove only that the test agrees with itself. /// /// /// Offline: no docker, no external services. Loopback Kestrel with h2c. /// /// public abstract class LocalDbSitePairHarness : IAsyncLifetime { private const string SharedApiKey = "site-pair-convergence-key"; /// How long a scenario waits for the pair to agree before failing. protected static readonly TimeSpan ConvergeTimeout = TimeSpan.FromSeconds(30); private readonly string _pathA = Path.Combine(Path.GetTempPath(), $"sitepairA-{Guid.NewGuid():N}.db"); private readonly string _pathB = Path.Combine(Path.GetTempPath(), $"sitepairB-{Guid.NewGuid():N}.db"); // The databases are owned by the fixture, in their own providers, and registered into the // hosts as pre-constructed instances. MS.DI does not dispose instances it did not create, // so tearing a host down (the offline-peer scenario) leaves the databases intact and // writable — which is exactly what lets node A accumulate writes while B is down. private ServiceProvider _dbProviderA = null!; private ServiceProvider _dbProviderB = null!; private IHost? _serverHost; // node B — passive private IHost? _initiatorHost; // node A — dials the peer static LocalDbSitePairHarness() => // Grpc.Net.Client dials the loopback server over HTTP/2 cleartext. AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); /// Node A — the initiator, which dials the peer. protected ILocalDb A => _dbProviderA.GetRequiredService(); /// Node B — the passive node, which listens. protected ILocalDb B => _dbProviderB.GetRequiredService(); public async Task InitializeAsync() { _dbProviderA = BuildDatabaseProvider(_pathA, "node-a"); _dbProviderB = BuildDatabaseProvider(_pathB, "node-b"); // Force construction (and therefore OnReady) before anything replicates. _ = A; _ = B; await StartPassiveAsync(); await StartInitiatorAsync(); } public async Task DisposeAsync() { await StopHostAsync(_initiatorHost); await StopHostAsync(_serverHost); await _dbProviderA.DisposeAsync(); await _dbProviderB.DisposeAsync(); Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); foreach (var path in new[] { _pathA, _pathB }) { foreach (var suffix in new[] { "", "-wal", "-shm" }) { try { File.Delete(path + suffix); } catch { /* best effort */ } } } } // ---- fixture internals ------------------------------------------------------------ /// /// A provider owning one consolidated site database, initialized through the host's own /// — same schema, same registration order. /// private static ServiceProvider BuildDatabaseProvider(string path, string nodeName) { var config = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = path, ["ScadaBridge:Node:NodeName"] = nodeName, // Point the legacy migrators at paths that do not exist, so they no-op rather // than picking up stray files from the test working directory. The two Phase 2 // defaults matter most: unlike the Phase 1 pair they resolve inside ./data/, // and a migration would also RENAME whatever it found. ["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db")}", ["ScadaBridge:SiteEventLog:DatabasePath"] = Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"), ["ScadaBridge:StoreAndForward:SqliteDbPath"] = Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"), ["ScadaBridge:Database:SiteDbPath"] = Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db"), }) .Build(); return new ServiceCollection() .AddZbLocalDb(config, db => { SiteLocalDbSetup.OnReady(db, config); RegisterPhase2TablesUntilCutover(db); }) .BuildServiceProvider(); } /// /// Registers the Phase 2 tables for capture, which production OnReady does not do /// until the Task 14 cutover. /// /// /// Delete this method at Task 14, along with its call above. Until the cutover the /// bespoke SiteReplicationActor and ReplicationService still own these /// tables, so OnReady deliberately leaves them unregistered — but the convergence /// specifications the cutover has to satisfy need capture triggers to mean anything. The /// tables are empty at this point, so registering here captures nothing retroactively; /// the ordering guarantee under test is unaffected. /// /// After Task 14 these registrations come from OnReady itself, and leaving this /// method behind would mask a cutover that forgot one — the whole point of these tests. /// /// private static void RegisterPhase2TablesUntilCutover(ILocalDb db) { foreach (var table in Phase2ReplicatedTables) db.RegisterReplicated(table); } /// /// The eight tables Task 14 registers. Listed literally rather than derived from /// production code, so that a cutover which registers the wrong set fails these tests /// instead of agreeing with itself. /// /// /// notification_lists and smtp_configurations are absent by design. They /// are permanently empty (no site writer since 2026-07-10, the migrator skips them, the /// active-node purge keeps them empty), so registering them would open a standing /// replication channel whose only historical payload was plaintext SMTP passwords. /// protected static readonly string[] Phase2ReplicatedTables = [ "sf_messages", "deployed_configurations", "static_attribute_overrides", "shared_scripts", "external_systems", "database_connections", "data_connection_definitions", "native_alarm_state", ]; private static IConfiguration ReplicationConfig(string? peerAddress) { var values = new Dictionary { // Tight flush + bounded reconnect backoff so convergence is observable well // inside the poll deadline. The 60 s production default would let the doubling // backoff overrun it after a peer outage. ["LocalDb:Replication:FlushInterval"] = "00:00:00.050", ["LocalDb:Replication:ReconnectBackoffMax"] = "00:00:02", // Both nodes share one key — the interceptor is fail-closed, so a mismatch here // turns every scenario below red (verified by deliberately breaking it). ["LocalDb:Replication:ApiKey"] = SharedApiKey, }; if (peerAddress is not null) values["LocalDb:Replication:PeerAddress"] = peerAddress; return new ConfigurationBuilder().AddInMemoryCollection(values).Build(); } /// Starts node B, the passive listener. protected async Task StartPassiveAsync() { var config = ReplicationConfig(peerAddress: null); _serverHost = await new HostBuilder() .ConfigureWebHost(web => { web.UseKestrel(o => o.Listen(IPAddress.Loopback, 0, listen => listen.Protocols = HttpProtocols.Http2)); web.ConfigureServices(services => { services.AddLogging(); services.AddRouting(); // The REAL interceptor, not a stand-in. If it rejected legitimate peer // traffic, every scenario below would fail — which is the point. services.AddGrpc(o => o.Interceptors.Add()); services.AddSingleton(B); services.AddZbLocalDbReplication(config); }); web.Configure(app => { app.UseRouting(); app.UseEndpoints(e => e.MapZbLocalDbSync()); }); }) .StartAsync(); } /// Starts node A, which dials the passive node. protected async Task StartInitiatorAsync() { var config = ReplicationConfig(PassiveAddress()); _initiatorHost = await new HostBuilder() .ConfigureServices(services => { services.AddLogging(); services.AddSingleton(A); services.AddZbLocalDbReplication(config); }) .StartAsync(); } /// Takes node B's listener down, leaving its database intact and writable. protected async Task StopPassiveAsync() { await StopHostAsync(_serverHost); _serverHost = null; } /// /// Brings node B back on a NEW loopback port and re-dials from A. The initiator's channel /// factory re-reads the peer address on each reconnect, so this is a genuine rejoin. /// protected async Task RestartPairAsync() { await StartPassiveAsync(); await StopHostAsync(_initiatorHost); await StartInitiatorAsync(); } private string PassiveAddress() => _serverHost!.Services.GetRequiredService() .Features.Get()!.Addresses.Single(); private static async Task StopHostAsync(IHost? host) { if (host is null) return; try { await host.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ } host.Dispose(); } /// Polls until true or the deadline passes. protected static async Task WaitUntilAsync(Func> condition, string because) { var deadline = DateTime.UtcNow + ConvergeTimeout; while (DateTime.UtcNow < deadline) { if (await condition()) return; await Task.Delay(50); } Assert.Fail($"Timed out after {ConvergeTimeout.TotalSeconds:0}s waiting for: {because}"); } }