using Microsoft.Data.Sqlite; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.ScadaBridge.Host; namespace ZB.MOM.WW.ScadaBridge.Host.Tests; /// /// LocalDb Phase 1 (Task 6) — the one-time copy of the pre-Phase-1 site databases into /// the consolidated one. /// /// /// This is a data migration that runs during host startup, so its failure modes are /// expensive: a duplicate-on-rerun bug silently doubles a site's event history, and a /// half-committed copy leaves an operator with no clean state to recover to. Every bullet /// of the plan's semantics gets its own test. /// public class SiteLocalDbLegacyMigratorTests : IDisposable { private readonly string _root; private readonly List _providers = []; public SiteLocalDbLegacyMigratorTests() { _root = Path.Combine(Path.GetTempPath(), $"localdb-migrator-{Guid.NewGuid():N}"); Directory.CreateDirectory(_root); } public void Dispose() { foreach (var provider in _providers) { try { provider.Dispose(); } catch { /* best effort */ } } try { Directory.Delete(_root, recursive: true); } catch { /* best effort */ } GC.SuppressFinalize(this); } private string Path_(string name) => System.IO.Path.Combine(_root, name); /// A consolidated database with the tables created and registered, as the host has it. /// /// sf_messages is registered here even though production OnReady does not /// register it until the Task 14 cutover. The oplog assertion below needs live capture /// triggers to mean anything, and the property under test — that the migrator runs after /// registration — is the same either way. The corollary is a real constraint on Task 14: /// Migrate must stay the LAST call in OnReady, after every registration. /// private ILocalDb CreateConsolidated(IConfiguration config) { var provider = new ServiceCollection() .AddZbLocalDb(config, db => { using var connection = db.CreateConnection(); ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking.OperationTrackingSchema.Apply(connection); ZB.MOM.WW.ScadaBridge.SiteEventLogging.SiteEventLogSchema.Apply(connection); ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardSchema.Apply(connection); ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection); db.RegisterReplicated("OperationTracking"); db.RegisterReplicated("site_events"); db.RegisterReplicated("sf_messages"); foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables) db.RegisterReplicated(table.Table); }) .BuildServiceProvider(); _providers.Add(provider); return provider.GetRequiredService(); } private IConfiguration Config( string? trackingPath = null, string? eventsPath = null, string nodeName = "node-a", string? storeAndForwardPath = null, string? siteDbPath = null) { var values = new Dictionary { ["LocalDb:Path"] = Path_("consolidated.db"), ["ScadaBridge:Node:NodeName"] = nodeName, // Always pinned inside the test's own directory, even when a test does not care // about it. Left unset, the resolver falls back to the CWD-relative code default // "./data/store-and-forward.db" — and the test run's CWD is the test binary's // output directory, so an unlucky run could migrate (and RENAME) a real file. ["ScadaBridge:StoreAndForward:SqliteDbPath"] = storeAndForwardPath ?? Path_("absent-store-and-forward.db"), // Same reasoning: the site-storage default is "./data/scadabridge.db". ["ScadaBridge:Database:SiteDbPath"] = siteDbPath ?? Path_("absent-scadabridge.db"), }; if (trackingPath is not null) values["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={trackingPath}"; if (eventsPath is not null) values["ScadaBridge:SiteEventLog:DatabasePath"] = eventsPath; return new ConfigurationBuilder().AddInMemoryCollection(values).Build(); } /// Seeds a legacy store-and-forward file with the CURRENT column set. private static void SeedLegacyStoreAndForward(string path, params string[] ids) { using var connection = new SqliteConnection($"Data Source={path}"); connection.Open(); ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardSchema.Apply(connection); foreach (var id in ids) { using var cmd = connection.CreateCommand(); cmd.CommandText = """ INSERT INTO sf_messages ( id, category, target, payload_json, retry_count, max_retries, retry_interval_ms, created_at, status, execution_id) VALUES ($id, 0, 'ERP', '{"order":1}', 0, 50, 30000, $now, 0, 'exec-1'); """; cmd.Parameters.AddWithValue("$id", id); cmd.Parameters.AddWithValue("$now", DateTime.UtcNow.ToString("o")); cmd.ExecuteNonQuery(); } } private static void SeedLegacyTracking(string path, params string[] ids) { using var connection = new SqliteConnection($"Data Source={path}"); connection.Open(); ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking.OperationTrackingSchema.Apply(connection); foreach (var id in ids) { using var cmd = connection.CreateCommand(); cmd.CommandText = """ INSERT INTO OperationTracking ( TrackedOperationId, Kind, TargetSummary, Status, RetryCount, CreatedAtUtc, UpdatedAtUtc) VALUES ($id, 'ApiCallCached', 'ERP.GetOrder', 'Submitted', 0, $now, $now); """; cmd.Parameters.AddWithValue("$id", id); cmd.Parameters.AddWithValue("$now", DateTime.UtcNow.ToString("o")); cmd.ExecuteNonQuery(); } } /// Seeds the LEGACY event schema: an autoincrement integer id, not a GUID. private static void SeedLegacyEvents(string path, int count) { using var connection = new SqliteConnection($"Data Source={path}"); connection.Open(); using (var ddl = connection.CreateCommand()) { ddl.CommandText = """ CREATE TABLE IF NOT EXISTS site_events ( id INTEGER PRIMARY KEY AUTOINCREMENT, timestamp TEXT NOT NULL, event_type TEXT NOT NULL, severity TEXT NOT NULL, instance_id TEXT, source TEXT NOT NULL, message TEXT NOT NULL, details TEXT ); """; ddl.ExecuteNonQuery(); } for (var i = 0; i < count; i++) { using var cmd = connection.CreateCommand(); cmd.CommandText = """ INSERT INTO site_events (timestamp, event_type, severity, source, message) VALUES ($ts, 'script', 'Info', 'src', $msg); """; cmd.Parameters.AddWithValue("$ts", DateTimeOffset.UtcNow.AddMinutes(-i).ToString("o")); cmd.Parameters.AddWithValue("$msg", $"legacy message {i}"); cmd.ExecuteNonQuery(); } } private static long CountRows(ILocalDb db, string table) { using var connection = db.CreateConnection(); using var cmd = connection.CreateCommand(); cmd.CommandText = $"SELECT COUNT(*) FROM {table}"; return (long)cmd.ExecuteScalar()!; } private static List SelectIds(ILocalDb db, string table, string idColumn) { using var connection = db.CreateConnection(); using var cmd = connection.CreateCommand(); cmd.CommandText = $"SELECT {idColumn} FROM {table} ORDER BY {idColumn}"; using var reader = cmd.ExecuteReader(); var ids = new List(); while (reader.Read()) ids.Add(reader.GetString(0)); return ids; } [Fact] public void MissingLegacyFiles_AreANoOp() { // The expected case on the docker rig: neither legacy key is set anywhere, and // the CWD-relative defaults point outside the data volume, so there is usually // nothing there at all. That must be a clean no-op, not a startup failure. var config = Config(Path_("absent-tracking.db"), Path_("absent-events.db")); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(0, CountRows(db, "OperationTracking")); Assert.Equal(0, CountRows(db, "site_events")); } [Fact] public void LegacyTrackingRows_AreCopiedAndTheFileIsRenamed() { var trackingPath = Path_("site-tracking.db"); SeedLegacyTracking(trackingPath, "op-1", "op-2", "op-3"); var config = Config(trackingPath, Path_("absent-events.db")); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(3, CountRows(db, "OperationTracking")); Assert.Equal(["op-1", "op-2", "op-3"], SelectIds(db, "OperationTracking", "TrackedOperationId")); // Renamed, not deleted — the operator can still inspect the original. Assert.False(File.Exists(trackingPath)); Assert.True(File.Exists(trackingPath + ".migrated")); } [Fact] public void LegacyEvents_GetDeterministicIds_NotFreshGuids() { // The whole reason a rerun cannot duplicate. Fresh GUIDs would make // INSERT OR IGNORE useless and double the history on every retry. var eventsPath = Path_("site_events.db"); SeedLegacyEvents(eventsPath, count: 3); var config = Config(Path_("absent-tracking.db"), eventsPath, nodeName: "node-b"); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal( ["mig-node-b-1", "mig-node-b-2", "mig-node-b-3"], SelectIds(db, "site_events", "id")); } [Fact] public void SecondBoot_IsANoOp_BecauseTheFileIsAlreadyMarkedMigrated() { var trackingPath = Path_("site-tracking.db"); var eventsPath = Path_("site_events.db"); SeedLegacyTracking(trackingPath, "op-1", "op-2"); SeedLegacyEvents(eventsPath, count: 2); var config = Config(trackingPath, eventsPath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(2, CountRows(db, "OperationTracking")); Assert.Equal(2, CountRows(db, "site_events")); } [Fact] public void RerunAgainstAnUnrenamedFile_DoesNotDuplicate() { // Simulates the crash window: the copy committed but the rename did not. On the // next boot the migrator sees the legacy file again and re-copies it. Both tables // must absorb that with no duplicates — tracking via its natural TEXT key, events // via the deterministic mig- id. var trackingPath = Path_("site-tracking.db"); var eventsPath = Path_("site_events.db"); SeedLegacyTracking(trackingPath, "op-1", "op-2"); SeedLegacyEvents(eventsPath, count: 2); var config = Config(trackingPath, eventsPath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); // Undo the rename, exactly as a crash between commit and rename would leave it. File.Move(trackingPath + ".migrated", trackingPath); File.Move(eventsPath + ".migrated", eventsPath); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(2, CountRows(db, "OperationTracking")); Assert.Equal(2, CountRows(db, "site_events")); } [Fact] public void MigratedRows_EnterTheOplog_SoTheyActuallyReplicate() { // The reason the migrator runs AFTER RegisterReplicated. Capture is trigger-based: // migrate before registration and the rows are invisible to the peer forever, with // no error anywhere. Asserting on the oplog is the only way to catch that ordering // being reversed — every other test here would still pass. var trackingPath = Path_("site-tracking.db"); SeedLegacyTracking(trackingPath, "op-1", "op-2"); var config = Config(trackingPath, Path_("absent-events.db")); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); using var connection = db.CreateConnection(); using var cmd = connection.CreateCommand(); cmd.CommandText = "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'OperationTracking'"; Assert.Equal(2L, (long)cmd.ExecuteScalar()!); } [Fact] public void SfMessages_AreCopiedFromTheLegacyFile() { // Phase 2 (Task 8). Unlike the two Phase 1 files, the store-and-forward default path // is inside the data volume, so on a real deployment this actually moves data: // undelivered messages that a lost migration would silently discard. var sfPath = Path_("store-and-forward.db"); SeedLegacyStoreAndForward(sfPath, "msg-1", "msg-2", "msg-3"); var config = Config(storeAndForwardPath: sfPath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(3, CountRows(db, "sf_messages")); Assert.Equal(["msg-1", "msg-2", "msg-3"], SelectIds(db, "sf_messages", "id")); // Renamed, not deleted — same contract as the Phase 1 files. Assert.False(File.Exists(sfPath)); Assert.True(File.Exists(sfPath + ".migrated")); } [Fact] public void SfMessages_Migration_IsIdempotent_WhenRerunAfterACrashBeforeRename() { // The crash window: copy committed, rename did not. sf_messages.id is a natural // TEXT key, so INSERT OR IGNORE absorbs the re-copy with no id synthesis needed. var sfPath = Path_("store-and-forward.db"); SeedLegacyStoreAndForward(sfPath, "msg-1", "msg-2"); var config = Config(storeAndForwardPath: sfPath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); File.Move(sfPath + ".migrated", sfPath); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(2, CountRows(db, "sf_messages")); } [Fact] public void MigratedSfMessages_EnterTheOplog_SoTheyActuallyReplicate() { // Same ordering trap as the Phase 1 tables: migrate before RegisterReplicated and // the rows never enter the oplog, never reach the peer, and nothing errors. Only an // assertion on __localdb_oplog catches the ordering being reversed. var sfPath = Path_("store-and-forward.db"); SeedLegacyStoreAndForward(sfPath, "msg-1", "msg-2"); var config = Config(storeAndForwardPath: sfPath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); using var connection = db.CreateConnection(); using var cmd = connection.CreateCommand(); cmd.CommandText = "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'sf_messages'"; Assert.Equal(2L, (long)cmd.ExecuteScalar()!); } [Fact] public void SfMessages_FromAnOlderBuild_MigrateDespiteMissingColumns() { // A legacy file written before execution_id / parent_execution_id / last_attempt_at_ms // existed. Naming a missing column in the SELECT throws "no such column", and the // reader treats that as an unrecognised shape — which would silently discard every // buffered message. The copy intersects the column sets instead. var sfPath = Path_("store-and-forward.db"); using (var legacy = new SqliteConnection($"Data Source={sfPath}")) { legacy.Open(); using var ddl = legacy.CreateCommand(); ddl.CommandText = """ CREATE TABLE sf_messages ( id TEXT PRIMARY KEY, category INTEGER NOT NULL, target TEXT NOT NULL, payload_json TEXT NOT NULL, retry_count INTEGER NOT NULL DEFAULT 0, max_retries INTEGER NOT NULL DEFAULT 50, retry_interval_ms INTEGER NOT NULL DEFAULT 30000, created_at TEXT NOT NULL, last_attempt_at TEXT, status INTEGER NOT NULL DEFAULT 0, last_error TEXT, origin_instance TEXT ); INSERT INTO sf_messages (id, category, target, payload_json, created_at) VALUES ('old-1', 0, 'ERP', '{}', '2026-01-01T00:00:00Z'); """; ddl.ExecuteNonQuery(); } var config = Config(storeAndForwardPath: sfPath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(1, CountRows(db, "sf_messages")); Assert.Equal(["old-1"], SelectIds(db, "sf_messages", "id")); } /// Seeds a legacy scadabridge.db with the CURRENT config schema and one row per table. private static void SeedLegacySiteStorage(string path) { using var connection = new SqliteConnection($"Data Source={path}"); connection.Open(); ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection); using var cmd = connection.CreateCommand(); cmd.CommandText = """ INSERT INTO deployed_configurations (instance_unique_name, config_json, deployment_id, revision_hash, is_enabled, deployed_at) VALUES ('inst-1', '{"a":1}', 'dep-1', 'hash-1', 1, '2026-01-01T00:00:00Z'); INSERT INTO static_attribute_overrides (instance_unique_name, attribute_name, override_value, updated_at) VALUES ('inst-1', 'Setpoint', '42', '2026-01-01T00:00:00Z'); INSERT INTO shared_scripts (name, code, updated_at) VALUES ('helper', 'return 1;', '2026-01-01T00:00:00Z'); INSERT INTO external_systems (name, endpoint_url, auth_type, updated_at, timeout_seconds) VALUES ('ERP', 'http://erp:5200', 'None', '2026-01-01T00:00:00Z', 30); INSERT INTO database_connections (name, connection_string, updated_at) VALUES ('BT', 'Server=sql;Database=BT', '2026-01-01T00:00:00Z'); INSERT INTO data_connection_definitions (name, protocol, updated_at) VALUES ('opc-1', 'OpcUa', '2026-01-01T00:00:00Z'); INSERT INTO native_alarm_state (instance_unique_name, source_canonical_name, source_reference, condition_json, last_transition_at) VALUES ('inst-1', 'Area.Line', 'ref-1', '{"active":true}', '2026-01-01T00:00:00Z'); -- The two that must NOT be migrated. A pre-2026-07-10 file really can hold these. INSERT INTO notification_lists (name, recipient_emails, updated_at) VALUES ('ops', 'ops@example.com', '2026-01-01T00:00:00Z'); INSERT INTO smtp_configurations (name, server, port, auth_mode, from_address, username, password, updated_at) VALUES ('smtp', 'smtp.example.com', 587, 'Basic', 'a@b.c', 'user', 'PLAINTEXT-SECRET', '2026-01-01T00:00:00Z'); """; cmd.ExecuteNonQuery(); } [Fact] public void SiteConfigTables_AreCopiedFromTheLegacyFile() { // Phase 2 (Task 9). All seven migrated tables live in one legacy file and are copied // in a single transaction: a partial config migration would leave a site node running // against half its old configuration, which is worse than failing startup. var sitePath = Path_("scadabridge.db"); SeedLegacySiteStorage(sitePath); var config = Config(siteDbPath: sitePath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables) Assert.Equal(1, CountRows(db, table.Table)); Assert.False(File.Exists(sitePath)); Assert.True(File.Exists(sitePath + ".migrated")); } [Fact] public void Migration_DoesNotCopyNotificationOrSmtpRows_EvenWhenTheLegacyFileHasThem() { // smtp_configurations.password is PLAINTEXT. Both tables are purged on every deploy // and permanently empty by design since the site write paths were removed // (2026-07-10), but a pre-fix legacy file can still hold rows — and Task 14 makes // these tables REPLICATED. Migrating them would push plaintext SMTP passwords across // a replication channel whose only historical payload was exactly that. var sitePath = Path_("scadabridge.db"); SeedLegacySiteStorage(sitePath); var config = Config(siteDbPath: sitePath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(0, CountRows(db, "notification_lists")); Assert.Equal(0, CountRows(db, "smtp_configurations")); // Belt and braces: the secret must not be anywhere in the consolidated file, // including the oplog, whose row_json is a json_object copy of every captured row. using var connection = db.CreateConnection(); using var cmd = connection.CreateCommand(); cmd.CommandText = "SELECT COUNT(*) FROM __localdb_oplog WHERE row_json LIKE '%PLAINTEXT-SECRET%'"; Assert.Equal(0L, (long)cmd.ExecuteScalar()!); } [Fact] public void MigratedConfigRows_EnterTheOplog_SoTheyActuallyReplicate() { var sitePath = Path_("scadabridge.db"); SeedLegacySiteStorage(sitePath); var config = Config(siteDbPath: sitePath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); using var connection = db.CreateConnection(); using var cmd = connection.CreateCommand(); cmd.CommandText = "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'deployed_configurations'"; Assert.Equal(1L, (long)cmd.ExecuteScalar()!); } [Fact] public void SiteConfigMigration_IsIdempotent_WhenRerunAfterACrashBeforeRename() { var sitePath = Path_("scadabridge.db"); SeedLegacySiteStorage(sitePath); var config = Config(siteDbPath: sitePath); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); File.Move(sitePath + ".migrated", sitePath); SiteLocalDbLegacyMigrator.Migrate(db, config); foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables) Assert.Equal(1, CountRows(db, table.Table)); } [Fact] public void MigratorColumnLists_MatchTheLiveSchema() { // A column named here but absent from the schema fails at runtime on a real // deployment and NOWHERE else — the intersection logic hides a typo (the column is // simply dropped from the copy) and every other test still passes. A column in the // schema but missing here is worse: that data is silently left behind. // // So this asserts set equality against the schema the host actually applies. using var connection = new SqliteConnection($"Data Source={Path_("schema-probe.db")}"); connection.Open(); ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection); foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables) { using var probe = connection.CreateCommand(); probe.CommandText = $"SELECT name FROM pragma_table_info('{table.Table}')"; using var reader = probe.ExecuteReader(); var actual = new List(); while (reader.Read()) actual.Add(reader.GetString(0)); Assert.Equal( actual.OrderBy(c => c, StringComparer.Ordinal).ToArray(), table.Columns.OrderBy(c => c, StringComparer.Ordinal).ToArray()); } } [Fact] public void LegacyFileWithoutTheExpectedTable_IsTreatedAsEmpty_NotAFailure() { // A file predating the table (or an unrecognised shape) must not fail host boot. var trackingPath = Path_("site-tracking.db"); using (var connection = new SqliteConnection($"Data Source={trackingPath}")) { connection.Open(); using var ddl = connection.CreateCommand(); ddl.CommandText = "CREATE TABLE something_else (x INTEGER);"; ddl.ExecuteNonQuery(); } var config = Config(trackingPath, Path_("absent-events.db")); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(0, CountRows(db, "OperationTracking")); Assert.True(File.Exists(trackingPath + ".migrated")); } [Fact] public void InMemoryLegacyConnectionStrings_AreSkipped() { // Test/dev configurations point the legacy stores at in-memory databases. There is // nothing durable to migrate, and Path.GetFullPath on ":memory:" would be nonsense. var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = Path_("consolidated.db"), ["ScadaBridge:Node:NodeName"] = "node-a", ["ScadaBridge:OperationTracking:ConnectionString"] = "Data Source=file:legacy?mode=memory&cache=shared", ["ScadaBridge:SiteEventLog:DatabasePath"] = ":memory:", }).Build(); Assert.Equal(string.Empty, SiteLocalDbLegacyMigrator.ResolveTrackingPath(config)); Assert.Equal(string.Empty, SiteLocalDbLegacyMigrator.ResolveEventLogPath(config)); var db = CreateConsolidated(config); SiteLocalDbLegacyMigrator.Migrate(db, config); Assert.Equal(0, CountRows(db, "site_events")); } [Fact] public void UnsetLegacyKeys_FallBackToTheCwdRelativeCodeDefaults() { // Neither key is set in any docker appsettings, so the OLD code used these // CWD-relative defaults. Resolving them anywhere else (e.g. against the data // volume) would look for the legacy data in a directory it was never written to. var config = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary { ["LocalDb:Path"] = Path_("consolidated.db"), }).Build(); Assert.Equal( System.IO.Path.GetFullPath("site-tracking.db"), SiteLocalDbLegacyMigrator.ResolveTrackingPath(config)); Assert.Equal( System.IO.Path.GetFullPath("site_events.db"), SiteLocalDbLegacyMigrator.ResolveEventLogPath(config)); } }