diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/SiteLocalDbLegacyMigrator.cs b/src/ZB.MOM.WW.ScadaBridge.Host/SiteLocalDbLegacyMigrator.cs index 1abb67fc..42bf6e42 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/SiteLocalDbLegacyMigrator.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/SiteLocalDbLegacyMigrator.cs @@ -60,6 +60,9 @@ public static class SiteLocalDbLegacyMigrator /// Default legacy store-and-forward path (StoreAndForwardOptions.SqliteDbPath). private const string DefaultStoreAndForwardPath = "./data/store-and-forward.db"; + /// Default legacy site configuration path (appsettings.Site.json). + private const string DefaultSiteStoragePath = "./data/scadabridge.db"; + /// /// Every column of the current sf_messages schema, in a fixed order. Columns /// absent from an older legacy file are dropped from the copy rather than failing it — @@ -73,6 +76,55 @@ public static class SiteLocalDbLegacyMigrator "execution_id", "source_script", "parent_execution_id", "last_attempt_at_ms", ]; + /// + /// The site configuration tables copied out of the legacy scadabridge.db, with + /// the current schema's columns for each. + /// + /// + /// notification_lists and smtp_configurations are deliberately absent. + /// Both are purged on every deploy and are permanently empty by design — the site-side + /// write paths were removed on 2026-07-10. A pre-fix legacy file can still hold rows, + /// and smtp_configurations.password is plaintext, so migrating them would + /// resurrect plaintext SMTP passwords into a table that Task 14 makes REPLICATED. The + /// tables themselves are still created (see SiteStorageSchema); only their + /// historical contents are left behind. + /// + internal static readonly LegacyTable[] SiteStorageTables = + [ + new("deployed_configurations", "instance_unique_name", + [ + "instance_unique_name", "config_json", "deployment_id", "revision_hash", + "is_enabled", "deployed_at", + ]), + new("static_attribute_overrides", "instance_unique_name", + [ + "instance_unique_name", "attribute_name", "override_value", "updated_at", + ]), + new("shared_scripts", "name", + [ + "name", "code", "parameter_definitions", "return_definition", "updated_at", + ]), + new("external_systems", "name", + [ + "name", "endpoint_url", "auth_type", "auth_configuration", "method_definitions", + "updated_at", "timeout_seconds", + ]), + new("database_connections", "name", + [ + "name", "connection_string", "max_retries", "retry_delay_ms", "updated_at", + ]), + new("data_connection_definitions", "name", + [ + "name", "protocol", "configuration", "backup_configuration", + "failover_retry_count", "updated_at", + ]), + new("native_alarm_state", "instance_unique_name", + [ + "instance_unique_name", "source_canonical_name", "source_reference", + "condition_json", "last_transition_at", "metadata_json", + ]), + ]; + /// /// Copies any legacy site databases into , then renames them. /// @@ -88,6 +140,7 @@ public static class SiteLocalDbLegacyMigrator MigrateTracking(db, ResolveTrackingPath(config)); MigrateEvents(db, ResolveEventLogPath(config), nodeName); MigrateStoreAndForward(db, ResolveStoreAndForwardPath(config)); + MigrateSiteStorage(db, ResolveSiteStoragePath(config)); } /// @@ -158,6 +211,25 @@ public static class SiteLocalDbLegacyMigrator return Path.GetFullPath(path); } + /// + /// Resolves the legacy site configuration database path from the OLD key, falling back + /// to the code default. Like store-and-forward — and unlike the two Phase 1 paths — this + /// default is inside the mounted data volume, so a real deployment has real config here. + /// + internal static string ResolveSiteStoragePath(IConfiguration config) + { + var path = config["ScadaBridge:Database:SiteDbPath"] ?? DefaultSiteStoragePath; + + if (string.IsNullOrWhiteSpace(path) || + path.Equals(":memory:", StringComparison.OrdinalIgnoreCase) || + path.StartsWith("file:", StringComparison.OrdinalIgnoreCase)) + { + return string.Empty; + } + + return Path.GetFullPath(path); + } + /// /// Copies buffered store-and-forward messages out of the legacy file. /// @@ -172,7 +244,19 @@ public static class SiteLocalDbLegacyMigrator /// /// private static void MigrateStoreAndForward(ILocalDb db, string legacyPath) - => MigrateTable(db, legacyPath, "sf_messages", "id", StoreAndForwardColumns); + => MigrateFile(db, legacyPath, [new LegacyTable("sf_messages", "id", StoreAndForwardColumns)]); + + /// + /// Copies the site's configuration tables out of the legacy scadabridge.db. + /// + /// + /// All seven migrated tables live in one file, so they are copied inside a single + /// transaction and the file is renamed once: a partial config migration would leave a + /// site node running against half its old configuration, which is worse than failing + /// startup outright. + /// + private static void MigrateSiteStorage(ILocalDb db, string legacyPath) + => MigrateFile(db, legacyPath, SiteStorageTables); /// /// Copies one table from a legacy file into the consolidated database, then renames the @@ -187,37 +271,41 @@ public static class SiteLocalDbLegacyMigrator /// /// The consolidated database. /// The legacy file, which may not exist. - /// Table name, identical on both sides. - /// - /// A column without which the table is not the one we mean — normally the primary key. - /// Copying rows with a NULL PK would be worse than copying nothing. - /// - /// The current schema's full column list, in a fixed order. - private static void MigrateTable( - ILocalDb db, string legacyPath, string table, string requiredColumn, IReadOnlyList columns) + /// Every table to copy out of this file, in order. + private static void MigrateFile(ILocalDb db, string legacyPath, IReadOnlyList tables) { if (!ShouldMigrate(legacyPath)) return; using (var legacy = OpenLegacyReadOnly(legacyPath)) { - var present = PresentColumns(legacy, table, columns); + using var connection = db.CreateConnection(); + using var transaction = connection.BeginTransaction(); - // An absent table probes as zero columns, so this one guard covers both "old - // file predating the table" and "file we do not recognise". - if (present.Contains(requiredColumn)) + foreach (var table in tables) { - using var connection = db.CreateConnection(); - using var transaction = connection.BeginTransaction(); + var present = PresentColumns(legacy, table.Table, table.Columns); - CopyRows(legacy, connection, transaction, table, present); - - transaction.Commit(); + // An absent table probes as zero columns, so this one guard covers both + // "old file predating the table" and "file we do not recognise". + if (present.Contains(table.RequiredColumn)) + CopyRows(legacy, connection, transaction, table.Table, present); } + + transaction.Commit(); } MarkMigrated(legacyPath); } + /// One table to copy out of a legacy file. + /// Table name, identical on both sides. + /// + /// A column without which the table is not the one we mean — normally the primary key. + /// Copying rows with a NULL PK would be worse than copying nothing. + /// + /// The current schema's full column list, in a fixed order. + internal sealed record LegacyTable(string Table, string RequiredColumn, string[] Columns); + private static void MigrateTracking(ILocalDb db, string legacyPath) { if (!ShouldMigrate(legacyPath)) return; diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbLegacyMigratorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbLegacyMigratorTests.cs index e12e6be8..834aa36a 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbLegacyMigratorTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbLegacyMigratorTests.cs @@ -57,9 +57,12 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking.OperationTrackingSchema.Apply(connection); ZB.MOM.WW.ScadaBridge.SiteEventLogging.SiteEventLogSchema.Apply(connection); ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardSchema.Apply(connection); + ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection); db.RegisterReplicated("OperationTracking"); db.RegisterReplicated("site_events"); db.RegisterReplicated("sf_messages"); + foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables) + db.RegisterReplicated(table.Table); }) .BuildServiceProvider(); _providers.Add(provider); @@ -71,7 +74,8 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable string? trackingPath = null, string? eventsPath = null, string nodeName = "node-a", - string? storeAndForwardPath = null) + string? storeAndForwardPath = null, + string? siteDbPath = null) { var values = new Dictionary { @@ -84,6 +88,9 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable // output directory, so an unlucky run could migrate (and RENAME) a real file. ["ScadaBridge:StoreAndForward:SqliteDbPath"] = storeAndForwardPath ?? Path_("absent-store-and-forward.db"), + + // Same reasoning: the site-storage default is "./data/scadabridge.db". + ["ScadaBridge:Database:SiteDbPath"] = siteDbPath ?? Path_("absent-scadabridge.db"), }; if (trackingPath is not null) @@ -395,6 +402,151 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable Assert.Equal(["old-1"], SelectIds(db, "sf_messages", "id")); } + /// Seeds a legacy scadabridge.db with the CURRENT config schema and one row per table. + private static void SeedLegacySiteStorage(string path) + { + using var connection = new SqliteConnection($"Data Source={path}"); + connection.Open(); + ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = """ + INSERT INTO deployed_configurations + (instance_unique_name, config_json, deployment_id, revision_hash, is_enabled, deployed_at) + VALUES ('inst-1', '{"a":1}', 'dep-1', 'hash-1', 1, '2026-01-01T00:00:00Z'); + INSERT INTO static_attribute_overrides + (instance_unique_name, attribute_name, override_value, updated_at) + VALUES ('inst-1', 'Setpoint', '42', '2026-01-01T00:00:00Z'); + INSERT INTO shared_scripts (name, code, updated_at) + VALUES ('helper', 'return 1;', '2026-01-01T00:00:00Z'); + INSERT INTO external_systems (name, endpoint_url, auth_type, updated_at, timeout_seconds) + VALUES ('ERP', 'http://erp:5200', 'None', '2026-01-01T00:00:00Z', 30); + INSERT INTO database_connections (name, connection_string, updated_at) + VALUES ('BT', 'Server=sql;Database=BT', '2026-01-01T00:00:00Z'); + INSERT INTO data_connection_definitions (name, protocol, updated_at) + VALUES ('opc-1', 'OpcUa', '2026-01-01T00:00:00Z'); + INSERT INTO native_alarm_state + (instance_unique_name, source_canonical_name, source_reference, condition_json, last_transition_at) + VALUES ('inst-1', 'Area.Line', 'ref-1', '{"active":true}', '2026-01-01T00:00:00Z'); + + -- The two that must NOT be migrated. A pre-2026-07-10 file really can hold these. + INSERT INTO notification_lists (name, recipient_emails, updated_at) + VALUES ('ops', 'ops@example.com', '2026-01-01T00:00:00Z'); + INSERT INTO smtp_configurations + (name, server, port, auth_mode, from_address, username, password, updated_at) + VALUES ('smtp', 'smtp.example.com', 587, 'Basic', 'a@b.c', 'user', + 'PLAINTEXT-SECRET', '2026-01-01T00:00:00Z'); + """; + cmd.ExecuteNonQuery(); + } + + [Fact] + public void SiteConfigTables_AreCopiedFromTheLegacyFile() + { + // Phase 2 (Task 9). All seven migrated tables live in one legacy file and are copied + // in a single transaction: a partial config migration would leave a site node running + // against half its old configuration, which is worse than failing startup. + var sitePath = Path_("scadabridge.db"); + SeedLegacySiteStorage(sitePath); + var config = Config(siteDbPath: sitePath); + var db = CreateConsolidated(config); + + SiteLocalDbLegacyMigrator.Migrate(db, config); + + foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables) + Assert.Equal(1, CountRows(db, table.Table)); + + Assert.False(File.Exists(sitePath)); + Assert.True(File.Exists(sitePath + ".migrated")); + } + + [Fact] + public void Migration_DoesNotCopyNotificationOrSmtpRows_EvenWhenTheLegacyFileHasThem() + { + // smtp_configurations.password is PLAINTEXT. Both tables are purged on every deploy + // and permanently empty by design since the site write paths were removed + // (2026-07-10), but a pre-fix legacy file can still hold rows — and Task 14 makes + // these tables REPLICATED. Migrating them would push plaintext SMTP passwords across + // a replication channel whose only historical payload was exactly that. + var sitePath = Path_("scadabridge.db"); + SeedLegacySiteStorage(sitePath); + var config = Config(siteDbPath: sitePath); + var db = CreateConsolidated(config); + + SiteLocalDbLegacyMigrator.Migrate(db, config); + + Assert.Equal(0, CountRows(db, "notification_lists")); + Assert.Equal(0, CountRows(db, "smtp_configurations")); + + // Belt and braces: the secret must not be anywhere in the consolidated file, + // including the oplog, whose row_json is a json_object copy of every captured row. + using var connection = db.CreateConnection(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM __localdb_oplog WHERE row_json LIKE '%PLAINTEXT-SECRET%'"; + Assert.Equal(0L, (long)cmd.ExecuteScalar()!); + } + + [Fact] + public void MigratedConfigRows_EnterTheOplog_SoTheyActuallyReplicate() + { + var sitePath = Path_("scadabridge.db"); + SeedLegacySiteStorage(sitePath); + var config = Config(siteDbPath: sitePath); + var db = CreateConsolidated(config); + + SiteLocalDbLegacyMigrator.Migrate(db, config); + + using var connection = db.CreateConnection(); + using var cmd = connection.CreateCommand(); + cmd.CommandText = + "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'deployed_configurations'"; + Assert.Equal(1L, (long)cmd.ExecuteScalar()!); + } + + [Fact] + public void SiteConfigMigration_IsIdempotent_WhenRerunAfterACrashBeforeRename() + { + var sitePath = Path_("scadabridge.db"); + SeedLegacySiteStorage(sitePath); + var config = Config(siteDbPath: sitePath); + var db = CreateConsolidated(config); + + SiteLocalDbLegacyMigrator.Migrate(db, config); + File.Move(sitePath + ".migrated", sitePath); + SiteLocalDbLegacyMigrator.Migrate(db, config); + + foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables) + Assert.Equal(1, CountRows(db, table.Table)); + } + + [Fact] + public void MigratorColumnLists_MatchTheLiveSchema() + { + // A column named here but absent from the schema fails at runtime on a real + // deployment and NOWHERE else — the intersection logic hides a typo (the column is + // simply dropped from the copy) and every other test still passes. A column in the + // schema but missing here is worse: that data is silently left behind. + // + // So this asserts set equality against the schema the host actually applies. + using var connection = new SqliteConnection($"Data Source={Path_("schema-probe.db")}"); + connection.Open(); + ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection); + + foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables) + { + using var probe = connection.CreateCommand(); + probe.CommandText = $"SELECT name FROM pragma_table_info('{table.Table}')"; + using var reader = probe.ExecuteReader(); + + var actual = new List(); + while (reader.Read()) actual.Add(reader.GetString(0)); + + Assert.Equal( + actual.OrderBy(c => c, StringComparer.Ordinal).ToArray(), + table.Columns.OrderBy(c => c, StringComparer.Ordinal).ToArray()); + } + } + [Fact] public void LegacyFileWithoutTheExpectedTable_IsTreatedAsEmpty_NotAFailure() {