dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(localdb): migrate legacy scadabridge.db config tables into the consolidated DB

Browse Source 
Task 9. Seven of the nine site configuration tables are copied out of the
legacy scadabridge.db in a single transaction, with one rename at the end: a
partial config migration would leave a site node running against half its old
configuration, which is worse than failing startup outright.

notification_lists and smtp_configurations are deliberately NOT migrated.
Both are purged on every deploy and permanently empty by design since the
site write paths were removed (2026-07-10), but a pre-fix legacy file can
still hold rows — and smtp_configurations.password is plaintext. Task 14
makes these tables replicated, so migrating them would push plaintext SMTP
passwords across a channel whose only historical payload was exactly that.
The tables are still created; only their historical contents stay behind.

Generalizes Task 8's MigrateTable into MigrateFile(many tables, one
transaction, one rename), keeping the legacy/current column intersection.

Five tests, including a column-parity test asserting each declared column
list equals the live SiteStorageSchema's. That mismatch is otherwise
invisible: a typo'd column is silently dropped by the intersection, and a
missing one silently leaves data behind. Non-vacuity verified twice — once by
removing the Migrate call (3 fail), once by wrongly adding notification_lists
and smtp_configurations to the table map (the skip test fails).

Verified: solution build 0 warnings; Host 329, SiteRuntime 532,
StoreAndForward 153 — all pass.

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
2026-07-20 03:40:31 -04:00
parent bdc0dffea2
commit 5ddc7eed6d
2 changed files with 259 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbLegacyMigratorTests.cs
+153 -1
View File
@@ -57,9 +57,12 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking.OperationTrackingSchema.Apply(connection);
ZB.MOM.WW.ScadaBridge.SiteEventLogging.SiteEventLogSchema.Apply(connection);
ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardSchema.Apply(connection);
ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection);
db.RegisterReplicated("OperationTracking");
db.RegisterReplicated("site_events");
db.RegisterReplicated("sf_messages");
foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables)
db.RegisterReplicated(table.Table);
})
.BuildServiceProvider();
_providers.Add(provider);
@@ -71,7 +74,8 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
string? trackingPath = null,
string? eventsPath = null,
string nodeName = "node-a",
string? storeAndForwardPath = null)
string? storeAndForwardPath = null,
string? siteDbPath = null)
{
var values = new Dictionary<string, string?>
{
@@ -84,6 +88,9 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
// output directory, so an unlucky run could migrate (and RENAME) a real file.
["ScadaBridge:StoreAndForward:SqliteDbPath"] =
storeAndForwardPath ?? Path_("absent-store-and-forward.db"),
// Same reasoning: the site-storage default is "./data/scadabridge.db".
["ScadaBridge:Database:SiteDbPath"] = siteDbPath ?? Path_("absent-scadabridge.db"),
};
if (trackingPath is not null)
@@ -395,6 +402,151 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
Assert.Equal(["old-1"], SelectIds(db, "sf_messages", "id"));
}
/// <summary>Seeds a legacy scadabridge.db with the CURRENT config schema and one row per table.</summary>
private static void SeedLegacySiteStorage(string path)
{
using var connection = new SqliteConnection($"Data Source={path}");
connection.Open();
ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection);
using var cmd = connection.CreateCommand();
cmd.CommandText = """
INSERT INTO deployed_configurations
(instance_unique_name, config_json, deployment_id, revision_hash, is_enabled, deployed_at)
VALUES ('inst-1', '{"a":1}', 'dep-1', 'hash-1', 1, '2026-01-01T00:00:00Z');
INSERT INTO static_attribute_overrides
(instance_unique_name, attribute_name, override_value, updated_at)
VALUES ('inst-1', 'Setpoint', '42', '2026-01-01T00:00:00Z');
INSERT INTO shared_scripts (name, code, updated_at)
VALUES ('helper', 'return 1;', '2026-01-01T00:00:00Z');
INSERT INTO external_systems (name, endpoint_url, auth_type, updated_at, timeout_seconds)
VALUES ('ERP', 'http://erp:5200', 'None', '2026-01-01T00:00:00Z', 30);
INSERT INTO database_connections (name, connection_string, updated_at)
VALUES ('BT', 'Server=sql;Database=BT', '2026-01-01T00:00:00Z');
INSERT INTO data_connection_definitions (name, protocol, updated_at)
VALUES ('opc-1', 'OpcUa', '2026-01-01T00:00:00Z');
INSERT INTO native_alarm_state
(instance_unique_name, source_canonical_name, source_reference, condition_json, last_transition_at)
VALUES ('inst-1', 'Area.Line', 'ref-1', '{"active":true}', '2026-01-01T00:00:00Z');
-- The two that must NOT be migrated. A pre-2026-07-10 file really can hold these.
INSERT INTO notification_lists (name, recipient_emails, updated_at)
VALUES ('ops', 'ops@example.com', '2026-01-01T00:00:00Z');
INSERT INTO smtp_configurations
(name, server, port, auth_mode, from_address, username, password, updated_at)
VALUES ('smtp', 'smtp.example.com', 587, 'Basic', 'a@b.c', 'user',
'PLAINTEXT-SECRET', '2026-01-01T00:00:00Z');
""";
cmd.ExecuteNonQuery();
}
[Fact]
public void SiteConfigTables_AreCopiedFromTheLegacyFile()
{
// Phase 2 (Task 9). All seven migrated tables live in one legacy file and are copied
// in a single transaction: a partial config migration would leave a site node running
// against half its old configuration, which is worse than failing startup.
var sitePath = Path_("scadabridge.db");
SeedLegacySiteStorage(sitePath);
var config = Config(siteDbPath: sitePath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables)
Assert.Equal(1, CountRows(db, table.Table));
Assert.False(File.Exists(sitePath));
Assert.True(File.Exists(sitePath + ".migrated"));
}
[Fact]
public void Migration_DoesNotCopyNotificationOrSmtpRows_EvenWhenTheLegacyFileHasThem()
{
// smtp_configurations.password is PLAINTEXT. Both tables are purged on every deploy
// and permanently empty by design since the site write paths were removed
// (2026-07-10), but a pre-fix legacy file can still hold rows — and Task 14 makes
// these tables REPLICATED. Migrating them would push plaintext SMTP passwords across
// a replication channel whose only historical payload was exactly that.
var sitePath = Path_("scadabridge.db");
SeedLegacySiteStorage(sitePath);
var config = Config(siteDbPath: sitePath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
Assert.Equal(0, CountRows(db, "notification_lists"));
Assert.Equal(0, CountRows(db, "smtp_configurations"));
// Belt and braces: the secret must not be anywhere in the consolidated file,
// including the oplog, whose row_json is a json_object copy of every captured row.
using var connection = db.CreateConnection();
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM __localdb_oplog WHERE row_json LIKE '%PLAINTEXT-SECRET%'";
Assert.Equal(0L, (long)cmd.ExecuteScalar()!);
}
[Fact]
public void MigratedConfigRows_EnterTheOplog_SoTheyActuallyReplicate()
{
var sitePath = Path_("scadabridge.db");
SeedLegacySiteStorage(sitePath);
var config = Config(siteDbPath: sitePath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
using var connection = db.CreateConnection();
using var cmd = connection.CreateCommand();
cmd.CommandText =
"SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'deployed_configurations'";
Assert.Equal(1L, (long)cmd.ExecuteScalar()!);
}
[Fact]
public void SiteConfigMigration_IsIdempotent_WhenRerunAfterACrashBeforeRename()
{
var sitePath = Path_("scadabridge.db");
SeedLegacySiteStorage(sitePath);
var config = Config(siteDbPath: sitePath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
File.Move(sitePath + ".migrated", sitePath);
SiteLocalDbLegacyMigrator.Migrate(db, config);
foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables)
Assert.Equal(1, CountRows(db, table.Table));
}
[Fact]
public void MigratorColumnLists_MatchTheLiveSchema()
{
// A column named here but absent from the schema fails at runtime on a real
// deployment and NOWHERE else — the intersection logic hides a typo (the column is
// simply dropped from the copy) and every other test still passes. A column in the
// schema but missing here is worse: that data is silently left behind.
//
// So this asserts set equality against the schema the host actually applies.
using var connection = new SqliteConnection($"Data Source={Path_("schema-probe.db")}");
connection.Open();
ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection);
foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables)
{
using var probe = connection.CreateCommand();
probe.CommandText = $"SELECT name FROM pragma_table_info('{table.Table}')";
using var reader = probe.ExecuteReader();
var actual = new List<string>();
while (reader.Read()) actual.Add(reader.GetString(0));
Assert.Equal(
actual.OrderBy(c => c, StringComparer.Ordinal).ToArray(),
table.Columns.OrderBy(c => c, StringComparer.Ordinal).ToArray());
}
}
[Fact]
public void LegacyFileWithoutTheExpectedTable_IsTreatedAsEmpty_NotAFailure()
{