dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(localdb): migrate legacy scadabridge.db config tables into the consolidated DB

Browse Source 
Task 9. Seven of the nine site configuration tables are copied out of the
legacy scadabridge.db in a single transaction, with one rename at the end: a
partial config migration would leave a site node running against half its old
configuration, which is worse than failing startup outright.

notification_lists and smtp_configurations are deliberately NOT migrated.
Both are purged on every deploy and permanently empty by design since the
site write paths were removed (2026-07-10), but a pre-fix legacy file can
still hold rows — and smtp_configurations.password is plaintext. Task 14
makes these tables replicated, so migrating them would push plaintext SMTP
passwords across a channel whose only historical payload was exactly that.
The tables are still created; only their historical contents stay behind.

Generalizes Task 8's MigrateTable into MigrateFile(many tables, one
transaction, one rename), keeping the legacy/current column intersection.

Five tests, including a column-parity test asserting each declared column
list equals the live SiteStorageSchema's. That mismatch is otherwise
invisible: a typo'd column is silently dropped by the intersection, and a
missing one silently leaves data behind. Non-vacuity verified twice — once by
removing the Migrate call (3 fail), once by wrongly adding notification_lists
and smtp_configurations to the table map (the skip test fails).

Verified: solution build 0 warnings; Host 329, SiteRuntime 532,
StoreAndForward 153 — all pass.

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
2026-07-20 03:40:31 -04:00
parent bdc0dffea2
commit 5ddc7eed6d
2 changed files with 259 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Host/SiteLocalDbLegacyMigrator.cs
+106 -18
View File
@@ -60,6 +60,9 @@ public static class SiteLocalDbLegacyMigrator
/// <summary>Default legacy store-and-forward path (<c>StoreAndForwardOptions.SqliteDbPath</c>).</summary>
private const string DefaultStoreAndForwardPath = "./data/store-and-forward.db";
/// <summary>Default legacy site configuration path (<c>appsettings.Site.json</c>).</summary>
private const string DefaultSiteStoragePath = "./data/scadabridge.db";
/// <summary>
/// Every column of the current <c>sf_messages</c> schema, in a fixed order. Columns
/// absent from an older legacy file are dropped from the copy rather than failing it —
@@ -73,6 +76,55 @@ public static class SiteLocalDbLegacyMigrator
"execution_id", "source_script", "parent_execution_id", "last_attempt_at_ms",
];
/// <summary>
/// The site configuration tables copied out of the legacy <c>scadabridge.db</c>, with
/// the current schema's columns for each.
/// </summary>
/// <remarks>
/// <b><c>notification_lists</c> and <c>smtp_configurations</c> are deliberately absent.</b>
/// Both are purged on every deploy and are permanently empty by design — the site-side
/// write paths were removed on 2026-07-10. A pre-fix legacy file can still hold rows,
/// and <c>smtp_configurations.password</c> is plaintext, so migrating them would
/// resurrect plaintext SMTP passwords into a table that Task 14 makes REPLICATED. The
/// tables themselves are still created (see <c>SiteStorageSchema</c>); only their
/// historical contents are left behind.
/// </remarks>
internal static readonly LegacyTable[] SiteStorageTables =
[
new("deployed_configurations", "instance_unique_name",
[
"instance_unique_name", "config_json", "deployment_id", "revision_hash",
"is_enabled", "deployed_at",
]),
new("static_attribute_overrides", "instance_unique_name",
[
"instance_unique_name", "attribute_name", "override_value", "updated_at",
]),
new("shared_scripts", "name",
[
"name", "code", "parameter_definitions", "return_definition", "updated_at",
]),
new("external_systems", "name",
[
"name", "endpoint_url", "auth_type", "auth_configuration", "method_definitions",
"updated_at", "timeout_seconds",
]),
new("database_connections", "name",
[
"name", "connection_string", "max_retries", "retry_delay_ms", "updated_at",
]),
new("data_connection_definitions", "name",
[
"name", "protocol", "configuration", "backup_configuration",
"failover_retry_count", "updated_at",
]),
new("native_alarm_state", "instance_unique_name",
[
"instance_unique_name", "source_canonical_name", "source_reference",
"condition_json", "last_transition_at", "metadata_json",
]),
];
/// <summary>
/// Copies any legacy site databases into <paramref name="db"/>, then renames them.
/// </summary>
@@ -88,6 +140,7 @@ public static class SiteLocalDbLegacyMigrator
MigrateTracking(db, ResolveTrackingPath(config));
MigrateEvents(db, ResolveEventLogPath(config), nodeName);
MigrateStoreAndForward(db, ResolveStoreAndForwardPath(config));
MigrateSiteStorage(db, ResolveSiteStoragePath(config));
}
/// <summary>
@@ -158,6 +211,25 @@ public static class SiteLocalDbLegacyMigrator
return Path.GetFullPath(path);
}
/// <summary>
/// Resolves the legacy site configuration database path from the OLD key, falling back
/// to the code default. Like store-and-forward — and unlike the two Phase 1 paths — this
/// default is inside the mounted data volume, so a real deployment has real config here.
/// </summary>
internal static string ResolveSiteStoragePath(IConfiguration config)
{
var path = config["ScadaBridge:Database:SiteDbPath"] ?? DefaultSiteStoragePath;
if (string.IsNullOrWhiteSpace(path) ||
path.Equals(":memory:", StringComparison.OrdinalIgnoreCase) ||
path.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
{
return string.Empty;
}
return Path.GetFullPath(path);
}
/// <summary>
/// Copies buffered store-and-forward messages out of the legacy file.
/// </summary>
@@ -172,7 +244,19 @@ public static class SiteLocalDbLegacyMigrator
/// </para>
/// </remarks>
private static void MigrateStoreAndForward(ILocalDb db, string legacyPath)
=> MigrateTable(db, legacyPath, "sf_messages", "id", StoreAndForwardColumns);
=> MigrateFile(db, legacyPath, [new LegacyTable("sf_messages", "id", StoreAndForwardColumns)]);
/// <summary>
/// Copies the site's configuration tables out of the legacy <c>scadabridge.db</c>.
/// </summary>
/// <remarks>
/// All seven migrated tables live in one file, so they are copied inside a single
/// transaction and the file is renamed once: a partial config migration would leave a
/// site node running against half its old configuration, which is worse than failing
/// startup outright.
/// </remarks>
private static void MigrateSiteStorage(ILocalDb db, string legacyPath)
=> MigrateFile(db, legacyPath, SiteStorageTables);
/// <summary>
/// Copies one table from a legacy file into the consolidated database, then renames the
@@ -187,37 +271,41 @@ public static class SiteLocalDbLegacyMigrator
/// </remarks>
/// <param name="db">The consolidated database.</param>
/// <param name="legacyPath">The legacy file, which may not exist.</param>
/// <param name="table">Table name, identical on both sides.</param>
/// <param name="requiredColumn">
/// A column without which the table is not the one we mean — normally the primary key.
/// Copying rows with a NULL PK would be worse than copying nothing.
/// </param>
/// <param name="columns">The current schema's full column list, in a fixed order.</param>
private static void MigrateTable(
ILocalDb db, string legacyPath, string table, string requiredColumn, IReadOnlyList<string> columns)
/// <param name="tables">Every table to copy out of this file, in order.</param>
private static void MigrateFile(ILocalDb db, string legacyPath, IReadOnlyList<LegacyTable> tables)
{
if (!ShouldMigrate(legacyPath)) return;
using (var legacy = OpenLegacyReadOnly(legacyPath))
{
var present = PresentColumns(legacy, table, columns);
using var connection = db.CreateConnection();
using var transaction = connection.BeginTransaction();
// An absent table probes as zero columns, so this one guard covers both "old
// file predating the table" and "file we do not recognise".
if (present.Contains(requiredColumn))
foreach (var table in tables)
{
using var connection = db.CreateConnection();
using var transaction = connection.BeginTransaction();
var present = PresentColumns(legacy, table.Table, table.Columns);
CopyRows(legacy, connection, transaction, table, present);
transaction.Commit();
// An absent table probes as zero columns, so this one guard covers both
// "old file predating the table" and "file we do not recognise".
if (present.Contains(table.RequiredColumn))
CopyRows(legacy, connection, transaction, table.Table, present);
}
transaction.Commit();
}
MarkMigrated(legacyPath);
}
/// <summary>One table to copy out of a legacy file.</summary>
/// <param name="Table">Table name, identical on both sides.</param>
/// <param name="RequiredColumn">
/// A column without which the table is not the one we mean — normally the primary key.
/// Copying rows with a NULL PK would be worse than copying nothing.
/// </param>
/// <param name="Columns">The current schema's full column list, in a fixed order.</param>
internal sealed record LegacyTable(string Table, string RequiredColumn, string[] Columns);
private static void MigrateTracking(ILocalDb db, string legacyPath)
{
if (!ShouldMigrate(legacyPath)) return;
tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbLegacyMigratorTests.cs
+153 -1
View File
@@ -57,9 +57,12 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking.OperationTrackingSchema.Apply(connection);
ZB.MOM.WW.ScadaBridge.SiteEventLogging.SiteEventLogSchema.Apply(connection);
ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardSchema.Apply(connection);
ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection);
db.RegisterReplicated("OperationTracking");
db.RegisterReplicated("site_events");
db.RegisterReplicated("sf_messages");
foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables)
db.RegisterReplicated(table.Table);
})
.BuildServiceProvider();
_providers.Add(provider);
@@ -71,7 +74,8 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
string? trackingPath = null,
string? eventsPath = null,
string nodeName = "node-a",
string? storeAndForwardPath = null)
string? storeAndForwardPath = null,
string? siteDbPath = null)
{
var values = new Dictionary<string, string?>
{
@@ -84,6 +88,9 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
// output directory, so an unlucky run could migrate (and RENAME) a real file.
["ScadaBridge:StoreAndForward:SqliteDbPath"] =
storeAndForwardPath ?? Path_("absent-store-and-forward.db"),
// Same reasoning: the site-storage default is "./data/scadabridge.db".
["ScadaBridge:Database:SiteDbPath"] = siteDbPath ?? Path_("absent-scadabridge.db"),
};
if (trackingPath is not null)
@@ -395,6 +402,151 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
Assert.Equal(["old-1"], SelectIds(db, "sf_messages", "id"));
}
/// <summary>Seeds a legacy scadabridge.db with the CURRENT config schema and one row per table.</summary>
private static void SeedLegacySiteStorage(string path)
{
using var connection = new SqliteConnection($"Data Source={path}");
connection.Open();
ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection);
using var cmd = connection.CreateCommand();
cmd.CommandText = """
INSERT INTO deployed_configurations
(instance_unique_name, config_json, deployment_id, revision_hash, is_enabled, deployed_at)
VALUES ('inst-1', '{"a":1}', 'dep-1', 'hash-1', 1, '2026-01-01T00:00:00Z');
INSERT INTO static_attribute_overrides
(instance_unique_name, attribute_name, override_value, updated_at)
VALUES ('inst-1', 'Setpoint', '42', '2026-01-01T00:00:00Z');
INSERT INTO shared_scripts (name, code, updated_at)
VALUES ('helper', 'return 1;', '2026-01-01T00:00:00Z');
INSERT INTO external_systems (name, endpoint_url, auth_type, updated_at, timeout_seconds)
VALUES ('ERP', 'http://erp:5200', 'None', '2026-01-01T00:00:00Z', 30);
INSERT INTO database_connections (name, connection_string, updated_at)
VALUES ('BT', 'Server=sql;Database=BT', '2026-01-01T00:00:00Z');
INSERT INTO data_connection_definitions (name, protocol, updated_at)
VALUES ('opc-1', 'OpcUa', '2026-01-01T00:00:00Z');
INSERT INTO native_alarm_state
(instance_unique_name, source_canonical_name, source_reference, condition_json, last_transition_at)
VALUES ('inst-1', 'Area.Line', 'ref-1', '{"active":true}', '2026-01-01T00:00:00Z');
-- The two that must NOT be migrated. A pre-2026-07-10 file really can hold these.
INSERT INTO notification_lists (name, recipient_emails, updated_at)
VALUES ('ops', 'ops@example.com', '2026-01-01T00:00:00Z');
INSERT INTO smtp_configurations
(name, server, port, auth_mode, from_address, username, password, updated_at)
VALUES ('smtp', 'smtp.example.com', 587, 'Basic', 'a@b.c', 'user',
'PLAINTEXT-SECRET', '2026-01-01T00:00:00Z');
""";
cmd.ExecuteNonQuery();
}
[Fact]
public void SiteConfigTables_AreCopiedFromTheLegacyFile()
{
// Phase 2 (Task 9). All seven migrated tables live in one legacy file and are copied
// in a single transaction: a partial config migration would leave a site node running
// against half its old configuration, which is worse than failing startup.
var sitePath = Path_("scadabridge.db");
SeedLegacySiteStorage(sitePath);
var config = Config(siteDbPath: sitePath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables)
Assert.Equal(1, CountRows(db, table.Table));
Assert.False(File.Exists(sitePath));
Assert.True(File.Exists(sitePath + ".migrated"));
}
[Fact]
public void Migration_DoesNotCopyNotificationOrSmtpRows_EvenWhenTheLegacyFileHasThem()
{
// smtp_configurations.password is PLAINTEXT. Both tables are purged on every deploy
// and permanently empty by design since the site write paths were removed
// (2026-07-10), but a pre-fix legacy file can still hold rows — and Task 14 makes
// these tables REPLICATED. Migrating them would push plaintext SMTP passwords across
// a replication channel whose only historical payload was exactly that.
var sitePath = Path_("scadabridge.db");
SeedLegacySiteStorage(sitePath);
var config = Config(siteDbPath: sitePath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
Assert.Equal(0, CountRows(db, "notification_lists"));
Assert.Equal(0, CountRows(db, "smtp_configurations"));
// Belt and braces: the secret must not be anywhere in the consolidated file,
// including the oplog, whose row_json is a json_object copy of every captured row.
using var connection = db.CreateConnection();
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM __localdb_oplog WHERE row_json LIKE '%PLAINTEXT-SECRET%'";
Assert.Equal(0L, (long)cmd.ExecuteScalar()!);
}
[Fact]
public void MigratedConfigRows_EnterTheOplog_SoTheyActuallyReplicate()
{
var sitePath = Path_("scadabridge.db");
SeedLegacySiteStorage(sitePath);
var config = Config(siteDbPath: sitePath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
using var connection = db.CreateConnection();
using var cmd = connection.CreateCommand();
cmd.CommandText =
"SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'deployed_configurations'";
Assert.Equal(1L, (long)cmd.ExecuteScalar()!);
}
[Fact]
public void SiteConfigMigration_IsIdempotent_WhenRerunAfterACrashBeforeRename()
{
var sitePath = Path_("scadabridge.db");
SeedLegacySiteStorage(sitePath);
var config = Config(siteDbPath: sitePath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
File.Move(sitePath + ".migrated", sitePath);
SiteLocalDbLegacyMigrator.Migrate(db, config);
foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables)
Assert.Equal(1, CountRows(db, table.Table));
}
[Fact]
public void MigratorColumnLists_MatchTheLiveSchema()
{
// A column named here but absent from the schema fails at runtime on a real
// deployment and NOWHERE else — the intersection logic hides a typo (the column is
// simply dropped from the copy) and every other test still passes. A column in the
// schema but missing here is worse: that data is silently left behind.
//
// So this asserts set equality against the schema the host actually applies.
using var connection = new SqliteConnection($"Data Source={Path_("schema-probe.db")}");
connection.Open();
ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence.SiteStorageSchema.Apply(connection);
foreach (var table in SiteLocalDbLegacyMigrator.SiteStorageTables)
{
using var probe = connection.CreateCommand();
probe.CommandText = $"SELECT name FROM pragma_table_info('{table.Table}')";
using var reader = probe.ExecuteReader();
var actual = new List<string>();
while (reader.Read()) actual.Add(reader.GetString(0));
Assert.Equal(
actual.OrderBy(c => c, StringComparer.Ordinal).ToArray(),
table.Columns.OrderBy(c => c, StringComparer.Ordinal).ToArray());
}
}
[Fact]
public void LegacyFileWithoutTheExpectedTable_IsTreatedAsEmpty_NotAFailure()
{