dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(localdb): migrate legacy scadabridge.db config tables into the consolidated DB

Browse Source 
Task 9. Seven of the nine site configuration tables are copied out of the
legacy scadabridge.db in a single transaction, with one rename at the end: a
partial config migration would leave a site node running against half its old
configuration, which is worse than failing startup outright.

notification_lists and smtp_configurations are deliberately NOT migrated.
Both are purged on every deploy and permanently empty by design since the
site write paths were removed (2026-07-10), but a pre-fix legacy file can
still hold rows — and smtp_configurations.password is plaintext. Task 14
makes these tables replicated, so migrating them would push plaintext SMTP
passwords across a channel whose only historical payload was exactly that.
The tables are still created; only their historical contents stay behind.

Generalizes Task 8's MigrateTable into MigrateFile(many tables, one
transaction, one rename), keeping the legacy/current column intersection.

Five tests, including a column-parity test asserting each declared column
list equals the live SiteStorageSchema's. That mismatch is otherwise
invisible: a typo'd column is silently dropped by the intersection, and a
missing one silently leaves data behind. Non-vacuity verified twice — once by
removing the Migrate call (3 fail), once by wrongly adding notification_lists
and smtp_configurations to the table map (the skip test fails).

Verified: solution build 0 warnings; Host 329, SiteRuntime 532,
StoreAndForward 153 — all pass.

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
2026-07-20 03:40:31 -04:00
parent bdc0dffea2
commit 5ddc7eed6d
2 changed files with 259 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Host/SiteLocalDbLegacyMigrator.cs
+106 -18
View File
@@ -60,6 +60,9 @@ public static class SiteLocalDbLegacyMigrator
/// <summary>Default legacy store-and-forward path (<c>StoreAndForwardOptions.SqliteDbPath</c>).</summary>
private const string DefaultStoreAndForwardPath = "./data/store-and-forward.db";
/// <summary>Default legacy site configuration path (<c>appsettings.Site.json</c>).</summary>
private const string DefaultSiteStoragePath = "./data/scadabridge.db";
/// <summary>
/// Every column of the current <c>sf_messages</c> schema, in a fixed order. Columns
/// absent from an older legacy file are dropped from the copy rather than failing it —
@@ -73,6 +76,55 @@ public static class SiteLocalDbLegacyMigrator
"execution_id", "source_script", "parent_execution_id", "last_attempt_at_ms",
];
/// <summary>
/// The site configuration tables copied out of the legacy <c>scadabridge.db</c>, with
/// the current schema's columns for each.
/// </summary>
/// <remarks>
/// <b><c>notification_lists</c> and <c>smtp_configurations</c> are deliberately absent.</b>
/// Both are purged on every deploy and are permanently empty by design — the site-side
/// write paths were removed on 2026-07-10. A pre-fix legacy file can still hold rows,
/// and <c>smtp_configurations.password</c> is plaintext, so migrating them would
/// resurrect plaintext SMTP passwords into a table that Task 14 makes REPLICATED. The
/// tables themselves are still created (see <c>SiteStorageSchema</c>); only their
/// historical contents are left behind.
/// </remarks>
internal static readonly LegacyTable[] SiteStorageTables =
[
new("deployed_configurations", "instance_unique_name",
[
"instance_unique_name", "config_json", "deployment_id", "revision_hash",
"is_enabled", "deployed_at",
]),
new("static_attribute_overrides", "instance_unique_name",
[
"instance_unique_name", "attribute_name", "override_value", "updated_at",
]),
new("shared_scripts", "name",
[
"name", "code", "parameter_definitions", "return_definition", "updated_at",
]),
new("external_systems", "name",
[
"name", "endpoint_url", "auth_type", "auth_configuration", "method_definitions",
"updated_at", "timeout_seconds",
]),
new("database_connections", "name",
[
"name", "connection_string", "max_retries", "retry_delay_ms", "updated_at",
]),
new("data_connection_definitions", "name",
[
"name", "protocol", "configuration", "backup_configuration",
"failover_retry_count", "updated_at",
]),
new("native_alarm_state", "instance_unique_name",
[
"instance_unique_name", "source_canonical_name", "source_reference",
"condition_json", "last_transition_at", "metadata_json",
]),
];
/// <summary>
/// Copies any legacy site databases into <paramref name="db"/>, then renames them.
/// </summary>
@@ -88,6 +140,7 @@ public static class SiteLocalDbLegacyMigrator
MigrateTracking(db, ResolveTrackingPath(config));
MigrateEvents(db, ResolveEventLogPath(config), nodeName);
MigrateStoreAndForward(db, ResolveStoreAndForwardPath(config));
MigrateSiteStorage(db, ResolveSiteStoragePath(config));
}
/// <summary>
@@ -158,6 +211,25 @@ public static class SiteLocalDbLegacyMigrator
return Path.GetFullPath(path);
}
/// <summary>
/// Resolves the legacy site configuration database path from the OLD key, falling back
/// to the code default. Like store-and-forward — and unlike the two Phase 1 paths — this
/// default is inside the mounted data volume, so a real deployment has real config here.
/// </summary>
internal static string ResolveSiteStoragePath(IConfiguration config)
{
var path = config["ScadaBridge:Database:SiteDbPath"] ?? DefaultSiteStoragePath;
if (string.IsNullOrWhiteSpace(path) ||
path.Equals(":memory:", StringComparison.OrdinalIgnoreCase) ||
path.StartsWith("file:", StringComparison.OrdinalIgnoreCase))
{
return string.Empty;
}
return Path.GetFullPath(path);
}
/// <summary>
/// Copies buffered store-and-forward messages out of the legacy file.
/// </summary>
@@ -172,7 +244,19 @@ public static class SiteLocalDbLegacyMigrator
/// </para>
/// </remarks>
private static void MigrateStoreAndForward(ILocalDb db, string legacyPath)
=> MigrateTable(db, legacyPath, "sf_messages", "id", StoreAndForwardColumns);
=> MigrateFile(db, legacyPath, [new LegacyTable("sf_messages", "id", StoreAndForwardColumns)]);
/// <summary>
/// Copies the site's configuration tables out of the legacy <c>scadabridge.db</c>.
/// </summary>
/// <remarks>
/// All seven migrated tables live in one file, so they are copied inside a single
/// transaction and the file is renamed once: a partial config migration would leave a
/// site node running against half its old configuration, which is worse than failing
/// startup outright.
/// </remarks>
private static void MigrateSiteStorage(ILocalDb db, string legacyPath)
=> MigrateFile(db, legacyPath, SiteStorageTables);
/// <summary>
/// Copies one table from a legacy file into the consolidated database, then renames the
@@ -187,37 +271,41 @@ public static class SiteLocalDbLegacyMigrator
/// </remarks>
/// <param name="db">The consolidated database.</param>
/// <param name="legacyPath">The legacy file, which may not exist.</param>
/// <param name="table">Table name, identical on both sides.</param>
/// <param name="requiredColumn">
/// A column without which the table is not the one we mean — normally the primary key.
/// Copying rows with a NULL PK would be worse than copying nothing.
/// </param>
/// <param name="columns">The current schema's full column list, in a fixed order.</param>
private static void MigrateTable(
ILocalDb db, string legacyPath, string table, string requiredColumn, IReadOnlyList<string> columns)
/// <param name="tables">Every table to copy out of this file, in order.</param>
private static void MigrateFile(ILocalDb db, string legacyPath, IReadOnlyList<LegacyTable> tables)
{
if (!ShouldMigrate(legacyPath)) return;
using (var legacy = OpenLegacyReadOnly(legacyPath))
{
var present = PresentColumns(legacy, table, columns);
using var connection = db.CreateConnection();
using var transaction = connection.BeginTransaction();
// An absent table probes as zero columns, so this one guard covers both "old
// file predating the table" and "file we do not recognise".
if (present.Contains(requiredColumn))
foreach (var table in tables)
{
using var connection = db.CreateConnection();
using var transaction = connection.BeginTransaction();
var present = PresentColumns(legacy, table.Table, table.Columns);
CopyRows(legacy, connection, transaction, table, present);
transaction.Commit();
// An absent table probes as zero columns, so this one guard covers both
// "old file predating the table" and "file we do not recognise".
if (present.Contains(table.RequiredColumn))
CopyRows(legacy, connection, transaction, table.Table, present);
}
transaction.Commit();
}
MarkMigrated(legacyPath);
}
/// <summary>One table to copy out of a legacy file.</summary>
/// <param name="Table">Table name, identical on both sides.</param>
/// <param name="RequiredColumn">
/// A column without which the table is not the one we mean — normally the primary key.
/// Copying rows with a NULL PK would be worse than copying nothing.
/// </param>
/// <param name="Columns">The current schema's full column list, in a fixed order.</param>
internal sealed record LegacyTable(string Table, string RequiredColumn, string[] Columns);
private static void MigrateTracking(ILocalDb db, string legacyPath)
{
if (!ShouldMigrate(legacyPath)) return;