dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(localdb): migrate legacy store-and-forward.db into the consolidated DB

Browse Source 
Task 8. Adds ResolveStoreAndForwardPath + MigrateStoreAndForward, wired as a
third call in Migrate alongside the two Phase 1 files.

Unlike the Phase 1 paths, the store-and-forward default sits INSIDE the data
volume, so a real deployment has a real file here holding undelivered
messages. This migration genuinely moves data; losing it would discard exactly
the buffered calls store-and-forward exists to protect.

No id synthesis: sf_messages.id is already a caller-assigned TEXT primary key,
so INSERT OR IGNORE is idempotent across a crash-then-rerun.

The copy intersects the legacy column set with the current one rather than
naming all 16 columns outright. A file from an older build predates
execution_id / parent_execution_id / last_attempt_at_ms, and naming a missing
column throws 'no such column' — which the existing reader treats as an
unrecognised shape and silently discards every row. A required-column (PK)
guard keeps that tolerance from degrading into copying NULL-keyed rows.

Four tests: copy+rename, crash-before-rename idempotence, the __localdb_oplog
assertion that catches migrate-before-register, and the older-column-set case.
All four verified to fail without the Migrate call.

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
Joseph Doherty
2026-07-20 03:32:10 -04:00
parent f8aa02e2a9
commit bdc0dffea2
2 changed files with 301 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbLegacyMigratorTests.cs
+136 -2
View File
@@ -40,7 +40,14 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
private string Path_(string name) => System.IO.Path.Combine(_root, name);
/// <summary>A consolidated database with both tables created and registered, as the host has it.</summary>
/// <summary>A consolidated database with the tables created and registered, as the host has it.</summary>
/// <remarks>
/// <c>sf_messages</c> is registered here even though production <c>OnReady</c> does not
/// register it until the Task 14 cutover. The oplog assertion below needs live capture
/// triggers to mean anything, and the property under test — that the migrator runs after
/// registration — is the same either way. The corollary is a real constraint on Task 14:
/// <c>Migrate</c> must stay the LAST call in <c>OnReady</c>, after every registration.
/// </remarks>
private ILocalDb CreateConsolidated(IConfiguration config)
{
var provider = new ServiceCollection()
@@ -49,8 +56,10 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
using var connection = db.CreateConnection();
ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking.OperationTrackingSchema.Apply(connection);
ZB.MOM.WW.ScadaBridge.SiteEventLogging.SiteEventLogSchema.Apply(connection);
ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardSchema.Apply(connection);
db.RegisterReplicated("OperationTracking");
db.RegisterReplicated("site_events");
db.RegisterReplicated("sf_messages");
})
.BuildServiceProvider();
_providers.Add(provider);
@@ -59,12 +68,22 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
}
private IConfiguration Config(
string? trackingPath = null, string? eventsPath = null, string nodeName = "node-a")
string? trackingPath = null,
string? eventsPath = null,
string nodeName = "node-a",
string? storeAndForwardPath = null)
{
var values = new Dictionary<string, string?>
{
["LocalDb:Path"] = Path_("consolidated.db"),
["ScadaBridge:Node:NodeName"] = nodeName,
// Always pinned inside the test's own directory, even when a test does not care
// about it. Left unset, the resolver falls back to the CWD-relative code default
// "./data/store-and-forward.db" — and the test run's CWD is the test binary's
// output directory, so an unlucky run could migrate (and RENAME) a real file.
["ScadaBridge:StoreAndForward:SqliteDbPath"] =
storeAndForwardPath ?? Path_("absent-store-and-forward.db"),
};
if (trackingPath is not null)
@@ -75,6 +94,28 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
}
/// <summary>Seeds a legacy store-and-forward file with the CURRENT column set.</summary>
private static void SeedLegacyStoreAndForward(string path, params string[] ids)
{
using var connection = new SqliteConnection($"Data Source={path}");
connection.Open();
ZB.MOM.WW.ScadaBridge.StoreAndForward.StoreAndForwardSchema.Apply(connection);
foreach (var id in ids)
{
using var cmd = connection.CreateCommand();
cmd.CommandText = """
INSERT INTO sf_messages (
id, category, target, payload_json, retry_count, max_retries,
retry_interval_ms, created_at, status, execution_id)
VALUES ($id, 0, 'ERP', '{"order":1}', 0, 50, 30000, $now, 0, 'exec-1');
""";
cmd.Parameters.AddWithValue("$id", id);
cmd.Parameters.AddWithValue("$now", DateTime.UtcNow.ToString("o"));
cmd.ExecuteNonQuery();
}
}
private static void SeedLegacyTracking(string path, params string[] ids)
{
using var connection = new SqliteConnection($"Data Source={path}");
@@ -261,6 +302,99 @@ public class SiteLocalDbLegacyMigratorTests : IDisposable
Assert.Equal(2L, (long)cmd.ExecuteScalar()!);
}
[Fact]
public void SfMessages_AreCopiedFromTheLegacyFile()
{
// Phase 2 (Task 8). Unlike the two Phase 1 files, the store-and-forward default path
// is inside the data volume, so on a real deployment this actually moves data:
// undelivered messages that a lost migration would silently discard.
var sfPath = Path_("store-and-forward.db");
SeedLegacyStoreAndForward(sfPath, "msg-1", "msg-2", "msg-3");
var config = Config(storeAndForwardPath: sfPath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
Assert.Equal(3, CountRows(db, "sf_messages"));
Assert.Equal(["msg-1", "msg-2", "msg-3"], SelectIds(db, "sf_messages", "id"));
// Renamed, not deleted — same contract as the Phase 1 files.
Assert.False(File.Exists(sfPath));
Assert.True(File.Exists(sfPath + ".migrated"));
}
[Fact]
public void SfMessages_Migration_IsIdempotent_WhenRerunAfterACrashBeforeRename()
{
// The crash window: copy committed, rename did not. sf_messages.id is a natural
// TEXT key, so INSERT OR IGNORE absorbs the re-copy with no id synthesis needed.
var sfPath = Path_("store-and-forward.db");
SeedLegacyStoreAndForward(sfPath, "msg-1", "msg-2");
var config = Config(storeAndForwardPath: sfPath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
File.Move(sfPath + ".migrated", sfPath);
SiteLocalDbLegacyMigrator.Migrate(db, config);
Assert.Equal(2, CountRows(db, "sf_messages"));
}
[Fact]
public void MigratedSfMessages_EnterTheOplog_SoTheyActuallyReplicate()
{
// Same ordering trap as the Phase 1 tables: migrate before RegisterReplicated and
// the rows never enter the oplog, never reach the peer, and nothing errors. Only an
// assertion on __localdb_oplog catches the ordering being reversed.
var sfPath = Path_("store-and-forward.db");
SeedLegacyStoreAndForward(sfPath, "msg-1", "msg-2");
var config = Config(storeAndForwardPath: sfPath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
using var connection = db.CreateConnection();
using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'sf_messages'";
Assert.Equal(2L, (long)cmd.ExecuteScalar()!);
}
[Fact]
public void SfMessages_FromAnOlderBuild_MigrateDespiteMissingColumns()
{
// A legacy file written before execution_id / parent_execution_id / last_attempt_at_ms
// existed. Naming a missing column in the SELECT throws "no such column", and the
// reader treats that as an unrecognised shape — which would silently discard every
// buffered message. The copy intersects the column sets instead.
var sfPath = Path_("store-and-forward.db");
using (var legacy = new SqliteConnection($"Data Source={sfPath}"))
{
legacy.Open();
using var ddl = legacy.CreateCommand();
ddl.CommandText = """
CREATE TABLE sf_messages (
id TEXT PRIMARY KEY, category INTEGER NOT NULL, target TEXT NOT NULL,
payload_json TEXT NOT NULL, retry_count INTEGER NOT NULL DEFAULT 0,
max_retries INTEGER NOT NULL DEFAULT 50,
retry_interval_ms INTEGER NOT NULL DEFAULT 30000,
created_at TEXT NOT NULL, last_attempt_at TEXT,
status INTEGER NOT NULL DEFAULT 0, last_error TEXT, origin_instance TEXT
);
INSERT INTO sf_messages (id, category, target, payload_json, created_at)
VALUES ('old-1', 0, 'ERP', '{}', '2026-01-01T00:00:00Z');
""";
ddl.ExecuteNonQuery();
}
var config = Config(storeAndForwardPath: sfPath);
var db = CreateConsolidated(config);
SiteLocalDbLegacyMigrator.Migrate(db, config);
Assert.Equal(1, CountRows(db, "sf_messages"));
Assert.Equal(["old-1"], SelectIds(db, "sf_messages", "id"));
}
[Fact]
public void LegacyFileWithoutTheExpectedTable_IsTreatedAsEmpty_NotAFailure()
{