Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbWiringTests.cs
T
Joseph Doherty 95c108f1d7 fix(localdb): allowlist the replication meter + record the Task 12 live gate
Task 12 (live gate on the docker rig) found one real defect, which this fixes.

THE DEFECT. The plan asserted LocalDb metrics "need no work - the
ZB.MOM.WW.LocalDb.Replication meter flows through the existing AddZbTelemetry
Prometheus export". It does not. ZbTelemetryOptions.Meters is an ALLOWLIST: an
unlisted meter's instruments are never observed, with no error, no warning, and
no missing-metric signal anywhere. The rig's /metrics scrape returned 301 lines
and zero localdb_* series. Only looking for them found it.

Fixed by hoisting the allowlist to SiteServiceRegistration.ObservedMeters and
adding LocalDbMetrics.MeterName. The pin test asserts on that constant rather
than on OTel's observed set, because AddZbTelemetry applies its options to a
local instance and never registers IOptions - there is nothing resolvable to
interrogate. The end-to-end proof is the live scrape below; the test exists to
stop the entry being dropped again. Called out as such in the test.

LIVE GATE EVIDENCE (docker rig, 8 nodes, rebuilt via docker/deploy.sh):

1. Consolidated DB created on every site node at /app/data/site-localdb.db -
   inside the mounted volume, unlike the legacy CWD-relative databases.

2. Replication proven REAL, not two independent local writes. Both site-a nodes
   hold the identical row and, decisively, the identical originating node_id and
   HLC in __localdb_row_version:
     site_events|{"id":"8b06b37c..."}|116946936661147648|65b00319-...|0
   __localdb_peer_state shows a completed handshake in both directions.

3. Convergence: 5/5 events on both nodes, __localdb_row_version byte-identical
   across the pair, oplog drained to 0, dead_letter 0 on both.

4. SITE failover drill. Note the repo's failover-drill.sh targets CENTRAL nodes
   and does not exercise this claim, so the site-pair flip was run directly:
   stopped site-a-a (the node holding the history); site-a-b survived with the
   complete pre-flip history (3/3 events) plus its own takeover event. Restarted
   site-a-a; it caught up on what it missed and both nodes reconverged to the
   same 5 ids. This is the failure Phase 1 exists to prevent.

5. localdb_* now exported after the fix:
     localdb_sync_reconnects_total{...} 1
     localdb_oplog_depth{...} 0

6. DEFAULT-OFF pin, live and side-by-side: site-b (no Replication section) boots
   identically, creates its consolidated DB, registers the engine - and stays
   inert. Zero replication log lines, no reconnect counter at all, so it never
   dialled anything.

One transient WRN at startup on site-a-a ("Connection refused", reconnecting) is
node A dialling before node B's listener was up; it reconnected within a second.
Expected for a pair that starts together.

Verified: build 0 warnings; Host wiring tests 11/11.

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-19 09:50:11 -04:00

229 lines
9.7 KiB
C#

using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using ZB.MOM.WW.Telemetry;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.ScadaBridge.Host;
using ZB.MOM.WW.ScadaBridge.Host.Actors;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// <summary>
/// LocalDb Phase 1 (Task 3) — locks the consolidated site database into the REAL site
/// composition root.
/// <para>
/// These tests build the actual container from
/// <see cref="SiteServiceRegistration.Configure"/> rather than asserting on a
/// hand-assembled <c>ServiceCollection</c>. That is deliberate and load-bearing: this
/// family has now shipped three wiring defects that only a container-built graph would
/// have caught — the Secrets Akka replicator that registered into a no-op sink (0.2.0),
/// the singleton cycle that deadlocked a real host (0.2.2), and Secrets.Ui's missing
/// <c>IAuditWriter</c> that 500'd only behind a login wall (ScadaBridge#22). A test that
/// merely checks "AddZbLocalDb was called" would have passed in every one of those cases.
/// </para>
/// </summary>
public class SiteLocalDbWiringTests : IDisposable
{
private readonly WebApplication _host;
private readonly string _tempDbPath;
private readonly string _tempSiteDbPath;
private readonly string _tempTrackingDbPath;
public SiteLocalDbWiringTests()
{
var stamp = Guid.NewGuid();
_tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_localdb_{stamp}.db");
_tempSiteDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_site_{stamp}.db");
_tempTrackingDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_track_{stamp}.db");
var builder = WebApplication.CreateBuilder();
builder.Configuration.Sources.Clear();
builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
{
["ScadaBridge:Node:Role"] = "Site",
["ScadaBridge:Node:NodeName"] = "node-a",
["ScadaBridge:Node:NodeHostname"] = "test-site",
["ScadaBridge:Node:SiteId"] = "TestSite",
["ScadaBridge:Node:RemotingPort"] = "0",
["ScadaBridge:Node:GrpcPort"] = "0",
["ScadaBridge:Database:SiteDbPath"] = _tempSiteDbPath,
["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={_tempTrackingDbPath}",
["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:2551",
["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:2552",
// The consolidated site database. No Replication section at all — this
// fixture is also the default-OFF pin: storage must work standalone.
["LocalDb:Path"] = _tempDbPath,
});
builder.Services.AddGrpc();
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
SiteServiceRegistration.Configure(builder.Services, builder.Configuration);
AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly(builder.Services);
_host = builder.Build();
}
public void Dispose()
{
(_host as IDisposable)?.Dispose();
foreach (var path in new[] { _tempDbPath, _tempSiteDbPath, _tempTrackingDbPath })
{
try { File.Delete(path); } catch { /* best effort */ }
}
GC.SuppressFinalize(this);
}
[Fact]
public void Site_Resolves_ILocalDb()
{
Assert.NotNull(_host.Services.GetService<ILocalDb>());
}
[Fact]
public void Site_LocalDb_RegistersBothReplicatedTables()
{
// The whole point of Phase 1. If onReady silently failed to register these,
// storage would still work and every other test would pass — the pair would
// just never replicate anything. Assert on the library's own view of what is
// registered, not on our belief that we called it.
var db = _host.Services.GetRequiredService<ILocalDb>();
Assert.Contains("OperationTracking", db.ReplicatedTables.Keys);
Assert.Contains("site_events", db.ReplicatedTables.Keys);
}
[Fact]
public void Site_LocalDb_ReplicatedTablesHaveExplicitPrimaryKeys()
{
// RegisterReplicated refuses a table with no explicit PK, so reaching this
// assertion at all proves the DDL ran before registration. The PK names are
// pinned because they are what last-writer-wins conflict resolution keys on.
var db = _host.Services.GetRequiredService<ILocalDb>();
Assert.Equal(
["TrackedOperationId"], db.ReplicatedTables["OperationTracking"].PkColumns);
Assert.Equal(["id"], db.ReplicatedTables["site_events"].PkColumns);
}
[Fact]
public void Site_LocalDb_IsASingleton()
{
// AddZbLocalDb is one-DB-per-process and first-registration-wins. Two
// instances would mean two SQLite handles and two trigger installations
// against the same file.
var first = _host.Services.GetRequiredService<ILocalDb>();
var second = _host.Services.GetRequiredService<ILocalDb>();
Assert.Same(first, second);
}
[Fact]
public void Site_Replication_IsRegistered_AndInertWithNoPeer()
{
// Task 7's default-OFF pin. This fixture configures NO
// LocalDb:Replication:PeerAddress, which is the shipping default, so the
// engine must resolve cleanly and sit idle rather than throw, retry, or
// report a connection. "Default-off" here means inert, NOT unregistered —
// the services are always in the graph.
var status = _host.Services.GetService<ISyncStatus>();
Assert.NotNull(status);
Assert.False(status!.Connected);
Assert.Null(status.PeerNodeId);
Assert.Null(status.LastSyncUtc);
}
[Fact]
public void Site_Replication_OplogBacklog_IsZero_NotNull_OnAHealthyIdleNode()
{
// OplogBacklog is deliberately nullable: null means "unknown" (no provider
// wired, or the poll failed) and must never be reported as a healthy 0. On a
// node whose local database is present and readable, a real 0 proves the
// backlog provider is actually wired to the oplog — a null here would mean
// Task 9's health signal is about to publish "unknown" forever.
var status = _host.Services.GetRequiredService<ISyncStatus>();
Assert.Equal(0L, status.OplogBacklog);
}
[Fact]
public void Site_Replication_HealthBridge_IsRegisteredAsAHostedService()
{
// Task 9. Registered via a factory lambda, so ImplementationType is null and the
// only honest assertion is on the resolved instance.
var hosted = _host.Services.GetServices<IHostedService>();
Assert.Contains(hosted, h => h is LocalDbReplicationStatusReporter);
}
[Fact]
public void Site_Replication_HealthBridge_PublishesStatusOntoTheHealthReport()
{
// End-to-end through the REAL collector: probe once, then collect. This is what
// catches a bridge that is registered but wired to the wrong provider — the
// registration test above would pass either way.
var reporter = _host.Services.GetServices<IHostedService>()
.OfType<LocalDbReplicationStatusReporter>()
.Single();
var collector = _host.Services.GetRequiredService<ISiteHealthCollector>();
reporter.Probe();
var report = collector.CollectReport("TestSite");
// No peer configured: not connected, and a REAL 0 backlog rather than null.
Assert.False(report.LocalDbReplicationConnected);
Assert.Equal(0L, report.LocalDbOplogBacklog);
}
[Fact]
public void Site_HealthReport_LocalDbFields_AreNull_BeforeTheReporterHasRun()
{
// Null means "no data yet", and must be distinguishable from a real
// "disconnected, zero backlog". A collector that defaulted these to false/0 would
// report an unwired node as a healthy connected one.
var collector = new SiteHealthCollector();
var report = collector.CollectReport("TestSite");
Assert.Null(report.LocalDbReplicationConnected);
Assert.Null(report.LocalDbOplogBacklog);
}
[Fact]
public void Site_Telemetry_Allowlists_TheLocalDbReplicationMeter()
{
// ZbTelemetryOptions.Meters is an ALLOWLIST: an unlisted Meter's instruments are
// never observed, with no error and no missing-metric warning anywhere. The
// replication meter was silently absent from the rig's /metrics scrape until the
// live gate looked for it, so pin the allowlist rather than trusting the next
// reader to remember.
//
// This asserts on the allowlist constant, not on OTel's observed-meter set:
// AddZbTelemetry applies its options to a local instance and never registers
// IOptions, so there is nothing resolvable to interrogate. The end-to-end proof
// that localdb_* actually reaches /metrics is the Task 12 live gate; this test
// exists to stop the entry being dropped again.
Assert.Contains(LocalDbMetrics.MeterName, SiteServiceRegistration.ObservedMeters);
}
[Fact]
public void Site_LocalDb_CreatesTheConfiguredFile()
{
// Resolving is what triggers onReady; the file should exist afterwards at the
// configured path and not somewhere CWD-relative.
_ = _host.Services.GetRequiredService<ILocalDb>();
Assert.True(File.Exists(_tempDbPath),
$"Expected the consolidated site database at '{_tempDbPath}'.");
}
}