feat(localdb): replication health check + meter export

LocalDbReplicationHealthCheck reports Healthy when replication is default-OFF or LocalDb is absent (admin-only graphs) - so a plain node is never degraded by it - Degraded when configured-but-disconnected, connected-with-unknown-backlog (a failed oplog poll must not read as zero), or connected-with-backlog past the threshold. Never Unhealthy: a replication fault does not stop the node serving its address space. Pure decision core is table-tested; registered unconditionally via a factory so AddOtOpcUaHealth keeps its no-arg signature and resolves ISyncStatus optionally. Adds LocalDbMetrics.MeterName to the observability allowlist. o.Meters is a strict allowlist with no wildcard, so the localdb.sync.* / localdb.oplog.depth series were otherwise silently absent from /metrics - the omission a ScadaBridge live gate caught.
2026-07-20 21:48:47 -04:00
parent abe10dbbd7
commit b350fba233
4 changed files with 273 additions and 2 deletions
@@ -0,0 +1,122 @@
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.LocalDb.Replication;
+using ZB.MOM.WW.OtOpcUa.Host.Health;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
+
+/// <summary>
+///     LocalDb Phase 1 (Task 10) — the replication health check.
+/// </summary>
+/// <remarks>
+///     The load-bearing case is the default-OFF one: a plain driver node with no peer and no
+///     listener must report <b>Healthy</b>, or every unreplicated node in the fleet degrades the
+///     moment this check ships. Beyond that, "connected" is not sufficient for healthy — an unknown
+///     backlog (a failed oplog poll) must not read as zero.
+/// </remarks>
+public sealed class LocalDbReplicationHealthCheckTests
+{
+    // ---- Pure decision core (table-tested) --------------------------------------------------
+
+    [Fact]
+    public void Unconfigured_IsHealthy_SoDefaultOffNodesDoNotDegrade()
+    {
+        LocalDbReplicationDecision.Evaluate(
+            peerConfigured: false, connected: false, oplogBacklog: null, degradedThreshold: 100_000)
+            .Status.ShouldBe(HealthStatus.Healthy);
+    }
+
+    [Fact]
+    public void ConfiguredButNotConnected_IsDegraded()
+    {
+        LocalDbReplicationDecision.Evaluate(
+            peerConfigured: true, connected: false, oplogBacklog: null, degradedThreshold: 100_000)
+            .Status.ShouldBe(HealthStatus.Degraded);
+    }
+
+    [Fact]
+    public void ConnectedButBacklogUnknown_IsDegraded()
+    {
+        // null backlog = the oplog poll failed. Reporting Healthy here would call a pair that cannot
+        // read its own oplog perfectly fine.
+        LocalDbReplicationDecision.Evaluate(
+            peerConfigured: true, connected: true, oplogBacklog: null, degradedThreshold: 100_000)
+            .Status.ShouldBe(HealthStatus.Degraded);
+    }
+
+    [Fact]
+    public void ConnectedWithSmallBacklog_IsHealthy()
+    {
+        LocalDbReplicationDecision.Evaluate(
+            peerConfigured: true, connected: true, oplogBacklog: 12, degradedThreshold: 100_000)
+            .Status.ShouldBe(HealthStatus.Healthy);
+    }
+
+    [Fact]
+    public void ConnectedWithBacklogOverThreshold_IsDegraded()
+    {
+        // A backlog past the threshold means the peer is falling behind faster than it drains —
+        // connected, but not keeping up.
+        LocalDbReplicationDecision.Evaluate(
+            peerConfigured: true, connected: true, oplogBacklog: 100_001, degradedThreshold: 100_000)
+            .Status.ShouldBe(HealthStatus.Degraded);
+    }
+
+    // ---- IHealthCheck wrapper ---------------------------------------------------------------
+
+    [Fact]
+    public async Task Wrapper_WithNoSyncStatusRegistered_IsHealthy()
+    {
+        // Admin-only graphs never register the replication engine. The check is registered
+        // unconditionally (AddOtOpcUaHealth takes no role argument), so it must no-op to Healthy
+        // rather than throw or degrade when LocalDb is simply absent.
+        var check = new LocalDbReplicationHealthCheck(
+            syncStatus: null,
+            options: Options.Create(new ReplicationOptions()),
+            syncListenPort: 0);
+
+        var result = await check.CheckHealthAsync(new HealthCheckContext(), TestContext.Current.CancellationToken);
+
+        result.Status.ShouldBe(HealthStatus.Healthy);
+    }
+
+    [Fact]
+    public async Task Wrapper_ListenerConfiguredButNotConnected_IsDegraded()
+    {
+        // A passive node (no PeerAddress) still counts as "replication configured" when it is
+        // listening — otherwise a passive node that never gets dialled would look healthy while
+        // silently accepting nothing.
+        var check = new LocalDbReplicationHealthCheck(
+            syncStatus: new StubSyncStatus { Connected = false },
+            options: Options.Create(new ReplicationOptions { PeerAddress = "" }),
+            syncListenPort: 9001);
+
+        var result = await check.CheckHealthAsync(new HealthCheckContext(), TestContext.Current.CancellationToken);
+
+        result.Status.ShouldBe(HealthStatus.Degraded);
+    }
+
+    [Fact]
+    public async Task Wrapper_PeerConfiguredAndConnectedAndDraining_IsHealthy()
+    {
+        var check = new LocalDbReplicationHealthCheck(
+            syncStatus: new StubSyncStatus { Connected = true, OplogBacklog = 3 },
+            options: Options.Create(new ReplicationOptions { PeerAddress = "https://peer:9001" }),
+            syncListenPort: 9001);
+
+        var result = await check.CheckHealthAsync(new HealthCheckContext(), TestContext.Current.CancellationToken);
+
+        result.Status.ShouldBe(HealthStatus.Healthy);
+    }
+
+    private sealed class StubSyncStatus : ISyncStatus
+    {
+        public bool Connected { get; init; }
+        public string? PeerNodeId { get; init; }
+        public DateTimeOffset? LastSyncUtc { get; init; }
+        public long? OplogBacklog { get; init; }
+        public long ConnectionAttempts { get; init; }
+    }
+}