b350fba233
LocalDbReplicationHealthCheck reports Healthy when replication is default-OFF or LocalDb is absent (admin-only graphs) - so a plain node is never degraded by it - Degraded when configured-but-disconnected, connected-with-unknown-backlog (a failed oplog poll must not read as zero), or connected-with-backlog past the threshold. Never Unhealthy: a replication fault does not stop the node serving its address space. Pure decision core is table-tested; registered unconditionally via a factory so AddOtOpcUaHealth keeps its no-arg signature and resolves ISyncStatus optionally. Adds LocalDbMetrics.MeterName to the observability allowlist. o.Meters is a strict allowlist with no wildcard, so the localdb.sync.* / localdb.oplog.depth series were otherwise silently absent from /metrics - the omission a ScadaBridge live gate caught.
123 lines
4.9 KiB
C#
123 lines
4.9 KiB
C#
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
|
using Microsoft.Extensions.Options;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.LocalDb.Replication;
|
|
using ZB.MOM.WW.OtOpcUa.Host.Health;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
|
|
|
/// <summary>
|
|
/// LocalDb Phase 1 (Task 10) — the replication health check.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The load-bearing case is the default-OFF one: a plain driver node with no peer and no
|
|
/// listener must report <b>Healthy</b>, or every unreplicated node in the fleet degrades the
|
|
/// moment this check ships. Beyond that, "connected" is not sufficient for healthy — an unknown
|
|
/// backlog (a failed oplog poll) must not read as zero.
|
|
/// </remarks>
|
|
public sealed class LocalDbReplicationHealthCheckTests
|
|
{
|
|
// ---- Pure decision core (table-tested) --------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Unconfigured_IsHealthy_SoDefaultOffNodesDoNotDegrade()
|
|
{
|
|
LocalDbReplicationDecision.Evaluate(
|
|
peerConfigured: false, connected: false, oplogBacklog: null, degradedThreshold: 100_000)
|
|
.Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Fact]
|
|
public void ConfiguredButNotConnected_IsDegraded()
|
|
{
|
|
LocalDbReplicationDecision.Evaluate(
|
|
peerConfigured: true, connected: false, oplogBacklog: null, degradedThreshold: 100_000)
|
|
.Status.ShouldBe(HealthStatus.Degraded);
|
|
}
|
|
|
|
[Fact]
|
|
public void ConnectedButBacklogUnknown_IsDegraded()
|
|
{
|
|
// null backlog = the oplog poll failed. Reporting Healthy here would call a pair that cannot
|
|
// read its own oplog perfectly fine.
|
|
LocalDbReplicationDecision.Evaluate(
|
|
peerConfigured: true, connected: true, oplogBacklog: null, degradedThreshold: 100_000)
|
|
.Status.ShouldBe(HealthStatus.Degraded);
|
|
}
|
|
|
|
[Fact]
|
|
public void ConnectedWithSmallBacklog_IsHealthy()
|
|
{
|
|
LocalDbReplicationDecision.Evaluate(
|
|
peerConfigured: true, connected: true, oplogBacklog: 12, degradedThreshold: 100_000)
|
|
.Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Fact]
|
|
public void ConnectedWithBacklogOverThreshold_IsDegraded()
|
|
{
|
|
// A backlog past the threshold means the peer is falling behind faster than it drains —
|
|
// connected, but not keeping up.
|
|
LocalDbReplicationDecision.Evaluate(
|
|
peerConfigured: true, connected: true, oplogBacklog: 100_001, degradedThreshold: 100_000)
|
|
.Status.ShouldBe(HealthStatus.Degraded);
|
|
}
|
|
|
|
// ---- IHealthCheck wrapper ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Wrapper_WithNoSyncStatusRegistered_IsHealthy()
|
|
{
|
|
// Admin-only graphs never register the replication engine. The check is registered
|
|
// unconditionally (AddOtOpcUaHealth takes no role argument), so it must no-op to Healthy
|
|
// rather than throw or degrade when LocalDb is simply absent.
|
|
var check = new LocalDbReplicationHealthCheck(
|
|
syncStatus: null,
|
|
options: Options.Create(new ReplicationOptions()),
|
|
syncListenPort: 0);
|
|
|
|
var result = await check.CheckHealthAsync(new HealthCheckContext(), TestContext.Current.CancellationToken);
|
|
|
|
result.Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Wrapper_ListenerConfiguredButNotConnected_IsDegraded()
|
|
{
|
|
// A passive node (no PeerAddress) still counts as "replication configured" when it is
|
|
// listening — otherwise a passive node that never gets dialled would look healthy while
|
|
// silently accepting nothing.
|
|
var check = new LocalDbReplicationHealthCheck(
|
|
syncStatus: new StubSyncStatus { Connected = false },
|
|
options: Options.Create(new ReplicationOptions { PeerAddress = "" }),
|
|
syncListenPort: 9001);
|
|
|
|
var result = await check.CheckHealthAsync(new HealthCheckContext(), TestContext.Current.CancellationToken);
|
|
|
|
result.Status.ShouldBe(HealthStatus.Degraded);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Wrapper_PeerConfiguredAndConnectedAndDraining_IsHealthy()
|
|
{
|
|
var check = new LocalDbReplicationHealthCheck(
|
|
syncStatus: new StubSyncStatus { Connected = true, OplogBacklog = 3 },
|
|
options: Options.Create(new ReplicationOptions { PeerAddress = "https://peer:9001" }),
|
|
syncListenPort: 9001);
|
|
|
|
var result = await check.CheckHealthAsync(new HealthCheckContext(), TestContext.Current.CancellationToken);
|
|
|
|
result.Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
private sealed class StubSyncStatus : ISyncStatus
|
|
{
|
|
public bool Connected { get; init; }
|
|
public string? PeerNodeId { get; init; }
|
|
public DateTimeOffset? LastSyncUtc { get; init; }
|
|
public long? OplogBacklog { get; init; }
|
|
public long ConnectionAttempts { get; init; }
|
|
}
|
|
}
|