Merge rows 80+73+71: nullable LocalDb health fields on unreplicated sites (Host supplies ReplicationIsConfigured), StartupReconcilerActive on the health report (proto field 29) with central No-self-heal badge, standby active-node ERR downgraded via narrowly-keyed Serilog filter
# Conflicts: # CLAUDE.md
This commit is contained in:
@@ -14,6 +14,7 @@ using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
@@ -443,6 +444,120 @@ public class HealthPageTests : BunitContext
|
||||
Assert.Contains("LocalDb Replication Backlog", cut.Markup));
|
||||
}
|
||||
|
||||
// ----- Register row 73: the "No self-heal" badge -----------------------------------
|
||||
//
|
||||
// Tri-state, and only the explicit false is badged. Null means "not reported" — the
|
||||
// central self-report row, and every site node on a pre-row-73 build — so badging null
|
||||
// would light up the whole fleet the moment central is upgraded ahead of the sites.
|
||||
|
||||
[Fact]
|
||||
public void Renders_StartupReconcilerBadge_WhenTheSiteReportsItIsMissing()
|
||||
{
|
||||
SeedStates(new SiteHealthState
|
||||
{
|
||||
SiteId = "site-a",
|
||||
IsOnline = true,
|
||||
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
||||
LatestReport = ReportWithReconciler("site-a", active: false),
|
||||
});
|
||||
|
||||
var cut = RenderHealthPage();
|
||||
|
||||
cut.WaitForAssertion(() =>
|
||||
Assert.Contains("data-test=\"startup-reconciler-badge\"", cut.Markup));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotRender_StartupReconcilerBadge_WhenTheReconcilerIsRunning()
|
||||
{
|
||||
SeedStates(new SiteHealthState
|
||||
{
|
||||
SiteId = "site-a",
|
||||
IsOnline = true,
|
||||
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
||||
LatestReport = ReportWithReconciler("site-a", active: true),
|
||||
});
|
||||
|
||||
var cut = RenderHealthPage();
|
||||
|
||||
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
||||
Assert.DoesNotContain("data-test=\"startup-reconciler-badge\"", cut.Markup);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotRender_StartupReconcilerBadge_WhenNotReported()
|
||||
{
|
||||
// Null = "not reported", never "missing". This is central's own self-report row and
|
||||
// any site that has not yet been upgraded.
|
||||
SeedStates(new SiteHealthState
|
||||
{
|
||||
SiteId = "site-a",
|
||||
IsOnline = true,
|
||||
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
||||
LatestReport = ReportWithReconciler("site-a", active: null),
|
||||
});
|
||||
|
||||
var cut = RenderHealthPage();
|
||||
|
||||
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
||||
Assert.DoesNotContain("data-test=\"startup-reconciler-badge\"", cut.Markup);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotRender_StartupReconcilerBadge_WhenTheSiteHasNoReportYet()
|
||||
{
|
||||
SeedStates(new SiteHealthState
|
||||
{
|
||||
SiteId = "site-a",
|
||||
IsOnline = true,
|
||||
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
||||
LatestReport = null,
|
||||
});
|
||||
|
||||
var cut = RenderHealthPage();
|
||||
|
||||
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
||||
Assert.DoesNotContain("data-test=\"startup-reconciler-badge\"", cut.Markup);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DoesNotRender_StartupReconcilerBadge_ForAnOfflineSite()
|
||||
{
|
||||
// Same rule as the replication-backlog badge: the Offline badge is the signal that
|
||||
// matters, and stacking a stale warning on top of it is noise.
|
||||
SeedStates(new SiteHealthState
|
||||
{
|
||||
SiteId = "site-a",
|
||||
IsOnline = false,
|
||||
LastHeartbeatAt = DateTimeOffset.UtcNow.AddMinutes(-30),
|
||||
LatestReport = ReportWithReconciler("site-a", active: false),
|
||||
});
|
||||
|
||||
var cut = RenderHealthPage();
|
||||
|
||||
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
||||
Assert.DoesNotContain("data-test=\"startup-reconciler-badge\"", cut.Markup);
|
||||
}
|
||||
|
||||
private static SiteHealthReport ReportWithReconciler(string siteId, bool? active) =>
|
||||
new(
|
||||
SiteId: siteId,
|
||||
SequenceNumber: 1,
|
||||
ReportTimestamp: DateTimeOffset.UtcNow,
|
||||
DataConnectionStatuses:
|
||||
new Dictionary<string, Commons.Types.Enums.ConnectionHealth>(),
|
||||
TagResolutionCounts: new Dictionary<string, TagResolutionStatus>(),
|
||||
ScriptErrorCount: 0,
|
||||
AlarmEvaluationErrorCount: 0,
|
||||
StoreAndForwardBufferDepths: new Dictionary<string, int>(),
|
||||
DeadLetterCount: 0,
|
||||
DeployedInstanceCount: 0,
|
||||
EnabledInstanceCount: 0,
|
||||
DisabledInstanceCount: 0)
|
||||
{
|
||||
StartupReconcilerActive = active,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Renders_OfflineSince_ForOfflineSiteWithStatusChange()
|
||||
{
|
||||
|
||||
@@ -368,6 +368,7 @@ public class CentralControlDtoMapperTests
|
||||
Assert.Null(roundTripped.ScriptOldestBusyAgeSeconds);
|
||||
Assert.Null(roundTripped.LocalDbReplicationConnected);
|
||||
Assert.Null(roundTripped.LocalDbOplogBacklog);
|
||||
Assert.Null(roundTripped.StartupReconcilerActive);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -407,12 +408,16 @@ public class CentralControlDtoMapperTests
|
||||
LocalDbOplogBacklog = 0,
|
||||
OldestParkedMessageAgeSeconds = 0d,
|
||||
ScriptOldestBusyAgeSeconds = 0d,
|
||||
// Register row 73: false ("this node booted with no self-heal") is the value
|
||||
// central badges on, so it must not decay into null ("not reported") en route.
|
||||
StartupReconcilerActive = false,
|
||||
};
|
||||
|
||||
var roundTripped = CentralControlDtoMapper.FromDto(Wire(CentralControlDtoMapper.ToDto(original)));
|
||||
|
||||
Assert.False(roundTripped.LocalDbReplicationConnected);
|
||||
Assert.Equal(0L, roundTripped.LocalDbOplogBacklog);
|
||||
Assert.False(roundTripped.StartupReconcilerActive);
|
||||
Assert.Equal(0d, roundTripped.OldestParkedMessageAgeSeconds);
|
||||
Assert.Equal(0d, roundTripped.ScriptOldestBusyAgeSeconds);
|
||||
}
|
||||
@@ -548,6 +553,7 @@ public class CentralControlDtoMapperTests
|
||||
ScriptOldestBusyAgeSeconds = 12.25d,
|
||||
LocalDbReplicationConnected = true,
|
||||
LocalDbOplogBacklog = 250L,
|
||||
StartupReconcilerActive = true,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
@@ -596,6 +602,7 @@ public class CentralControlDtoMapperTests
|
||||
Assert.Equal(expected.ScriptOldestBusyAgeSeconds, actual.ScriptOldestBusyAgeSeconds);
|
||||
Assert.Equal(expected.LocalDbReplicationConnected, actual.LocalDbReplicationConnected);
|
||||
Assert.Equal(expected.LocalDbOplogBacklog, actual.LocalDbOplogBacklog);
|
||||
Assert.Equal(expected.StartupReconcilerActive, actual.StartupReconcilerActive);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Deferred-work register row 80 — <see cref="LocalDbReplicationStatusReporter"/> must
|
||||
/// distinguish "replication is not configured on this node" (null, no reading) from
|
||||
/// "replication is configured and the peer session is down" (false, a genuine fault).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The replication engine is registered unconditionally on every site node and sits inert
|
||||
/// without a peer, so <c>ISyncStatus</c> answers <c>Connected=false</c> / <c>OplogBacklog=0</c>
|
||||
/// on a site that was never meant to replicate. Passing that straight through made rig
|
||||
/// site-b/site-c publish "disconnected, empty backlog" — a healthy unreplicated site reading
|
||||
/// as a broken replicated one, and central's null branches (conditional KPI emission,
|
||||
/// sustained-badge anchor exclusion, per register row 33) never receiving the null they exist
|
||||
/// for. The gate is supplied by the Host from <c>SiteLocalDbSetup.ReplicationIsConfigured</c>,
|
||||
/// mirroring <c>SiteEventLogReplicationCheck</c>.
|
||||
/// </remarks>
|
||||
public class LocalDbReplicationStatusReporterTests
|
||||
{
|
||||
private static LocalDbReplicationStatusReporter Build(
|
||||
SiteHealthCollector collector,
|
||||
bool replicationConfigured,
|
||||
bool connected = false,
|
||||
long? backlog = 0L) =>
|
||||
new(
|
||||
connectedProvider: () => connected,
|
||||
oplogBacklogProvider: () => backlog,
|
||||
replicationConfiguredProvider: () => replicationConfigured,
|
||||
collector: collector,
|
||||
logger: NullLogger<LocalDbReplicationStatusReporter>.Instance,
|
||||
refreshInterval: TimeSpan.FromHours(1));
|
||||
|
||||
[Fact]
|
||||
public void Probe_ReplicationUnconfigured_PublishesNullForBothFields()
|
||||
{
|
||||
// The row-80 regression. The inert engine's false/0 must NOT reach the report.
|
||||
var collector = new SiteHealthCollector();
|
||||
using var reporter = Build(collector, replicationConfigured: false);
|
||||
|
||||
reporter.Probe();
|
||||
var report = collector.CollectReport("TestSite");
|
||||
|
||||
Assert.Null(report.LocalDbReplicationConnected);
|
||||
Assert.Null(report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Probe_ReplicationUnconfigured_DoesNotReadTheSyncStatusProviders()
|
||||
{
|
||||
// The gate short-circuits rather than reading and discarding: an unconfigured node
|
||||
// should not be polling a replication engine it never uses. Also guards against a
|
||||
// future provider that throws when replication is off.
|
||||
var connectedReads = 0;
|
||||
var backlogReads = 0;
|
||||
var collector = new SiteHealthCollector();
|
||||
using var reporter = new LocalDbReplicationStatusReporter(
|
||||
connectedProvider: () => { connectedReads++; return false; },
|
||||
oplogBacklogProvider: () => { backlogReads++; return 0L; },
|
||||
replicationConfiguredProvider: () => false,
|
||||
collector: collector,
|
||||
logger: NullLogger<LocalDbReplicationStatusReporter>.Instance,
|
||||
refreshInterval: TimeSpan.FromHours(1));
|
||||
|
||||
reporter.Probe();
|
||||
|
||||
Assert.Equal(0, connectedReads);
|
||||
Assert.Equal(0, backlogReads);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Probe_ReplicationConfiguredButDisconnected_PublishesFalse_NotNull()
|
||||
{
|
||||
// The other half of the contract, and the reason the fix is a gate rather than a
|
||||
// blanket "never report false": on a node that IS configured to replicate, a down
|
||||
// peer session is a real fault and must stay visible as false.
|
||||
var collector = new SiteHealthCollector();
|
||||
using var reporter = Build(
|
||||
collector, replicationConfigured: true, connected: false, backlog: 4200L);
|
||||
|
||||
reporter.Probe();
|
||||
var report = collector.CollectReport("TestSite");
|
||||
|
||||
Assert.False(report.LocalDbReplicationConnected);
|
||||
Assert.Equal(4200L, report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Probe_ReplicationConfiguredAndConnected_PublishesTrue()
|
||||
{
|
||||
var collector = new SiteHealthCollector();
|
||||
using var reporter = Build(
|
||||
collector, replicationConfigured: true, connected: true, backlog: 0L);
|
||||
|
||||
reporter.Probe();
|
||||
var report = collector.CollectReport("TestSite");
|
||||
|
||||
Assert.True(report.LocalDbReplicationConnected);
|
||||
Assert.Equal(0L, report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Probe_ReplicationConfigured_PreservesANullBacklogAsUnknown()
|
||||
{
|
||||
// A failed oplog poll on a REPLICATED node is "unknown", never 0 — rendering it as a
|
||||
// healthy empty backlog would report a pair that cannot read its own oplog as fine.
|
||||
// Connected stays false (the session really is down); only the backlog is unknown.
|
||||
var collector = new SiteHealthCollector();
|
||||
using var reporter = Build(
|
||||
collector, replicationConfigured: true, connected: false, backlog: null);
|
||||
|
||||
reporter.Probe();
|
||||
var report = collector.CollectReport("TestSite");
|
||||
|
||||
Assert.False(report.LocalDbReplicationConnected);
|
||||
Assert.Null(report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Probe_ProviderThrows_LeavesThePreviousSnapshotIntact()
|
||||
{
|
||||
// Health reporting must never take the node down, and a failed probe must not
|
||||
// silently downgrade a good reading to null.
|
||||
var shouldThrow = false;
|
||||
var collector = new SiteHealthCollector();
|
||||
using var reporter = new LocalDbReplicationStatusReporter(
|
||||
connectedProvider: () => shouldThrow
|
||||
? throw new InvalidOperationException("sync status unavailable")
|
||||
: true,
|
||||
oplogBacklogProvider: () => 7L,
|
||||
replicationConfiguredProvider: () => true,
|
||||
collector: collector,
|
||||
logger: NullLogger<LocalDbReplicationStatusReporter>.Instance,
|
||||
refreshInterval: TimeSpan.FromHours(1));
|
||||
|
||||
reporter.Probe();
|
||||
shouldThrow = true;
|
||||
reporter.Probe();
|
||||
|
||||
var report = collector.CollectReport("TestSite");
|
||||
Assert.True(report.LocalDbReplicationConnected);
|
||||
Assert.Equal(7L, report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Probe_GateFlippedOn_StopsReportingNull()
|
||||
{
|
||||
// Turning replication on for a previously-unreplicated site is a both-nodes-together
|
||||
// restart in practice, but the gate is read per probe, so pin that it is not latched.
|
||||
var configured = false;
|
||||
var collector = new SiteHealthCollector();
|
||||
using var reporter = new LocalDbReplicationStatusReporter(
|
||||
connectedProvider: () => true,
|
||||
oplogBacklogProvider: () => 12L,
|
||||
replicationConfiguredProvider: () => configured,
|
||||
collector: collector,
|
||||
logger: NullLogger<LocalDbReplicationStatusReporter>.Instance,
|
||||
refreshInterval: TimeSpan.FromHours(1));
|
||||
|
||||
reporter.Probe();
|
||||
Assert.Null(collector.CollectReport("TestSite").LocalDbReplicationConnected);
|
||||
|
||||
configured = true;
|
||||
reporter.Probe();
|
||||
|
||||
var report = collector.CollectReport("TestSite");
|
||||
Assert.True(report.LocalDbReplicationConnected);
|
||||
Assert.Equal(12L, report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Constructor_NullReplicationConfiguredProvider_Throws()
|
||||
{
|
||||
Assert.Throws<ArgumentNullException>(() => new LocalDbReplicationStatusReporter(
|
||||
connectedProvider: () => false,
|
||||
oplogBacklogProvider: () => 0L,
|
||||
replicationConfiguredProvider: null!,
|
||||
collector: new SiteHealthCollector(),
|
||||
logger: NullLogger<LocalDbReplicationStatusReporter>.Instance));
|
||||
}
|
||||
}
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Deferred-work register row 73 — <c>SiteHealthReport.StartupReconcilerActive</c> surfaces
|
||||
/// whether a site node spawned its <c>SiteReconciliationActor</c> at boot.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The skip used to be a single boot Warning and nothing else, so a node that lost its whole
|
||||
/// self-heal path went on reporting itself perfectly healthy. The field is tri-state and the
|
||||
/// three cases must stay distinct: null = not reported (central self-report, or a pre-row-73
|
||||
/// site build), true = running, false = skipped because no <c>IDeploymentConfigFetcher</c>
|
||||
/// was available. Collapsing null into false would badge every site the moment central is
|
||||
/// upgraded ahead of them.
|
||||
/// </remarks>
|
||||
public class StartupReconcilerHealthFieldTests
|
||||
{
|
||||
[Fact]
|
||||
public void CollectReport_NeverTold_LeavesTheFieldNull()
|
||||
{
|
||||
// "Not reported" — NOT "the reconciler is missing". This is the state of central's
|
||||
// own self-report and of every test host that never calls the setter.
|
||||
var collector = new SiteHealthCollector();
|
||||
|
||||
var report = collector.CollectReport("TestSite");
|
||||
|
||||
Assert.Null(report.StartupReconcilerActive);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_ReconcilerSpawned_ReportsTrue()
|
||||
{
|
||||
var collector = new SiteHealthCollector();
|
||||
|
||||
collector.SetStartupReconcilerActive(true);
|
||||
|
||||
Assert.True(collector.CollectReport("TestSite").StartupReconcilerActive);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_ReconcilerSkipped_ReportsFalse_NotNull()
|
||||
{
|
||||
// The case the row exists for: false must be distinguishable from "not reported",
|
||||
// because false is what central badges.
|
||||
var collector = new SiteHealthCollector();
|
||||
|
||||
collector.SetStartupReconcilerActive(false);
|
||||
|
||||
var report = collector.CollectReport("TestSite");
|
||||
Assert.NotNull(report.StartupReconcilerActive);
|
||||
Assert.False(report.StartupReconcilerActive);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CollectReport_IsPointInTime_NotDrainedOnCollect()
|
||||
{
|
||||
// A latched boot fact, not a per-interval counter: every subsequent report must
|
||||
// carry it, or the badge would flicker off after the first report.
|
||||
var collector = new SiteHealthCollector();
|
||||
collector.SetStartupReconcilerActive(false);
|
||||
|
||||
collector.CollectReport("TestSite");
|
||||
var second = collector.CollectReport("TestSite");
|
||||
|
||||
Assert.False(second.StartupReconcilerActive);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SetStartupReconcilerActive_IsNotWriteOnce()
|
||||
{
|
||||
// Nothing in the Host calls this twice today, but a latch that silently ignored a
|
||||
// correction would be a nasty way to find that out.
|
||||
var collector = new SiteHealthCollector();
|
||||
|
||||
collector.SetStartupReconcilerActive(false);
|
||||
collector.SetStartupReconcilerActive(true);
|
||||
|
||||
Assert.True(collector.CollectReport("TestSite").StartupReconcilerActive);
|
||||
}
|
||||
}
|
||||
@@ -178,6 +178,18 @@ public class SiteLocalDbWiringTests : IDisposable
|
||||
// End-to-end through the REAL collector: probe once, then collect. This is what
|
||||
// catches a bridge that is registered but wired to the wrong provider — the
|
||||
// registration test above would pass either way.
|
||||
//
|
||||
// This fixture IS a replicating node: it carries LocalDb:Replication:ApiKey (the
|
||||
// passive half of the rig's site-a pair), and configured-ness is PeerAddress OR
|
||||
// ApiKey. So false/0 is the correct, meaningful answer here — "replication is on,
|
||||
// no session is up yet, and the oplog really is empty". Register row 80 is about
|
||||
// the OTHER case, which is pinned by UnreplicatedSiteLocalDbHealthTests: a node
|
||||
// with NEITHER key must report null/null, because the engine's inert false/0 there
|
||||
// means "not applicable", not "disconnected".
|
||||
//
|
||||
// Do not weaken this assertion to null without checking which fixture you are in —
|
||||
// conflating "no PeerAddress" with "replication unconfigured" is precisely the
|
||||
// mistake that produced row 80.
|
||||
var reporter = _host.Services.GetServices<IHostedService>()
|
||||
.OfType<LocalDbReplicationStatusReporter>()
|
||||
.Single();
|
||||
@@ -186,7 +198,6 @@ public class SiteLocalDbWiringTests : IDisposable
|
||||
reporter.Probe();
|
||||
var report = collector.CollectReport("TestSite");
|
||||
|
||||
// No peer configured: not connected, and a REAL 0 backlog rather than null.
|
||||
Assert.False(report.LocalDbReplicationConnected);
|
||||
Assert.Equal(0L, report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Serilog;
|
||||
using Serilog.Events;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Deferred-work register row 71 — a standby node logged the correct-by-design
|
||||
/// <c>active-node</c> health-check verdict at ERROR on every probe.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <c>DefaultHealthCheckService</c> picks the log level from the verdict (Debug/Warning/Error
|
||||
/// for Healthy/Degraded/Unhealthy). Both active-node checks answer Unhealthy on the standby
|
||||
/// because 503 is the contract <c>/health/active</c> publishes, so a healthy standby logged
|
||||
/// ERR every 10 s on a site node and every 5 s on a central node behind Traefik's probe.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The constraint on the fix is that a GENUINE failure must still log at Error, which is why
|
||||
/// the filter is keyed on the check NAME rather than on the logger category.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class StandbyHealthCheckLogNoiseTests
|
||||
{
|
||||
private const string HealthCheckCategory =
|
||||
"Microsoft.Extensions.Diagnostics.HealthChecks.DefaultHealthCheckService";
|
||||
|
||||
private const string Template =
|
||||
"Health check {HealthCheckName} with status {HealthStatus} completed after "
|
||||
+ "{ElapsedMilliseconds}ms with message '{HealthCheckDescription}'";
|
||||
|
||||
private static ILogger BuildLogger(InMemorySink sink)
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["ScadaBridge:Logging:MinimumLevel"] = "Debug",
|
||||
})
|
||||
.Build();
|
||||
|
||||
return LoggerConfigurationFactory
|
||||
.Build(config, "Site", "site-a", "site-a-b")
|
||||
.WriteTo.Sink(sink)
|
||||
.CreateLogger();
|
||||
}
|
||||
|
||||
/// <summary>Emits one event shaped exactly like the framework's per-check line.</summary>
|
||||
private static void EmitHealthCheckLine(
|
||||
ILogger logger, string category, LogEventLevel level, string checkName, string status) =>
|
||||
logger
|
||||
.ForContext("SourceContext", category)
|
||||
.Write(level, Template, checkName, status, 1.0, "some message");
|
||||
|
||||
[Fact]
|
||||
public void StandbyActiveNodeVerdict_IsNotLogged()
|
||||
{
|
||||
// The row-71 regression: "Health check active-node with status Unhealthy … 'not the
|
||||
// site pair primary (standby)'" at Error, once per probe, forever.
|
||||
var sink = new InMemorySink();
|
||||
var logger = BuildLogger(sink);
|
||||
|
||||
EmitHealthCheckLine(
|
||||
logger, HealthCheckCategory, LogEventLevel.Error, "active-node", "Unhealthy");
|
||||
|
||||
Assert.Empty(sink.LogEvents);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("localdb")]
|
||||
[InlineData("akka-cluster")]
|
||||
[InlineData("database")]
|
||||
[InlineData("required-singletons")]
|
||||
public void GenuineCheckFailure_StillLogsAtError(string checkName)
|
||||
{
|
||||
// The constraint. The filter is name-scoped, so every OTHER check on the same logger
|
||||
// category keeps its Error line — a blanket category override would have silenced
|
||||
// these too, which is exactly the trap this test exists to catch.
|
||||
var sink = new InMemorySink();
|
||||
var logger = BuildLogger(sink);
|
||||
|
||||
EmitHealthCheckLine(
|
||||
logger, HealthCheckCategory, LogEventLevel.Error, checkName, "Unhealthy");
|
||||
|
||||
var evt = Assert.Single(sink.LogEvents);
|
||||
Assert.Equal(LogEventLevel.Error, evt.Level);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ActiveNodeHealthyVerdict_IsUnaffected()
|
||||
{
|
||||
// The active node reports Healthy, which the framework logs at Debug. Nothing about
|
||||
// the filter should touch it — it only ever matches Error.
|
||||
var sink = new InMemorySink();
|
||||
var logger = BuildLogger(sink);
|
||||
|
||||
EmitHealthCheckLine(
|
||||
logger, HealthCheckCategory, LogEventLevel.Debug, "active-node", "Healthy");
|
||||
|
||||
Assert.Single(sink.LogEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ActiveNodeDegradedVerdict_IsUnaffected()
|
||||
{
|
||||
// Degraded logs at Warning. If a future check shape ever reports Degraded for
|
||||
// active-node, that should still be visible — the filter is Error-only.
|
||||
var sink = new InMemorySink();
|
||||
var logger = BuildLogger(sink);
|
||||
|
||||
EmitHealthCheckLine(
|
||||
logger, HealthCheckCategory, LogEventLevel.Warning, "active-node", "Degraded");
|
||||
|
||||
Assert.Single(sink.LogEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AnErrorMentioningActiveNode_FromAnotherCategory_IsNotSuppressed()
|
||||
{
|
||||
// The predicate requires BOTH the health-check category and the check name, so
|
||||
// application code logging about the active node keeps its Error.
|
||||
var sink = new InMemorySink();
|
||||
var logger = BuildLogger(sink);
|
||||
|
||||
EmitHealthCheckLine(
|
||||
logger,
|
||||
"ZB.MOM.WW.ScadaBridge.Host.Actors.AkkaHostedService",
|
||||
LogEventLevel.Error,
|
||||
"active-node",
|
||||
"Unhealthy");
|
||||
|
||||
Assert.Single(sink.LogEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OrdinaryErrorsWithNoHealthCheckProperties_AreNotSuppressed()
|
||||
{
|
||||
var sink = new InMemorySink();
|
||||
var logger = BuildLogger(sink);
|
||||
|
||||
logger.Error("something genuinely broke");
|
||||
|
||||
Assert.Single(sink.LogEvents);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Predicate_RejectsANullEvent()
|
||||
{
|
||||
// Defensive: Serilog never hands the filter a null, but the predicate is internal
|
||||
// and reachable, and returning "suppress" for null would be the wrong default.
|
||||
Assert.False(LoggerConfigurationFactory.IsExpectedStandbyActiveNodeProbe(null!));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Deferred-work register row 80 — a site node with replication UNCONFIGURED must report
|
||||
/// <c>LocalDbReplicationConnected</c> and <c>LocalDbOplogBacklog</c> as <c>null</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The sibling <see cref="SiteLocalDbWiringTests"/> fixture is a <em>replicating</em> node
|
||||
/// (it sets <c>LocalDb:Replication:ApiKey</c>, modelling the passive half of the rig's
|
||||
/// site-a pair), so false/0 is the right answer there. This fixture is the other posture —
|
||||
/// the rig's site-b/site-c, and the shipping default — with NEITHER <c>PeerAddress</c> nor
|
||||
/// <c>ApiKey</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The replication engine is registered unconditionally and idles, so <c>ISyncStatus</c>
|
||||
/// still answers <c>Connected=false</c> with a real <c>0</c> backlog. Publishing that pair
|
||||
/// is the row-80 bug: a dashboard reader sees <c>connected: false</c> on a perfectly healthy
|
||||
/// unreplicated site and reads a fault, and central's null branches — the conditional KPI
|
||||
/// emission and the sustained-badge anchor exclusion built for register row 33 — never
|
||||
/// receive the null they exist for, so unreplicated sites emitted a misleading
|
||||
/// <c>localDbReplicationConnected=0</c> series.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Built from the REAL composition root for the same reason the sibling fixture is: the
|
||||
/// defect is in the wiring, and a hand-assembled <c>ServiceCollection</c> would prove
|
||||
/// nothing about what the Host actually passes.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class UnreplicatedSiteLocalDbHealthTests : IDisposable
|
||||
{
|
||||
private readonly WebApplication _host;
|
||||
private readonly string _tempDbPath;
|
||||
private readonly string _tempSiteDbPath;
|
||||
private readonly string _tempTrackingDbPath;
|
||||
|
||||
public UnreplicatedSiteLocalDbHealthTests()
|
||||
{
|
||||
var stamp = Guid.NewGuid();
|
||||
_tempDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_unrep_localdb_{stamp}.db");
|
||||
_tempSiteDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_unrep_site_{stamp}.db");
|
||||
_tempTrackingDbPath = Path.Combine(Path.GetTempPath(), $"scadabridge_unrep_track_{stamp}.db");
|
||||
|
||||
var builder = WebApplication.CreateBuilder();
|
||||
builder.Configuration.Sources.Clear();
|
||||
builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["ScadaBridge:Node:Role"] = "Site",
|
||||
["ScadaBridge:Node:NodeName"] = "node-a",
|
||||
["ScadaBridge:Node:NodeHostname"] = "test-site-b",
|
||||
["ScadaBridge:Node:SiteId"] = "TestSiteB",
|
||||
["ScadaBridge:Node:RemotingPort"] = "0",
|
||||
["ScadaBridge:Node:GrpcPort"] = "0",
|
||||
["ScadaBridge:Database:SiteDbPath"] = _tempSiteDbPath,
|
||||
["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={_tempTrackingDbPath}",
|
||||
["ScadaBridge:Cluster:SeedNodes:0"] = "akka.tcp://scadabridge@localhost:2551",
|
||||
["ScadaBridge:Cluster:SeedNodes:1"] = "akka.tcp://scadabridge@localhost:2552",
|
||||
|
||||
// The point of this fixture: LocalDb:Path only. No PeerAddress, no ApiKey —
|
||||
// SiteLocalDbSetup.ReplicationIsConfigured is false, matching rig site-b/site-c.
|
||||
["LocalDb:Path"] = _tempDbPath,
|
||||
});
|
||||
|
||||
builder.Services.AddGrpc();
|
||||
builder.Services.AddSingleton<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
|
||||
|
||||
SiteServiceRegistration.Configure(builder.Services, builder.Configuration);
|
||||
|
||||
AkkaHostedServiceRemover.RemoveAkkaHostedServiceOnly(builder.Services);
|
||||
|
||||
_host = builder.Build();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
(_host as IDisposable)?.Dispose();
|
||||
foreach (var path in new[] { _tempDbPath, _tempSiteDbPath, _tempTrackingDbPath })
|
||||
{
|
||||
try { File.Delete(path); } catch { /* best effort */ }
|
||||
}
|
||||
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnreplicatedSite_HealthReport_ReportsNullForBothLocalDbFields()
|
||||
{
|
||||
// The row-80 regression, end-to-end through the real container and the real
|
||||
// collector. Before the fix this produced false/0.
|
||||
var reporter = _host.Services.GetServices<IHostedService>()
|
||||
.OfType<LocalDbReplicationStatusReporter>()
|
||||
.Single();
|
||||
var collector = _host.Services.GetRequiredService<ISiteHealthCollector>();
|
||||
|
||||
reporter.Probe();
|
||||
var report = collector.CollectReport("TestSiteB");
|
||||
|
||||
Assert.Null(report.LocalDbReplicationConnected);
|
||||
Assert.Null(report.LocalDbOplogBacklog);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnreplicatedSite_StillRegistersTheHealthBridge()
|
||||
{
|
||||
// The fix is a GATE on what the reporter publishes, not a skipped registration.
|
||||
// Keeping the hosted service present means turning replication on is a config
|
||||
// change rather than a different service graph, and it keeps one code path for
|
||||
// both postures.
|
||||
var hosted = _host.Services.GetServices<IHostedService>();
|
||||
|
||||
Assert.Contains(hosted, h => h is LocalDbReplicationStatusReporter);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnreplicatedSite_ReplicationEngine_StillResolvesAndIdles()
|
||||
{
|
||||
// Establishes the premise the row-80 fix rests on: the engine really does answer
|
||||
// false/0 here. If this ever starts returning null on its own, the gate becomes
|
||||
// redundant rather than wrong — but the test above still pins the observable
|
||||
// contract either way.
|
||||
var status = _host.Services.GetRequiredService<ISyncStatus>();
|
||||
|
||||
Assert.False(status.Connected);
|
||||
Assert.Equal(0L, status.OplogBacklog);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user