8b761841bc
A healthy standby logged "Health check active-node with status Unhealthy … 'not the site pair primary (standby)'" at Error on every probe — every 10s on a site node, and (discovered while fixing this) every 5s on a central standby, which Traefik health-probes at that cadence. Correct-by-design state at an alarming severity: it floods the log and trains operators to ignore ERR. Mechanism: DefaultHealthCheckService logs one line per check per probe and picks the level from the verdict — Debug/Warning/Error for Healthy/Degraded/Unhealthy. Both active-node checks answer Unhealthy on the standby because 503 IS the contract /health/active publishes. There is no IHealthCheckPublisher in this repo, so each line is one inbound HTTP probe. Fix: one Serilog exclusion filter in LoggerConfigurationFactory. A level override could not work — Serilog's MinimumLevel.Override only RAISES a per-source floor, and a LogEvent's Level is immutable, so neither an override nor an enricher can rewrite Error as Information. Changing the check to report Degraded was rejected: it would alter the HTTP status that /health/active publishes, which is load-bearing for Traefik and the family dashboard. The constraint (genuine failures must still log at Error) is preserved by keying the predicate on the check NAME rather than the logger category: localdb, akka-cluster, database and required-singletons keep their Error lines, and the active-node check has no error path of its own — the active node always answers Healthy — so the only event this can ever suppress is a standby reporting that it is a standby. The verdict stays fully observable via /health/active's own HTTP status, the readiness probe's cluster view, and the health dashboard. Also corrects a stale CLAUDE.md claim found while verifying what consumes this check: "/health/active is central-only (site nodes map no /health/* at all)" is wrong — SiteServiceRegistration registers akka-cluster + localdb + the role-scoped SitePairActiveNodeHealthCheck, and Program.cs's Site branch calls MapZbHealth() on the HTTP/1.1 listener. Fail-before evidence: with the filter behaviourally disabled, StandbyActiveNodeVerdict_IsNotLogged fails "Collection was not empty" while the other 9 tests in the file — the must-still-log constraints — pass either way, which is what proves the filter is narrow rather than a blanket suppression. Tests: Host 518 green (10 new).
153 lines
5.5 KiB
C#
153 lines
5.5 KiB
C#
using Microsoft.Extensions.Configuration;
|
|
using Serilog;
|
|
using Serilog.Events;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
|
|
|
/// <summary>
|
|
/// Deferred-work register row 71 — a standby node logged the correct-by-design
|
|
/// <c>active-node</c> health-check verdict at ERROR on every probe.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// <c>DefaultHealthCheckService</c> picks the log level from the verdict (Debug/Warning/Error
|
|
/// for Healthy/Degraded/Unhealthy). Both active-node checks answer Unhealthy on the standby
|
|
/// because 503 is the contract <c>/health/active</c> publishes, so a healthy standby logged
|
|
/// ERR every 10 s on a site node and every 5 s on a central node behind Traefik's probe.
|
|
/// </para>
|
|
/// <para>
|
|
/// The constraint on the fix is that a GENUINE failure must still log at Error, which is why
|
|
/// the filter is keyed on the check NAME rather than on the logger category.
|
|
/// </para>
|
|
/// </remarks>
|
|
public class StandbyHealthCheckLogNoiseTests
|
|
{
|
|
private const string HealthCheckCategory =
|
|
"Microsoft.Extensions.Diagnostics.HealthChecks.DefaultHealthCheckService";
|
|
|
|
private const string Template =
|
|
"Health check {HealthCheckName} with status {HealthStatus} completed after "
|
|
+ "{ElapsedMilliseconds}ms with message '{HealthCheckDescription}'";
|
|
|
|
private static ILogger BuildLogger(InMemorySink sink)
|
|
{
|
|
var config = new ConfigurationBuilder()
|
|
.AddInMemoryCollection(new Dictionary<string, string?>
|
|
{
|
|
["ScadaBridge:Logging:MinimumLevel"] = "Debug",
|
|
})
|
|
.Build();
|
|
|
|
return LoggerConfigurationFactory
|
|
.Build(config, "Site", "site-a", "site-a-b")
|
|
.WriteTo.Sink(sink)
|
|
.CreateLogger();
|
|
}
|
|
|
|
/// <summary>Emits one event shaped exactly like the framework's per-check line.</summary>
|
|
private static void EmitHealthCheckLine(
|
|
ILogger logger, string category, LogEventLevel level, string checkName, string status) =>
|
|
logger
|
|
.ForContext("SourceContext", category)
|
|
.Write(level, Template, checkName, status, 1.0, "some message");
|
|
|
|
[Fact]
|
|
public void StandbyActiveNodeVerdict_IsNotLogged()
|
|
{
|
|
// The row-71 regression: "Health check active-node with status Unhealthy … 'not the
|
|
// site pair primary (standby)'" at Error, once per probe, forever.
|
|
var sink = new InMemorySink();
|
|
var logger = BuildLogger(sink);
|
|
|
|
EmitHealthCheckLine(
|
|
logger, HealthCheckCategory, LogEventLevel.Error, "active-node", "Unhealthy");
|
|
|
|
Assert.Empty(sink.LogEvents);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("localdb")]
|
|
[InlineData("akka-cluster")]
|
|
[InlineData("database")]
|
|
[InlineData("required-singletons")]
|
|
public void GenuineCheckFailure_StillLogsAtError(string checkName)
|
|
{
|
|
// The constraint. The filter is name-scoped, so every OTHER check on the same logger
|
|
// category keeps its Error line — a blanket category override would have silenced
|
|
// these too, which is exactly the trap this test exists to catch.
|
|
var sink = new InMemorySink();
|
|
var logger = BuildLogger(sink);
|
|
|
|
EmitHealthCheckLine(
|
|
logger, HealthCheckCategory, LogEventLevel.Error, checkName, "Unhealthy");
|
|
|
|
var evt = Assert.Single(sink.LogEvents);
|
|
Assert.Equal(LogEventLevel.Error, evt.Level);
|
|
}
|
|
|
|
[Fact]
|
|
public void ActiveNodeHealthyVerdict_IsUnaffected()
|
|
{
|
|
// The active node reports Healthy, which the framework logs at Debug. Nothing about
|
|
// the filter should touch it — it only ever matches Error.
|
|
var sink = new InMemorySink();
|
|
var logger = BuildLogger(sink);
|
|
|
|
EmitHealthCheckLine(
|
|
logger, HealthCheckCategory, LogEventLevel.Debug, "active-node", "Healthy");
|
|
|
|
Assert.Single(sink.LogEvents);
|
|
}
|
|
|
|
[Fact]
|
|
public void ActiveNodeDegradedVerdict_IsUnaffected()
|
|
{
|
|
// Degraded logs at Warning. If a future check shape ever reports Degraded for
|
|
// active-node, that should still be visible — the filter is Error-only.
|
|
var sink = new InMemorySink();
|
|
var logger = BuildLogger(sink);
|
|
|
|
EmitHealthCheckLine(
|
|
logger, HealthCheckCategory, LogEventLevel.Warning, "active-node", "Degraded");
|
|
|
|
Assert.Single(sink.LogEvents);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnErrorMentioningActiveNode_FromAnotherCategory_IsNotSuppressed()
|
|
{
|
|
// The predicate requires BOTH the health-check category and the check name, so
|
|
// application code logging about the active node keeps its Error.
|
|
var sink = new InMemorySink();
|
|
var logger = BuildLogger(sink);
|
|
|
|
EmitHealthCheckLine(
|
|
logger,
|
|
"ZB.MOM.WW.ScadaBridge.Host.Actors.AkkaHostedService",
|
|
LogEventLevel.Error,
|
|
"active-node",
|
|
"Unhealthy");
|
|
|
|
Assert.Single(sink.LogEvents);
|
|
}
|
|
|
|
[Fact]
|
|
public void OrdinaryErrorsWithNoHealthCheckProperties_AreNotSuppressed()
|
|
{
|
|
var sink = new InMemorySink();
|
|
var logger = BuildLogger(sink);
|
|
|
|
logger.Error("something genuinely broke");
|
|
|
|
Assert.Single(sink.LogEvents);
|
|
}
|
|
|
|
[Fact]
|
|
public void Predicate_RejectsANullEvent()
|
|
{
|
|
// Defensive: Serilog never hands the filter a null, but the predicate is internal
|
|
// and reachable, and returning "suppress" for null would be the wrong default.
|
|
Assert.False(LoggerConfigurationFactory.IsExpectedStandbyActiveNodeProbe(null!));
|
|
}
|
|
}
|