Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs
T
Joseph Doherty 494dcb5b87 fix(controlplane,runtime): #868 second review — final-outcome node rows, unpersisted acks survive deadline and restart
M1 DriverHostActor wrote its own NodeDeploymentState row Applied/null as soon
   as the revision advanced, up to ~105 s before the address-space outcome.
   The coordinator's recovery counts any non-Applying row as an ack, so a
   coordinator restart in that window sealed CLEAN while the node was about to
   report degraded (#601). The row now stays Applying until ReleasePendingAck
   (or the no-publish-actor path) writes the final Applied, clean or with the
   degraded reason. A node restart in the window replays the orphan Applying
   row; /fleet shows Applying until the outcome (docs/Redundancy.md).
M2 Acks whose persist failed are tracked in _unpersisted (newest per node).
   HandleDeadline persists that backlog first and defers its decision while
   any remain, so a node that acked is never timed out as silent; PreRestart
   re-sends them to Self so a restart (timers lost) does not drop them.
L3 A successful persist cancels the node's retry timer and drops its older
   pending ack; a retry only fires for the node's current pending ack.
L4 A dispatch queued during Loading is sent with a fresh deadline however long
   the outage lasted (never closed "Dispatch was lost"); the inline re-drive of
   a genuinely lost dispatch arms the REMAINING deadline.
L5 A recovered row with a blank RevisionHash (re-dispatch) or blank NodeId is
   closed PartiallyFailed with a reason instead of failing every load.
L6 Readiness carries PendingAckPersists + LastPersistError; the config-publish
   health check is Degraded while Ready with a backlog.
L7 Ack-persist retries back off (shared Backoff); stack trace on the first
   failure only.
L8 Terminal writes (Sealed / PartiallyFailed / TimedOut / failure reason) only
   apply to a row still Dispatching/AwaitingApplyAcks.

Tests (all fail on the previous code): ControlPlane DbOutage — deadline does
not time out an unpersisted ack; unpersisted ack survives a restart; stale
retry does not overwrite a newer ack; queued dispatch through an outage longer
than the deadline is sent; malformed row closed and coordinator carries on;
readiness reports the persist backlog. Runtime DegradedAck — own row stays
Applying until the outcome. Host — Ready + backlog is Degraded.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 12:54:30 -05:00

81 lines
2.9 KiB
C#

using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.Health;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.Host.Health;
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
/// <summary>lmxopcua#868: the deploy coordinator reports Degraded (never Unhealthy) while it cannot load from the ConfigDb.</summary>
public sealed class ConfigPublishCoordinatorHealthCheckTests
{
private static async Task<HealthCheckResult> Check(ConfigPublishCoordinatorReadiness? readiness) =>
await new ConfigPublishCoordinatorHealthCheck(readiness).CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
[Fact]
public async Task Loading_after_failed_attempts_is_degraded_and_names_the_error()
{
var r = new ConfigPublishCoordinatorReadiness();
r.MarkLoading(3, "connection refused");
var result = await Check(r);
result.Status.ShouldBe(HealthStatus.Degraded);
result.Description.ShouldNotBeNull();
result.Description.ShouldContain("connection refused");
result.Description.ShouldContain("queued");
}
[Fact]
public async Task Ready_is_healthy()
{
var r = new ConfigPublishCoordinatorReadiness();
r.MarkLoading(2, "x");
r.MarkReady();
(await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
}
[Fact]
public async Task Ready_with_an_ack_persist_backlog_is_degraded_and_clears()
{
var r = new ConfigPublishCoordinatorReadiness();
r.MarkReady();
r.MarkAckPersistBacklog(2, "timeout");
var result = await Check(r);
result.Status.ShouldBe(HealthStatus.Degraded);
result.Description.ShouldNotBeNull();
result.Description.ShouldContain("timeout");
r.MarkAckPersistBacklog(0, null);
(await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
}
[Fact]
public async Task Not_hosted_and_untracked_are_healthy()
{
(await Check(new ConfigPublishCoordinatorReadiness())).Status.ShouldBe(HealthStatus.Healthy);
(await Check(null)).Status.ShouldBe(HealthStatus.Healthy);
}
[Theory]
[InlineData(true, true)]
[InlineData(false, false)]
public void It_is_registered_on_the_ready_tier_only_on_admin_nodes(bool hasAdmin, bool registered)
{
var registrations = new ServiceCollection()
.AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: true)
.BuildServiceProvider()
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
.Value.Registrations
.Where(x => x.Name == ConfigPublishCoordinatorHealthCheck.Name)
.ToList();
if (!registered)
{
registrations.ShouldBeEmpty();
return;
}
registrations.ShouldHaveSingleItem().Tags.ShouldBe([ZbHealthTags.Ready]);
}
}