494dcb5b87
M1 DriverHostActor wrote its own NodeDeploymentState row Applied/null as soon as the revision advanced, up to ~105 s before the address-space outcome. The coordinator's recovery counts any non-Applying row as an ack, so a coordinator restart in that window sealed CLEAN while the node was about to report degraded (#601). The row now stays Applying until ReleasePendingAck (or the no-publish-actor path) writes the final Applied, clean or with the degraded reason. A node restart in the window replays the orphan Applying row; /fleet shows Applying until the outcome (docs/Redundancy.md). M2 Acks whose persist failed are tracked in _unpersisted (newest per node). HandleDeadline persists that backlog first and defers its decision while any remain, so a node that acked is never timed out as silent; PreRestart re-sends them to Self so a restart (timers lost) does not drop them. L3 A successful persist cancels the node's retry timer and drops its older pending ack; a retry only fires for the node's current pending ack. L4 A dispatch queued during Loading is sent with a fresh deadline however long the outage lasted (never closed "Dispatch was lost"); the inline re-drive of a genuinely lost dispatch arms the REMAINING deadline. L5 A recovered row with a blank RevisionHash (re-dispatch) or blank NodeId is closed PartiallyFailed with a reason instead of failing every load. L6 Readiness carries PendingAckPersists + LastPersistError; the config-publish health check is Degraded while Ready with a backlog. L7 Ack-persist retries back off (shared Backoff); stack trace on the first failure only. L8 Terminal writes (Sealed / PartiallyFailed / TimedOut / failure reason) only apply to a row still Dispatching/AwaitingApplyAcks. Tests (all fail on the previous code): ControlPlane DbOutage — deadline does not time out an unpersisted ack; unpersisted ack survives a restart; stale retry does not overwrite a newer ack; queued dispatch through an outage longer than the deadline is sent; malformed row closed and coordinator carries on; readiness reports the persist backlog. Runtime DegradedAck — own row stays Applying until the outcome. Host — Ready + backlog is Degraded. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
81 lines
2.9 KiB
C#
81 lines
2.9 KiB
C#
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
|
using Microsoft.Extensions.Options;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.Health;
|
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
|
using ZB.MOM.WW.OtOpcUa.Host.Health;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
|
|
|
|
/// <summary>lmxopcua#868: the deploy coordinator reports Degraded (never Unhealthy) while it cannot load from the ConfigDb.</summary>
|
|
public sealed class ConfigPublishCoordinatorHealthCheckTests
|
|
{
|
|
private static async Task<HealthCheckResult> Check(ConfigPublishCoordinatorReadiness? readiness) =>
|
|
await new ConfigPublishCoordinatorHealthCheck(readiness).CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
|
|
|
|
[Fact]
|
|
public async Task Loading_after_failed_attempts_is_degraded_and_names_the_error()
|
|
{
|
|
var r = new ConfigPublishCoordinatorReadiness();
|
|
r.MarkLoading(3, "connection refused");
|
|
var result = await Check(r);
|
|
result.Status.ShouldBe(HealthStatus.Degraded);
|
|
result.Description.ShouldNotBeNull();
|
|
result.Description.ShouldContain("connection refused");
|
|
result.Description.ShouldContain("queued");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Ready_is_healthy()
|
|
{
|
|
var r = new ConfigPublishCoordinatorReadiness();
|
|
r.MarkLoading(2, "x");
|
|
r.MarkReady();
|
|
(await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Ready_with_an_ack_persist_backlog_is_degraded_and_clears()
|
|
{
|
|
var r = new ConfigPublishCoordinatorReadiness();
|
|
r.MarkReady();
|
|
r.MarkAckPersistBacklog(2, "timeout");
|
|
var result = await Check(r);
|
|
result.Status.ShouldBe(HealthStatus.Degraded);
|
|
result.Description.ShouldNotBeNull();
|
|
result.Description.ShouldContain("timeout");
|
|
|
|
r.MarkAckPersistBacklog(0, null);
|
|
(await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Not_hosted_and_untracked_are_healthy()
|
|
{
|
|
(await Check(new ConfigPublishCoordinatorReadiness())).Status.ShouldBe(HealthStatus.Healthy);
|
|
(await Check(null)).Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(true, true)]
|
|
[InlineData(false, false)]
|
|
public void It_is_registered_on_the_ready_tier_only_on_admin_nodes(bool hasAdmin, bool registered)
|
|
{
|
|
var registrations = new ServiceCollection()
|
|
.AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: true)
|
|
.BuildServiceProvider()
|
|
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
|
|
.Value.Registrations
|
|
.Where(x => x.Name == ConfigPublishCoordinatorHealthCheck.Name)
|
|
.ToList();
|
|
if (!registered)
|
|
{
|
|
registrations.ShouldBeEmpty();
|
|
return;
|
|
}
|
|
registrations.ShouldHaveSingleItem().Tags.ShouldBe([ZbHealthTags.Ready]);
|
|
}
|
|
}
|