02503a4a9f
PreStart ran the in-flight recovery query synchronously. With the ConfigDb unreachable when the singleton started, the throw became an ActorInitializationException, the default strategy stopped the actor, the ClusterSingletonManager never re-created it, and every later deploy hung. - PreStart now only subscribes and kicks off an async recovery load piped back to the actor. A failure retries with exponential backoff (1 s doubling to 30 s, forever) and never stops the actor. - Until the load succeeds the coordinator is in a Loading behaviour that stashes every message (DispatchDeployment logged as queued, ApplyAck too); they are replayed in order after the recovered state is applied. Recovery decisions (re-dispatch / time out) now run inline before the replay instead of via self-tells, and a duplicate dispatch of the in-flight deployment with a seeded expected set is ignored so queued acks are never reset. - New ConfigPublishCoordinatorReadiness (admin-node DI singleton) + "config-publish" ready-tier health check: Degraded while loading (with the attempt count and last error), Healthy when ready or not hosted here. Tests: ConfigPublishCoordinatorDbOutageTests (DB down at start -> actor alive, Loading reported, deploy queued, DB returns -> load succeeds, queued deploy dispatched and sealed; queued ack applied to the recovered in-flight set; NotHosted after stop) and ConfigPublishCoordinatorHealthCheckTests. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
66 lines
2.4 KiB
C#
66 lines
2.4 KiB
C#
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
|
using Microsoft.Extensions.Options;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.Health;
|
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
|
|
using ZB.MOM.WW.OtOpcUa.Host.Health;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
|
|
|
|
/// <summary>lmxopcua#868: the deploy coordinator reports Degraded (never Unhealthy) while it cannot load from the ConfigDb.</summary>
|
|
public sealed class ConfigPublishCoordinatorHealthCheckTests
|
|
{
|
|
private static async Task<HealthCheckResult> Check(ConfigPublishCoordinatorReadiness? readiness) =>
|
|
await new ConfigPublishCoordinatorHealthCheck(readiness).CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
|
|
|
|
[Fact]
|
|
public async Task Loading_after_failed_attempts_is_degraded_and_names_the_error()
|
|
{
|
|
var r = new ConfigPublishCoordinatorReadiness();
|
|
r.MarkLoading(3, "connection refused");
|
|
var result = await Check(r);
|
|
result.Status.ShouldBe(HealthStatus.Degraded);
|
|
result.Description.ShouldNotBeNull();
|
|
result.Description.ShouldContain("connection refused");
|
|
result.Description.ShouldContain("queued");
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Ready_is_healthy()
|
|
{
|
|
var r = new ConfigPublishCoordinatorReadiness();
|
|
r.MarkLoading(2, "x");
|
|
r.MarkReady();
|
|
(await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Not_hosted_and_untracked_are_healthy()
|
|
{
|
|
(await Check(new ConfigPublishCoordinatorReadiness())).Status.ShouldBe(HealthStatus.Healthy);
|
|
(await Check(null)).Status.ShouldBe(HealthStatus.Healthy);
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(true, true)]
|
|
[InlineData(false, false)]
|
|
public void It_is_registered_on_the_ready_tier_only_on_admin_nodes(bool hasAdmin, bool registered)
|
|
{
|
|
var registrations = new ServiceCollection()
|
|
.AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: true)
|
|
.BuildServiceProvider()
|
|
.GetRequiredService<IOptions<HealthCheckServiceOptions>>()
|
|
.Value.Registrations
|
|
.Where(x => x.Name == ConfigPublishCoordinatorHealthCheck.Name)
|
|
.ToList();
|
|
if (!registered)
|
|
{
|
|
registrations.ShouldBeEmpty();
|
|
return;
|
|
}
|
|
registrations.ShouldHaveSingleItem().Tags.ShouldBe([ZbHealthTags.Ready]);
|
|
}
|
|
}
|