Merge #867 Heidenhain integer leaves keep their declared type; #868 ConfigPublishCoordinator survives ConfigDb outage at start

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 12:29:00 -05:00
parent 233e4bab25 02503a4a9f
commit cbd24d7a01
11 changed files with 720 additions and 41 deletions
@@ -0,0 +1,185 @@
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+/// <summary>
+///     lmxopcua#868: the coordinator's recovery read used to run synchronously in <c>PreStart</c>. With the
+///     ConfigDb unreachable when the singleton started, the actor died on initialization, was never re-created,
+///     and every deployment hung. It now loads asynchronously with backoff, queues what arrives meanwhile and
+///     reports <see cref="ConfigPublishCoordinatorState.Loading"/> until the load succeeds.
+/// </summary>
+public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTestBase
+{
+    private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('d', 64));
+    private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
+    private static readonly TimeSpan RetryMin = TimeSpan.FromMilliseconds(50);
+    private static readonly TimeSpan RetryMax = TimeSpan.FromMilliseconds(200);
+    private const string Node = "site-a-1:4053";
+
+    [Fact]
+    public void Db_unreachable_at_start_does_not_kill_the_coordinator_and_a_deploy_sent_during_the_outage_completes_once_it_returns()
+    {
+        var dbName = Guid.NewGuid().ToString("N");
+        var direct = NewInMemoryDbFactory(dbName);
+        var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
+        SeedNodes(direct, Node);
+        var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
+        var readiness = new ConfigPublishCoordinatorReadiness();
+
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+            outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+        Watch(coordinator);
+
+        // Several failed loads: the actor keeps retrying (it is alive) and reports Loading with the error.
+        AwaitAssert(() =>
+        {
+            outage.Attempts.ShouldBeGreaterThanOrEqualTo(3);
+            readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Loading);
+            readiness.Current.FailedLoadAttempts.ShouldBeGreaterThanOrEqualTo(2);
+            readiness.Current.LastError.ShouldNotBeNull();
+            readiness.Current.LastError.ShouldContain("unreachable");
+        }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+        // A deployment dispatched during the outage is queued, not processed and not dropped.
+        coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
+        ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
+        using (var db = direct.CreateDbContext())
+        {
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching);
+            db.NodeDeploymentStates.ShouldBeEmpty();
+        }
+
+        // The DB comes back: the load succeeds, the queued dispatch runs, and the node's ack seals it.
+        outage.Down = false;
+        AwaitAssert(() =>
+        {
+            readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
+            using var db = direct.CreateDbContext();
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+            db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { Node });
+        }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+        coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+        AwaitAssert(() =>
+        {
+            using var db = direct.CreateDbContext();
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+        }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+        ExpectNoMsg(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); // never Terminated
+    }
+
+    [Fact]
+    public void In_flight_deployment_is_recovered_after_the_outage_and_an_ack_queued_during_it_is_not_lost()
+    {
+        var dbName = Guid.NewGuid().ToString("N");
+        var direct = NewInMemoryDbFactory(dbName);
+        var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
+        SeedNodes(direct, Node, "site-a-2:4053");
+        var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow);
+        using (var db = direct.CreateDbContext())
+        {
+            db.NodeDeploymentStates.Add(new NodeDeploymentState
+                { NodeId = Node, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying });
+            db.NodeDeploymentStates.Add(new NodeDeploymentState
+                { NodeId = "site-a-2:4053", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied, AppliedAtUtc = DateTime.UtcNow });
+            db.SaveChanges();
+        }
+
+        var readiness = new ConfigPublishCoordinatorReadiness();
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+            outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+        AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2),
+            duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+        // The last outstanding ack arrives while the coordinator is still loading: it is queued, then applied
+        // against the recovered expected set (site-a-2 already Applied in the DB) and seals the deployment.
+        coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+        outage.Down = false;
+
+        AwaitAssert(() =>
+        {
+            readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
+            using var db = direct.CreateDbContext();
+            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+        }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+    }
+
+    [Fact]
+    public void Readiness_is_NotHosted_after_the_coordinator_stops()
+    {
+        var readiness = new ConfigPublishCoordinatorReadiness();
+        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(NewInMemoryDbFactory(), NoTimeout, readiness: readiness));
+        AwaitAssert(() => readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready),
+            duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+        Watch(coordinator);
+        Sys.Stop(coordinator);
+        ExpectTerminated(coordinator, cancellationToken: TestContext.Current.CancellationToken);
+        readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.NotHosted);
+    }
+
+    private static DeploymentId SeedDeployment(
+        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus status, DateTime createdAtUtc)
+    {
+        var id = DeploymentId.NewId();
+        using var db = dbFactory.CreateDbContext();
+        db.Deployments.Add(new Deployment
+        {
+            DeploymentId = id.Value,
+            RevisionHash = TestRevision.Value,
+            Status = status,
+            CreatedBy = "test",
+            CreatedAtUtc = createdAtUtc,
+        });
+        db.SaveChanges();
+        return id;
+    }
+
+    private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodeIds)
+    {
+        using var db = dbFactory.CreateDbContext();
+        db.ServerClusters.Add(new ServerCluster
+        {
+            ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
+            NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
+        });
+        foreach (var nodeId in nodeIds)
+        {
+            db.ClusterNodes.Add(new ClusterNode
+            {
+                NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0],
+                ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test",
+            });
+        }
+        db.SaveChanges();
+    }
+
+    /// <summary>A factory that throws while <see cref="Down"/> — a ConfigDb that is unreachable.</summary>
+    private sealed class OutageDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner) : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        private volatile bool _down;
+        private int _attempts;
+
+        public bool Down { get => _down; set => _down = value; }
+
+        public int Attempts => Volatile.Read(ref _attempts);
+
+        public OtOpcUaConfigDbContext CreateDbContext()
+        {
+            Interlocked.Increment(ref _attempts);
+            if (_down) throw new InvalidOperationException("ConfigDb unreachable (simulated outage)");
+            return inner.CreateDbContext();
+        }
+    }
+}
@@ -136,9 +136,9 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes
        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout));
        var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty<IDriverProbe>()));

-        // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and
-        // the actor would be stopped, not restarted). Identify is processed after creation, so a
-        // resolved selection proves PreStart completed.
+        // Arm the fault AFTER PreStart has run. Since lmxopcua#868 the recovery read is asynchronous and
+        // retried, so a fault that lands on it instead of on HandleDispatch is also recovered (the dispatch is
+        // queued until the retried load succeeds) — either way the deployment must complete.
        await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget, TestContext.Current.CancellationToken);
        faulting.FailNext();
        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
@@ -0,0 +1,65 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.Health;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
+using ZB.MOM.WW.OtOpcUa.Host.Health;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
+
+/// <summary>lmxopcua#868: the deploy coordinator reports Degraded (never Unhealthy) while it cannot load from the ConfigDb.</summary>
+public sealed class ConfigPublishCoordinatorHealthCheckTests
+{
+    private static async Task<HealthCheckResult> Check(ConfigPublishCoordinatorReadiness? readiness) =>
+        await new ConfigPublishCoordinatorHealthCheck(readiness).CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
+
+    [Fact]
+    public async Task Loading_after_failed_attempts_is_degraded_and_names_the_error()
+    {
+        var r = new ConfigPublishCoordinatorReadiness();
+        r.MarkLoading(3, "connection refused");
+        var result = await Check(r);
+        result.Status.ShouldBe(HealthStatus.Degraded);
+        result.Description.ShouldNotBeNull();
+        result.Description.ShouldContain("connection refused");
+        result.Description.ShouldContain("queued");
+    }
+
+    [Fact]
+    public async Task Ready_is_healthy()
+    {
+        var r = new ConfigPublishCoordinatorReadiness();
+        r.MarkLoading(2, "x");
+        r.MarkReady();
+        (await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
+    }
+
+    [Fact]
+    public async Task Not_hosted_and_untracked_are_healthy()
+    {
+        (await Check(new ConfigPublishCoordinatorReadiness())).Status.ShouldBe(HealthStatus.Healthy);
+        (await Check(null)).Status.ShouldBe(HealthStatus.Healthy);
+    }
+
+    [Theory]
+    [InlineData(true, true)]
+    [InlineData(false, false)]
+    public void It_is_registered_on_the_ready_tier_only_on_admin_nodes(bool hasAdmin, bool registered)
+    {
+        var registrations = new ServiceCollection()
+            .AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: true)
+            .BuildServiceProvider()
+            .GetRequiredService<IOptions<HealthCheckServiceOptions>>()
+            .Value.Registrations
+            .Where(x => x.Name == ConfigPublishCoordinatorHealthCheck.Name)
+            .ToList();
+        if (!registered)
+        {
+            registrations.ShouldBeEmpty();
+            return;
+        }
+        registrations.ShouldHaveSingleItem().Tags.ShouldBe([ZbHealthTags.Ready]);
+    }
+}