lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs

using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;

namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;

/// <summary>
///     Review 03 §2.2: a <c>Deployment</c> row left in <c>Dispatching</c> with NO <c>NodeDeploymentState</c>
///     rows — <c>HandleDispatch</c> threw before its <c>SaveChanges</c>, or the process died between the
///     AdminOps row write and the coordinator's <c>Tell</c> — used to be recovered with an EMPTY expected
///     set, at which point the deadline hit the <c>0 == 0</c> "race" branch and returned without touching
///     the row. Every later <c>StartDeployment</c> was then refused <c>AnotherDeploymentInFlight</c> forever.
/// </summary>
public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase
{
    private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64));
    private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);

    /// <summary>
    ///     A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it,
    ///     the expected set is seeded from the enabled rows, and the node's ack seals it.
    /// </summary>
    [Fact]
    public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
        SeedNodes(dbFactory, "site-a-1:4053");

        // No DispatchDeployment is ever sent — PreStart alone has to recover it.
        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" });
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
        }, duration: PresenceBudget);

        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
        }, duration: PresenceBudget);
    }

    /// <summary>
    ///     A lost dispatch that is already past the apply deadline is closed as <c>TimedOut</c> with a reason
    ///     naming the condition — never left in <c>Dispatching</c> — and the next deployment proceeds
    ///     (positive control: it is dispatched, acked and sealed through the same coordinator).
    /// </summary>
    [Fact]
    public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged()
    {
        var dbFactory = NewInMemoryDbFactory();
        var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10));
        SeedNodes(dbFactory, "site-a-1:4053");

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value);
            row.Status.ShouldBe(DeploymentStatus.TimedOut);
            row.FailureReason.ShouldNotBeNull();
            row.FailureReason.ShouldContain("lost");
        }, duration: PresenceBudget);

        // Positive control: the coordinator is not wedged — a fresh deployment runs end to end.
        var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow);
        actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId()));
        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
        }, duration: PresenceBudget);

        actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed);
        }, duration: PresenceBudget);
    }

    /// <summary>
    ///     An <c>AwaitingApplyAcks</c> row with no rows (the process died between the status flip and the
    ///     empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh
    ///     dispatch would — the recovery keys on the missing expected set, not on the status value.
    /// </summary>
    [Fact]
    public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty()
    {
        var dbFactory = NewInMemoryDbFactory();
        SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow);
        Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
            db.NodeDeploymentStates.ShouldBeEmpty();
        }, duration: PresenceBudget);
    }

    /// <summary>
    ///     The brief's scenario end to end: <c>HandleDispatch</c> throws once (a transient fault seeding the
    ///     expected set). The coordinator restarts, recovers the <c>Dispatching</c> row, re-dispatches it and
    ///     the deployment completes; a second <c>StartDeployment</c> is then Accepted (positive control: a
    ///     normal deploy seals).
    /// </summary>
    [Fact]
    public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds()
    {
        var dbName = Guid.NewGuid().ToString("N");
        var adminFactory = NewInMemoryDbFactory(dbName);
        var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName));
        SeedNodes(adminFactory, "site-a-1:4053");

        var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout));
        var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty<IDriverProbe>()));

        // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and
        // the actor would be stopped, not restarted). Identify is processed after creation, so a
        // resolved selection proves PreStart completed.
        await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget);
        faulting.FailNext();
        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
        var first = ExpectMsg<StartDeploymentResult>(PresenceBudget);
        first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
        first.DeploymentId.ShouldNotBeNull();

        // The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands.
        AwaitAssert(() =>
        {
            faulting.Faulted.ShouldBeTrue();
            using var db = adminFactory.CreateDbContext();
            db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1);
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
        }, duration: PresenceBudget);

        coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
        AwaitAssert(() =>
        {
            using var db = adminFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
        }, duration: PresenceBudget);

        // Positive control: nothing is wedged — the next deploy is Accepted and seals too.
        admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
        var second = ExpectMsg<StartDeploymentResult>(PresenceBudget);
        second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted);
        coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
        AwaitAssert(() =>
        {
            using var db = adminFactory.CreateDbContext();
            db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2);
        }, duration: PresenceBudget);
    }

    private static DeploymentId SeedDeployment(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentStatus status, DateTime createdAtUtc)
    {
        var id = DeploymentId.NewId();
        using var db = dbFactory.CreateDbContext();
        db.Deployments.Add(new Deployment
        {
            DeploymentId = id.Value,
            RevisionHash = TestRevision.Value,
            Status = status,
            CreatedBy = "test",
            CreatedAtUtc = createdAtUtc,
        });
        db.SaveChanges();
        return id;
    }

    private static void SeedNodes(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] nodeIds)
    {
        using var db = dbFactory.CreateDbContext();
        db.ServerClusters.Add(new ServerCluster
        {
            ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
            NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
        });
        foreach (var nodeId in nodeIds)
        {
            db.ClusterNodes.Add(new ClusterNode
            {
                NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0],
                ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test",
            });
        }
        db.SaveChanges();
    }

    /// <summary>Throws from exactly one <c>CreateDbContext</c> call after <see cref="FailNext"/>.</summary>
    private sealed class OneShotFaultingDbFactory(IDbContextFactory<OtOpcUaConfigDbContext> inner)
        : IDbContextFactory<OtOpcUaConfigDbContext>
    {
        private int _armed;
        public bool Faulted { get; private set; }

        public void FailNext() => Interlocked.Exchange(ref _armed, 1);

        public OtOpcUaConfigDbContext CreateDbContext()
        {
            if (Interlocked.Exchange(ref _armed, 0) == 1)
            {
                Faulted = true;
                throw new InvalidOperationException("injected transient fault seeding the expected set");
            }
            return inner.CreateDbContext();
        }
    }
}