lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorTimeoutTests.cs

using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;

namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;

public sealed class ConfigPublishCoordinatorTimeoutTests : ControlPlaneActorTestBase
{
    private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('b', 64));

    /// <summary>Verifies that DeadlineElapsed marks the current deployment as timed out.</summary>
    [Fact]
    public void DeadlineElapsed_for_current_deployment_marks_TimedOut()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);

        // Seed the NodeDeploymentState row BEFORE the actor is spawned. PreStart reads
        // NodeDeploymentStates to recover the expected-ack set, and it runs as soon as Sys.ActorOf
        // schedules the actor — so seeding afterwards is a race: if PreStart wins, the coordinator
        // sees zero expected acks and can short-circuit straight to Sealed, and the assertion below
        // never sees TimedOut. We pretend a driver node exists in the cluster.
        using (var db = dbFactory.CreateDbContext())
        {
            db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
            {
                NodeId = "phantom-driver",
                DeploymentId = deploymentId.Value,
                Status = NodeDeploymentStatus.Applying,
            });
            db.SaveChanges();
        }

        // Deliberately un-hittable wall-clock deadline (matching the other tests in this class).
        // The apply deadline is a real Akka timer: a short one can self-fire under load and race
        // the explicit DeadlineElapsed below, so the test would sometimes be measuring the timer
        // rather than the message handler. Five minutes guarantees the ONLY deadline signal the
        // coordinator ever receives is the one this test sends.
        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(5)));

        // Drive the deadline ourselves rather than waiting for the cluster's empty driver-set
        // bypass. Tell the actor the deadline elapsed for this id.
        actor.Tell(new ConfigPublishCoordinator.DeadlineElapsed(deploymentId));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.TimedOut);
        }, duration: TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken);
    }

    /// <summary>Verifies that stale DeadlineElapsed for other deployments is ignored.</summary>
    [Fact]
    public void Stale_DeadlineElapsed_for_other_deployment_is_ignored()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory);
        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(1)));

        // Tell the actor a deadline elapsed for a completely different deployment id.
        actor.Tell(new ConfigPublishCoordinator.DeadlineElapsed(DeploymentId.NewId()));

        // The seeded one should remain in its starting state (no transition triggered).
        ExpectNoMsg(TimeSpan.FromMilliseconds(250), TestContext.Current.CancellationToken);
        using var db = dbFactory.CreateDbContext();
        var status = db.Deployments.Single().Status;
        status.ShouldBeOneOf(DeploymentStatus.Dispatching, DeploymentStatus.AwaitingApplyAcks, DeploymentStatus.Sealed);
        status.ShouldNotBe(DeploymentStatus.TimedOut);
    }

    /// <summary>Verifies that PreStart recovers in-flight deployment state.</summary>
    [Fact]
    public void PreStart_recovers_inflight_deployment_state()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory, status: DeploymentStatus.AwaitingApplyAcks);

        // Seed two NodeDeploymentState rows — one already Applied, one still Applying.
        using (var db = dbFactory.CreateDbContext())
        {
            db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
            {
                NodeId = "driver-a", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied,
            });
            db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
            {
                NodeId = "driver-b", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying,
            });
            db.SaveChanges();
        }

        // Start a fresh coordinator — simulates singleton failover to this node.
        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(5)));

        // Send the missing ACK; the recovered state should expect exactly that node, and the
        // deployment should now seal (both nodes acked Applied).
        actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("driver-b"),
            ApplyAckOutcome.Applied, null, CorrelationId.NewId()));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
        }, duration: TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken);
    }

    /// <summary>
    ///     lmxopcua#784 — a deployment that timed out with SOME acks landed must persist WHY: the status API
    ///     returned TimedOut with a null reason, and only the central log named the silent node. The reason
    ///     names the node(s) that never acked and not the one that did.
    /// </summary>
    [Fact]
    public void Timeout_with_some_acks_landed_persists_a_reason_naming_the_silent_nodes()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory, status: DeploymentStatus.AwaitingApplyAcks);
        SeedNodeRows(dbFactory, deploymentId, applied: new[] { "site-a-1:4053" }, applying: new[] { "site-b-1:4053" });

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(5)));
        actor.Tell(new ConfigPublishCoordinator.DeadlineElapsed(deploymentId));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            var row = db.Deployments.Single();
            row.Status.ShouldBe(DeploymentStatus.TimedOut);
            row.FailureReason.ShouldNotBeNull();
            row.FailureReason.ShouldContain("No ack from: site-b-1:4053.");
            row.FailureReason.ShouldContain("(1/2 landed)");
            row.FailureReason.ShouldNotContain("site-a-1");
        }, duration: TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken);
    }

    /// <summary>lmxopcua#784 — the timeout reason stays bounded on a large fleet: it names 20 nodes and counts the rest.</summary>
    [Fact]
    public void Timeout_reason_is_bounded_when_many_nodes_are_silent()
    {
        var dbFactory = NewInMemoryDbFactory();
        var deploymentId = SeedDispatchingDeployment(dbFactory, status: DeploymentStatus.AwaitingApplyAcks);
        var silent = Enumerable.Range(0, 25).Select(i => $"node-{i:D2}-with-a-fairly-long-host-name:4053").ToArray();
        SeedNodeRows(dbFactory, deploymentId, applied: new[] { "acked-node:4053" }, applying: silent);

        var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, TimeSpan.FromMinutes(5)));
        actor.Tell(new ConfigPublishCoordinator.DeadlineElapsed(deploymentId));

        AwaitAssert(() =>
        {
            using var db = dbFactory.CreateDbContext();
            var reason = db.Deployments.Single().FailureReason;
            reason.ShouldNotBeNull();
            reason.ShouldContain("node-19-with-a-fairly-long-host-name:4053");
            reason.ShouldNotContain("node-20-");
            reason.ShouldContain("(and 5 more)");
            reason.Length.ShouldBeLessThanOrEqualTo(2048);
        }, duration: TimeSpan.FromSeconds(3), cancellationToken: TestContext.Current.CancellationToken);
    }

    private static void SeedNodeRows(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, DeploymentId deploymentId, string[] applied, string[] applying)
    {
        using var db = dbFactory.CreateDbContext();
        foreach (var n in applied)
            db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
            {
                NodeId = n, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied,
            });
        foreach (var n in applying)
            db.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
            {
                NodeId = n, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying,
            });
        db.SaveChanges();
    }

    private static DeploymentId SeedDispatchingDeployment(
        IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
        DeploymentStatus status = DeploymentStatus.Dispatching)
    {
        var id = DeploymentId.NewId();
        using var db = dbFactory.CreateDbContext();
        db.Deployments.Add(new Configuration.Entities.Deployment
        {
            DeploymentId = id.Value,
            RevisionHash = TestRevision.Value,
            Status = status,
            CreatedBy = "test",
        });
        db.SaveChanges();
        return id;
    }
}