lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs

using System.Collections.Concurrent;
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;

namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;

/// <summary>
///     v3 Batch 4 (B4-WP4) — the inbound native-condition <b>acknowledge</b> routing wired into
///     <see cref="DriverHostActor"/> re-expressed for the v3 raw-condition model. An OPC UA client Acknowledges
///     a NATIVE condition (materialised at the raw tag, ConditionId = its RawPath); the node manager invokes
///     <c>NativeAlarmAckRouter</c> and the host receives a <see cref="DriverHostActor.RouteNativeAlarmAck"/>.
///     The host resolves the condition NodeId (== the RawPath) → owning <c>(DriverInstanceId, RawPath)</c> via
///     the <c>_driverRefByAlarmNodeId</c> inverse map (built alongside the alarm forward map in
///     <c>PushDesiredSubscriptions</c> from the alarm-bearing <c>composition.RawTags</c>), applies the SAME
///     primary gate the inbound write path uses, and routes to the owning driver child's
///     <see cref="IAlarmSource.AcknowledgeAsync"/> carrying the principal.
///
///     <para>
///         Mirrors <c>DriverHostActorLiveValueTests</c>: a real apply through the harness spawns a real
///         (non-stubbed) <see cref="DriverInstanceActor"/> child backed by a recording <see cref="IAlarmSource"/>
///         driver (DriverType "GalaxyMxGateway", Enabled), so the inverse map is populated authentically and the
///         forwarded acknowledge request can be observed. The seeded raw tag carries an <c>alarm</c> object so it
///         materialises as a Part 9 condition at its RawPath, not a value variable.
///     </para>
/// </summary>
public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTestBase
{
    private static readonly NodeId TestNode = NodeId.Parse("driver-ack-test");
    private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
    private static readonly TimeSpan Timeout = PresenceBudget;

    // The v3 RawPath for the seeded alarm tag: RawFolder "Plant" / DriverName "gw" / Device "dev1" / Tag.
    private const string AlarmRawPath = "Plant/gw/dev1/temp_hi";

    /// <summary>On the PRIMARY (role unknown ⇒ Primary), a RouteNativeAlarmAck for a mapped raw condition NodeId
    /// (== the RawPath) forwards exactly one <see cref="AlarmAcknowledgeRequest"/> to the owning driver's
    /// <see cref="IAlarmSource.AcknowledgeAsync"/>, correlated on the RawPath, with the operator principal + the
    /// comment.</summary>
    [Fact]
    public void RouteNativeAlarmAck_routes_to_driver_AcknowledgeAsync_with_principal()
    {
        var db = NewInMemoryDbFactory();
        var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway");
        var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");

        var actor = SpawnHostAndApply(db, deploymentId, recorder);

        // Local role unknown ⇒ treated as Primary ⇒ ack allowed (default-allow semantics).
        actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice", ApiKeyId: "0a1b2c"));

        // The driver received exactly one acknowledge, correlated on its wire-ref RawPath, with principal
        // and the session's API key id (lmxopcua#749 — the Galaxy driver resolves the key's ack identity).
        AwaitAssert(() =>
        {
            recorder.Acks.Count.ShouldBe(1);
            recorder.Acks[0].ConditionId.ShouldBe(AlarmRawPath);
            recorder.Acks[0].SourceNodeId.ShouldBe(AlarmRawPath);
            recorder.Acks[0].Comment.ShouldBe("cmt");
            recorder.Acks[0].OperatorUser.ShouldBe("alice");
            recorder.Acks[0].ApiKeyId.ShouldBe("0a1b2c");
        }, duration: Timeout);
    }

    /// <summary>An ack for an unmapped condition NodeId is dropped: no throw, and the driver's
    /// <see cref="IAlarmSource.AcknowledgeAsync"/> is never called.</summary>
    [Fact]
    public void RouteNativeAlarmAck_unknown_node_is_dropped()
    {
        var db = NewInMemoryDbFactory();
        var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway");
        var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");

        var actor = SpawnHostAndApply(db, deploymentId, recorder);

        actor.Tell(new DriverHostActor.RouteNativeAlarmAck("Plant/gw/dev1/does-not-exist", "cmt", "alice"));

        // POSITIVE CONTROL, not a sleep (#501). `Acks` is empty at t=0, so any assertion made before the
        // host has processed the message above passes without testing anything — and AwaitAssert returns on
        // its FIRST success, so it spends none of its duration and is the wrong tool for an absence.
        //
        // Instead send a MAPPED ack afterwards. The host processes its mailbox in order and forwards to the
        // child via Tell, and the child handles RouteAlarmAck with ReceiveAsync (which suspends its own
        // mailbox), so by the time this one reaches the driver the unmapped one has provably already been
        // handled. Whatever `Acks` holds then is the complete answer.
        actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "control", "alice"));

        AwaitAssert(() => recorder.Acks.ShouldContain(a => a.Comment == "control"), duration: Timeout);
        recorder.Acks.Count.ShouldBe(1,
            "only the control ack may reach the driver — the unmapped condition NodeId must have been dropped");
    }

    /// <summary>On a SECONDARY node the ack is gated off (same primary gate as the inbound write path): the
    /// driver's <see cref="IAlarmSource.AcknowledgeAsync"/> is NOT called.</summary>
    [Fact]
    public void RouteNativeAlarmAck_on_non_primary_is_dropped()
    {
        var db = NewInMemoryDbFactory();
        var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway");
        var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");

        var actor = SpawnHostAndApply(db, deploymentId, recorder);

        // Force this node Secondary so the primary gate rejects the ack.
        actor.Tell(new RedundancyStateChanged(
            new[]
            {
                new NodeRedundancyState(TestNode, RedundancyRole.Secondary,
                    IsClusterLeader: false, IsDriverPrimary: false, AsOfUtc: DateTime.UtcNow),
            },
            CorrelationId.NewId()));

        actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "gated", "alice"));

        // POSITIVE CONTROL (#501) — same reasoning as the unmapped-node test, with the role itself as the
        // control: return the node to PRIMARY and re-send the SAME mapped ack. Mailbox ordering means that
        // once this one reaches the driver, the Secondary-gated one has already been processed. The
        // discriminator is the comment, so a leaked ack is visible rather than merged into the count.
        actor.Tell(new RedundancyStateChanged(
            new[]
            {
                new NodeRedundancyState(TestNode, RedundancyRole.Primary,
                    IsClusterLeader: true, IsDriverPrimary: true, AsOfUtc: DateTime.UtcNow),
            },
            CorrelationId.NewId()));
        actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "control", "alice"));

        AwaitAssert(() => recorder.Acks.ShouldContain(a => a.Comment == "control"), duration: Timeout);
        recorder.Acks.ShouldNotContain(a => a.Comment == "gated",
            "the Primary gate must have dropped the ack sent while this node was Secondary");
        recorder.Acks.Count.ShouldBe(1);
    }

    /// <summary>
    ///     lmxopcua#760 — the driver REFUSES the acknowledge (no Galaxy ack credential): the host reverts the
    ///     condition the node manager optimistically acknowledged, re-projecting it Active + UNacknowledged with
    ///     its message and severity intact, so the address space stops claiming an ack Galaxy never received.
    /// </summary>
    [Fact]
    public void A_refused_ack_reverts_the_condition_to_unacknowledged()
    {
        var db = NewInMemoryDbFactory();
        var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
        {
            AckFault = new DriverOperationRefusedException("no Galaxy ack credential"),
        };
        var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
        var publish = this.CreateCleanOutcomePublishProbe();
        var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref);

        RaiseAndExpectUnacked(actor, publish);

        actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));

        var reverted = NextAlarmUpdate(publish);
        reverted.AlarmNodeId.ShouldBe(AlarmRawPath);
        reverted.Realm.ShouldBe(AddressSpaceRealm.Raw);
        reverted.State.Active.ShouldBeTrue();
        reverted.State.Acknowledged.ShouldBeFalse();
        reverted.State.Message.ShouldBe("temperature high");
        reverted.State.Severity.ShouldBe((ushort)700);
        recorder.Acks.Count.ShouldBe(1, "the ack did reach the driver, which refused it");
    }

    /// <summary>
    ///     lmxopcua#760 — any backend failure (here: the gateway rejecting the ack) reverts the same way, not just
    ///     a deliberate refusal.
    /// </summary>
    [Fact]
    public void A_failed_ack_reverts_the_condition_to_unacknowledged()
    {
        var db = NewInMemoryDbFactory();
        var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
        {
            AckFault = new InvalidOperationException("gateway hresult 0x80070005"),
        };
        var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
        var publish = this.CreateCleanOutcomePublishProbe();
        var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref);

        RaiseAndExpectUnacked(actor, publish);
        actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));

        NextAlarmUpdate(publish).State.Acknowledged.ShouldBeFalse();
    }

    /// <summary>
    ///     lmxopcua#760 — a SUCCESSFUL ack reverts nothing. Positive control: the next driver Clear projects
    ///     inactive + ACKED, which it only can when the local ack stayed in the projector (a revert would have
    ///     left it unacked) — and it must be the very next condition update, so no revert slipped in between.
    /// </summary>
    [Fact]
    public void A_successful_ack_is_not_reverted()
    {
        var db = NewInMemoryDbFactory();
        var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway");
        var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
        var publish = this.CreateCleanOutcomePublishProbe();
        var actor = SpawnHostAndApply(db, deploymentId, recorder, publish.Ref);

        RaiseAndExpectUnacked(actor, publish);
        actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
        AwaitAssert(() => recorder.Acks.Count.ShouldBe(1), duration: Timeout);

        actor.Tell(new DriverInstanceActor.AttributeAlarmPublished("drv-1", Evt(AlarmTransitionKind.Clear)));
        var next = NextAlarmUpdate(publish);
        next.State.Active.ShouldBeFalse();
        next.State.Acknowledged.ShouldBeTrue("the successful ack must have survived — no revert");
    }

    /// <summary>
    ///     lmxopcua#760 — the host publishes the condition → owning-driver pre-check snapshot the OPC UA node
    ///     manager uses to refuse an acknowledge synchronously: the driver's own verdict for the condition it
    ///     owns, nothing for an unknown condition.
    /// </summary>
    [Fact]
    public void The_ack_preflight_view_answers_with_the_owning_drivers_verdict()
    {
        var db = NewInMemoryDbFactory();
        var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway")
        {
            PreflightRefusal = key => key is null ? "anonymous sessions cannot acknowledge" : null,
        };
        var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
        var view = new NativeAlarmAckPreflightView();
        SpawnHostAndApply(db, deploymentId, recorder, ackPreflightView: view);

        AwaitAssert(() => view.Check(AlarmRawPath, apiKeyId: null).ShouldBe("anonymous sessions cannot acknowledge"),
            duration: Timeout);
        view.Check(AlarmRawPath, apiKeyId: "0a1b2c").ShouldBeNull();
        view.Check("Plant/gw/dev1/does-not-exist", apiKeyId: null).ShouldBeNull();
    }

    private static AlarmEventArgs Evt(AlarmTransitionKind kind) => new(
        new StubAlarmHandleForTests(), SourceNodeId: "Temp", ConditionId: AlarmRawPath, AlarmType: "OffNormalAlarm",
        Message: "temperature high", Severity: AlarmSeverity.High, SourceTimestampUtc: DateTime.UtcNow, Kind: kind);

    private void RaiseAndExpectUnacked(IActorRef actor, TestProbe publish)
    {
        actor.Tell(new DriverInstanceActor.AttributeAlarmPublished("drv-1", Evt(AlarmTransitionKind.Raise)));
        var raised = NextAlarmUpdate(publish);
        raised.State.Active.ShouldBeTrue();
        raised.State.Acknowledged.ShouldBeFalse();
    }

    private static OpcUaPublishActor.AlarmStateUpdate NextAlarmUpdate(TestProbe publish) =>
        publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, Timeout);

    private sealed class StubAlarmHandleForTests : IAlarmSubscriptionHandle
    {
        public string DiagnosticId => "test-alarm-sub";
    }

    /// <summary>Spawns the host with the recording alarm-driver factory, dispatches the deployment, and waits
    /// for the Applied ACK so the apply (and thus the inverse-map build in PushDesiredSubscriptions) has
    /// completed before the test routes an ack.</summary>
    private IActorRef SpawnHostAndApply(
        IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory factory,
        IActorRef? opcUaPublishActor = null, NativeAlarmAckPreflightView? ackPreflightView = null)
    {
        var coordinator = CreateTestProbe();
        var actor = Sys.ActorOf(DriverHostActor.Props(
            db, TestNode, coordinator.Ref,
            driverFactory: factory,
            localRoles: new HashSet<string> { "driver" },
            opcUaPublishActor: opcUaPublishActor,
            ackPreflightView: ackPreflightView));

        actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);

        // Applied does NOT mean the driver child is Connected: the apply acks once the child is spawned,
        // and a RouteAlarmAck reaching a still-Connecting child is dropped by design (fire-and-forget, no
        // upstream to reach). Each test sends its positive-control ack exactly once, so on a loaded runner
        // the control itself was dropped and the wait read [] — 4 of 20 runs under thread-pool starvation,
        // and on Gitea (#606). SubscribeAlarmsAsync is only ever called from the Connected state (a
        // pre-connect request is swallowed and re-issued on the Connected entry), so wait for it.
        if (factory is RecordingAlarmDriverFactory recording)
            AwaitCondition(() => recording.AlarmsSubscribed, Timeout);

        return actor;
    }

    /// <summary>
    ///     Seeds a Sealed deployment whose artifact carries the v3 raw-tag chain (RawFolder "Plant" →
    ///     DriverInstance(RawFolderId, Name "gw", DriverType "GalaxyMxGateway", Enabled) → Device "dev1" → Tag)
    ///     with the tag's <c>TagConfig</c> carrying an <c>alarm</c> object so the composer projects a non-null
    ///     <c>EquipmentTagAlarmInfo</c> — making the raw tag a Part 9 condition at its RawPath. The non-Windows
    ///     <c>DriverType</c> + Enabled flag spawn a REAL <see cref="DriverInstanceActor"/> child.
    /// </summary>
    private static DeploymentId SeedV3AlarmDeployment(
        IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev, string Driver, string Tag)
    {
        var artifact = JsonSerializer.SerializeToUtf8Bytes(new
        {
            RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
            DriverInstances = new[]
            {
                new
                {
                    DriverInstanceRowId = Guid.NewGuid(),
                    DriverInstanceId = Driver,
                    RawFolderId = "rf-plant",
                    Name = "gw",
                    DriverType = "GalaxyMxGateway", // not Windows-only ⇒ a real child is spawned (not stubbed)
                    Enabled = true,
                    DriverConfig = "{}",
                    ClusterId = "c1",
                },
            },
            Devices = new[]
            {
                new { DeviceId = $"{Driver}:dev1", DriverInstanceId = Driver, Name = "dev1", DeviceConfig = "{}" },
            },
            TagGroups = Array.Empty<object>(),
            Tags = new[]
            {
                new
                {
                    TagId = "tag-0",
                    DeviceId = $"{Driver}:dev1",
                    TagGroupId = (string?)null,
                    Name = Tag,
                    DataType = "Boolean",
                    AccessLevel = 0, // TagAccessLevel.Read
                    TagConfig = JsonSerializer.Serialize(new { alarm = new { alarmType = "OffNormalAlarm", severity = 700 } }),
                },
            },
        });

        var id = DeploymentId.NewId();
        using var ctx = db.CreateDbContext();
        ctx.Deployments.Add(new Deployment
        {
            DeploymentId = id.Value,
            RevisionHash = rev.Value,
            Status = DeploymentStatus.Sealed,
            CreatedBy = "test",
            SealedAtUtc = DateTime.UtcNow,
            ArtifactBlob = artifact,
        });
        ctx.SaveChanges();
        return id;
    }

    /// <summary>Factory producing a single <see cref="RecordingAlarmDriver"/> for the supported type, whose
    /// recorded acknowledge list is exposed for assertions.</summary>
    private sealed class RecordingAlarmDriverFactory : IDriverFactory
    {
        private readonly string _supportedType;
        private readonly RecordingAlarmDriver _driver = new();
        public RecordingAlarmDriverFactory(string supportedType) { _supportedType = supportedType; }

        /// <summary>The acknowledge requests the spawned driver received (thread-safe — AcknowledgeAsync runs
        /// off the actor thread).</summary>
        public IReadOnlyList<AlarmAcknowledgeRequest> Acks => _driver.Acks;

        /// <summary>True once the driver child has reached Connected and opened its alarm subscription.</summary>
        public bool AlarmsSubscribed => _driver.AlarmsSubscribed;

        /// <summary>When set, every acknowledge is recorded and then fails with this exception.</summary>
        public Exception? AckFault { init => _driver.AckFault = value; }

        /// <summary>The driver's synchronous pre-check verdict (lmxopcua#760); null ⇒ always proceed.</summary>
        public Func<string?, string?>? PreflightRefusal { init => _driver.PreflightRefusal = value; }

        /// <inheritdoc />
        public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
        {
            if (!string.Equals(driverType, _supportedType, StringComparison.Ordinal)) return null;
            _driver.Bind(driverInstanceId, driverType);
            return _driver;
        }

        /// <inheritdoc />
        public IReadOnlyCollection<string> SupportedTypes => new[] { _supportedType };
    }

    /// <summary>An <see cref="IDriver"/> + <see cref="IAlarmSource"/> that records every acknowledge.</summary>
    private sealed class RecordingAlarmDriver : IDriver, IAlarmSource, IAlarmAcknowledgePreflight
    {
        /// <summary>When set, <see cref="AcknowledgeAsync"/> records the request and then throws this.</summary>
        public Exception? AckFault { get; set; }

        /// <summary>The <see cref="GetAcknowledgeRefusal"/> verdict; null ⇒ always proceed.</summary>
        public Func<string?, string?>? PreflightRefusal { get; set; }

        /// <inheritdoc />
        public string? GetAcknowledgeRefusal(string? apiKeyId) => PreflightRefusal?.Invoke(apiKeyId);

        private readonly ConcurrentQueue<AlarmAcknowledgeRequest> _acks = new();
        /// <inheritdoc />
        public string DriverInstanceId { get; private set; } = string.Empty;
        /// <inheritdoc />
        public string DriverType { get; private set; } = string.Empty;
        /// <summary>The acknowledge requests received so far.</summary>
        public IReadOnlyList<AlarmAcknowledgeRequest> Acks => _acks.ToArray();
        /// <summary>Sets the identity once the factory is asked to create it.</summary>
        public void Bind(string id, string type) { DriverInstanceId = id; DriverType = type; }
        /// <inheritdoc />
        public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
        /// <inheritdoc />
        public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
        /// <inheritdoc />
        public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
        /// <inheritdoc />
        public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, LastError: null);

        /// <inheritdoc />
        public Task<IAlarmSubscriptionHandle> SubscribeAlarmsAsync(
            IReadOnlyList<string> sourceNodeIds, CancellationToken cancellationToken)
        {
            Volatile.Write(ref _alarmsSubscribed, true);
            return Task.FromResult<IAlarmSubscriptionHandle>(new StubAlarmHandle());
        }

        private bool _alarmsSubscribed;

        /// <summary>True once <see cref="SubscribeAlarmsAsync"/> has been called (only ever from Connected).</summary>
        public bool AlarmsSubscribed => Volatile.Read(ref _alarmsSubscribed);

        /// <inheritdoc />
        public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) =>
            Task.CompletedTask;

        /// <inheritdoc />
        public Task AcknowledgeAsync(
            IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
        {
            foreach (var a in acknowledgements) _acks.Enqueue(a);
            return AckFault is { } fault ? Task.FromException(fault) : Task.CompletedTask;
        }

        /// <inheritdoc />
        public event EventHandler<AlarmEventArgs>? OnAlarmEvent
        {
            add { }
            remove { }
        }

        private sealed class StubAlarmHandle : IAlarmSubscriptionHandle
        {
            public string DiagnosticId => "stub-alarm-sub";
        }
    }
}