lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/NativeAlarmRestatementEndToEndTests.cs

using System.Runtime.CompilerServices;
using System.Text.Json;
using System.Threading.Channels;
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.TestKit;
using Google.Protobuf.WellKnownTypes;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
using ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using LocalizedText = Opc.Ua.LocalizedText;
using Proto = ZB.MOM.WW.MxGateway.Contracts.Proto;
using UaNodeId = Opc.Ua.NodeId;
using Variant = Opc.Ua.Variant;
using VariantCollection = Opc.Ua.VariantCollection;

namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;

/// <summary>
///     Gitea #656 — native-alarm RE-STATEMENTS end to end: the REAL Galaxy driver (over the REAL
///     <see cref="GatewayGalaxyAlarmFeed"/> reading a scripted gateway stream) and the REAL OpcUaClient driver raise
///     into the REAL <see cref="DriverHostActor"/>, and the test reads what its <c>ForwardNativeAlarm</c> +
///     <c>NativeAlarmProjector</c> turned them into: the Part 9 condition updates and the <c>/alerts</c> rows.
/// </summary>
/// <remarks>
///     Every "no row" assertion is followed by a transition that MUST write one — an absence check on its own passes
///     against a host that never publishes anything.
/// </remarks>
public sealed class NativeAlarmRestatementEndToEndTests : RuntimeActorTestBase
{
    private static readonly NodeId TestNode = NodeId.Parse("alarm-restatement-e2e");
    private static readonly RevisionHash Rev = RevisionHash.Parse(new string('a', 64));
    private static readonly TimeSpan Wait = TimeSpan.FromSeconds(5);
    private static readonly TimeSpan Quiet = TimeSpan.FromMilliseconds(600);

    // ---------------------------------------------------------------------------------------------------------
    // Galaxy
    // ---------------------------------------------------------------------------------------------------------

    private const string GalaxyDriverId = "drv-galaxy";
    private const string GalaxyAlarmRef = "Tank01.Level.HiHi";
    private const string GalaxyRawPath = "Plant/Galaxy/gw/tank01_hihi";

    /// <summary>
    ///     #656 claim 1 — a cold host receives an ActiveAcked snapshot entry (an alarm active AND acknowledged in
    ///     AVEVA before this process started). Pre-fix the feed sent Acknowledge, which the projector applied to its
    ///     inactive default: the condition was INACTIVE+acked — invisible to clients — with an "Acknowledged" row for
    ///     an alarm the server showed as inactive. Now: ONE Active+acked update, "Activated" (at the original raise
    ///     time, no operator comment) then "Acknowledged" (carrying the comment).
    /// </summary>
    [Fact]
    public async Task Galaxy_cold_host_ActiveAcked_snapshot_projects_active_and_acked_with_both_rows()
    {
        var rig = await GalaxyRig.StartAsync(this);
        try
        {
            var raisedAt = new DateTime(2026, 10, 8, 9, 0, 0, DateTimeKind.Utc);
            var ackedAt = raisedAt.AddMinutes(2);
            rig.Send(Snapshot(Proto.AlarmConditionState.ActiveAcked, raisedAt, ackedAt, "alice", "investigating"));

            var update = NextAlarmUpdate(rig.Publish);
            update.AlarmNodeId.ShouldBe(GalaxyRawPath);
            update.State.Active.ShouldBeTrue("an acked active alarm must project ACTIVE, not keep the cold default");
            update.State.Acknowledged.ShouldBeTrue();

            var activated = rig.Alerts.ExpectMsg<AlarmTransitionEvent>(Wait);
            activated.TransitionKind.ShouldBe("Activated");
            activated.Comment.ShouldBeNull("the operator comment belongs to the acknowledgement");
            activated.TimestampUtc.ShouldBe(raisedAt);
            var acked = rig.Alerts.ExpectMsg<AlarmTransitionEvent>(Wait);
            acked.TransitionKind.ShouldBe("Acknowledged");
            acked.Comment.ShouldBe("investigating");
            rig.Alerts.ExpectNoMsg(Quiet);

            // Positive control: a live Clear MUST write its row — and the NEXT condition update is the clear, so no
            // unacked intermediate update sat between the snapshot's projection and it.
            rig.Send(Live(Proto.AlarmTransitionKind.Clear, ackedAt.AddMinutes(1)));
            var cleared = NextAlarmUpdate(rig.Publish);
            cleared.State.Active.ShouldBeFalse();
            cleared.State.Acknowledged.ShouldBeTrue();
            rig.Alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Cleared");
        }
        finally
        {
            await rig.DisposeAsync();
        }
    }

    /// <summary>
    ///     #656 claim 2 — every gateway reconnect replays the active-alarm snapshot. Pre-fix an unacked active entry
    ///     (a Retrigger) wrote a second "Activated" row on EVERY reconnect, for a condition whose node did not change
    ///     (so clients saw no event: history and the node disagreed). Likewise an Inactive entry wrote a duplicate
    ///     "Cleared". Now a replay re-projects the condition but writes no row; a live transition still does.
    /// </summary>
    [Fact]
    public async Task Galaxy_reconnect_snapshot_replay_writes_no_duplicate_rows()
    {
        var rig = await GalaxyRig.StartAsync(this);
        try
        {
            var raisedAt = new DateTime(2026, 10, 8, 9, 0, 0, DateTimeKind.Utc);
            rig.Send(Snapshot(Proto.AlarmConditionState.Active, raisedAt, raisedAt));
            NextAlarmUpdate(rig.Publish).State.Active.ShouldBeTrue();
            rig.Alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Activated",
                "the first statement on a cold host is a real activation");

            // The gateway stream faults; the feed reopens and the gateway replays the same snapshot.
            rig.Fault();
            rig.Send(Snapshot(Proto.AlarmConditionState.Active, raisedAt, raisedAt));
            var replayed = NextAlarmUpdate(rig.Publish);
            replayed.State.Active.ShouldBeTrue();
            replayed.State.Acknowledged.ShouldBeFalse();
            rig.Alerts.ExpectNoMsg(Quiet);

            // Positive control: a live Clear MUST write its row.
            rig.Send(Live(Proto.AlarmTransitionKind.Clear, raisedAt.AddMinutes(1)));
            NextAlarmUpdate(rig.Publish).State.Active.ShouldBeFalse();
            rig.Alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Cleared");

            // Reconnect again; the snapshot now carries the alarm as Inactive — a re-stated Clear of an inactive
            // condition, no row.
            rig.Fault();
            rig.Send(Snapshot(Proto.AlarmConditionState.Inactive, raisedAt, raisedAt.AddMinutes(1)));
            NextAlarmUpdate(rig.Publish).State.Active.ShouldBeFalse();
            rig.Alerts.ExpectNoMsg(Quiet);

            // Positive control: a LIVE Retrigger (a genuine MXAccess re-occurrence) is never a re-statement.
            rig.Send(Live(Proto.AlarmTransitionKind.Retrigger, raisedAt.AddMinutes(3)));
            NextAlarmUpdate(rig.Publish).State.Active.ShouldBeTrue();
            rig.Alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Activated");
        }
        finally
        {
            await rig.DisposeAsync();
        }
    }

    private static Proto.AlarmFeedMessage Snapshot(
        Proto.AlarmConditionState state, DateTime raisedAt, DateTime lastTransition, string user = "", string comment = "")
        => new()
        {
            ActiveAlarm = new Proto.ActiveAlarmSnapshot
            {
                AlarmFullReference = GalaxyAlarmRef,
                SourceObjectReference = "Tank01",
                AlarmTypeName = "AnalogLimitAlarm.HiHi",
                Severity = 750,
                CurrentState = state,
                Category = "Process",
                Description = "Tank 01 high-high level",
                OriginalRaiseTimestamp = Timestamp.FromDateTime(raisedAt),
                LastTransitionTimestamp = Timestamp.FromDateTime(lastTransition),
                OperatorUser = user,
                OperatorComment = comment,
            },
        };

    private static Proto.AlarmFeedMessage Live(Proto.AlarmTransitionKind kind, DateTime at)
        => new()
        {
            Transition = new Proto.OnAlarmTransitionEvent
            {
                AlarmFullReference = GalaxyAlarmRef,
                SourceObjectReference = "Tank01",
                AlarmTypeName = "AnalogLimitAlarm.HiHi",
                TransitionKind = kind,
                Severity = 750,
                Category = "Process",
                Description = "Tank 01 high-high level",
                TransitionTimestamp = Timestamp.FromDateTime(at),
            },
        };

    /// <summary>The real Galaxy driver over the real alarm feed, reading a scripted gateway stream.</summary>
    private sealed class GalaxyRig : IAsyncDisposable
    {
        private readonly Channel<object> _stream = Channel.CreateUnbounded<object>();
        private GalaxyDriver? _driver;

        public TestProbe Publish { get; private set; } = null!;
        public TestProbe Alerts { get; private set; } = null!;

        public static async Task<GalaxyRig> StartAsync(NativeAlarmRestatementEndToEndTests t)
        {
            var rig = new GalaxyRig();
            var db = NewInMemoryDbFactory();
            var blob = BuildArtifact(GalaxyDriverId, DriverTypeNames.Galaxy, "Galaxy", "gw", "tank01_hihi",
                new { attributeRef = GalaxyAlarmRef, alarm = new { alarmType = "AlarmCondition", severity = 800 } });
            rig.Alerts = t.SubscribeAlerts();
            var (host, publish, _) = t.SpawnHostAndApply(db, Seed(db, blob, Rev));
            rig.Publish = publish;

            var rawTags = RawTagsFor(blob, GalaxyDriverId);
            rawTags.Select(r => r.RawPath).ShouldBe([GalaxyRawPath]);
            var feed = new GatewayGalaxyAlarmFeed(rig.OpenStream, clientName: "Restatement656",
                reconnectDelay: TimeSpan.FromMilliseconds(20));
            var options = new GalaxyDriverOptions(
                new GalaxyGatewayOptions("http://localhost:5000", "literal-api-key"),
                new GalaxyMxAccessOptions("Restatement656"),
                new GalaxyRepositoryOptions(),
                new GalaxyReconnectOptions()) { RawTags = rawTags };
            rig._driver = new GalaxyDriver(GalaxyDriverId, options, hierarchySource: null, alarmFeed: feed);
            rig._driver.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(GalaxyDriverId, e));
#pragma warning disable OTOPCUA0001 // the routing is under test, not resilience wrapping
            await rig._driver.SubscribeAlarmsAsync([GalaxyRawPath], CancellationToken.None);
#pragma warning restore OTOPCUA0001
            return rig;
        }

        public void Send(Proto.AlarmFeedMessage message) => _stream.Writer.TryWrite(message).ShouldBeTrue();

        /// <summary>Fault the current gateway stream; the feed reopens it (the next open reads on).</summary>
        public void Fault() => _stream.Writer.TryWrite(new IOException("synthetic gateway stream fault")).ShouldBeTrue();

        private async IAsyncEnumerable<Proto.AlarmFeedMessage> OpenStream(
            Proto.StreamAlarmsRequest request, [EnumeratorCancellation] CancellationToken ct)
        {
            while (true)
            {
                var item = await _stream.Reader.ReadAsync(ct);
                if (item is Exception ex) throw ex;
                yield return (Proto.AlarmFeedMessage)item;
            }
        }

        public async ValueTask DisposeAsync()
        {
            if (_driver is not null) await _driver.ShutdownAsync(CancellationToken.None);
            _driver?.Dispose();
        }
    }

    // ---------------------------------------------------------------------------------------------------------
    // OpcUaClient
    // ---------------------------------------------------------------------------------------------------------

    private const string OpcDriverId = "drv-opcua";
    private const string OpcRawPath = "Plant/Opc/plc/trip1";
    private const string UpstreamSource = "ns=3;s=Alarms.Trip1";

    /// <summary>
    ///     #656 claim 2 (OpcUaClient) — a deploy that changes the driver's config RESPAWNS it, and the fresh instance's
    ///     ConditionRefresh re-reads every retained condition with no record of what the host was told. Its first
    ///     sight of an unacked active alarm is a Retrigger (#647: upstream's unack is authoritative); pre-fix that
    ///     wrote a second "Activated" row for a condition already showing exactly that. Now it is a flagged
    ///     re-statement: the condition is re-projected (still unacked) and no row is written.
    /// </summary>
    [Fact]
    public async Task OpcUaClient_respawn_restating_an_unacked_active_alarm_writes_no_second_Activated_row()
    {
        var db = NewInMemoryDbFactory();
        var blob = BuildOpcUaArtifact();
        var alerts = SubscribeAlerts();
        var (host, publish, coordinator) = SpawnHostAndApply(db, Seed(db, blob, Rev));
        var handle = new Handle();

        using (var a = NewOpcUaDriver(host, "a"))
        {
            a.RouteAlarmEvent(ConditionFields(active: true, acked: false), SourceIndex, handle);
            var raised = NextAlarmUpdate(publish);
            raised.AlarmNodeId.ShouldBe(OpcRawPath);
            raised.State.Active.ShouldBeTrue();
            raised.State.Acknowledged.ShouldBeFalse();
            alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Activated");
        }

        // A config-changing deploy: the host keeps the still-deployed condition's state, the driver is respawned.
        var revB = RevisionHash.Parse(new string('b', 64));
        host.Tell(new DispatchDeployment(Seed(db, blob, revB), revB, CorrelationId.NewId()));
        coordinator.FishForMessage<ApplyAck>(x => x.Outcome == ApplyAckOutcome.Applied, Wait);
        publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, Wait);

        using var b = NewOpcUaDriver(host, "b");
        b.RouteAlarmEvent(ConditionFields(active: true, acked: false), SourceIndex, handle); // ConditionRefresh copy
        var restated = NextAlarmUpdate(publish);
        restated.State.Active.ShouldBeTrue();
        restated.State.Acknowledged.ShouldBeFalse("upstream says unacknowledged");
        alerts.ExpectNoMsg(Quiet);

        // Positive control: the upstream operator acknowledges — that transition MUST write its row.
        b.RouteAlarmEvent(ConditionFields(active: true, acked: true), SourceIndex, handle);
        var acked = NextAlarmUpdate(publish);
        acked.State.Acknowledged.ShouldBeTrue();
        alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Acknowledged");
    }

    /// <summary>
    ///     #656 related item 1 (OpcUaClient) — an upstream condition first seen Active+acked used to go out as
    ///     Raise-then-Acknowledge: TWO condition updates, the first Active+UNacked (a client-visible un-ack window).
    ///     Now it is ONE ActiveAcknowledged: the first condition update is already acked, with "Activated" then
    ///     "Acknowledged" rows.
    /// </summary>
    [Fact]
    public void OpcUaClient_first_sight_of_an_acked_active_alarm_is_one_active_and_acked_update()
    {
        var db = NewInMemoryDbFactory();
        var alerts = SubscribeAlerts();
        var (host, publish, _) = SpawnHostAndApply(db, Seed(db, BuildOpcUaArtifact(), Rev));
        var handle = new Handle();
        using var drv = NewOpcUaDriver(host, "cold");

        drv.RouteAlarmEvent(ConditionFields(active: true, acked: true), SourceIndex, handle);
        var first = NextAlarmUpdate(publish);
        first.State.Active.ShouldBeTrue();
        first.State.Acknowledged.ShouldBeTrue("no Active+unacked intermediate may reach clients");
        alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Activated");
        alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Acknowledged");
        alerts.ExpectNoMsg(Quiet);

        // Positive control: the next update is the clear (not a leftover second update), and it writes its row.
        drv.RouteAlarmEvent(ConditionFields(active: false, acked: true), SourceIndex, handle);
        var cleared = NextAlarmUpdate(publish);
        cleared.State.Active.ShouldBeFalse();
        alerts.ExpectMsg<AlarmTransitionEvent>(Wait).TransitionKind.ShouldBe("Cleared");
    }

    private static readonly Dictionary<string, string> SourceIndex = new() { [UpstreamSource] = OpcRawPath };

    private static OpcUaClientDriver NewOpcUaDriver(IActorRef host, string suffix)
    {
        var drv = new OpcUaClientDriver(new OpcUaClientDriverOptions(), $"{OpcDriverId}-{suffix}");
        drv.OnAlarmEvent += (_, e) => host.Tell(new DriverInstanceActor.AttributeAlarmPublished(OpcDriverId, e));
        return drv;
    }

    /// <summary>An upstream A&amp;C condition event, in the driver's select-clause order.</summary>
    private static VariantCollection ConditionFields(bool active, bool acked) =>
    [
        new Variant(new byte[] { 1 }),                         // EventId
        new Variant(new UaNodeId("TripAlarmType", 3)),         // EventType
        new Variant(UaNodeId.Parse(UpstreamSource)),           // SourceNode
        new Variant(new LocalizedText("tripped")),             // Message
        new Variant((ushort)700),                              // Severity
        new Variant(DateTime.UtcNow),                          // Time
        new Variant(UaNodeId.Parse(UpstreamSource + ".Cond")), // ConditionId
        new Variant(active),                                   // ActiveState/Id
        new Variant(acked),                                    // AckedState/Id
        Variant.Null,                                          // BranchId
    ];

    private static byte[] BuildOpcUaArtifact() => BuildArtifact(OpcDriverId, DriverTypeNames.OpcUaClient, "Opc", "plc", "trip1",
        new { nodeId = UpstreamSource, alarm = new { alarmType = "AlarmCondition", severity = 700 } });

    private sealed class Handle : IAlarmSubscriptionHandle
    {
        public string DiagnosticId => "e2e";
    }

    // ---------------------------------------------------------------------------------------------------------
    // Shared host harness (as FocasNativeAlarmEndToEndTests)
    // ---------------------------------------------------------------------------------------------------------

    /// <summary>The next condition update the host sent to the publish actor (skipping anything else).</summary>
    private static OpcUaPublishActor.AlarmStateUpdate NextAlarmUpdate(TestProbe publish) =>
        publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(_ => true, Wait);

    private TestProbe SubscribeAlerts()
    {
        var alerts = CreateTestProbe();
        DistributedPubSub.Get(Sys).Mediator.Tell(new Subscribe(ScriptedAlarmHostActor.AlertsTopic, alerts.Ref), alerts.Ref);
        alerts.ExpectMsg<SubscribeAck>(Wait);
        return alerts;
    }

    private static List<RawTagEntry> RawTagsFor(byte[] blob, string driverInstanceId)
    {
        using var artifact = DeploymentArtifact.Open(blob);
        var spec = artifact.ParseDriverInstances().Single(s => s.DriverInstanceId == driverInstanceId);
        using var doc = JsonDocument.Parse(spec.DriverConfig);
        return JsonSerializer.Deserialize<List<RawTagEntry>>(
            doc.RootElement.GetProperty("RawTags").GetRawText(),
            new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
    }

    private static byte[] BuildArtifact(
        string driverId, string driverType, string driverName, string deviceName, string tagName, object tagConfig) =>
        JsonSerializer.SerializeToUtf8Bytes(new
        {
            RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
            DriverInstances = new[]
            {
                // Enabled=false: the host must not spawn a real wire driver — the test drives its own instance.
                new { DriverInstanceId = driverId, RawFolderId = "rf-plant", Name = driverName, DriverType = driverType, DriverConfig = "{}", ClusterId = "c1", Enabled = false },
            },
            Devices = new[]
            {
                new { DeviceId = "dev-1", DriverInstanceId = driverId, Name = deviceName, DeviceConfig = "{}" },
            },
            TagGroups = Array.Empty<object>(),
            Tags = new[]
            {
                new
                {
                    TagId = "tag-alarm",
                    DeviceId = "dev-1",
                    TagGroupId = (string?)null,
                    Name = tagName,
                    DataType = "Boolean",
                    AccessLevel = 0,
                    TagConfig = JsonSerializer.Serialize(tagConfig),
                },
            },
        });

    private static DeploymentId Seed(IDbContextFactory<OtOpcUaConfigDbContext> db, byte[] blob, RevisionHash rev)
    {
        var id = DeploymentId.NewId();
        using var ctx = db.CreateDbContext();
        ctx.Deployments.Add(new Deployment
        {
            DeploymentId = id.Value,
            RevisionHash = rev.Value,
            Status = DeploymentStatus.Sealed,
            CreatedBy = "test",
            SealedAtUtc = DateTime.UtcNow,
            ArtifactBlob = blob,
        });
        ctx.SaveChanges();
        return id;
    }

    private (IActorRef Actor, TestProbe Publish, TestProbe Coordinator) SpawnHostAndApply(
        IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId)
    {
        var coordinator = CreateTestProbe();
        var publish = this.CreateCleanOutcomePublishProbe();
        var actor = Sys.ActorOf(DriverHostActor.Props(
            db, TestNode, coordinator.Ref,
            localRoles: new HashSet<string> { "driver" },
            dependencyMux: CreateTestProbe().Ref,
            opcUaPublishActor: publish.Ref,
            virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
            virtualTagHostOverride: CreateTestProbe().Ref));

        actor.Tell(new DispatchDeployment(deploymentId, Rev, CorrelationId.NewId()));
        coordinator.ExpectMsg<ApplyAck>(Wait).Outcome.ShouldBe(ApplyAckOutcome.Applied);
        publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(Wait);
        return (actor, publish, coordinator);
    }
}