lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/WarmRedundancyPooledWireTests.cs

using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig;
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.WarmRedundancyWireTests;

namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;

/// <summary>
///     Process-wide state these tests observe — SDK 2.0.1's notification-payload pool and the Warm instruments — is
///     shared by every server and client in the test process, so they run alone.
/// </summary>
[CollectionDefinition(Name, DisableParallelization = true)]
public sealed class WarmRedundancyProcessWideCollection
{
    /// <summary>The collection name.</summary>
    public const string Name = "warm-redundancy-process-wide";
}

/// <summary>
///     Gitea #717 over the wire, where the observation is process-wide: a monitored-item ClientHandle reused after a
///     delete while the SDK's payload pool hands out notifications that still carry an old ordinary-tag marker (T7),
///     and the two OpcUaServer-side Warm instruments (T10), <c>standby.projected_notifications</c> and
///     <c>standby.suppressed_alarm_events</c>, each selected by instrument INSTANCE (#503).
/// </summary>
[Trait("Category", "Integration")]
[Collection(WarmRedundancyProcessWideCollection.Name)]
public sealed class WarmRedundancyPooledWireTests
{
    private const NodePermissions All =
        NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead
        | NodePermissions.AlarmAcknowledge | NodePermissions.AlarmConfirm | NodePermissions.AlarmShelve
        | NodePermissions.MethodCall | NodePermissions.WriteOperate;

    /// <summary>
    ///     SDK 2.0.1 returns acknowledged (and evicted) notification payloads to a process-wide pool, and
    ///     <c>Initialize()</c> clears only ClientHandle and Value, so a payload stamped for an ordinary tag goes back out
    ///     to the next item that samples — with its old marker. Here an ordinary-tag item's payloads are delivered and
    ///     acknowledged in one go, the item is deleted, and its ClientHandle is reused by an item on a NON-ordinary node
    ///     (<c>i=2267</c>): on the Standby every value of that item must stay Good, because the stale marker names
    ///     another value. The test first proves the pool really does hold a marked payload at that moment, and an
    ///     ordinary-tag item reusing the handle afterwards is projected (positive control).
    /// </summary>
    [Fact]
    public async Task A_client_handle_reused_after_delete_by_a_non_ordinary_item_is_never_projected_from_a_recycled_payload()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717Reuse", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        var raw = rig.EnsureRawVariable("reuse", "Double", writable: false);
        rig.Sink.WriteValue(raw, 0.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        Promote(projection);
        var serviceLevel = new SdkServiceLevelPublisher(rig.Server.CurrentInstance!,
            Microsoft.Extensions.Logging.Abstractions.NullLogger<SdkServiceLevelPublisher>.Instance);
        const uint handle = 7;

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        var client = (SessionClient)session;
        var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
        var tagItem = await AddItemAsync(client, sub, Raw(session, raw), handle, 1, ct);

        // Deliver a run of marked notifications WITHOUT acknowledging, then acknowledge them all at once, so a batch of
        // marked payloads is back in the pool together (evicted ones are returned too).
        var delivered = new List<uint>();
        for (var i = 1; i <= 24; i++)
        {
            rig.Sink.WriteValue(raw, i, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
            var r = await PublishDataAsync(client, ct);
            delivered.Add(r.NotificationMessage.SequenceNumber);
        }

        StatusCode.IsGood((await client.DeleteMonitoredItemsAsync(H(), sub, new[] { tagItem }, ct)).Results[0]).ShouldBeTrue();
        await client.PublishAsync(H(), delivered.Select(seq => new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = seq }).ToArray(), ct);

        var levelItem = await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, handle, 10, ct);
        await CollectAsync(client, ct, TimeSpan.FromMilliseconds(500)); // the initial sample, delivered while Active
        DemoteDeliveryOnly(projection);

        // Artifact before outcome: the pool's next payload really is a recycled ordinary-tag one. Peek and put it back.
        var peeked = (MonitoredItemNotification)MonitoredItemNotificationActivator.Instance.CreateInstance();
        peeked.Message.ShouldBeOfType<OrdinaryTagMarker>("the precondition: the SDK pool hands out a payload still carrying a marker");
        peeked.Reuse();

        var levels = new byte[] { 91, 92, 93, 94, 95, 96, 97, 98 };
        foreach (var level in levels)
        {
            serviceLevel.Publish(level);
            await Task.Delay(60, ct);
        }

        var standby = await CollectAsync(client, ct, TimeSpan.FromSeconds(1));
        standby.Where(x => x.Handle == handle).Select(x => Convert.ToByte(x.Value.ClrValue())).ShouldBe(levels,
            "every i=2267 change reaches the client Good: a recycled payload's stale marker never validates");
        standby.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);

        // Positive control: an ordinary-tag item reusing the handle IS projected on the same Standby.
        StatusCode.IsGood((await client.DeleteMonitoredItemsAsync(H(), sub, new[] { levelItem }, ct)).Results[0]).ShouldBeTrue();
        rig.Sink.WriteValue(raw, 99.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        await AddItemAsync(client, sub, Raw(session, raw), handle, 10, ct);
        var control = await CollectAsync(client, ct, TimeSpan.FromSeconds(1));
        control.Where(x => x.Handle == handle).ShouldNotBeEmpty();
        control.Where(x => x.Handle == handle).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
    }

    /// <summary>Sums every measurement of ONE counter instance.</summary>
    private sealed class CounterTotal : IDisposable
    {
        private readonly MeterListener _listener = new();
        private long _total;

        public CounterTotal(Instrument instrument)
        {
            _listener.InstrumentPublished = (published, l) =>
            {
                if (ReferenceEquals(published, instrument)) l.EnableMeasurementEvents(published);
            };
            _listener.SetMeasurementEventCallback<long>((_, value, _, _) => Interlocked.Add(ref _total, value));
            _listener.Start();
        }

        public long Total => Interlocked.Read(ref _total);

        public void Dispose() => _listener.Dispose();
    }

    /// <summary>
    ///     <c>standby.projected_notifications</c> counts each notification the Standby rewrites, at Publish and again at
    ///     Republish (each is a delivery), and nothing while Active.
    /// </summary>
    [Fact]
    public async Task Projected_notifications_counts_each_rewritten_notification_per_delivery_and_nothing_while_Active()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717ProjMetric", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        var raw = rig.EnsureRawVariable("m", "Double", writable: false);
        rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        Promote(projection);
        using var counter = new CounterTotal(OtOpcUaTelemetry.StandbyProjectedNotifications);

        using var session = await rig.ConnectAsync(anonymous: true, ct);
        var client = (SessionClient)session;
        var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
        await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
        await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, 2, 1, ct);
        await PublishDataAsync(client, ct);
        counter.Total.ShouldBe(0, "nothing is rewritten while Active");

        await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
        rig.Sink.WriteValue(raw, 10.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        rig.Sink.WriteValue(raw, 11.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        rig.Sink.WriteValue(raw, 12.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
        new SdkServiceLevelPublisher(rig.Server.CurrentInstance!,
            Microsoft.Extensions.Logging.Abstractions.NullLogger<SdkServiceLevelPublisher>.Instance).Publish(90);
        DemoteDeliveryOnly(projection);
        await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);

        var msg = await PublishDataAsync(client, ct);
        var changes = DataChanges(msg.NotificationMessage);
        var rewritten = changes.Count(x => x.Handle == 1 && x.Value.StatusCode == StatusCodes.BadOutOfService);
        rewritten.ShouldBe(3, "the three queued Good samples");
        changes.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good, "i=2267 is not rewritten, nor counted");
        counter.Total.ShouldBe(rewritten);

        await client.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct);
        counter.Total.ShouldBe(2 * rewritten, "a Republish is another delivery, rewritten and counted again");
    }

    /// <summary>
    ///     <c>standby.suppressed_alarm_events</c> counts one per new alarm event withheld on the Standby, and none once
    ///     Active (where the same transition is delivered).
    /// </summary>
    [Fact]
    public async Task Suppressed_alarm_events_counts_each_withheld_event_on_the_Standby_and_none_once_Active()
    {
        var ct = Deadline();
        await using var rig = await StartModeAsync("Warm717AlarmMetric", "Warm", ct);
        var projection = rig.Server.CreateStandbyProjection()!;
        rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
        rig.MaterialiseScripted();
        using var counter = new CounterTotal(OtOpcUaTelemetry.StandbySuppressedAlarmEvents);

        using var session = await rig.ConnectAsync(anonymous: false, ct);
        var events = new RecordedEvents();
        var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
        var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
        await subscription.ApplyChangesAsync(ct);

        rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "standby-1"));
        rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 600, Message: "standby-2"));
        counter.Total.ShouldBe(2, "one per withheld transition");

        Promote(projection);
        await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType));
        rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
            Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "active"));
        await RecordedEvents.WaitUntilAsync(() => events.Has(serverItem, "active"));
        counter.Total.ShouldBe(2, "a delivered event is not counted");
        events.Has(serverItem, "standby-1").ShouldBeFalse("withheld events are never delivered as transitions");
        await subscription.DeleteAsync(true, ct);
    }
}