using System.Collections.Concurrent; using System.Diagnostics.Metrics; using Opc.Ua; using Opc.Ua.Client; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig; using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.WarmRedundancyWireTests; namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// Process-wide state these tests observe — SDK 2.0.1's notification-payload pool and the Warm instruments — is /// shared by every server and client in the test process, so they run alone. /// [CollectionDefinition(Name, DisableParallelization = true)] public sealed class WarmRedundancyProcessWideCollection { /// The collection name. public const string Name = "warm-redundancy-process-wide"; } /// /// Gitea #717 over the wire, where the observation is process-wide: a monitored-item ClientHandle reused after a /// delete while the SDK's payload pool hands out notifications that still carry an old ordinary-tag marker (T7), /// and the two OpcUaServer-side Warm instruments (T10), standby.projected_notifications and /// standby.suppressed_alarm_events, each selected by instrument INSTANCE (#503). /// [Trait("Category", "Integration")] [Collection(WarmRedundancyProcessWideCollection.Name)] public sealed class WarmRedundancyPooledWireTests { private const NodePermissions All = NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead | NodePermissions.AlarmAcknowledge | NodePermissions.AlarmConfirm | NodePermissions.AlarmShelve | NodePermissions.MethodCall | NodePermissions.WriteOperate; /// /// SDK 2.0.1 returns acknowledged (and evicted) notification payloads to a process-wide pool, and /// Initialize() clears only ClientHandle and Value, so a payload stamped for an ordinary tag goes back out /// to the next item that samples — with its old marker. Here an ordinary-tag item's payloads are delivered and /// acknowledged in one go, the item is deleted, and its ClientHandle is reused by an item on a NON-ordinary node /// (i=2267): on the Standby every value of that item must stay Good, because the stale marker names /// another value. The test first proves the pool really does hold a marked payload at that moment, and an /// ordinary-tag item reusing the handle afterwards is projected (positive control). /// [Fact] public async Task A_client_handle_reused_after_delete_by_a_non_ordinary_item_is_never_projected_from_a_recycled_payload() { var ct = Deadline(); await using var rig = await StartModeAsync("Warm717Reuse", "Warm", ct); var projection = rig.Server.CreateStandbyProjection()!; var raw = rig.EnsureRawVariable("reuse", "Double", writable: false); rig.Sink.WriteValue(raw, 0.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); Promote(projection); var serviceLevel = new SdkServiceLevelPublisher(rig.Server.CurrentInstance!, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); const uint handle = 7; using var session = await rig.ConnectAsync(anonymous: true, ct); var client = (SessionClient)session; var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId; var tagItem = await AddItemAsync(client, sub, Raw(session, raw), handle, 1, ct); // Deliver a run of marked notifications WITHOUT acknowledging, then acknowledge them all at once, so a batch of // marked payloads is back in the pool together (evicted ones are returned too). var delivered = new List(); for (var i = 1; i <= 24; i++) { rig.Sink.WriteValue(raw, i, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); var r = await PublishDataAsync(client, ct); delivered.Add(r.NotificationMessage.SequenceNumber); } StatusCode.IsGood((await client.DeleteMonitoredItemsAsync(H(), sub, new[] { tagItem }, ct)).Results[0]).ShouldBeTrue(); await client.PublishAsync(H(), delivered.Select(seq => new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = seq }).ToArray(), ct); var levelItem = await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, handle, 10, ct); await CollectAsync(client, ct, TimeSpan.FromMilliseconds(500)); // the initial sample, delivered while Active DemoteDeliveryOnly(projection); // Artifact before outcome: the pool's next payload really is a recycled ordinary-tag one. Peek and put it back. var peeked = (MonitoredItemNotification)MonitoredItemNotificationActivator.Instance.CreateInstance(); peeked.Message.ShouldBeOfType("the precondition: the SDK pool hands out a payload still carrying a marker"); peeked.Reuse(); var levels = new byte[] { 91, 92, 93, 94, 95, 96, 97, 98 }; foreach (var level in levels) { serviceLevel.Publish(level); await Task.Delay(60, ct); } var standby = await CollectAsync(client, ct, TimeSpan.FromSeconds(1)); standby.Where(x => x.Handle == handle).Select(x => Convert.ToByte(x.Value.ClrValue())).ShouldBe(levels, "every i=2267 change reaches the client Good: a recycled payload's stale marker never validates"); standby.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good); // Positive control: an ordinary-tag item reusing the handle IS projected on the same Standby. StatusCode.IsGood((await client.DeleteMonitoredItemsAsync(H(), sub, new[] { levelItem }, ct)).Results[0]).ShouldBeTrue(); rig.Sink.WriteValue(raw, 99.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); await AddItemAsync(client, sub, Raw(session, raw), handle, 10, ct); var control = await CollectAsync(client, ct, TimeSpan.FromSeconds(1)); control.Where(x => x.Handle == handle).ShouldNotBeEmpty(); control.Where(x => x.Handle == handle).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService); } /// Sums every measurement of ONE counter instance. private sealed class CounterTotal : IDisposable { private readonly MeterListener _listener = new(); private long _total; public CounterTotal(Instrument instrument) { _listener.InstrumentPublished = (published, l) => { if (ReferenceEquals(published, instrument)) l.EnableMeasurementEvents(published); }; _listener.SetMeasurementEventCallback((_, value, _, _) => Interlocked.Add(ref _total, value)); _listener.Start(); } public long Total => Interlocked.Read(ref _total); public void Dispose() => _listener.Dispose(); } /// /// standby.projected_notifications counts each notification the Standby rewrites, at Publish and again at /// Republish (each is a delivery), and nothing while Active. /// [Fact] public async Task Projected_notifications_counts_each_rewritten_notification_per_delivery_and_nothing_while_Active() { var ct = Deadline(); await using var rig = await StartModeAsync("Warm717ProjMetric", "Warm", ct); var projection = rig.Server.CreateStandbyProjection()!; var raw = rig.EnsureRawVariable("m", "Double", writable: false); rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); Promote(projection); using var counter = new CounterTotal(OtOpcUaTelemetry.StandbyProjectedNotifications); using var session = await rig.ConnectAsync(anonymous: true, ct); var client = (SessionClient)session; var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId; await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct); await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, 2, 1, ct); await PublishDataAsync(client, ct); counter.Total.ShouldBe(0, "nothing is rewritten while Active"); await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct); rig.Sink.WriteValue(raw, 10.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); rig.Sink.WriteValue(raw, 11.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); rig.Sink.WriteValue(raw, 12.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw); new SdkServiceLevelPublisher(rig.Server.CurrentInstance!, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance).Publish(90); DemoteDeliveryOnly(projection); await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct); var msg = await PublishDataAsync(client, ct); var changes = DataChanges(msg.NotificationMessage); var rewritten = changes.Count(x => x.Handle == 1 && x.Value.StatusCode == StatusCodes.BadOutOfService); rewritten.ShouldBe(3, "the three queued Good samples"); changes.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good, "i=2267 is not rewritten, nor counted"); counter.Total.ShouldBe(rewritten); await client.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct); counter.Total.ShouldBe(2 * rewritten, "a Republish is another delivery, rewritten and counted again"); } /// /// standby.suppressed_alarm_events counts one per new alarm event withheld on the Standby, and none once /// Active (where the same transition is delivered). /// [Fact] public async Task Suppressed_alarm_events_counts_each_withheld_event_on_the_Standby_and_none_once_Active() { var ct = Deadline(); await using var rig = await StartModeAsync("Warm717AlarmMetric", "Warm", ct); var projection = rig.Server.CreateStandbyProjection()!; rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All)); rig.MaterialiseScripted(); using var counter = new CounterTotal(OtOpcUaTelemetry.StandbySuppressedAlarmEvents); using var session = await rig.ConnectAsync(anonymous: false, ct); var events = new RecordedEvents(); var subscription = await RecordedEvents.SubscribeAsync(session, events, ct); var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server); await subscription.ApplyChangesAsync(ct); rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "standby-1")); rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, Shelving: AlarmShelvingKind.Unshelved, Severity: 600, Message: "standby-2")); counter.Total.ShouldBe(2, "one per withheld transition"); Promote(projection); await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType)); rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true, Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "active")); await RecordedEvents.WaitUntilAsync(() => events.Has(serverItem, "active")); counter.Total.ShouldBe(2, "a delivered event is not counted"); events.Has(serverItem, "standby-1").ShouldBeFalse("withheld events are never delivered as transitions"); await subscription.DeleteAsync(true, ct); } }