using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig;
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.WarmRedundancyWireTests;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
///
/// Process-wide state these tests observe — SDK 2.0.1's notification-payload pool and the Warm instruments — is
/// shared by every server and client in the test process, so they run alone.
///
[CollectionDefinition(Name, DisableParallelization = true)]
public sealed class WarmRedundancyProcessWideCollection
{
/// The collection name.
public const string Name = "warm-redundancy-process-wide";
}
///
/// Gitea #717 over the wire, where the observation is process-wide: a monitored-item ClientHandle reused after a
/// delete while the SDK's payload pool hands out notifications that still carry an old ordinary-tag marker (T7),
/// and the two OpcUaServer-side Warm instruments (T10), standby.projected_notifications and
/// standby.suppressed_alarm_events, each selected by instrument INSTANCE (#503).
///
[Trait("Category", "Integration")]
[Collection(WarmRedundancyProcessWideCollection.Name)]
public sealed class WarmRedundancyPooledWireTests
{
private const NodePermissions All =
NodePermissions.Browse | NodePermissions.Read | NodePermissions.Subscribe | NodePermissions.AlarmRead
| NodePermissions.AlarmAcknowledge | NodePermissions.AlarmConfirm | NodePermissions.AlarmShelve
| NodePermissions.MethodCall | NodePermissions.WriteOperate;
///
/// SDK 2.0.1 returns acknowledged (and evicted) notification payloads to a process-wide pool, and
/// Initialize() clears only ClientHandle and Value, so a payload stamped for an ordinary tag goes back out
/// to the next item that samples — with its old marker. Here an ordinary-tag item's payloads are delivered and
/// acknowledged in one go, the item is deleted, and its ClientHandle is reused by an item on a NON-ordinary node
/// (i=2267): on the Standby every value of that item must stay Good, because the stale marker names
/// another value. The test first proves the pool really does hold a marked payload at that moment, and an
/// ordinary-tag item reusing the handle afterwards is projected (positive control).
///
[Fact]
public async Task A_client_handle_reused_after_delete_by_a_non_ordinary_item_is_never_projected_from_a_recycled_payload()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717Reuse", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("reuse", "Double", writable: false);
rig.Sink.WriteValue(raw, 0.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
var serviceLevel = new SdkServiceLevelPublisher(rig.Server.CurrentInstance!,
Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance);
const uint handle = 7;
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
var tagItem = await AddItemAsync(client, sub, Raw(session, raw), handle, 1, ct);
// Deliver a run of marked notifications WITHOUT acknowledging, then acknowledge them all at once, so a batch of
// marked payloads is back in the pool together (evicted ones are returned too).
var delivered = new List();
for (var i = 1; i <= 24; i++)
{
rig.Sink.WriteValue(raw, i, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
var r = await PublishDataAsync(client, ct);
delivered.Add(r.NotificationMessage.SequenceNumber);
}
StatusCode.IsGood((await client.DeleteMonitoredItemsAsync(H(), sub, new[] { tagItem }, ct)).Results[0]).ShouldBeTrue();
await client.PublishAsync(H(), delivered.Select(seq => new SubscriptionAcknowledgement { SubscriptionId = sub, SequenceNumber = seq }).ToArray(), ct);
var levelItem = await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, handle, 10, ct);
await CollectAsync(client, ct, TimeSpan.FromMilliseconds(500)); // the initial sample, delivered while Active
DemoteDeliveryOnly(projection);
// Artifact before outcome: the pool's next payload really is a recycled ordinary-tag one. Peek and put it back.
var peeked = (MonitoredItemNotification)MonitoredItemNotificationActivator.Instance.CreateInstance();
peeked.Message.ShouldBeOfType("the precondition: the SDK pool hands out a payload still carrying a marker");
peeked.Reuse();
var levels = new byte[] { 91, 92, 93, 94, 95, 96, 97, 98 };
foreach (var level in levels)
{
serviceLevel.Publish(level);
await Task.Delay(60, ct);
}
var standby = await CollectAsync(client, ct, TimeSpan.FromSeconds(1));
standby.Where(x => x.Handle == handle).Select(x => Convert.ToByte(x.Value.ClrValue())).ShouldBe(levels,
"every i=2267 change reaches the client Good: a recycled payload's stale marker never validates");
standby.ShouldAllBe(x => x.Value.StatusCode == StatusCodes.Good);
// Positive control: an ordinary-tag item reusing the handle IS projected on the same Standby.
StatusCode.IsGood((await client.DeleteMonitoredItemsAsync(H(), sub, new[] { levelItem }, ct)).Results[0]).ShouldBeTrue();
rig.Sink.WriteValue(raw, 99.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
await AddItemAsync(client, sub, Raw(session, raw), handle, 10, ct);
var control = await CollectAsync(client, ct, TimeSpan.FromSeconds(1));
control.Where(x => x.Handle == handle).ShouldNotBeEmpty();
control.Where(x => x.Handle == handle).ShouldAllBe(x => x.Value.StatusCode == StatusCodes.BadOutOfService);
}
/// Sums every measurement of ONE counter instance.
private sealed class CounterTotal : IDisposable
{
private readonly MeterListener _listener = new();
private long _total;
public CounterTotal(Instrument instrument)
{
_listener.InstrumentPublished = (published, l) =>
{
if (ReferenceEquals(published, instrument)) l.EnableMeasurementEvents(published);
};
_listener.SetMeasurementEventCallback((_, value, _, _) => Interlocked.Add(ref _total, value));
_listener.Start();
}
public long Total => Interlocked.Read(ref _total);
public void Dispose() => _listener.Dispose();
}
///
/// standby.projected_notifications counts each notification the Standby rewrites, at Publish and again at
/// Republish (each is a delivery), and nothing while Active.
///
[Fact]
public async Task Projected_notifications_counts_each_rewritten_notification_per_delivery_and_nothing_while_Active()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717ProjMetric", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
var raw = rig.EnsureRawVariable("m", "Double", writable: false);
rig.Sink.WriteValue(raw, 1.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
Promote(projection);
using var counter = new CounterTotal(OtOpcUaTelemetry.StandbyProjectedNotifications);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var client = (SessionClient)session;
var sub = (await client.CreateSubscriptionAsync(H(), 50, 300, 10, 0, true, 0, ct)).SubscriptionId;
await AddItemAsync(client, sub, Raw(session, raw), 1, 10, ct);
await AddItemAsync(client, sub, VariableIds.Server_ServiceLevel, 2, 1, ct);
await PublishDataAsync(client, ct);
counter.Total.ShouldBe(0, "nothing is rewritten while Active");
await client.SetPublishingModeAsync(H(), false, new[] { sub }, ct);
rig.Sink.WriteValue(raw, 10.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 11.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
rig.Sink.WriteValue(raw, 12.0, OpcUaStatus.Good, DateTime.UtcNow, AddressSpaceRealm.Raw);
new SdkServiceLevelPublisher(rig.Server.CurrentInstance!,
Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance).Publish(90);
DemoteDeliveryOnly(projection);
await client.SetPublishingModeAsync(H(), true, new[] { sub }, ct);
var msg = await PublishDataAsync(client, ct);
var changes = DataChanges(msg.NotificationMessage);
var rewritten = changes.Count(x => x.Handle == 1 && x.Value.StatusCode == StatusCodes.BadOutOfService);
rewritten.ShouldBe(3, "the three queued Good samples");
changes.ShouldContain(x => x.Handle == 2 && x.Value.StatusCode == StatusCodes.Good, "i=2267 is not rewritten, nor counted");
counter.Total.ShouldBe(rewritten);
await client.RepublishAsync(H(), sub, msg.NotificationMessage.SequenceNumber, ct);
counter.Total.ShouldBe(2 * rewritten, "a Republish is another delivery, rewritten and counted again");
}
///
/// standby.suppressed_alarm_events counts one per new alarm event withheld on the Standby, and none once
/// Active (where the same transition is delivered).
///
[Fact]
public async Task Suppressed_alarm_events_counts_each_withheld_event_on_the_Standby_and_none_once_Active()
{
var ct = Deadline();
await using var rig = await StartModeAsync("Warm717AlarmMetric", "Warm", ct);
var projection = rig.Server.CreateStandbyProjection()!;
rig.Enforce(false, [Key()], ClusterGrant(KeyPrincipal, "acl-key", All));
rig.MaterialiseScripted();
using var counter = new CounterTotal(OtOpcUaTelemetry.StandbySuppressedAlarmEvents);
using var session = await rig.ConnectAsync(anonymous: false, ct);
var events = new RecordedEvents();
var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
var serverItem = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
await subscription.ApplyChangesAsync(ct);
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 500, Message: "standby-1"));
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 600, Message: "standby-2"));
counter.Total.ShouldBe(2, "one per withheld transition");
Promote(projection);
await RecordedEvents.WaitUntilAsync(() => events.HasType(serverItem, ObjectTypeIds.RefreshEndEventType));
rig.Project(native: false, new AlarmConditionSnapshot(Active: true, Acknowledged: false, Confirmed: true, Enabled: true,
Shelving: AlarmShelvingKind.Unshelved, Severity: 700, Message: "active"));
await RecordedEvents.WaitUntilAsync(() => events.Has(serverItem, "active"));
counter.Total.ShouldBe(2, "a delivered event is not counted");
events.Has(serverItem, "standby-1").ShouldBeFalse("withheld events are never delivered as transitions");
await subscription.DeleteAsync(true, ct);
}
}