using System.Collections.Concurrent; using System.Diagnostics.Metrics; using Akka.Actor; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy; using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa; /// /// The Warm instruments are process-wide statics (the role gauge's backing value too), so these tests run alone: /// another Warm actor test running at the same time would move what is asserted. /// [CollectionDefinition(Name, DisableParallelization = true)] public sealed class WarmRedundancyMetricsCollection { /// The collection name. public const string Name = "warm-redundancy-metrics"; } /// /// Gitea #717 T10: each Runtime-side Warm instrument fires on its own path โ€” warm_transition with its /// direction/reason tags, the warm_role gauge, and snapshot_rejected per reason from both /// subscribers ( and ). Listeners select by /// instrument INSTANCE, never by meter name (#503). mode_mismatch lives in /// . /// [Collection(WarmRedundancyMetricsCollection.Name)] public sealed class WarmRedundancyMetricsTests : RuntimeActorTestBase { private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9"); private readonly ConcurrentQueue _log = new(); private long _seq; private NodeId Self { get { var cluster = Akka.Cluster.Cluster.Get(Sys); AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget); return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}"); } } private IActorRef StartWarm() => Sys.ActorOf(OpcUaPublishActor.PropsForTests( serviceLevel: new LoggingPublisher(_log), localNode: Self, dbLess: true, healthTickInterval: TimeSpan.FromMinutes(5), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm), standbyProjection: new RecordingProjection(_log))); private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) => new(new[] { new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary, false, primary == Self, DateTime.UtcNow), new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary, false, primary == Other, DateTime.UtcNow), }, CorrelationId.NewId(), revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null); /// Records every measurement of ONE instrument, with its tags. private sealed class Recorder : IDisposable where T : struct { private readonly MeterListener _listener = new(); private readonly ConcurrentQueue<(T Value, Dictionary Tags)> _seen = new(); public Recorder(Instrument instrument) { _listener.InstrumentPublished = (published, l) => { if (ReferenceEquals(published, instrument)) l.EnableMeasurementEvents(published); }; _listener.SetMeasurementEventCallback((_, value, tags, _) => { var copy = new Dictionary(); foreach (var tag in tags) copy[tag.Key] = tag.Value; _seen.Enqueue((value, copy)); }); _listener.Start(); } public IReadOnlyList<(T Value, Dictionary Tags)> Seen => _seen.ToArray(); public void Observe() => _listener.RecordObservableInstruments(); public void Dispose() => _listener.Dispose(); } private void AwaitLogEndsWith(params string[] tail) => AwaitAssert(() => _log.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", _log)), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); [Fact] public void Warm_transition_counts_each_promotion_and_demotion_with_direction_and_reason() { using var transitions = new Recorder(OtOpcUaTelemetry.WarmTransition); var actor = StartWarm(); actor.Tell(Snapshot(Self)); AwaitLogEndsWith("SL:250"); AwaitAssert(() => transitions.Seen.Count.ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); transitions.Seen[0].Value.ShouldBe(1); transitions.Seen[0].Tags["direction"].ShouldBe("promote"); transitions.Seen[0].Tags["reason"].ShouldBe("agreed", "membership and the accepted snapshot both name this node"); actor.Tell(Snapshot(Other)); AwaitLogEndsWith("queue:True"); AwaitAssert(() => transitions.Seen.Count.ShouldBe(2), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); transitions.Seen[1].Tags["direction"].ShouldBe("demote"); transitions.Seen[1].Tags["reason"].ShouldBe("snapshot", "membership still names this node; the snapshot does not"); // A same-role snapshot (a heartbeat) is not a transition. actor.Tell(Snapshot(Other)); ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken); transitions.Seen.Count.ShouldBe(2); } [Fact] public void Warm_role_gauge_reads_Standby_at_boot_Active_after_promotion_and_Standby_after_demotion() { using var role = new Recorder(OtOpcUaTelemetry.WarmRole); int Current() { var before = role.Seen.Count; role.Observe(); return role.Seen.Skip(before).Single().Value; } OtOpcUaTelemetry.SetWarmRole(-1); Current().ShouldBe(-1, "not Warm until a Warm actor starts"); var actor = StartWarm(); AwaitAssert(() => Current().ShouldBe(0), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); actor.Tell(Snapshot(Self)); AwaitLogEndsWith("SL:250"); AwaitAssert(() => Current().ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); actor.Tell(Snapshot(Other)); AwaitLogEndsWith("queue:True"); AwaitAssert(() => Current().ShouldBe(0), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); Sys.Stop(actor); OtOpcUaTelemetry.SetWarmRole(-1); } [Fact] public void Snapshot_rejected_counts_older_revisions_and_unrevisioned_snapshots_by_reason_at_the_publish_actor() { using var rejected = new Recorder(OtOpcUaTelemetry.SnapshotRejected); var actor = StartWarm(); actor.Tell(Snapshot(Self, upNumber: 5)); AwaitLogEndsWith("SL:250"); rejected.Seen.ShouldBeEmpty("an accepted snapshot is not counted"); actor.Tell(Snapshot(Other, upNumber: 4)); AwaitAssert(() => rejected.Seen.Count.ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); rejected.Seen[0].Tags["reason"].ShouldBe("older_revision"); actor.Tell(Snapshot(Other, revisioned: false)); AwaitAssert(() => rejected.Seen.Count.ShouldBe(2), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); rejected.Seen[1].Tags["reason"].ShouldBe("unrevisioned"); rejected.Seen.ShouldAllBe(m => m.Value == 1); } /// /// The half of spec ยง4.2 (its Primary gates): under Warm a stale snapshot is /// dropped โ€” the role view does not move โ€” and counted; under Unset (no ordering) the same sequence moves it. /// The view publishes once per accepted snapshot, so Primary@5, Secondary@4 (stale), Primary@6, Secondary@7 /// publishes exactly [true, true, false] when the stale one is dropped and [true, false, true, false] when not. /// [Theory] [InlineData(true)] [InlineData(false)] public void Snapshot_rejected_counts_an_older_revision_at_the_driver_host_and_its_roles_do_not_move(bool warm) { using var rejected = new Recorder(OtOpcUaTelemetry.SnapshotRejected); var node = NodeId.Parse("driver-warm-order:4053"); var peer = NodeId.Parse("the-peer:4053"); var view = new PublishedRoleView(); var host = Sys.ActorOf(DriverHostActor.Props( NewInMemoryDbFactory(), node, ackRouter: null, driverMemberCountProvider: () => 2, redundancyRoleView: view, replicationPeerHost: "the-peer", enforceSnapshotOrdering: warm)); RedundancyStateChanged Snap(bool primary, long upNumber) => new( [ new NodeRedundancyState(node, primary ? RedundancyRole.Primary : RedundancyRole.Secondary, primary, primary, DateTime.UtcNow), new NodeRedundancyState(peer, primary ? RedundancyRole.Secondary : RedundancyRole.Primary, !primary, !primary, DateTime.UtcNow), ], CorrelationId.NewId(), new SnapshotRevision(upNumber, 1, upNumber)); host.Tell(Snap(primary: true, 5)); host.Tell(Snap(primary: false, 4)); // a departed older incarnation's late heartbeat host.Tell(Snap(primary: true, 6)); host.Tell(Snap(primary: false, 7)); bool[] expected = warm ? [true, true, false] : [true, false, true, false]; AwaitAssert(() => view.Published.ShouldBe(expected), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken); ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken); view.Published.ShouldBe(expected); rejected.Seen.Count.ShouldBe(warm ? 1 : 0); if (warm) rejected.Seen[0].Tags["reason"].ShouldBe("older_revision"); } private sealed class PublishedRoleView : IRedundancyRoleView { private readonly List _published = []; private readonly Lock _gate = new(); public IReadOnlyList Published { get { lock (_gate) return _published.ToArray(); } } public bool ShouldDrainAlarmHistory { get { lock (_gate) return _published.Count == 0 || _published[^1]; } } public void Publish(bool shouldDrainAlarmHistory) { lock (_gate) _published.Add(shouldDrainAlarmHistory); } } }