Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/OpcUa/WarmRedundancyMetricsTests.cs
T
Joseph Doherty 687256d68e
v2-ci / unit-tests (push) Has been cancelled
test(717): deferred Warm coverage - 2-node handover, DataEncoding, deleted/reused items, clone matrix, ACL stand-down, per-counter metrics
- T3 WarmSingletonHandoverTests: real 2-node cluster, singleton under ClusterSingletonManager on both nodes; the old
  incarnation's held-back publish (older revision, higher sequence) reaches a real Warm OpcUaPublishActor after the
  handover and never demotes the new Primary. Pin: a removed node cannot rejoin from its old ActorSystem; a new one
  rejoins younger (higher up-number).
- T6 DataEncoding: Binary/XML encoded reads of a structured tag projected on the Standby; scalar encoding error kept;
  a DefaultXml item is projected, then promoted through its own encoding (TypeId i=888).
- T7 item deleted between queue and publish; ClientHandle reused by i=2267 on a payload recycled through the SDK 2.0.1
  pool (precondition asserted); WarmStandbyCloneMatrixTests over DataChange+DiagnosticInfos/EventNotificationList/
  StatusChange, clone/cached immutability, encode round-trip, real-pool stale marker.
- T8 ACL-stood-down item receives nothing from demotion or promotion; restored grant reaches it again.
- T10 per-counter MeterListener tests (by instrument instance): warm_transition, warm_role gauge, snapshot_rejected
  (publish actor both reasons + DriverHostActor's ordering gate, first actor test of it), projected_notifications,
  suppressed_alarm_events. Process-wide state tests run in non-parallel collections.
- Every test mutation-checked (26 mutations killed); no product bug found. Plan status updated.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 04:31:11 -05:00

235 lines
11 KiB
C#

using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
/// <summary>
/// The Warm instruments are process-wide statics (the role gauge's backing value too), so these tests run alone:
/// another Warm actor test running at the same time would move what is asserted.
/// </summary>
[CollectionDefinition(Name, DisableParallelization = true)]
public sealed class WarmRedundancyMetricsCollection
{
/// <summary>The collection name.</summary>
public const string Name = "warm-redundancy-metrics";
}
/// <summary>
/// Gitea #717 T10: each Runtime-side Warm instrument fires on its own path — <c>warm_transition</c> with its
/// direction/reason tags, the <c>warm_role</c> gauge, and <c>snapshot_rejected</c> per reason from both
/// subscribers (<see cref="OpcUaPublishActor"/> and <see cref="DriverHostActor"/>). Listeners select by
/// instrument INSTANCE, never by meter name (#503). <c>mode_mismatch</c> lives in
/// <see cref="OpcUaPublishActorWarmTests"/>.
/// </summary>
[Collection(WarmRedundancyMetricsCollection.Name)]
public sealed class WarmRedundancyMetricsTests : RuntimeActorTestBase
{
private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9");
private readonly ConcurrentQueue<string> _log = new();
private long _seq;
private NodeId Self
{
get
{
var cluster = Akka.Cluster.Cluster.Get(Sys);
AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget);
return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}");
}
}
private IActorRef StartWarm() =>
Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: new LoggingPublisher(_log), localNode: Self, dbLess: true,
healthTickInterval: TimeSpan.FromMinutes(5), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm),
standbyProjection: new RecordingProjection(_log)));
private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) =>
new(new[]
{
new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Self, DateTime.UtcNow),
new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Other, DateTime.UtcNow),
},
CorrelationId.NewId(),
revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null);
/// <summary>Records every measurement of ONE instrument, with its tags.</summary>
private sealed class Recorder<T> : IDisposable where T : struct
{
private readonly MeterListener _listener = new();
private readonly ConcurrentQueue<(T Value, Dictionary<string, object?> Tags)> _seen = new();
public Recorder(Instrument instrument)
{
_listener.InstrumentPublished = (published, l) =>
{
if (ReferenceEquals(published, instrument)) l.EnableMeasurementEvents(published);
};
_listener.SetMeasurementEventCallback<T>((_, value, tags, _) =>
{
var copy = new Dictionary<string, object?>();
foreach (var tag in tags) copy[tag.Key] = tag.Value;
_seen.Enqueue((value, copy));
});
_listener.Start();
}
public IReadOnlyList<(T Value, Dictionary<string, object?> Tags)> Seen => _seen.ToArray();
public void Observe() => _listener.RecordObservableInstruments();
public void Dispose() => _listener.Dispose();
}
private void AwaitLogEndsWith(params string[] tail) =>
AwaitAssert(() => _log.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", _log)), PresenceBudget,
cancellationToken: TestContext.Current.CancellationToken);
[Fact]
public void Warm_transition_counts_each_promotion_and_demotion_with_direction_and_reason()
{
using var transitions = new Recorder<long>(OtOpcUaTelemetry.WarmTransition);
var actor = StartWarm();
actor.Tell(Snapshot(Self));
AwaitLogEndsWith("SL:250");
AwaitAssert(() => transitions.Seen.Count.ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
transitions.Seen[0].Value.ShouldBe(1);
transitions.Seen[0].Tags["direction"].ShouldBe("promote");
transitions.Seen[0].Tags["reason"].ShouldBe("agreed", "membership and the accepted snapshot both name this node");
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("queue:True");
AwaitAssert(() => transitions.Seen.Count.ShouldBe(2), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
transitions.Seen[1].Tags["direction"].ShouldBe("demote");
transitions.Seen[1].Tags["reason"].ShouldBe("snapshot", "membership still names this node; the snapshot does not");
// A same-role snapshot (a heartbeat) is not a transition.
actor.Tell(Snapshot(Other));
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
transitions.Seen.Count.ShouldBe(2);
}
[Fact]
public void Warm_role_gauge_reads_Standby_at_boot_Active_after_promotion_and_Standby_after_demotion()
{
using var role = new Recorder<int>(OtOpcUaTelemetry.WarmRole);
int Current()
{
var before = role.Seen.Count;
role.Observe();
return role.Seen.Skip(before).Single().Value;
}
OtOpcUaTelemetry.SetWarmRole(-1);
Current().ShouldBe(-1, "not Warm until a Warm actor starts");
var actor = StartWarm();
AwaitAssert(() => Current().ShouldBe(0), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
actor.Tell(Snapshot(Self));
AwaitLogEndsWith("SL:250");
AwaitAssert(() => Current().ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("queue:True");
AwaitAssert(() => Current().ShouldBe(0), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
Sys.Stop(actor);
OtOpcUaTelemetry.SetWarmRole(-1);
}
[Fact]
public void Snapshot_rejected_counts_older_revisions_and_unrevisioned_snapshots_by_reason_at_the_publish_actor()
{
using var rejected = new Recorder<long>(OtOpcUaTelemetry.SnapshotRejected);
var actor = StartWarm();
actor.Tell(Snapshot(Self, upNumber: 5));
AwaitLogEndsWith("SL:250");
rejected.Seen.ShouldBeEmpty("an accepted snapshot is not counted");
actor.Tell(Snapshot(Other, upNumber: 4));
AwaitAssert(() => rejected.Seen.Count.ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
rejected.Seen[0].Tags["reason"].ShouldBe("older_revision");
actor.Tell(Snapshot(Other, revisioned: false));
AwaitAssert(() => rejected.Seen.Count.ShouldBe(2), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
rejected.Seen[1].Tags["reason"].ShouldBe("unrevisioned");
rejected.Seen.ShouldAllBe(m => m.Value == 1);
}
/// <summary>
/// The <see cref="DriverHostActor"/> half of spec §4.2 (its Primary gates): under Warm a stale snapshot is
/// dropped — the role view does not move — and counted; under Unset (no ordering) the same sequence moves it.
/// The view publishes once per accepted snapshot, so Primary@5, Secondary@4 (stale), Primary@6, Secondary@7
/// publishes exactly [true, true, false] when the stale one is dropped and [true, false, true, false] when not.
/// </summary>
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Snapshot_rejected_counts_an_older_revision_at_the_driver_host_and_its_roles_do_not_move(bool warm)
{
using var rejected = new Recorder<long>(OtOpcUaTelemetry.SnapshotRejected);
var node = NodeId.Parse("driver-warm-order:4053");
var peer = NodeId.Parse("the-peer:4053");
var view = new PublishedRoleView();
var host = Sys.ActorOf(DriverHostActor.Props(
NewInMemoryDbFactory(), node, ackRouter: null, driverMemberCountProvider: () => 2,
redundancyRoleView: view, replicationPeerHost: "the-peer", enforceSnapshotOrdering: warm));
RedundancyStateChanged Snap(bool primary, long upNumber) => new(
[
new NodeRedundancyState(node, primary ? RedundancyRole.Primary : RedundancyRole.Secondary, primary, primary, DateTime.UtcNow),
new NodeRedundancyState(peer, primary ? RedundancyRole.Secondary : RedundancyRole.Primary, !primary, !primary, DateTime.UtcNow),
],
CorrelationId.NewId(), new SnapshotRevision(upNumber, 1, upNumber));
host.Tell(Snap(primary: true, 5));
host.Tell(Snap(primary: false, 4)); // a departed older incarnation's late heartbeat
host.Tell(Snap(primary: true, 6));
host.Tell(Snap(primary: false, 7));
bool[] expected = warm ? [true, true, false] : [true, false, true, false];
AwaitAssert(() => view.Published.ShouldBe(expected), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken);
view.Published.ShouldBe(expected);
rejected.Seen.Count.ShouldBe(warm ? 1 : 0);
if (warm) rejected.Seen[0].Tags["reason"].ShouldBe("older_revision");
}
private sealed class PublishedRoleView : IRedundancyRoleView
{
private readonly List<bool> _published = [];
private readonly Lock _gate = new();
public IReadOnlyList<bool> Published
{
get { lock (_gate) return _published.ToArray(); }
}
public bool ShouldDrainAlarmHistory
{
get { lock (_gate) return _published.Count == 0 || _published[^1]; }
}
public void Publish(bool shouldDrainAlarmHistory)
{
lock (_gate) _published.Add(shouldDrainAlarmHistory);
}
}
}