using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
///
/// The Warm instruments are process-wide statics (the role gauge's backing value too), so these tests run alone:
/// another Warm actor test running at the same time would move what is asserted.
///
[CollectionDefinition(Name, DisableParallelization = true)]
public sealed class WarmRedundancyMetricsCollection
{
/// The collection name.
public const string Name = "warm-redundancy-metrics";
}
///
/// Gitea #717 T10: each Runtime-side Warm instrument fires on its own path โ warm_transition with its
/// direction/reason tags, the warm_role gauge, and snapshot_rejected per reason from both
/// subscribers ( and ). Listeners select by
/// instrument INSTANCE, never by meter name (#503). mode_mismatch lives in
/// .
///
[Collection(WarmRedundancyMetricsCollection.Name)]
public sealed class WarmRedundancyMetricsTests : RuntimeActorTestBase
{
private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9");
private readonly ConcurrentQueue _log = new();
private long _seq;
private NodeId Self
{
get
{
var cluster = Akka.Cluster.Cluster.Get(Sys);
AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget);
return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}");
}
}
private IActorRef StartWarm() =>
Sys.ActorOf(OpcUaPublishActor.PropsForTests(
serviceLevel: new LoggingPublisher(_log), localNode: Self, dbLess: true,
healthTickInterval: TimeSpan.FromMinutes(5), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm),
standbyProjection: new RecordingProjection(_log)));
private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) =>
new(new[]
{
new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Self, DateTime.UtcNow),
new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary,
false, primary == Other, DateTime.UtcNow),
},
CorrelationId.NewId(),
revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null);
/// Records every measurement of ONE instrument, with its tags.
private sealed class Recorder : IDisposable where T : struct
{
private readonly MeterListener _listener = new();
private readonly ConcurrentQueue<(T Value, Dictionary Tags)> _seen = new();
public Recorder(Instrument instrument)
{
_listener.InstrumentPublished = (published, l) =>
{
if (ReferenceEquals(published, instrument)) l.EnableMeasurementEvents(published);
};
_listener.SetMeasurementEventCallback((_, value, tags, _) =>
{
var copy = new Dictionary();
foreach (var tag in tags) copy[tag.Key] = tag.Value;
_seen.Enqueue((value, copy));
});
_listener.Start();
}
public IReadOnlyList<(T Value, Dictionary Tags)> Seen => _seen.ToArray();
public void Observe() => _listener.RecordObservableInstruments();
public void Dispose() => _listener.Dispose();
}
private void AwaitLogEndsWith(params string[] tail) =>
AwaitAssert(() => _log.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", _log)), PresenceBudget,
cancellationToken: TestContext.Current.CancellationToken);
[Fact]
public void Warm_transition_counts_each_promotion_and_demotion_with_direction_and_reason()
{
using var transitions = new Recorder(OtOpcUaTelemetry.WarmTransition);
var actor = StartWarm();
actor.Tell(Snapshot(Self));
AwaitLogEndsWith("SL:250");
AwaitAssert(() => transitions.Seen.Count.ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
transitions.Seen[0].Value.ShouldBe(1);
transitions.Seen[0].Tags["direction"].ShouldBe("promote");
transitions.Seen[0].Tags["reason"].ShouldBe("agreed", "membership and the accepted snapshot both name this node");
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("queue:True");
AwaitAssert(() => transitions.Seen.Count.ShouldBe(2), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
transitions.Seen[1].Tags["direction"].ShouldBe("demote");
transitions.Seen[1].Tags["reason"].ShouldBe("snapshot", "membership still names this node; the snapshot does not");
// A same-role snapshot (a heartbeat) is not a transition.
actor.Tell(Snapshot(Other));
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
transitions.Seen.Count.ShouldBe(2);
}
[Fact]
public void Warm_role_gauge_reads_Standby_at_boot_Active_after_promotion_and_Standby_after_demotion()
{
using var role = new Recorder(OtOpcUaTelemetry.WarmRole);
int Current()
{
var before = role.Seen.Count;
role.Observe();
return role.Seen.Skip(before).Single().Value;
}
OtOpcUaTelemetry.SetWarmRole(-1);
Current().ShouldBe(-1, "not Warm until a Warm actor starts");
var actor = StartWarm();
AwaitAssert(() => Current().ShouldBe(0), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
actor.Tell(Snapshot(Self));
AwaitLogEndsWith("SL:250");
AwaitAssert(() => Current().ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
actor.Tell(Snapshot(Other));
AwaitLogEndsWith("queue:True");
AwaitAssert(() => Current().ShouldBe(0), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
Sys.Stop(actor);
OtOpcUaTelemetry.SetWarmRole(-1);
}
[Fact]
public void Snapshot_rejected_counts_older_revisions_and_unrevisioned_snapshots_by_reason_at_the_publish_actor()
{
using var rejected = new Recorder(OtOpcUaTelemetry.SnapshotRejected);
var actor = StartWarm();
actor.Tell(Snapshot(Self, upNumber: 5));
AwaitLogEndsWith("SL:250");
rejected.Seen.ShouldBeEmpty("an accepted snapshot is not counted");
actor.Tell(Snapshot(Other, upNumber: 4));
AwaitAssert(() => rejected.Seen.Count.ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
rejected.Seen[0].Tags["reason"].ShouldBe("older_revision");
actor.Tell(Snapshot(Other, revisioned: false));
AwaitAssert(() => rejected.Seen.Count.ShouldBe(2), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
rejected.Seen[1].Tags["reason"].ShouldBe("unrevisioned");
rejected.Seen.ShouldAllBe(m => m.Value == 1);
}
///
/// The half of spec ยง4.2 (its Primary gates): under Warm a stale snapshot is
/// dropped โ the role view does not move โ and counted; under Unset (no ordering) the same sequence moves it.
/// The view publishes once per accepted snapshot, so Primary@5, Secondary@4 (stale), Primary@6, Secondary@7
/// publishes exactly [true, true, false] when the stale one is dropped and [true, false, true, false] when not.
///
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Snapshot_rejected_counts_an_older_revision_at_the_driver_host_and_its_roles_do_not_move(bool warm)
{
using var rejected = new Recorder(OtOpcUaTelemetry.SnapshotRejected);
var node = NodeId.Parse("driver-warm-order:4053");
var peer = NodeId.Parse("the-peer:4053");
var view = new PublishedRoleView();
var host = Sys.ActorOf(DriverHostActor.Props(
NewInMemoryDbFactory(), node, ackRouter: null, driverMemberCountProvider: () => 2,
redundancyRoleView: view, replicationPeerHost: "the-peer", enforceSnapshotOrdering: warm));
RedundancyStateChanged Snap(bool primary, long upNumber) => new(
[
new NodeRedundancyState(node, primary ? RedundancyRole.Primary : RedundancyRole.Secondary, primary, primary, DateTime.UtcNow),
new NodeRedundancyState(peer, primary ? RedundancyRole.Secondary : RedundancyRole.Primary, !primary, !primary, DateTime.UtcNow),
],
CorrelationId.NewId(), new SnapshotRevision(upNumber, 1, upNumber));
host.Tell(Snap(primary: true, 5));
host.Tell(Snap(primary: false, 4)); // a departed older incarnation's late heartbeat
host.Tell(Snap(primary: true, 6));
host.Tell(Snap(primary: false, 7));
bool[] expected = warm ? [true, true, false] : [true, false, true, false];
AwaitAssert(() => view.Published.ShouldBe(expected), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken);
view.Published.ShouldBe(expected);
rejected.Seen.Count.ShouldBe(warm ? 1 : 0);
if (warm) rejected.Seen[0].Tags["reason"].ShouldBe("older_revision");
}
private sealed class PublishedRoleView : IRedundancyRoleView
{
private readonly List _published = [];
private readonly Lock _gate = new();
public IReadOnlyList Published
{
get { lock (_gate) return _published.ToArray(); }
}
public bool ShouldDrainAlarmHistory
{
get { lock (_gate) return _published.Count == 0 || _published[^1]; }
}
public void Publish(bool shouldDrainAlarmHistory)
{
lock (_gate) _published.Add(shouldDrainAlarmHistory);
}
}
}