687256d68e
v2-ci / unit-tests (push) Has been cancelled
- T3 WarmSingletonHandoverTests: real 2-node cluster, singleton under ClusterSingletonManager on both nodes; the old incarnation's held-back publish (older revision, higher sequence) reaches a real Warm OpcUaPublishActor after the handover and never demotes the new Primary. Pin: a removed node cannot rejoin from its old ActorSystem; a new one rejoins younger (higher up-number). - T6 DataEncoding: Binary/XML encoded reads of a structured tag projected on the Standby; scalar encoding error kept; a DefaultXml item is projected, then promoted through its own encoding (TypeId i=888). - T7 item deleted between queue and publish; ClientHandle reused by i=2267 on a payload recycled through the SDK 2.0.1 pool (precondition asserted); WarmStandbyCloneMatrixTests over DataChange+DiagnosticInfos/EventNotificationList/ StatusChange, clone/cached immutability, encode round-trip, real-pool stale marker. - T8 ACL-stood-down item receives nothing from demotion or promotion; restored grant reaches it again. - T10 per-counter MeterListener tests (by instrument instance): warm_transition, warm_role gauge, snapshot_rejected (publish actor both reasons + DriverHostActor's ordering gate, first actor test of it), projected_notifications, suppressed_alarm_events. Process-wide state tests run in non-parallel collections. - Every test mutation-checked (26 mutations killed); no product bug found. Plan status updated. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
235 lines
11 KiB
C#
235 lines
11 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Diagnostics.Metrics;
|
|
using Akka.Actor;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Redundancy;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Redundancy;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
|
|
|
|
/// <summary>
|
|
/// The Warm instruments are process-wide statics (the role gauge's backing value too), so these tests run alone:
|
|
/// another Warm actor test running at the same time would move what is asserted.
|
|
/// </summary>
|
|
[CollectionDefinition(Name, DisableParallelization = true)]
|
|
public sealed class WarmRedundancyMetricsCollection
|
|
{
|
|
/// <summary>The collection name.</summary>
|
|
public const string Name = "warm-redundancy-metrics";
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gitea #717 T10: each Runtime-side Warm instrument fires on its own path — <c>warm_transition</c> with its
|
|
/// direction/reason tags, the <c>warm_role</c> gauge, and <c>snapshot_rejected</c> per reason from both
|
|
/// subscribers (<see cref="OpcUaPublishActor"/> and <see cref="DriverHostActor"/>). Listeners select by
|
|
/// instrument INSTANCE, never by meter name (#503). <c>mode_mismatch</c> lives in
|
|
/// <see cref="OpcUaPublishActorWarmTests"/>.
|
|
/// </summary>
|
|
[Collection(WarmRedundancyMetricsCollection.Name)]
|
|
public sealed class WarmRedundancyMetricsTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId Other = NodeId.Parse("10.255.255.1:9");
|
|
|
|
private readonly ConcurrentQueue<string> _log = new();
|
|
private long _seq;
|
|
|
|
private NodeId Self
|
|
{
|
|
get
|
|
{
|
|
var cluster = Akka.Cluster.Cluster.Get(Sys);
|
|
AwaitCondition(() => cluster.SelfMember.Status == Akka.Cluster.MemberStatus.Up, PresenceBudget);
|
|
return NodeId.Parse($"{cluster.SelfAddress.Host}:{cluster.SelfAddress.Port}");
|
|
}
|
|
}
|
|
|
|
private IActorRef StartWarm() =>
|
|
Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
|
serviceLevel: new LoggingPublisher(_log), localNode: Self, dbLess: true,
|
|
healthTickInterval: TimeSpan.FromMinutes(5), redundancy: new WarmRedundancySettings(OpcUaRedundancyMode.Warm),
|
|
standbyProjection: new RecordingProjection(_log)));
|
|
|
|
private RedundancyStateChanged Snapshot(NodeId primary, bool revisioned = true, long upNumber = 1) =>
|
|
new(new[]
|
|
{
|
|
new NodeRedundancyState(Self, primary == Self ? RedundancyRole.Primary : RedundancyRole.Secondary,
|
|
false, primary == Self, DateTime.UtcNow),
|
|
new NodeRedundancyState(Other, primary == Other ? RedundancyRole.Primary : RedundancyRole.Secondary,
|
|
false, primary == Other, DateTime.UtcNow),
|
|
},
|
|
CorrelationId.NewId(),
|
|
revisioned ? new SnapshotRevision(upNumber, 1, Interlocked.Increment(ref _seq)) : null);
|
|
|
|
/// <summary>Records every measurement of ONE instrument, with its tags.</summary>
|
|
private sealed class Recorder<T> : IDisposable where T : struct
|
|
{
|
|
private readonly MeterListener _listener = new();
|
|
private readonly ConcurrentQueue<(T Value, Dictionary<string, object?> Tags)> _seen = new();
|
|
|
|
public Recorder(Instrument instrument)
|
|
{
|
|
_listener.InstrumentPublished = (published, l) =>
|
|
{
|
|
if (ReferenceEquals(published, instrument)) l.EnableMeasurementEvents(published);
|
|
};
|
|
_listener.SetMeasurementEventCallback<T>((_, value, tags, _) =>
|
|
{
|
|
var copy = new Dictionary<string, object?>();
|
|
foreach (var tag in tags) copy[tag.Key] = tag.Value;
|
|
_seen.Enqueue((value, copy));
|
|
});
|
|
_listener.Start();
|
|
}
|
|
|
|
public IReadOnlyList<(T Value, Dictionary<string, object?> Tags)> Seen => _seen.ToArray();
|
|
|
|
public void Observe() => _listener.RecordObservableInstruments();
|
|
|
|
public void Dispose() => _listener.Dispose();
|
|
}
|
|
|
|
private void AwaitLogEndsWith(params string[] tail) =>
|
|
AwaitAssert(() => _log.TakeLast(tail.Length).ShouldBe(tail, string.Join(" ", _log)), PresenceBudget,
|
|
cancellationToken: TestContext.Current.CancellationToken);
|
|
|
|
[Fact]
|
|
public void Warm_transition_counts_each_promotion_and_demotion_with_direction_and_reason()
|
|
{
|
|
using var transitions = new Recorder<long>(OtOpcUaTelemetry.WarmTransition);
|
|
var actor = StartWarm();
|
|
|
|
actor.Tell(Snapshot(Self));
|
|
AwaitLogEndsWith("SL:250");
|
|
AwaitAssert(() => transitions.Seen.Count.ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
transitions.Seen[0].Value.ShouldBe(1);
|
|
transitions.Seen[0].Tags["direction"].ShouldBe("promote");
|
|
transitions.Seen[0].Tags["reason"].ShouldBe("agreed", "membership and the accepted snapshot both name this node");
|
|
|
|
actor.Tell(Snapshot(Other));
|
|
AwaitLogEndsWith("queue:True");
|
|
AwaitAssert(() => transitions.Seen.Count.ShouldBe(2), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
transitions.Seen[1].Tags["direction"].ShouldBe("demote");
|
|
transitions.Seen[1].Tags["reason"].ShouldBe("snapshot", "membership still names this node; the snapshot does not");
|
|
|
|
// A same-role snapshot (a heartbeat) is not a transition.
|
|
actor.Tell(Snapshot(Other));
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
|
|
transitions.Seen.Count.ShouldBe(2);
|
|
}
|
|
|
|
[Fact]
|
|
public void Warm_role_gauge_reads_Standby_at_boot_Active_after_promotion_and_Standby_after_demotion()
|
|
{
|
|
using var role = new Recorder<int>(OtOpcUaTelemetry.WarmRole);
|
|
int Current()
|
|
{
|
|
var before = role.Seen.Count;
|
|
role.Observe();
|
|
return role.Seen.Skip(before).Single().Value;
|
|
}
|
|
|
|
OtOpcUaTelemetry.SetWarmRole(-1);
|
|
Current().ShouldBe(-1, "not Warm until a Warm actor starts");
|
|
var actor = StartWarm();
|
|
AwaitAssert(() => Current().ShouldBe(0), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
|
|
actor.Tell(Snapshot(Self));
|
|
AwaitLogEndsWith("SL:250");
|
|
AwaitAssert(() => Current().ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
|
|
actor.Tell(Snapshot(Other));
|
|
AwaitLogEndsWith("queue:True");
|
|
AwaitAssert(() => Current().ShouldBe(0), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
Sys.Stop(actor);
|
|
OtOpcUaTelemetry.SetWarmRole(-1);
|
|
}
|
|
|
|
[Fact]
|
|
public void Snapshot_rejected_counts_older_revisions_and_unrevisioned_snapshots_by_reason_at_the_publish_actor()
|
|
{
|
|
using var rejected = new Recorder<long>(OtOpcUaTelemetry.SnapshotRejected);
|
|
var actor = StartWarm();
|
|
|
|
actor.Tell(Snapshot(Self, upNumber: 5));
|
|
AwaitLogEndsWith("SL:250");
|
|
rejected.Seen.ShouldBeEmpty("an accepted snapshot is not counted");
|
|
|
|
actor.Tell(Snapshot(Other, upNumber: 4));
|
|
AwaitAssert(() => rejected.Seen.Count.ShouldBe(1), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
rejected.Seen[0].Tags["reason"].ShouldBe("older_revision");
|
|
|
|
actor.Tell(Snapshot(Other, revisioned: false));
|
|
AwaitAssert(() => rejected.Seen.Count.ShouldBe(2), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
rejected.Seen[1].Tags["reason"].ShouldBe("unrevisioned");
|
|
rejected.Seen.ShouldAllBe(m => m.Value == 1);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The <see cref="DriverHostActor"/> half of spec §4.2 (its Primary gates): under Warm a stale snapshot is
|
|
/// dropped — the role view does not move — and counted; under Unset (no ordering) the same sequence moves it.
|
|
/// The view publishes once per accepted snapshot, so Primary@5, Secondary@4 (stale), Primary@6, Secondary@7
|
|
/// publishes exactly [true, true, false] when the stale one is dropped and [true, false, true, false] when not.
|
|
/// </summary>
|
|
[Theory]
|
|
[InlineData(true)]
|
|
[InlineData(false)]
|
|
public void Snapshot_rejected_counts_an_older_revision_at_the_driver_host_and_its_roles_do_not_move(bool warm)
|
|
{
|
|
using var rejected = new Recorder<long>(OtOpcUaTelemetry.SnapshotRejected);
|
|
var node = NodeId.Parse("driver-warm-order:4053");
|
|
var peer = NodeId.Parse("the-peer:4053");
|
|
var view = new PublishedRoleView();
|
|
var host = Sys.ActorOf(DriverHostActor.Props(
|
|
NewInMemoryDbFactory(), node, ackRouter: null, driverMemberCountProvider: () => 2,
|
|
redundancyRoleView: view, replicationPeerHost: "the-peer", enforceSnapshotOrdering: warm));
|
|
|
|
RedundancyStateChanged Snap(bool primary, long upNumber) => new(
|
|
[
|
|
new NodeRedundancyState(node, primary ? RedundancyRole.Primary : RedundancyRole.Secondary, primary, primary, DateTime.UtcNow),
|
|
new NodeRedundancyState(peer, primary ? RedundancyRole.Secondary : RedundancyRole.Primary, !primary, !primary, DateTime.UtcNow),
|
|
],
|
|
CorrelationId.NewId(), new SnapshotRevision(upNumber, 1, upNumber));
|
|
|
|
host.Tell(Snap(primary: true, 5));
|
|
host.Tell(Snap(primary: false, 4)); // a departed older incarnation's late heartbeat
|
|
host.Tell(Snap(primary: true, 6));
|
|
host.Tell(Snap(primary: false, 7));
|
|
|
|
bool[] expected = warm ? [true, true, false] : [true, false, true, false];
|
|
AwaitAssert(() => view.Published.ShouldBe(expected), PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken);
|
|
view.Published.ShouldBe(expected);
|
|
rejected.Seen.Count.ShouldBe(warm ? 1 : 0);
|
|
if (warm) rejected.Seen[0].Tags["reason"].ShouldBe("older_revision");
|
|
}
|
|
|
|
private sealed class PublishedRoleView : IRedundancyRoleView
|
|
{
|
|
private readonly List<bool> _published = [];
|
|
private readonly Lock _gate = new();
|
|
|
|
public IReadOnlyList<bool> Published
|
|
{
|
|
get { lock (_gate) return _published.ToArray(); }
|
|
}
|
|
|
|
public bool ShouldDrainAlarmHistory
|
|
{
|
|
get { lock (_gate) return _published.Count == 0 || _published[^1]; }
|
|
}
|
|
|
|
public void Publish(bool shouldDrainAlarmHistory)
|
|
{
|
|
lock (_gate) _published.Add(shouldDrainAlarmHistory);
|
|
}
|
|
}
|
|
}
|