dohertj2/lmxopcua
1
0
Fork 0
Code Issues 25 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(historian): alarm adapter drains a bounded queue; drop the senderless receive (lmxopcua#746)

Browse Source 
HistorianAdapterActor fired `_ = EnqueueAsync(...)` per alarm transition
with no in-flight cap, so an alarm storm (a native source re-raising
hundreds of conditions on reconnect) spawned hundreds of concurrent SQLite
inserts that serialised on the file lock and held pool threads. It also kept
a Receive<AlarmHistorianEvent> with no production sender: a transition told
directly AND published on `alerts` would have been historized twice.

Transitions now go to a bounded channel (10000) drained by one loop, so the
sink sees one enqueue at a time in arrival order; a full queue holds the
mailbox (backpressure, nothing dropped); PostStop flushes the queue within
the shared 5 s stop budget. The direct receive is deleted. The gate tests
drive transitions instead; a directly-told AlarmHistorianEvent is shown to
be unhandled against a positive control.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-10-22 17:34:34 -04:00
parent a5557375ee
commit 64269573e7
3 changed files with 154 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Health/HealthProbeActorTests.cs
+10 -9
View File
@@ -77,18 +77,19 @@ public sealed class HealthProbeActorTests : RuntimeActorTestBase
var sink = new RecordingSink();
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink));
// Transitions arrive as AlarmTransitionEvent (the alerts-topic shape); the direct AlarmHistorianEvent receive
// had no production sender and was removed.
for (var i = 0; i < 5; i++)
actor.Tell(new AlarmHistorianEvent(
actor.Tell(new ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts.AlarmTransitionEvent(
AlarmId: $"alm-{i}",
EquipmentPath: "Plant/LineA",
AlarmName: $"Alarm{i}",
AlarmTypeName: "LimitAlarm",
Severity: AlarmSeverity.High,
EventKind: "Activated",
TransitionKind: "Activated",
Severity: 700,
Message: $"Test alarm {i}",
User: "system",
Comment: null,
TimestampUtc: DateTime.UtcNow));
TimestampUtc: DateTime.UtcNow,
AlarmTypeName: "LimitAlarm"));
AwaitCondition(() => sink.Enqueued.Count == 5, TimeSpan.FromSeconds(2));
sink.Enqueued.Select(e => e.AlarmId).OrderBy(s => s).ShouldBe(
@@ -102,9 +103,9 @@ public sealed class HealthProbeActorTests : RuntimeActorTestBase
var sink = new RecordingSink();
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink));
actor.Tell(new AlarmHistorianEvent(
"alm-x", "Plant/LineB", "OffNormal", "OffNormalAlarm",
AlarmSeverity.Low, "Activated", "msg", "system", null, DateTime.UtcNow));
actor.Tell(new ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts.AlarmTransitionEvent(
AlarmId: "alm-x", EquipmentPath: "Plant/LineB", AlarmName: "OffNormal", TransitionKind: "Activated",
Severity: 200, Message: "msg", User: "system", TimestampUtc: DateTime.UtcNow, AlarmTypeName: "OffNormalAlarm"));
AwaitCondition(() => sink.Enqueued.Count == 1, TimeSpan.FromSeconds(2));
var status = await actor.Ask<HistorianSinkStatus>(
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/HistorianAdapterActorTests.cs
+82 -18
View File
@@ -80,19 +80,6 @@ public sealed class HistorianAdapterActorTests : RuntimeActorTestBase
DrainState: HistorianDrainState.Idle);
}
/// <summary>Builds a minimal <see cref="AlarmHistorianEvent"/> for the gate tests.</summary>
private static AlarmHistorianEvent SampleEvent() => new(
AlarmId: "alm-1",
EquipmentPath: "Area/Line/Equip",
AlarmName: "HiHi",
AlarmTypeName: "LimitAlarm",
Severity: AlarmSeverity.High,
EventKind: "Activated",
Message: "level high",
User: "system",
Comment: null,
TimestampUtc: DateTime.UtcNow);
/// <summary>Tell <paramref name="actor"/> a <see cref="RedundancyStateChanged"/> snapshot marking
/// <see cref="LocalNode"/> with <paramref name="role"/> so the gate observes the local role.</summary>
private static void TellRedundancyRole(IActorRef actor, RedundancyRole role) =>
@@ -117,7 +104,7 @@ public sealed class HistorianAdapterActorTests : RuntimeActorTestBase
var sink = new RecordingSink();
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink, LocalNode));
actor.Tell(SampleEvent());
actor.Tell(SampleTransition());
AwaitAssert(() => sink.EnqueueCount.ShouldBe(1), AssertTimeout);
}
@@ -131,7 +118,7 @@ public sealed class HistorianAdapterActorTests : RuntimeActorTestBase
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink, LocalNode));
TellRedundancyRole(actor, RedundancyRole.Secondary);
actor.Tell(SampleEvent());
actor.Tell(SampleTransition());
// Give the (suppressed) fire-and-forget a stable window, then assert nothing landed.
ExpectNoMsg(Settle);
@@ -146,7 +133,7 @@ public sealed class HistorianAdapterActorTests : RuntimeActorTestBase
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink, LocalNode));
TellRedundancyRole(actor, RedundancyRole.Detached);
actor.Tell(SampleEvent());
actor.Tell(SampleTransition());
ExpectNoMsg(Settle);
sink.EnqueueCount.ShouldBe(0);
@@ -161,7 +148,7 @@ public sealed class HistorianAdapterActorTests : RuntimeActorTestBase
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink, LocalNode));
TellRedundancyRole(actor, RedundancyRole.Primary);
actor.Tell(SampleEvent());
actor.Tell(SampleTransition());
AwaitAssert(() => sink.EnqueueCount.ShouldBe(1), AssertTimeout);
}
@@ -189,7 +176,7 @@ public sealed class HistorianAdapterActorTests : RuntimeActorTestBase
},
CorrelationId.NewId()));
actor.Tell(SampleEvent());
actor.Tell(SampleTransition());
// Local role is still unknown ⇒ default-historize path: sink must record exactly one enqueue.
AwaitAssert(() => sink.EnqueueCount.ShouldBe(1), AssertTimeout);
@@ -346,4 +333,81 @@ public sealed class HistorianAdapterActorTests : RuntimeActorTestBase
() => sink.Events.ShouldHaveSingleItem().AlarmTypeName.ShouldBe("AlarmCondition"),
AssertTimeout);
}
/// <summary>
/// The direct <see cref="AlarmHistorianEvent"/> receive is gone (it had no production sender and was a latent
/// double-historize trap: a transition told directly AND published on <c>alerts</c> would be written twice). A
/// directly-told event is unhandled; the positive control — a transition after it — is historized exactly once.
/// </summary>
[Fact]
public void A_directly_told_AlarmHistorianEvent_is_not_historized()
{
var sink = new RecordingSink();
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink));
actor.Tell(new AlarmHistorianEvent("alm-1", "Area/Line/Equip", "HiHi", "LimitAlarm", AlarmSeverity.High,
"Activated", "level high", "system", null, DateTime.UtcNow));
actor.Tell(SampleTransition());
AwaitAssert(() => sink.EnqueueCount.ShouldBe(1), AssertTimeout);
ExpectNoMsg(Settle);
sink.Events.ShouldHaveSingleItem().AlarmId.ShouldBe("alm-9");
}
/// <summary>A sink whose enqueue takes time and reports how many enqueues overlap.</summary>
private sealed class SlowSink(TimeSpan delay) : IAlarmHistorianSink
{
private int _inFlight;
private int _maxInFlight;
private readonly object _lock = new();
private readonly List<string> _order = new();
public int MaxInFlight => Volatile.Read(ref _maxInFlight);
public IReadOnlyList<string> Order { get { lock (_lock) return _order.ToArray(); } }
public async Task EnqueueAsync(AlarmHistorianEvent evt, CancellationToken cancellationToken)
{
var now = Interlocked.Increment(ref _inFlight);
int seen;
while ((seen = Volatile.Read(ref _maxInFlight)) < now && Interlocked.CompareExchange(ref _maxInFlight, now, seen) != seen) { }
await Task.Delay(delay, cancellationToken);
lock (_lock) _order.Add(evt.Message ?? "");
Interlocked.Decrement(ref _inFlight);
}
public HistorianSinkStatus GetStatus() => new(0, 0, null, null, null, HistorianDrainState.Idle);
}
/// <summary>
/// An alarm storm no longer fans into one concurrent sink insert per transition: the adapter drains a bounded
/// queue with one loop, so the sink sees one enqueue at a time, in arrival order, and every event lands.
/// </summary>
[Fact]
public void An_alarm_storm_is_enqueued_one_at_a_time_in_order()
{
var sink = new SlowSink(TimeSpan.FromMilliseconds(5));
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink));
for (var i = 0; i < 40; i++) actor.Tell(SampleTransition() with { Message = $"m{i:00}" });
AwaitAssert(() => sink.Order.Count.ShouldBe(40), TimeSpan.FromSeconds(10));
sink.MaxInFlight.ShouldBe(1, "one loop drains the queue — never one concurrent insert per transition");
sink.Order.ShouldBe(Enumerable.Range(0, 40).Select(i => $"m{i:00}"));
}
/// <summary>Stopping the adapter flushes the events still queued within the stop budget.</summary>
[Fact]
public void Stopping_flushes_the_queued_events()
{
var sink = new SlowSink(TimeSpan.FromMilliseconds(20));
var actor = Sys.ActorOf(HistorianAdapterActor.Props(sink));
for (var i = 0; i < 10; i++) actor.Tell(SampleTransition() with { Message = $"m{i:00}" });
// PoisonPill queues behind the transitions (Sys.Stop would overtake them in the mailbox): all ten are accepted
// into the adapter's queue, most still waiting on the slow sink when the actor stops — and they are flushed.
Watch(actor);
actor.Tell(PoisonPill.Instance);
ExpectTerminated(actor, TimeSpan.FromSeconds(10));
sink.Order.Count.ShouldBe(10);
}
}