dohertj2/lmxopcua
1
0
Fork 0
Code Issues 31 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(historian): alarm adapter drains a bounded queue; drop the senderless receive (lmxopcua#746)

Browse Source 
HistorianAdapterActor fired `_ = EnqueueAsync(...)` per alarm transition
with no in-flight cap, so an alarm storm (a native source re-raising
hundreds of conditions on reconnect) spawned hundreds of concurrent SQLite
inserts that serialised on the file lock and held pool threads. It also kept
a Receive<AlarmHistorianEvent> with no production sender: a transition told
directly AND published on `alerts` would have been historized twice.

Transitions now go to a bounded channel (10000) drained by one loop, so the
sink sees one enqueue at a time in arrival order; a full queue holds the
mailbox (backpressure, nothing dropped); PostStop flushes the queue within
the shared 5 s stop budget. The direct receive is deleted. The gate tests
drive transitions instead; a directly-told AlarmHistorianEvent is shown to
be unhandled against a positive control.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-10-22 17:34:34 -04:00
parent a5557375ee
commit 64269573e7
3 changed files with 154 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianAdapterActor.cs
+62 -9
View File
@@ -1,3 +1,4 @@
using System.Threading.Channels;
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Event;
@@ -12,9 +13,12 @@ using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Historian;
/// <summary>
/// Thin actor wrapper around <see cref="IAlarmHistorianSink"/>. Engine code (ScriptedAlarmActor,
/// Galaxy native alarm bridge, AB CIP ALMD reader) tells <see cref="AlarmHistorianEvent"/>s to this
/// actor; the actor enqueues them on the sink fire-and-forget. Production deployments register
/// Thin actor wrapper around <see cref="IAlarmHistorianSink"/>. Live alarm transitions arrive off the cluster
/// <c>alerts</c> topic; the Primary's adapter translates each to an <see cref="AlarmHistorianEvent"/> and queues it
/// for the sink. The queue is bounded and drained by ONE loop, so the sink sees one enqueue at a time in arrival order
/// (an alarm storm used to fan into one concurrent SQLite insert per transition, serialising on the file lock and
/// holding pool threads); a full queue holds the mailbox until room frees (backpressure, nothing dropped), and stopping
/// flushes the queue within the shared 5 s stop budget. Production deployments register
/// <see cref="LocalDbStoreAndForwardSink"/> against <c>IAlarmHistorianSink</c>; the sink owns the
/// durable queue + drain-to-HistorianGateway-SendEvent loop. The actor here owns nothing operational beyond
/// the message contract — its job is to keep the engine actors on Akka's mailbox without blocking
@@ -34,6 +38,18 @@ public sealed class HistorianAdapterActor : ReceiveActor
private readonly ILoggingAdapter _log = Context.GetLogger();
private IActorRef? _mediator;
/// <summary>The most events queued for the sink before a transition waits for room (backpressure).</summary>
public const int QueueCapacity = 10_000;
/// <summary>How long stopping waits for the queue to flush (the runtime's shared actor-stop budget).</summary>
public static readonly TimeSpan FlushBudget = TimeSpan.FromSeconds(5);
private readonly Channel<AlarmHistorianEvent> _queue = Channel.CreateBounded<AlarmHistorianEvent>(
new BoundedChannelOptions(QueueCapacity) { SingleReader = true, SingleWriter = true, FullMode = BoundedChannelFullMode.Wait });
private readonly CancellationTokenSource _lifetime = new();
private Task _drain = Task.CompletedTask;
/// <summary>Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
/// snapshot (null = unknown until the first snapshot arrives, or no <see cref="_localNode"/> wired). The
/// durable sink enqueue in the <see cref="AlarmHistorianEvent"/> handler is gated on this: only the
@@ -60,10 +76,8 @@ public sealed class HistorianAdapterActor : ReceiveActor
_sink = sink;
_localNode = localNode;
// A direct AlarmHistorianEvent source is kept for a future engine→historian path; it goes through the
// SAME ShouldHistorize gate. WARNING: do NOT Tell an AlarmHistorianEvent for a transition that is ALSO
// published on the `alerts` topic — the sink would then double-write (both handlers enqueue it).
Receive<AlarmHistorianEvent>(evt => { if (ShouldHistorize()) _ = EnqueueAsync(evt); });
// (The direct AlarmHistorianEvent receive is gone: it had no production sender, and a transition told directly
// AND published on `alerts` would have been written twice.)
// Live alarm transitions arrive off the cluster `alerts` DPS topic (subscribed in PreStart). The
// Primary ScriptedAlarmHostActor publishes each transition ONCE, but DistributedPubSub fans that
@@ -76,7 +90,21 @@ public sealed class HistorianAdapterActor : ReceiveActor
// an old-format message missing the field deserializes to null and is historized (default-on), so no
// audit row is dropped at a handover — same posture as the AlarmTypeName null-coalesce in Translate.
// The producer (ScriptedAlarmHostActor) always sets a concrete true/false.
Receive<AlarmTransitionEvent>(t => { if (ShouldHistorize() && t.HistorizeToAveva is not false) _ = EnqueueAsync(Translate(t)); });
ReceiveAsync<AlarmTransitionEvent>(async t =>
{
if (!ShouldHistorize() || t.HistorizeToAveva is false) return;
var evt = Translate(t);
// Off the mailbox unless the queue is full: then the mailbox waits for room (bounded memory, nothing dropped).
if (_queue.Writer.TryWrite(evt)) return;
try
{
await _queue.Writer.WriteAsync(evt, _lifetime.Token).ConfigureAwait(false);
}
catch (Exception ex) when (ex is OperationCanceledException or ChannelClosedException)
{
// Stopping raced the enqueue.
}
});
Receive<GetStatus>(_ => Sender.Tell(_sink.GetStatus()));
@@ -134,9 +162,30 @@ public sealed class HistorianAdapterActor : ReceiveActor
_mediator = DistributedPubSub.Get(Context.System).Mediator;
_mediator.Tell(new Subscribe(OpcUaPublishActor.RedundancyStateTopic, Self));
_mediator.Tell(new Subscribe(ScriptedAlarmHostActor.AlertsTopic, Self));
_drain = Task.Run(DrainAsync);
base.PreStart();
}
/// <inheritdoc />
protected override void PostStop()
{
// Flush what the Primary already accepted from the alerts topic, bounded by the shared stop budget.
_queue.Writer.TryComplete();
if (!_drain.Wait(FlushBudget))
_log.Warning("Historian adapter stopped with alarm events still queued for the sink; abandoned after {Seconds} s.",
FlushBudget.TotalSeconds);
_lifetime.Cancel();
_lifetime.Dispose();
base.PostStop();
}
/// <summary>The single loop feeding the sink, in arrival order. Never throws: a rejected event is logged.</summary>
private async Task DrainAsync()
{
await foreach (var evt in _queue.Reader.ReadAllAsync(CancellationToken.None).ConfigureAwait(false))
await EnqueueAsync(evt).ConfigureAwait(false);
}
/// <summary>Caches this node's <see cref="RedundancyRole"/> from a cluster redundancy snapshot so the
/// <see cref="AlarmHistorianEvent"/> handler can gate the durable sink enqueue to the Primary. A snapshot
/// that doesn't mention <see cref="_localNode"/> (or no local node wired) leaves the cached role unchanged
@@ -160,7 +209,11 @@ public sealed class HistorianAdapterActor : ReceiveActor
{
try
{
await _sink.EnqueueAsync(evt, CancellationToken.None);
await _sink.EnqueueAsync(evt, _lifetime.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (_lifetime.IsCancellationRequested)
{
// Stopped past the flush budget.
}
catch (Exception ex)
{