fix(runtime): restart-safe alarm host, mux last-value replay, VT log rate limit, Calculation remediation (E4)
- R02:H1 ScriptedAlarmHostActor gains a factory Props (Func<(upstream, engine)>) so every incarnation builds and owns its own engine; a supervision restart no longer re-creates the actor around a disposed engine. The legacy instance-taking Props stays source-compatible and is now restart-safe in the weaker sense (PreRestart marks the stop as a restart, PostStop then detaches instead of disposing). DriverHostActor still spawns through the legacy overload (not owned by this lane — F3). - R02:H3 OnAlarmCommand runs every engine op under a 10 s token linked to the actor's lifetime (DefaultCommandTimeout / Props commandTimeout) and logs a Warning on timeout instead of suspending the mailbox on a stalled store. - R02:L3 DependencyMuxTagUpstreamSource.RetainOnly(loaded dep-ref union), called after every successful load. - R02:P6 VirtualTagActor's per-failure ScriptLogEntry is rate-limited per tag (ScriptLogRateLimiter, 20/min + one summary); the Bad publish and the metric are untouched. - R07:H6 DependencyMuxActor retains the last (value, status, ts) per ever-registered ref (through unregisters) and replays them in one batch to a RegisterInterest(ReplayRetained: true) — VirtualTagActor, the Calculation mux adapter and the alarm host opt in; the historization recorder does not (a replay there would record a stale sample per deploy). CalculationDriver records inputs regardless of _running so a replay landing before InitializeAsync still satisfies the change-gate. - R07:P4 CalculationEvaluator negative-caches compile failures by source hash (one Roslyn attempt + one Warning per source until redeploy); the driver's script-log entry fires on the Good→Bad transition or a change of reason, matching the once-per-transition Bad publish. - R07:H8 array results dedup structurally. - R07:L2 ShutdownAsync drops the compiled scripts (collectible ALCs). - R07:U5 DraftValidator rule CalculationNoEffectiveTrigger: a tag with no timer whose change trigger is off or whose script has no literal ctx.GetTag read can never evaluate. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
+26
-2
@@ -42,8 +42,10 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
|
||||
// publish.
|
||||
private const uint StatusNeverPushed = OpcUaStatus.BadWaitingForInitialData;
|
||||
|
||||
// Intentionally never pruned: cold-start semantics depend on retained last values so
|
||||
// ReadTag can answer synchronously for any path that has ever been pushed.
|
||||
// Retained last values: ReadTag answers synchronously for any path that has been pushed. Pruned
|
||||
// ONLY through RetainOnly — the host calls it with the loaded dependency-ref union after every
|
||||
// (re)load, so a renamed/removed raw path (a RawTree rename is a surgical remove+add) stops
|
||||
// costing a boxed value + two DateTimes for the process lifetime (R02:L3).
|
||||
private readonly ConcurrentDictionary<string, DataValueSnapshot> _cache
|
||||
= new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, ImmutableList<Action<string, DataValueSnapshot>>> _observers
|
||||
@@ -91,6 +93,28 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drop every cached snapshot whose path is not in <paramref name="paths"/>. Observers are
|
||||
/// untouched. Called by the host after each successful engine (re)load with the union of the loaded
|
||||
/// alarms' dependency refs — exactly the set the mux will push from now on — so paths no alarm
|
||||
/// references any more stop being retained.
|
||||
/// </summary>
|
||||
/// <param name="paths">The paths to keep.</param>
|
||||
/// <returns>The number of cached snapshots dropped.</returns>
|
||||
public int RetainOnly(IReadOnlySet<string> paths)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(paths);
|
||||
var dropped = 0;
|
||||
foreach (var key in _cache.Keys)
|
||||
{
|
||||
if (!paths.Contains(key) && _cache.TryRemove(key, out _)) dropped++;
|
||||
}
|
||||
return dropped;
|
||||
}
|
||||
|
||||
/// <summary>Test/diagnostic: the number of paths currently retained.</summary>
|
||||
public int RetainedPathCount => _cache.Count;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public DataValueSnapshot ReadTag(string path)
|
||||
{
|
||||
|
||||
@@ -23,10 +23,16 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
||||
/// cluster <c>alerts</c> DistributedPubSub topic.
|
||||
///
|
||||
/// <para>
|
||||
/// The host is the engine's lifecycle owner: the caller (the driver-role startup) builds
|
||||
/// the <see cref="DependencyMuxTagUpstreamSource"/>, constructs a <see cref="ScriptedAlarmEngine"/>
|
||||
/// <b>around that same upstream</b>, and passes both here. The host disposes the engine in
|
||||
/// <see cref="PostStop"/>.
|
||||
/// The host is the engine's lifecycle owner. The restart-safe shape (R02:H1) is the
|
||||
/// <b>factory</b> <see cref="Props(IActorRef, IActorRef?, Func{ValueTuple{DependencyMuxTagUpstreamSource, ScriptedAlarmEngine}}, NodeId?, Func{int}?, ITelemetryLocalHub?, TimeSpan?)"/>:
|
||||
/// every incarnation builds its own <see cref="DependencyMuxTagUpstreamSource"/> + a
|
||||
/// <see cref="ScriptedAlarmEngine"/> around it, and disposes them in <see cref="PostStop"/>.
|
||||
/// Akka's default restart runs <c>PostStop</c> then re-creates the actor from the SAME Props, so the
|
||||
/// legacy instance-taking Props — a fixed engine captured in the Props — restarted around a
|
||||
/// <em>disposed</em> engine and every later <c>LoadAsync</c> threw: the node's scripted alarms
|
||||
/// silently died. The legacy overload is kept source-compatible but is restart-safe only in the
|
||||
/// weaker sense that a restart now DETACHES rather than disposes its engine (see
|
||||
/// <see cref="PreRestart"/>); production should spawn through the factory.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
@@ -99,6 +105,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
/// Warning on every clean shutdown.</summary>
|
||||
private sealed record AlarmsLoadCanceled;
|
||||
|
||||
/// <summary>Default bound on one inbound alarm command's engine op (<see cref="OnAlarmCommand"/>): 10 s.</summary>
|
||||
public static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(10);
|
||||
|
||||
private readonly IActorRef _publishActor;
|
||||
private readonly IActorRef? _mux;
|
||||
private readonly DependencyMuxTagUpstreamSource _upstream;
|
||||
@@ -107,6 +116,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private readonly EventHandler<ScriptedAlarmEvent> _onEngineEvent;
|
||||
private readonly TimeSpan _commandTimeout;
|
||||
|
||||
/// <summary>True when THIS incarnation built its engine (factory Props) and therefore disposes it on
|
||||
/// every stop, restart included. False for the legacy instance-taking Props: the engine outlives the
|
||||
/// incarnation, so a restart only detaches from it and a final stop disposes it.</summary>
|
||||
private readonly bool _ownsEngine;
|
||||
|
||||
/// <summary>Set by <see cref="PreRestart"/> so the <see cref="PostStop"/> it triggers can tell a restart
|
||||
/// (the next incarnation reuses a legacy engine) from a genuine stop.</summary>
|
||||
private bool _restarting;
|
||||
|
||||
/// <summary>Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
|
||||
/// snapshot (null = unknown until the first snapshot arrives, or no <see cref="_localNode"/> wired). The
|
||||
@@ -152,6 +171,8 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
/// <param name="telemetryHub">Optional node-local live-telemetry hub (Phase 5) each cluster-wide
|
||||
/// <c>alerts</c> transition is also emitted into; null (tests) skips the emit.</param>
|
||||
/// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
|
||||
/// <remarks>Legacy, instance-taking shape — see the class remarks: a supervision restart re-creates the
|
||||
/// actor around these SAME instances, so prefer the factory overload in production.</remarks>
|
||||
public static Props Props(
|
||||
IActorRef publishActor,
|
||||
IActorRef? mux,
|
||||
@@ -162,6 +183,60 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
ITelemetryLocalHub? telemetryHub = null) =>
|
||||
Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub));
|
||||
|
||||
/// <summary>
|
||||
/// Restart-safe factory Props (R02:H1): each incarnation of the actor calls
|
||||
/// <paramref name="engineFactory"/> in its constructor and owns what it returns — the engine must
|
||||
/// be constructed <b>around</b> the returned upstream — so a supervision restart gets a fresh,
|
||||
/// undisposed engine and the next <see cref="ApplyScriptedAlarms"/> loads cleanly.
|
||||
/// </summary>
|
||||
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
|
||||
/// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
|
||||
/// <param name="engineFactory">Builds a fresh (upstream, engine) pair; called once per incarnation.</param>
|
||||
/// <param name="localNode">The local cluster node id for the Primary alerts-emit gate; null ⇒ default-emit.</param>
|
||||
/// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
|
||||
/// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
|
||||
/// <param name="commandTimeout">Bound on one inbound alarm command's engine op; null ⇒ <see cref="DefaultCommandTimeout"/>.</param>
|
||||
/// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
|
||||
public static Props Props(
|
||||
IActorRef publishActor,
|
||||
IActorRef? mux,
|
||||
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
|
||||
NodeId? localNode = null,
|
||||
Func<int>? driverMemberCountProvider = null,
|
||||
ITelemetryLocalHub? telemetryHub = null,
|
||||
TimeSpan? commandTimeout = null) =>
|
||||
Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, engineFactory, localNode, driverMemberCountProvider, telemetryHub, commandTimeout));
|
||||
|
||||
/// <summary>Factory-built incarnation: owns (and on every stop disposes) the engine it builds.</summary>
|
||||
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
|
||||
/// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
|
||||
/// <param name="engineFactory">Builds this incarnation's (upstream, engine) pair.</param>
|
||||
/// <param name="localNode">The local cluster node id for the Primary alerts-emit gate.</param>
|
||||
/// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
|
||||
/// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
|
||||
/// <param name="commandTimeout">Bound on one inbound alarm command's engine op.</param>
|
||||
public ScriptedAlarmHostActor(
|
||||
IActorRef publishActor,
|
||||
IActorRef? mux,
|
||||
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
|
||||
NodeId? localNode = null,
|
||||
Func<int>? driverMemberCountProvider = null,
|
||||
ITelemetryLocalHub? telemetryHub = null,
|
||||
TimeSpan? commandTimeout = null)
|
||||
: this(publishActor, mux, Build(engineFactory), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: true, commandTimeout)
|
||||
{
|
||||
}
|
||||
|
||||
private static (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) Build(
|
||||
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(engineFactory);
|
||||
var pair = engineFactory();
|
||||
ArgumentNullException.ThrowIfNull(pair.Upstream, "engineFactory().Upstream");
|
||||
ArgumentNullException.ThrowIfNull(pair.Engine, "engineFactory().Engine");
|
||||
return pair;
|
||||
}
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="ScriptedAlarmHostActor"/> class.</summary>
|
||||
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
|
||||
/// <param name="mux">Optional dependency multiplexer the host registers dependency interest with.</param>
|
||||
@@ -183,14 +258,29 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
NodeId? localNode = null,
|
||||
Func<int>? driverMemberCountProvider = null,
|
||||
ITelemetryLocalHub? telemetryHub = null)
|
||||
: this(publishActor, mux, (upstream, engine), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: false, commandTimeout: null)
|
||||
{
|
||||
}
|
||||
|
||||
private ScriptedAlarmHostActor(
|
||||
IActorRef publishActor,
|
||||
IActorRef? mux,
|
||||
(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) pair,
|
||||
NodeId? localNode,
|
||||
Func<int>? driverMemberCountProvider,
|
||||
ITelemetryLocalHub? telemetryHub,
|
||||
bool ownsEngine,
|
||||
TimeSpan? commandTimeout)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(publishActor);
|
||||
ArgumentNullException.ThrowIfNull(upstream);
|
||||
ArgumentNullException.ThrowIfNull(engine);
|
||||
ArgumentNullException.ThrowIfNull(pair.Upstream);
|
||||
ArgumentNullException.ThrowIfNull(pair.Engine);
|
||||
_publishActor = publishActor;
|
||||
_mux = mux;
|
||||
_upstream = upstream;
|
||||
_engine = engine;
|
||||
_upstream = pair.Upstream;
|
||||
_engine = pair.Engine;
|
||||
_ownsEngine = ownsEngine;
|
||||
_commandTimeout = commandTimeout is { } t && t > TimeSpan.Zero ? t : DefaultCommandTimeout;
|
||||
_localNode = localNode;
|
||||
_driverMemberCountProvider = driverMemberCountProvider;
|
||||
_telemetryHub = telemetryHub;
|
||||
@@ -294,8 +384,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
// subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the
|
||||
// engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor),
|
||||
// so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed.
|
||||
_mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self));
|
||||
_log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs", msg.DepRefs.Count);
|
||||
// ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a
|
||||
// supervision restart builds one around an EMPTY upstream) converges immediately instead of
|
||||
// waiting for every input to change again.
|
||||
_mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true));
|
||||
// Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references
|
||||
// any more will never be pushed again, so its last value is dead weight for the process lifetime.
|
||||
var dropped = _upstream.RetainOnly(new HashSet<string>(msg.DepRefs, StringComparer.Ordinal));
|
||||
_log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs (pruned {Dropped} retained path(s))",
|
||||
msg.DepRefs.Count, dropped);
|
||||
|
||||
ReassertConditionNodes();
|
||||
}
|
||||
@@ -591,18 +688,25 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
return;
|
||||
}
|
||||
|
||||
// Bounded (R02:H3): every op ends in IAlarmStateStore.SaveAsync (LocalDb/SQLite), and this is a
|
||||
// ReceiveAsync — the mailbox is suspended until the op returns, so an unbounded await on a hung
|
||||
// file lock stalled dependency-value ingestion for the duration. Linked to the actor's lifetime
|
||||
// token so a stop still cancels promptly.
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
|
||||
cts.CancelAfter(_commandTimeout);
|
||||
var ct = cts.Token;
|
||||
try
|
||||
{
|
||||
switch (cmd.Operation)
|
||||
{
|
||||
case "Acknowledge":
|
||||
await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
|
||||
await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
|
||||
break;
|
||||
case "Confirm":
|
||||
await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
|
||||
await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
|
||||
break;
|
||||
case "OneShotShelve":
|
||||
await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
|
||||
await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, ct);
|
||||
break;
|
||||
case "TimedShelve":
|
||||
// A timed shelve needs the absolute unshelve instant. T18 derives it from the OPC UA
|
||||
@@ -614,16 +718,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
cmd.AlarmId);
|
||||
return;
|
||||
}
|
||||
await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, CancellationToken.None);
|
||||
await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, ct);
|
||||
break;
|
||||
case "Unshelve":
|
||||
await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
|
||||
await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, ct);
|
||||
break;
|
||||
case "Enable":
|
||||
await _engine.EnableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
|
||||
await _engine.EnableAsync(cmd.AlarmId, cmd.User, ct);
|
||||
break;
|
||||
case "Disable":
|
||||
await _engine.DisableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
|
||||
await _engine.DisableAsync(cmd.AlarmId, cmd.User, ct);
|
||||
break;
|
||||
case "AddComment":
|
||||
if (string.IsNullOrWhiteSpace(cmd.Comment))
|
||||
@@ -631,7 +735,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
_log.Warning("ScriptedAlarmHost: rejecting AddComment for {AlarmId} — empty comment text", cmd.AlarmId);
|
||||
return;
|
||||
}
|
||||
await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
|
||||
await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
|
||||
break;
|
||||
default:
|
||||
_log.Warning("ScriptedAlarmHost: ignoring AlarmCommand with unknown operation {Op} for {AlarmId}",
|
||||
@@ -639,6 +743,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (!_cts.IsCancellationRequested)
|
||||
{
|
||||
_log.Warning("ScriptedAlarmHost: engine op {Op} for alarm {AlarmId} timed out after {Timeout} — the condition-state store is stalled",
|
||||
cmd.Operation, cmd.AlarmId, _commandTimeout);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A failing engine op must not crash the actor — mirror OnLoadFailed's log-and-stay-inert style.
|
||||
@@ -672,16 +781,28 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
base.PreStart();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Akka's default <c>PreRestart</c> stops the children and calls <see cref="PostStop"/>. The flag
|
||||
/// lets that <see cref="PostStop"/> keep a legacy (Props-captured) engine alive for the next incarnation
|
||||
/// instead of disposing the instance it is about to be re-created around.</remarks>
|
||||
protected override void PreRestart(Exception reason, object message)
|
||||
{
|
||||
_restarting = true;
|
||||
base.PreRestart(reason, message);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
// Unregister mux interest first so no further DependencyValueChanged arrives while we tear down.
|
||||
_mux?.Tell(new DependencyMuxActor.UnregisterInterest(Self));
|
||||
// Cancel any in-flight LoadAsync, detach the OnEvent handler (so a late engine emission can't
|
||||
// Tell a stopping actor), then dispose the engine (which drains its background work + clears).
|
||||
// Tell a stopping actor), then dispose the engine (which drains its background work + clears) —
|
||||
// unless this is a RESTART around a legacy Props-captured engine, which the next incarnation will
|
||||
// re-attach to (R02:H1).
|
||||
_cts.Cancel();
|
||||
_engine.OnEvent -= _onEngineEvent;
|
||||
_engine.Dispose();
|
||||
if (_ownsEngine || !_restarting) _engine.Dispose();
|
||||
_cts.Dispose();
|
||||
base.PostStop();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user