Merge lane E: script runtime safety + scripted alarms (#734)

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 14:03:54 -04:00
44 changed files with 3826 additions and 264 deletions
@@ -31,8 +31,17 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines;
/// into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no
/// longer open this window at all.
/// </para>
/// <para>
/// <b>Run quarantine.</b> Every compiled run goes through one <see cref="ScriptRunGuard"/> keyed by
/// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous
/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive
/// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach. Both surface as <see cref="VirtualTagEvalResult.Failure"/>
/// so <c>VirtualTagActor</c> publishes Bad exactly as it does for a timeout. A quarantine lifts when the
/// expression changes, on <see cref="ClearCompiledScripts"/> / <see cref="ResetQuarantines"/> (the deploy
/// boundary), never by itself.
/// </para>
/// </summary>
public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable
public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IScriptQuarantineOwner, IDisposable
{
// Purpose-built source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and
// Clear()/Dispose() that dispose each evaluator's collectible AssemblyLoadContext. The apply-boundary
@@ -42,22 +51,29 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
private readonly ILogger<RoslynVirtualTagEvaluator> _logger;
private readonly SerilogLogger _scriptRoot;
private readonly TimeSpan _runTimeout;
private readonly ScriptRunGuard _guard;
private bool _disposed;
/// <summary>Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout.</summary>
/// <param name="logger">Logger for recording compilation and execution errors (host diagnostics).</param>
/// <param name="scriptRoot">Root script logger; user <c>ctx.Logger.*</c> output flows through this to the Script-log page.</param>
/// <param name="runTimeout">Maximum execution time for each script; defaults to 2 seconds if not specified.</param>
/// <param name="scripting">Run-guard budget + quarantine threshold (the <c>Scripting</c> section); null ⇒ defaults.</param>
public RoslynVirtualTagEvaluator(
ILogger<RoslynVirtualTagEvaluator> logger,
ScriptRootLogger scriptRoot,
TimeSpan? runTimeout = null)
TimeSpan? runTimeout = null,
ScriptingOptions? scripting = null)
{
_logger = logger;
_scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
_runTimeout = runTimeout ?? TimeSpan.FromSeconds(2);
_guard = new ScriptRunGuard(scripting, _scriptRoot);
}
/// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
public ScriptRunGuard RunGuard => _guard;
// Value-only overload. Every dependency reaches the script stamped Good with an
// evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot
// overload instead; this one remains for callers that genuinely hold only
@@ -150,13 +166,13 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
try
{
// Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't
// interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous
// CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when
// the wall-clock budget fires regardless of whether the inner task completes (the orphaned
// thread is the documented, accepted trade-off — TimedScriptEvaluator remarks).
var timed = new TimedScriptEvaluator<VirtualTagContext, object?>(evaluator, _runTimeout);
var raw = timed.RunAsync(context).GetAwaiter().GetResult();
// Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by virtual-tag id): a raw
// CancellationToken can't interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll
// it), so the wall-clock budget is what returns control — and the guard is what stops the
// abandoned run from being joined by another on the next trigger (one orphan per tag, then
// quarantine), which is the leak the old per-evaluation TimedScriptEvaluator allowed.
var raw = _guard.RunAsync(virtualTagId, expression, evaluator, context, _runTimeout)
.GetAwaiter().GetResult();
return VirtualTagEvalResult.Ok(raw);
}
catch (ScriptTimeoutException)
@@ -164,6 +180,14 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
return VirtualTagEvalResult.Failure(
$"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s");
}
catch (ScriptRunRefusedException ex)
{
return VirtualTagEvalResult.Failure(ex.Message);
}
catch (ScriptQuarantinedException ex)
{
return VirtualTagEvalResult.Failure(ex.Message);
}
catch (ObjectDisposedException) when (!_disposed && attempt == 0)
{
// S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our
@@ -185,7 +209,19 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
}
/// <inheritdoc />
public void ClearCompiledScripts() => _cache.Clear();
public void ClearCompiledScripts()
{
_cache.Clear();
// The compile-cache clear IS the deploy boundary for this evaluator: a redeploy lifts every quarantine
// (an edited expression already lifts its own through the guard's source identity).
_guard.ResetAll();
}
/// <inheritdoc />
public void ResetQuarantines() => _guard.ResetAll();
/// <inheritdoc />
public IReadOnlyCollection<string> QuarantinedScripts => _guard.QuarantinedKeys;
/// <summary>Disposes the evaluator and cleans up all cached script evaluators.</summary>
public void Dispose()
@@ -42,8 +42,10 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
// publish.
private const uint StatusNeverPushed = OpcUaStatus.BadWaitingForInitialData;
// Intentionally never pruned: cold-start semantics depend on retained last values so
// ReadTag can answer synchronously for any path that has ever been pushed.
// Retained last values: ReadTag answers synchronously for any path that has been pushed. Pruned
// ONLY through RetainOnly — the host calls it with the loaded dependency-ref union after every
// (re)load, so a renamed/removed raw path (a RawTree rename is a surgical remove+add) stops
// costing a boxed value + two DateTimes for the process lifetime (R02:L3).
private readonly ConcurrentDictionary<string, DataValueSnapshot> _cache
= new(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, ImmutableList<Action<string, DataValueSnapshot>>> _observers
@@ -91,6 +93,28 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
}
}
/// <summary>
/// Drop every cached snapshot whose path is not in <paramref name="paths"/>. Observers are
/// untouched. Called by the host after each successful engine (re)load with the union of the loaded
/// alarms' dependency refs — exactly the set the mux will push from now on — so paths no alarm
/// references any more stop being retained.
/// </summary>
/// <param name="paths">The paths to keep.</param>
/// <returns>The number of cached snapshots dropped.</returns>
public int RetainOnly(IReadOnlySet<string> paths)
{
ArgumentNullException.ThrowIfNull(paths);
var dropped = 0;
foreach (var key in _cache.Keys)
{
if (!paths.Contains(key) && _cache.TryRemove(key, out _)) dropped++;
}
return dropped;
}
/// <summary>Test/diagnostic: the number of paths currently retained.</summary>
public int RetainedPathCount => _cache.Count;
/// <inheritdoc/>
public DataValueSnapshot ReadTag(string path)
{
@@ -23,10 +23,16 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
/// cluster <c>alerts</c> DistributedPubSub topic.
///
/// <para>
/// The host is the engine's lifecycle owner: the caller (the driver-role startup) builds
/// the <see cref="DependencyMuxTagUpstreamSource"/>, constructs a <see cref="ScriptedAlarmEngine"/>
/// <b>around that same upstream</b>, and passes both here. The host disposes the engine in
/// <see cref="PostStop"/>.
/// The host is the engine's lifecycle owner. The restart-safe shape (R02:H1) is the
/// <b>factory</b> <see cref="Props(IActorRef, IActorRef?, Func{ValueTuple{DependencyMuxTagUpstreamSource, ScriptedAlarmEngine}}, NodeId?, Func{int}?, ITelemetryLocalHub?, TimeSpan?)"/>:
/// every incarnation builds its own <see cref="DependencyMuxTagUpstreamSource"/> + a
/// <see cref="ScriptedAlarmEngine"/> around it, and disposes them in <see cref="PostStop"/>.
/// Akka's default restart runs <c>PostStop</c> then re-creates the actor from the SAME Props, so the
/// legacy instance-taking Props — a fixed engine captured in the Props — restarted around a
/// <em>disposed</em> engine and every later <c>LoadAsync</c> threw: the node's scripted alarms
/// silently died. The legacy overload is kept source-compatible but is restart-safe only in the
/// weaker sense that a restart now DETACHES rather than disposes its engine (see
/// <see cref="PreRestart"/>); production should spawn through the factory.
/// </para>
///
/// <para>
@@ -99,6 +105,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
/// Warning on every clean shutdown.</summary>
private sealed record AlarmsLoadCanceled;
/// <summary>Default bound on one inbound alarm command's engine op (<see cref="OnAlarmCommand"/>): 10 s.</summary>
public static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(10);
private readonly IActorRef _publishActor;
private readonly IActorRef? _mux;
private readonly DependencyMuxTagUpstreamSource _upstream;
@@ -107,6 +116,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
private readonly ILoggingAdapter _log = Context.GetLogger();
private readonly CancellationTokenSource _cts = new();
private readonly EventHandler<ScriptedAlarmEvent> _onEngineEvent;
private readonly TimeSpan _commandTimeout;
/// <summary>True when THIS incarnation built its engine (factory Props) and therefore disposes it on
/// every stop, restart included. False for the legacy instance-taking Props: the engine outlives the
/// incarnation, so a restart only detaches from it and a final stop disposes it.</summary>
private readonly bool _ownsEngine;
/// <summary>Set by <see cref="PreRestart"/> so the <see cref="PostStop"/> it triggers can tell a restart
/// (the next incarnation reuses a legacy engine) from a genuine stop.</summary>
private bool _restarting;
/// <summary>Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
/// snapshot (null = unknown until the first snapshot arrives, or no <see cref="_localNode"/> wired). The
@@ -152,6 +171,8 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
/// <param name="telemetryHub">Optional node-local live-telemetry hub (Phase 5) each cluster-wide
/// <c>alerts</c> transition is also emitted into; null (tests) skips the emit.</param>
/// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
/// <remarks>Legacy, instance-taking shape — see the class remarks: a supervision restart re-creates the
/// actor around these SAME instances, so prefer the factory overload in production.</remarks>
public static Props Props(
IActorRef publishActor,
IActorRef? mux,
@@ -162,6 +183,60 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
ITelemetryLocalHub? telemetryHub = null) =>
Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub));
/// <summary>
/// Restart-safe factory Props (R02:H1): each incarnation of the actor calls
/// <paramref name="engineFactory"/> in its constructor and owns what it returns — the engine must
/// be constructed <b>around</b> the returned upstream — so a supervision restart gets a fresh,
/// undisposed engine and the next <see cref="ApplyScriptedAlarms"/> loads cleanly.
/// </summary>
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
/// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
/// <param name="engineFactory">Builds a fresh (upstream, engine) pair; called once per incarnation.</param>
/// <param name="localNode">The local cluster node id for the Primary alerts-emit gate; null ⇒ default-emit.</param>
/// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
/// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
/// <param name="commandTimeout">Bound on one inbound alarm command's engine op; null ⇒ <see cref="DefaultCommandTimeout"/>.</param>
/// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
public static Props Props(
IActorRef publishActor,
IActorRef? mux,
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
NodeId? localNode = null,
Func<int>? driverMemberCountProvider = null,
ITelemetryLocalHub? telemetryHub = null,
TimeSpan? commandTimeout = null) =>
Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, engineFactory, localNode, driverMemberCountProvider, telemetryHub, commandTimeout));
/// <summary>Factory-built incarnation: owns (and on every stop disposes) the engine it builds.</summary>
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
/// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
/// <param name="engineFactory">Builds this incarnation's (upstream, engine) pair.</param>
/// <param name="localNode">The local cluster node id for the Primary alerts-emit gate.</param>
/// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
/// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
/// <param name="commandTimeout">Bound on one inbound alarm command's engine op.</param>
public ScriptedAlarmHostActor(
IActorRef publishActor,
IActorRef? mux,
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
NodeId? localNode = null,
Func<int>? driverMemberCountProvider = null,
ITelemetryLocalHub? telemetryHub = null,
TimeSpan? commandTimeout = null)
: this(publishActor, mux, Build(engineFactory), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: true, commandTimeout)
{
}
private static (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) Build(
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory)
{
ArgumentNullException.ThrowIfNull(engineFactory);
var pair = engineFactory();
ArgumentNullException.ThrowIfNull(pair.Upstream, "engineFactory().Upstream");
ArgumentNullException.ThrowIfNull(pair.Engine, "engineFactory().Engine");
return pair;
}
/// <summary>Initializes a new instance of the <see cref="ScriptedAlarmHostActor"/> class.</summary>
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
/// <param name="mux">Optional dependency multiplexer the host registers dependency interest with.</param>
@@ -183,14 +258,29 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
NodeId? localNode = null,
Func<int>? driverMemberCountProvider = null,
ITelemetryLocalHub? telemetryHub = null)
: this(publishActor, mux, (upstream, engine), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: false, commandTimeout: null)
{
}
private ScriptedAlarmHostActor(
IActorRef publishActor,
IActorRef? mux,
(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) pair,
NodeId? localNode,
Func<int>? driverMemberCountProvider,
ITelemetryLocalHub? telemetryHub,
bool ownsEngine,
TimeSpan? commandTimeout)
{
ArgumentNullException.ThrowIfNull(publishActor);
ArgumentNullException.ThrowIfNull(upstream);
ArgumentNullException.ThrowIfNull(engine);
ArgumentNullException.ThrowIfNull(pair.Upstream);
ArgumentNullException.ThrowIfNull(pair.Engine);
_publishActor = publishActor;
_mux = mux;
_upstream = upstream;
_engine = engine;
_upstream = pair.Upstream;
_engine = pair.Engine;
_ownsEngine = ownsEngine;
_commandTimeout = commandTimeout is { } t && t > TimeSpan.Zero ? t : DefaultCommandTimeout;
_localNode = localNode;
_driverMemberCountProvider = driverMemberCountProvider;
_telemetryHub = telemetryHub;
@@ -294,8 +384,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
// subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the
// engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor),
// so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed.
_mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self));
_log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs", msg.DepRefs.Count);
// ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a
// supervision restart builds one around an EMPTY upstream) converges immediately instead of
// waiting for every input to change again.
_mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true));
// Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references
// any more will never be pushed again, so its last value is dead weight for the process lifetime.
var dropped = _upstream.RetainOnly(new HashSet<string>(msg.DepRefs, StringComparer.Ordinal));
_log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs (pruned {Dropped} retained path(s))",
msg.DepRefs.Count, dropped);
ReassertConditionNodes();
}
@@ -591,18 +688,25 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
return;
}
// Bounded (R02:H3): every op ends in IAlarmStateStore.SaveAsync (LocalDb/SQLite), and this is a
// ReceiveAsync — the mailbox is suspended until the op returns, so an unbounded await on a hung
// file lock stalled dependency-value ingestion for the duration. Linked to the actor's lifetime
// token so a stop still cancels promptly.
using var cts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
cts.CancelAfter(_commandTimeout);
var ct = cts.Token;
try
{
switch (cmd.Operation)
{
case "Acknowledge":
await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
break;
case "Confirm":
await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
break;
case "OneShotShelve":
await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, ct);
break;
case "TimedShelve":
// A timed shelve needs the absolute unshelve instant. T18 derives it from the OPC UA
@@ -614,16 +718,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
cmd.AlarmId);
return;
}
await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, CancellationToken.None);
await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, ct);
break;
case "Unshelve":
await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, ct);
break;
case "Enable":
await _engine.EnableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
await _engine.EnableAsync(cmd.AlarmId, cmd.User, ct);
break;
case "Disable":
await _engine.DisableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
await _engine.DisableAsync(cmd.AlarmId, cmd.User, ct);
break;
case "AddComment":
if (string.IsNullOrWhiteSpace(cmd.Comment))
@@ -631,7 +735,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
_log.Warning("ScriptedAlarmHost: rejecting AddComment for {AlarmId} — empty comment text", cmd.AlarmId);
return;
}
await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
break;
default:
_log.Warning("ScriptedAlarmHost: ignoring AlarmCommand with unknown operation {Op} for {AlarmId}",
@@ -639,6 +743,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
break;
}
}
catch (OperationCanceledException) when (!_cts.IsCancellationRequested)
{
_log.Warning("ScriptedAlarmHost: engine op {Op} for alarm {AlarmId} timed out after {Timeout} — the condition-state store is stalled",
cmd.Operation, cmd.AlarmId, _commandTimeout);
}
catch (Exception ex)
{
// A failing engine op must not crash the actor — mirror OnLoadFailed's log-and-stay-inert style.
@@ -672,16 +781,28 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
base.PreStart();
}
/// <inheritdoc />
/// <remarks>Akka's default <c>PreRestart</c> stops the children and calls <see cref="PostStop"/>. The flag
/// lets that <see cref="PostStop"/> keep a legacy (Props-captured) engine alive for the next incarnation
/// instead of disposing the instance it is about to be re-created around.</remarks>
protected override void PreRestart(Exception reason, object message)
{
_restarting = true;
base.PreRestart(reason, message);
}
/// <inheritdoc />
protected override void PostStop()
{
// Unregister mux interest first so no further DependencyValueChanged arrives while we tear down.
_mux?.Tell(new DependencyMuxActor.UnregisterInterest(Self));
// Cancel any in-flight LoadAsync, detach the OnEvent handler (so a late engine emission can't
// Tell a stopping actor), then dispose the engine (which drains its background work + clears).
// Tell a stopping actor), then dispose the engine (which drains its background work + clears) —
// unless this is a RESTART around a legacy Props-captured engine, which the next incarnation will
// re-attach to (R02:H1).
_cts.Cancel();
_engine.OnEvent -= _onEngineEvent;
_engine.Dispose();
if (_ownsEngine || !_restarting) _engine.Dispose();
_cts.Dispose();
base.PostStop();
}
@@ -62,7 +62,10 @@ public sealed class DependencyConsumerMuxAdapter : ReceiveActor
var refs = _consumer.DependencyRefs.ToList();
// A RegisterInterest replaces the prior interest set for this subscriber (mux is idempotent on
// re-register), so re-registering after an apply cleanly adopts an added/removed ref.
_mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self));
// ReplayRetained (R07:H6): a driver Restart is a NEW consumer behind a new adapter; the mux replays
// the last value of every ref it retained so the calc tags compute from current inputs at once
// instead of sitting BadWaitingForInitialData until each input happens to change.
_mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self, ReplayRetained: true));
_log.Debug("DependencyConsumerMuxAdapter: registered {Count} dependency ref(s)", refs.Count);
}
@@ -26,32 +26,134 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
/// <see cref="VirtualTagActor.DependencyValueChanged"/> receive as well, for direct senders and tests.
/// The mux is deliberately not a DPS subscriber — virtual-tag evaluation is local to each node and
/// would over-emit if it spanned the cluster.
///
/// <para>
/// <b>Last-value retention + replay (R07:H6, bounded in review fix round 1 / I2).</b> The mux
/// retains the last <c>(value, status, timestamp)</c> of a ref <b>only while at least one subscriber
/// registered with <see cref="RegisterInterest.ReplayRetained"/> names it</b> (the virtual-tag
/// actors, the Calculation mux adapter, the scripted-alarm host). A registration that asks receives
/// one batch of the retained values for its refs, in the shape a driver burst arrives in. The case
/// this serves is a subscriber that goes away and comes back: a Calculation driver Restart is a new
/// driver instance + a new mux adapter, and before this every calc tag sat
/// <c>BadWaitingForInitialData</c> until each of its inputs happened to change again. So when a
/// ref's last replay-opting subscriber leaves, its value is kept for <see cref="RetentionGrace"/>
/// (default <see cref="DefaultRetentionGrace"/>; a returning replay registration inside it cancels
/// the release), then dropped together with the ref's entry.
/// </para>
/// <para>
/// A subscriber that does NOT opt in retains nothing: the continuous-historization recorder
/// registers every historized ref on the node (a replay there would record a stale sample per tag
/// per deploy), and retaining on its behalf made the map grow with the historized address space and
/// never shrink. Memory is therefore bounded by the refs replay-opting subscribers currently name,
/// plus those released within the last grace window. A ref with no subscriber and nothing retained
/// has no entry at all.
/// </para>
/// <para>
/// <b>Source removal.</b> The mux records which driver instance last published each retained ref;
/// <see cref="SourceRemoved"/> evicts every value that driver produced, so a removed driver's last
/// Good is never replayed as current. A comms-loss fan (a Bad batch) already overwrites the retained
/// value with Bad. ⚠️ As of fix round 1 no production actor sends <see cref="SourceRemoved"/> yet —
/// <c>DriverHostActor</c> (outside the scripting lane) must Tell it when it stops / removes a driver
/// child. Until then a removed driver's last value can still be replayed to a replay-opting
/// subscriber that re-registers for that ref while another replay-opting subscriber holds it, or
/// within the grace window.
/// </para>
/// </summary>
public sealed class DependencyMuxActor : ReceiveActor
public sealed class DependencyMuxActor : ReceiveActor, IWithTimers
{
public const string ActorName = "dependency-mux";
/// <summary>How long a ref's retained value outlives its last replay-opting subscriber.</summary>
public static readonly TimeSpan DefaultRetentionGrace = TimeSpan.FromMinutes(2);
/// <summary>Register a subscriber's interest in a set of tag refs. Idempotent on re-register —
/// the second call replaces the prior interest set for that subscriber.</summary>
public sealed record RegisterInterest(IReadOnlyList<string> TagRefs, IActorRef Subscriber);
/// <param name="TagRefs">The refs the subscriber wants.</param>
/// <param name="Subscriber">The subscriber.</param>
/// <param name="ReplayRetained">When true, the mux retains the last value of every ref in
/// <paramref name="TagRefs"/> while this subscriber is registered, and immediately sends the subscriber one
/// <see cref="VirtualTagActor.DependencyValueBatch"/> holding the last retained value of every ref it has
/// one for (nothing when it has none).</param>
public sealed record RegisterInterest(IReadOnlyList<string> TagRefs, IActorRef Subscriber, bool ReplayRetained = false);
/// <summary>Unregister every interest held by <see cref="Subscriber"/>. Sent on PostStop by
/// the subscriber, or by Terminated handling when the mux watches.</summary>
public sealed record UnregisterInterest(IActorRef Subscriber);
private readonly Dictionary<string, HashSet<IActorRef>> _byRef = new(StringComparer.Ordinal);
private readonly Dictionary<IActorRef, HashSet<string>> _bySubscriber = new();
/// <summary>A driver instance was removed / stopped: evict every retained value it produced so it is
/// never replayed. Subscriptions are untouched.</summary>
/// <param name="DriverInstanceId">The removed driver instance.</param>
public sealed record SourceRemoved(string DriverInstanceId);
private sealed class SweepReleased
{
public static readonly SweepReleased Instance = new();
}
private const string SweepTimerKey = "sweep-released";
/// <summary>Per-ref state. One lookup on the hot path answers both "who wants it" and "retain it?".</summary>
private sealed class RefEntry
{
public readonly HashSet<IActorRef> Subscribers = new();
/// <summary>Registered subscribers of this ref that opted into replay.</summary>
public int Replayers;
/// <summary>When the last replay-opting subscriber left (retention in grace), or null.</summary>
public DateTime? ReleasedAtUtc;
public VirtualTagActor.DependencyValueChanged? Last;
public string? LastSource;
public bool Retains => Replayers > 0 || ReleasedAtUtc is not null;
public bool IsDead => Subscribers.Count == 0 && Replayers == 0 && Last is null;
public void Forget()
{
Last = null;
LastSource = null;
ReleasedAtUtc = null;
}
}
private sealed record Registration(HashSet<string> Refs, bool Replay);
private readonly Dictionary<string, RefEntry> _byRef = new(StringComparer.Ordinal);
private readonly Dictionary<IActorRef, Registration> _bySubscriber = new();
private readonly ILoggingAdapter _log = Context.GetLogger();
/// <summary>How long a released ref's value is kept.</summary>
public TimeSpan RetentionGrace { get; }
/// <inheritdoc />
public ITimerScheduler Timers { get; set; } = null!;
/// <summary>Test hook — refs that currently hold a retained value.</summary>
internal int RetainedRefCount => _byRef.Values.Count(e => e.Last is not null);
/// <summary>Test hook — refs the mux holds any state for (subscribers or a retained value).</summary>
internal int TrackedRefCount => _byRef.Count;
/// <summary>Creates props for the DependencyMuxActor.</summary>
/// <param name="retentionGrace">How long a ref's value outlives its last replay-opting subscriber;
/// null ⇒ <see cref="DefaultRetentionGrace"/>.</param>
/// <returns>The props for creating this actor.</returns>
public static Props Props() => Akka.Actor.Props.Create<DependencyMuxActor>();
public static Props Props(TimeSpan? retentionGrace = null)
{
var grace = retentionGrace ?? DefaultRetentionGrace;
return Akka.Actor.Props.Create(() => new DependencyMuxActor(grace));
}
/// <summary>Initializes a new instance of the <see cref="DependencyMuxActor"/> class.</summary>
public DependencyMuxActor()
/// <param name="retentionGrace">See <see cref="Props"/>; non-positive ⇒ <see cref="DefaultRetentionGrace"/>.</param>
public DependencyMuxActor(TimeSpan retentionGrace)
{
RetentionGrace = retentionGrace > TimeSpan.Zero ? retentionGrace : DefaultRetentionGrace;
Receive<RegisterInterest>(OnRegister);
Receive<UnregisterInterest>(msg => RemoveSubscriber(msg.Subscriber));
Receive<SourceRemoved>(OnSourceRemoved);
Receive<SweepReleased>(_ => OnSweep());
Receive<DriverInstanceActor.AttributeValuePublished>(OnAttributeValuePublished);
Receive<DriverInstanceActor.AttributeValueBatchPublished>(OnAttributeValueBatchPublished);
Receive<Terminated>(msg => RemoveSubscriber(msg.ActorRef));
@@ -59,60 +161,125 @@ public sealed class DependencyMuxActor : ReceiveActor
private void OnRegister(RegisterInterest msg)
{
// Replace any prior interest set so re-registers on actor restart don't leak old refs.
if (_bySubscriber.TryGetValue(msg.Subscriber, out var priorRefs))
{
foreach (var r in priorRefs)
{
if (_byRef.TryGetValue(r, out var set))
{
set.Remove(msg.Subscriber);
if (set.Count == 0) _byRef.Remove(r);
}
}
}
// Replace any prior interest set so re-registers on actor restart don't leak old refs. Detaching first
// may put a ref into grace; the re-attach below takes it straight back out.
Detach(msg.Subscriber);
var refs = new HashSet<string>(msg.TagRefs, StringComparer.Ordinal);
_bySubscriber[msg.Subscriber] = refs;
_bySubscriber[msg.Subscriber] = new Registration(refs, msg.ReplayRetained);
List<VirtualTagActor.DependencyValueChanged>? replay = null;
foreach (var r in refs)
{
if (!_byRef.TryGetValue(r, out var set))
if (!_byRef.TryGetValue(r, out var entry))
{
set = new HashSet<IActorRef>();
_byRef[r] = set;
entry = new RefEntry();
_byRef[r] = entry;
}
set.Add(msg.Subscriber);
entry.Subscribers.Add(msg.Subscriber);
if (!msg.ReplayRetained) continue;
entry.Replayers++;
entry.ReleasedAtUtc = null;
if (entry.Last is { } retained)
(replay ??= []).Add(retained);
}
Context.Watch(msg.Subscriber);
_log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count);
_log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs (replay {Replay})",
msg.Subscriber, refs.Count, msg.ReplayRetained);
// One batch, after the registration is complete, in exactly the shape a driver burst arrives in —
// so the subscriber's hot path handles it with no special case.
if (replay is not null)
{
msg.Subscriber.Tell(new VirtualTagActor.DependencyValueBatch(replay));
_log.Debug("DependencyMux: replayed {Count} retained value(s) to {Sub}", replay.Count, msg.Subscriber);
}
}
private void RemoveSubscriber(IActorRef subscriber)
{
if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return;
foreach (var r in refs)
{
if (_byRef.TryGetValue(r, out var set))
{
set.Remove(subscriber);
if (set.Count == 0) _byRef.Remove(r);
}
}
_bySubscriber.Remove(subscriber);
if (!Detach(subscriber)) return;
Context.Unwatch(subscriber);
_log.Debug("DependencyMux: subscriber {Sub} removed", subscriber);
}
/// <summary>Drop <paramref name="subscriber"/>'s registration. A ref losing its last replay-opting
/// subscriber goes into grace; a ref left with no state at all is removed.</summary>
private bool Detach(IActorRef subscriber)
{
if (!_bySubscriber.Remove(subscriber, out var registration)) return false;
var now = DateTime.UtcNow;
var released = false;
foreach (var r in registration.Refs)
{
if (!_byRef.TryGetValue(r, out var entry)) continue;
entry.Subscribers.Remove(subscriber);
if (registration.Replay && --entry.Replayers == 0 && entry.Last is not null)
{
entry.ReleasedAtUtc = now;
released = true;
}
if (entry.IsDead) _byRef.Remove(r);
}
if (released && !Timers.IsTimerActive(SweepTimerKey))
Timers.StartPeriodicTimer(SweepTimerKey, SweepReleased.Instance, RetentionGrace);
return true;
}
/// <summary>Drop retained values whose grace has run out — with a periodic sweep at the grace interval a
/// released value lives between one and two grace periods — and stop the sweep once nothing is in grace.</summary>
private void OnSweep()
{
var cutoff = DateTime.UtcNow - RetentionGrace;
var stillInGrace = false;
List<string>? dead = null;
foreach (var (r, entry) in _byRef)
{
if (entry.ReleasedAtUtc is not { } releasedAt) continue;
if (releasedAt > cutoff)
{
stillInGrace = true;
continue;
}
entry.Forget();
if (entry.IsDead) (dead ??= []).Add(r);
}
if (dead is not null)
foreach (var r in dead) _byRef.Remove(r);
if (!stillInGrace) Timers.Cancel(SweepTimerKey);
}
private void OnSourceRemoved(SourceRemoved msg)
{
List<string>? dead = null;
var evicted = 0;
foreach (var (r, entry) in _byRef)
{
if (entry.Last is null || !string.Equals(entry.LastSource, msg.DriverInstanceId, StringComparison.Ordinal))
continue;
entry.Forget();
evicted++;
if (entry.IsDead) (dead ??= []).Add(r);
}
if (dead is not null)
foreach (var r in dead) _byRef.Remove(r);
_log.Debug("DependencyMux: source {Driver} removed — evicted {Count} retained value(s)", msg.DriverInstanceId, evicted);
}
private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg)
{
if (!_byRef.TryGetValue(msg.FullReference, out var subscribers) || subscribers.Count == 0)
if (!_byRef.TryGetValue(msg.FullReference, out var entry))
{
// No virtual tag cares about this ref — drop. Common in normal operation; the address
// Nothing is registered for (or retaining) this ref — drop. Common in normal operation; the address
// space carries thousands of tags and only a fraction feed virtual-tag expressions.
return;
}
var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode);
foreach (var sub in subscribers)
if (entry.Retains)
{
entry.Last = dep;
entry.LastSource = msg.DriverInstanceId;
}
foreach (var sub in entry.Subscribers)
{
// A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs
// only the batch receive on its hot path. One list per subscriber: batch ownership transfers
@@ -122,18 +289,9 @@ public sealed class DependencyMuxActor : ReceiveActor
}
}
/// <summary>Filters one batched driver production burst per item and delivers each subscriber's matched
/// slice as ONE <see cref="VirtualTagActor.DependencyValueBatch"/>. Items nobody registered for are
/// dropped (the common case: only a fraction of an address space feeds virtual-tag expressions), so a
/// burst of N values costs one inbound message plus one outbound message per genuinely interested
/// SUBSCRIBER — not per (item, subscriber) pair as it did when this fanned out per value. That is what
/// lets <see cref="VirtualTagActor"/> evaluate its script once per burst instead of once per input.
/// <para>The registered set is NOT just virtual-tag inputs: <c>ContinuousHistorizationRecorder</c>
/// registers interest in every historized ref, so on a historizing node <see cref="_byRef"/> can be
/// large and a sizeable fraction of each burst resolves.</para></summary>
private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg)
{
// Nobody has registered anything — skip the whole burst rather than probing an empty map per item.
// Nothing registered and nothing retained — skip the whole burst rather than probing an empty map per item.
if (_byRef.Count == 0) return;
// Allocated lazily on the first matched item: a burst in which nothing is registered for — still
@@ -146,8 +304,15 @@ public sealed class DependencyMuxActor : ReceiveActor
for (var i = 0; i < count; i++)
{
var item = msg.Items[i];
if (!_byRef.TryGetValue(item.FullReference, out var subscribers) || subscribers.Count == 0) continue;
if (!_byRef.TryGetValue(item.FullReference, out var entry)) continue;
var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode);
if (entry.Retains)
{
entry.Last = dep;
entry.LastSource = msg.DriverInstanceId;
}
var subscribers = entry.Subscribers;
if (subscribers.Count == 0) continue; // retained (in grace) for a returning subscriber; nobody to forward to now
perSubscriber ??= new Dictionary<IActorRef, List<VirtualTagActor.DependencyValueChanged>>();
foreach (var sub in subscribers)
{
@@ -7,6 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
@@ -76,6 +77,10 @@ public sealed class VirtualTagActor : ReceiveActor
private readonly IReadOnlyList<string> _dependencyRefs;
private readonly IActorRef? _mux;
private readonly ITelemetryLocalHub? _telemetryHub;
// R02:P6 — a script failing on every trigger of a 1 Hz input published one cluster-wide DPS
// ScriptLogEntry per second per tag; 200 such tags shed everyone's entries out of the bounded
// telemetry channels. The per-failure entry is now rate-limited per tag (20/min + one summary).
private readonly ScriptLogRateLimiter _failureLogLimiter;
private readonly ILoggingAdapter _log = Context.GetLogger();
// Dependency SNAPSHOTS, not bare values (archreview 02/S14): the status code is what distinguishes a
// comms-lost input from a genuine Good null, and it is what the script reads through
@@ -104,6 +109,8 @@ public sealed class VirtualTagActor : ReceiveActor
/// <param name="mux">Optional reference to a dependency multiplexer actor.</param>
/// <param name="telemetryHub">Optional node-local live-telemetry hub each script-log entry is also
/// emitted into (Phase 5); null (tests) simply skips the emit.</param>
/// <param name="failureLogLimiter">Rate limiter for this tag's per-failure script-log entries; null ⇒
/// the default 20 per minute + one summary.</param>
/// <returns>A configured <see cref="Akka.Actor.Props"/> for creating the actor.</returns>
public static Props Props(
string virtualTagId,
@@ -113,7 +120,8 @@ public sealed class VirtualTagActor : ReceiveActor
Func<DPSPublisher>? publisherFactory = null,
IReadOnlyList<string>? dependencyRefs = null,
IActorRef? mux = null,
ITelemetryLocalHub? telemetryHub = null) =>
ITelemetryLocalHub? telemetryHub = null,
ScriptLogRateLimiter? failureLogLimiter = null) =>
Akka.Actor.Props.Create(() => new VirtualTagActor(
virtualTagId, expression,
evaluator ?? NullVirtualTagEvaluator.Instance,
@@ -121,7 +129,8 @@ public sealed class VirtualTagActor : ReceiveActor
publisherFactory,
dependencyRefs ?? Array.Empty<string>(),
mux,
telemetryHub));
telemetryHub,
failureLogLimiter));
/// <summary>Initializes a virtual tag actor with the given configuration and dependencies.</summary>
/// <param name="virtualTagId">Unique identifier for the virtual tag.</param>
@@ -133,6 +142,8 @@ public sealed class VirtualTagActor : ReceiveActor
/// <param name="mux">Optional reference to a dependency multiplexer actor.</param>
/// <param name="telemetryHub">Optional node-local live-telemetry hub each script-log entry is also
/// emitted into (Phase 5); null (tests) simply skips the emit.</param>
/// <param name="failureLogLimiter">Rate limiter for this tag's per-failure script-log entries; null ⇒
/// the default 20 per minute + one summary.</param>
public VirtualTagActor(
string virtualTagId,
string expression,
@@ -141,7 +152,8 @@ public sealed class VirtualTagActor : ReceiveActor
Func<DPSPublisher>? publisherFactory,
IReadOnlyList<string> dependencyRefs,
IActorRef? mux,
ITelemetryLocalHub? telemetryHub = null)
ITelemetryLocalHub? telemetryHub = null,
ScriptLogRateLimiter? failureLogLimiter = null)
{
_virtualTagId = virtualTagId;
_scriptId = scriptId;
@@ -151,6 +163,7 @@ public sealed class VirtualTagActor : ReceiveActor
_dependencyRefs = dependencyRefs;
_mux = mux;
_telemetryHub = telemetryHub;
_failureLogLimiter = failureLogLimiter ?? new ScriptLogRateLimiter();
Receive<DependencyValueChanged>(OnDependencyChanged);
Receive<DependencyValueBatch>(OnDependencyBatch);
@@ -162,7 +175,9 @@ public sealed class VirtualTagActor : ReceiveActor
{
if (_mux is not null && _dependencyRefs.Count > 0)
{
_mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self));
// ReplayRetained: a respawned child (plan edit, supervision restart) evaluates from the mux's
// last-known inputs at once instead of waiting for each to change again.
_mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self, ReplayRetained: true));
}
}
@@ -353,7 +368,18 @@ public sealed class VirtualTagActor : ReceiveActor
private void OnEvaluationFailed(string reason, string level, DateTime timestampUtc)
{
OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair<string, object?>("outcome", "fail"));
PublishLog(level, reason);
// Rate-limited per tag (R02:P6): the first 20 failures a minute publish as before; the rest are
// counted and surfaced as ONE summary entry when the window rolls. The metric above and the Bad
// publish below are not limited — only the cluster-wide log fan-out is.
var admission = _failureLogLimiter.Admit(_virtualTagId);
if (admission.SuppressedInPreviousWindow > 0)
{
PublishLog("Warning",
$"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " +
$"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {_failureLogLimiter.LimitPerWindow}/window); last: {reason}");
}
if (admission.Admitted) PublishLog(level, reason);
if ((_hasPublished && _lastPublishedStatus == OpcUaStatus.BadInternalError) || !AllDependenciesArrived())
{
@@ -3,6 +3,7 @@ using Akka.Event;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
@@ -112,6 +113,11 @@ public sealed class VirtualTagHostActor : ReceiveActor
(_evaluator as IScriptCacheOwner)?.ClearCompiledScripts();
}
// Every apply is a deploy, and a deploy lifts the run guard's quarantines unconditionally — "the
// operator redeployed to retry" is one of the two documented ways out of a quarantine (the other,
// an edited expression, the guard detects on its own). Cheap on a no-op redeploy (an empty map).
(_evaluator as IScriptQuarantineOwner)?.ResetQuarantines();
var desired = new HashSet<string>(msg.Plans.Select(p => p.VirtualTagId), StringComparer.Ordinal);
// Stop + forget children whose vtagId is no longer desired. Stopping the child triggers its