Merge lane E: script runtime safety + scripted alarms (#734)
Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
@@ -31,8 +31,17 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines;
|
||||
/// into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no
|
||||
/// longer open this window at all.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Run quarantine.</b> Every compiled run goes through one <see cref="ScriptRunGuard"/> keyed by
|
||||
/// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous
|
||||
/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive
|
||||
/// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach. Both surface as <see cref="VirtualTagEvalResult.Failure"/>
|
||||
/// so <c>VirtualTagActor</c> publishes Bad exactly as it does for a timeout. A quarantine lifts when the
|
||||
/// expression changes, on <see cref="ClearCompiledScripts"/> / <see cref="ResetQuarantines"/> (the deploy
|
||||
/// boundary), never by itself.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable
|
||||
public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IScriptQuarantineOwner, IDisposable
|
||||
{
|
||||
// Purpose-built source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and
|
||||
// Clear()/Dispose() that dispose each evaluator's collectible AssemblyLoadContext. The apply-boundary
|
||||
@@ -42,22 +51,29 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
private readonly ILogger<RoslynVirtualTagEvaluator> _logger;
|
||||
private readonly SerilogLogger _scriptRoot;
|
||||
private readonly TimeSpan _runTimeout;
|
||||
private readonly ScriptRunGuard _guard;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout.</summary>
|
||||
/// <param name="logger">Logger for recording compilation and execution errors (host diagnostics).</param>
|
||||
/// <param name="scriptRoot">Root script logger; user <c>ctx.Logger.*</c> output flows through this to the Script-log page.</param>
|
||||
/// <param name="runTimeout">Maximum execution time for each script; defaults to 2 seconds if not specified.</param>
|
||||
/// <param name="scripting">Run-guard budget + quarantine threshold (the <c>Scripting</c> section); null ⇒ defaults.</param>
|
||||
public RoslynVirtualTagEvaluator(
|
||||
ILogger<RoslynVirtualTagEvaluator> logger,
|
||||
ScriptRootLogger scriptRoot,
|
||||
TimeSpan? runTimeout = null)
|
||||
TimeSpan? runTimeout = null,
|
||||
ScriptingOptions? scripting = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger;
|
||||
_runTimeout = runTimeout ?? TimeSpan.FromSeconds(2);
|
||||
_guard = new ScriptRunGuard(scripting, _scriptRoot);
|
||||
}
|
||||
|
||||
/// <summary>The run guard every compiled evaluation goes through (diagnostics / tests).</summary>
|
||||
public ScriptRunGuard RunGuard => _guard;
|
||||
|
||||
// Value-only overload. Every dependency reaches the script stamped Good with an
|
||||
// evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot
|
||||
// overload instead; this one remains for callers that genuinely hold only
|
||||
@@ -150,13 +166,13 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
|
||||
try
|
||||
{
|
||||
// Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't
|
||||
// interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous
|
||||
// CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when
|
||||
// the wall-clock budget fires regardless of whether the inner task completes (the orphaned
|
||||
// thread is the documented, accepted trade-off — TimedScriptEvaluator remarks).
|
||||
var timed = new TimedScriptEvaluator<VirtualTagContext, object?>(evaluator, _runTimeout);
|
||||
var raw = timed.RunAsync(context).GetAwaiter().GetResult();
|
||||
// Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by virtual-tag id): a raw
|
||||
// CancellationToken can't interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll
|
||||
// it), so the wall-clock budget is what returns control — and the guard is what stops the
|
||||
// abandoned run from being joined by another on the next trigger (one orphan per tag, then
|
||||
// quarantine), which is the leak the old per-evaluation TimedScriptEvaluator allowed.
|
||||
var raw = _guard.RunAsync(virtualTagId, expression, evaluator, context, _runTimeout)
|
||||
.GetAwaiter().GetResult();
|
||||
return VirtualTagEvalResult.Ok(raw);
|
||||
}
|
||||
catch (ScriptTimeoutException)
|
||||
@@ -164,6 +180,14 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
return VirtualTagEvalResult.Failure(
|
||||
$"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s");
|
||||
}
|
||||
catch (ScriptRunRefusedException ex)
|
||||
{
|
||||
return VirtualTagEvalResult.Failure(ex.Message);
|
||||
}
|
||||
catch (ScriptQuarantinedException ex)
|
||||
{
|
||||
return VirtualTagEvalResult.Failure(ex.Message);
|
||||
}
|
||||
catch (ObjectDisposedException) when (!_disposed && attempt == 0)
|
||||
{
|
||||
// S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our
|
||||
@@ -185,7 +209,19 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void ClearCompiledScripts() => _cache.Clear();
|
||||
public void ClearCompiledScripts()
|
||||
{
|
||||
_cache.Clear();
|
||||
// The compile-cache clear IS the deploy boundary for this evaluator: a redeploy lifts every quarantine
|
||||
// (an edited expression already lifts its own through the guard's source identity).
|
||||
_guard.ResetAll();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void ResetQuarantines() => _guard.ResetAll();
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<string> QuarantinedScripts => _guard.QuarantinedKeys;
|
||||
|
||||
/// <summary>Disposes the evaluator and cleans up all cached script evaluators.</summary>
|
||||
public void Dispose()
|
||||
|
||||
+26
-2
@@ -42,8 +42,10 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
|
||||
// publish.
|
||||
private const uint StatusNeverPushed = OpcUaStatus.BadWaitingForInitialData;
|
||||
|
||||
// Intentionally never pruned: cold-start semantics depend on retained last values so
|
||||
// ReadTag can answer synchronously for any path that has ever been pushed.
|
||||
// Retained last values: ReadTag answers synchronously for any path that has been pushed. Pruned
|
||||
// ONLY through RetainOnly — the host calls it with the loaded dependency-ref union after every
|
||||
// (re)load, so a renamed/removed raw path (a RawTree rename is a surgical remove+add) stops
|
||||
// costing a boxed value + two DateTimes for the process lifetime (R02:L3).
|
||||
private readonly ConcurrentDictionary<string, DataValueSnapshot> _cache
|
||||
= new(StringComparer.Ordinal);
|
||||
private readonly ConcurrentDictionary<string, ImmutableList<Action<string, DataValueSnapshot>>> _observers
|
||||
@@ -91,6 +93,28 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drop every cached snapshot whose path is not in <paramref name="paths"/>. Observers are
|
||||
/// untouched. Called by the host after each successful engine (re)load with the union of the loaded
|
||||
/// alarms' dependency refs — exactly the set the mux will push from now on — so paths no alarm
|
||||
/// references any more stop being retained.
|
||||
/// </summary>
|
||||
/// <param name="paths">The paths to keep.</param>
|
||||
/// <returns>The number of cached snapshots dropped.</returns>
|
||||
public int RetainOnly(IReadOnlySet<string> paths)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(paths);
|
||||
var dropped = 0;
|
||||
foreach (var key in _cache.Keys)
|
||||
{
|
||||
if (!paths.Contains(key) && _cache.TryRemove(key, out _)) dropped++;
|
||||
}
|
||||
return dropped;
|
||||
}
|
||||
|
||||
/// <summary>Test/diagnostic: the number of paths currently retained.</summary>
|
||||
public int RetainedPathCount => _cache.Count;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public DataValueSnapshot ReadTag(string path)
|
||||
{
|
||||
|
||||
@@ -23,10 +23,16 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
||||
/// cluster <c>alerts</c> DistributedPubSub topic.
|
||||
///
|
||||
/// <para>
|
||||
/// The host is the engine's lifecycle owner: the caller (the driver-role startup) builds
|
||||
/// the <see cref="DependencyMuxTagUpstreamSource"/>, constructs a <see cref="ScriptedAlarmEngine"/>
|
||||
/// <b>around that same upstream</b>, and passes both here. The host disposes the engine in
|
||||
/// <see cref="PostStop"/>.
|
||||
/// The host is the engine's lifecycle owner. The restart-safe shape (R02:H1) is the
|
||||
/// <b>factory</b> <see cref="Props(IActorRef, IActorRef?, Func{ValueTuple{DependencyMuxTagUpstreamSource, ScriptedAlarmEngine}}, NodeId?, Func{int}?, ITelemetryLocalHub?, TimeSpan?)"/>:
|
||||
/// every incarnation builds its own <see cref="DependencyMuxTagUpstreamSource"/> + a
|
||||
/// <see cref="ScriptedAlarmEngine"/> around it, and disposes them in <see cref="PostStop"/>.
|
||||
/// Akka's default restart runs <c>PostStop</c> then re-creates the actor from the SAME Props, so the
|
||||
/// legacy instance-taking Props — a fixed engine captured in the Props — restarted around a
|
||||
/// <em>disposed</em> engine and every later <c>LoadAsync</c> threw: the node's scripted alarms
|
||||
/// silently died. The legacy overload is kept source-compatible but is restart-safe only in the
|
||||
/// weaker sense that a restart now DETACHES rather than disposes its engine (see
|
||||
/// <see cref="PreRestart"/>); production should spawn through the factory.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
@@ -99,6 +105,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
/// Warning on every clean shutdown.</summary>
|
||||
private sealed record AlarmsLoadCanceled;
|
||||
|
||||
/// <summary>Default bound on one inbound alarm command's engine op (<see cref="OnAlarmCommand"/>): 10 s.</summary>
|
||||
public static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(10);
|
||||
|
||||
private readonly IActorRef _publishActor;
|
||||
private readonly IActorRef? _mux;
|
||||
private readonly DependencyMuxTagUpstreamSource _upstream;
|
||||
@@ -107,6 +116,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private readonly EventHandler<ScriptedAlarmEvent> _onEngineEvent;
|
||||
private readonly TimeSpan _commandTimeout;
|
||||
|
||||
/// <summary>True when THIS incarnation built its engine (factory Props) and therefore disposes it on
|
||||
/// every stop, restart included. False for the legacy instance-taking Props: the engine outlives the
|
||||
/// incarnation, so a restart only detaches from it and a final stop disposes it.</summary>
|
||||
private readonly bool _ownsEngine;
|
||||
|
||||
/// <summary>Set by <see cref="PreRestart"/> so the <see cref="PostStop"/> it triggers can tell a restart
|
||||
/// (the next incarnation reuses a legacy engine) from a genuine stop.</summary>
|
||||
private bool _restarting;
|
||||
|
||||
/// <summary>Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
|
||||
/// snapshot (null = unknown until the first snapshot arrives, or no <see cref="_localNode"/> wired). The
|
||||
@@ -152,6 +171,8 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
/// <param name="telemetryHub">Optional node-local live-telemetry hub (Phase 5) each cluster-wide
|
||||
/// <c>alerts</c> transition is also emitted into; null (tests) skips the emit.</param>
|
||||
/// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
|
||||
/// <remarks>Legacy, instance-taking shape — see the class remarks: a supervision restart re-creates the
|
||||
/// actor around these SAME instances, so prefer the factory overload in production.</remarks>
|
||||
public static Props Props(
|
||||
IActorRef publishActor,
|
||||
IActorRef? mux,
|
||||
@@ -162,6 +183,60 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
ITelemetryLocalHub? telemetryHub = null) =>
|
||||
Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub));
|
||||
|
||||
/// <summary>
|
||||
/// Restart-safe factory Props (R02:H1): each incarnation of the actor calls
|
||||
/// <paramref name="engineFactory"/> in its constructor and owns what it returns — the engine must
|
||||
/// be constructed <b>around</b> the returned upstream — so a supervision restart gets a fresh,
|
||||
/// undisposed engine and the next <see cref="ApplyScriptedAlarms"/> loads cleanly.
|
||||
/// </summary>
|
||||
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
|
||||
/// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
|
||||
/// <param name="engineFactory">Builds a fresh (upstream, engine) pair; called once per incarnation.</param>
|
||||
/// <param name="localNode">The local cluster node id for the Primary alerts-emit gate; null ⇒ default-emit.</param>
|
||||
/// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
|
||||
/// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
|
||||
/// <param name="commandTimeout">Bound on one inbound alarm command's engine op; null ⇒ <see cref="DefaultCommandTimeout"/>.</param>
|
||||
/// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
|
||||
public static Props Props(
|
||||
IActorRef publishActor,
|
||||
IActorRef? mux,
|
||||
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
|
||||
NodeId? localNode = null,
|
||||
Func<int>? driverMemberCountProvider = null,
|
||||
ITelemetryLocalHub? telemetryHub = null,
|
||||
TimeSpan? commandTimeout = null) =>
|
||||
Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, engineFactory, localNode, driverMemberCountProvider, telemetryHub, commandTimeout));
|
||||
|
||||
/// <summary>Factory-built incarnation: owns (and on every stop disposes) the engine it builds.</summary>
|
||||
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
|
||||
/// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
|
||||
/// <param name="engineFactory">Builds this incarnation's (upstream, engine) pair.</param>
|
||||
/// <param name="localNode">The local cluster node id for the Primary alerts-emit gate.</param>
|
||||
/// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
|
||||
/// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
|
||||
/// <param name="commandTimeout">Bound on one inbound alarm command's engine op.</param>
|
||||
public ScriptedAlarmHostActor(
|
||||
IActorRef publishActor,
|
||||
IActorRef? mux,
|
||||
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
|
||||
NodeId? localNode = null,
|
||||
Func<int>? driverMemberCountProvider = null,
|
||||
ITelemetryLocalHub? telemetryHub = null,
|
||||
TimeSpan? commandTimeout = null)
|
||||
: this(publishActor, mux, Build(engineFactory), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: true, commandTimeout)
|
||||
{
|
||||
}
|
||||
|
||||
private static (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) Build(
|
||||
Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(engineFactory);
|
||||
var pair = engineFactory();
|
||||
ArgumentNullException.ThrowIfNull(pair.Upstream, "engineFactory().Upstream");
|
||||
ArgumentNullException.ThrowIfNull(pair.Engine, "engineFactory().Engine");
|
||||
return pair;
|
||||
}
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="ScriptedAlarmHostActor"/> class.</summary>
|
||||
/// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
|
||||
/// <param name="mux">Optional dependency multiplexer the host registers dependency interest with.</param>
|
||||
@@ -183,14 +258,29 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
NodeId? localNode = null,
|
||||
Func<int>? driverMemberCountProvider = null,
|
||||
ITelemetryLocalHub? telemetryHub = null)
|
||||
: this(publishActor, mux, (upstream, engine), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: false, commandTimeout: null)
|
||||
{
|
||||
}
|
||||
|
||||
private ScriptedAlarmHostActor(
|
||||
IActorRef publishActor,
|
||||
IActorRef? mux,
|
||||
(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) pair,
|
||||
NodeId? localNode,
|
||||
Func<int>? driverMemberCountProvider,
|
||||
ITelemetryLocalHub? telemetryHub,
|
||||
bool ownsEngine,
|
||||
TimeSpan? commandTimeout)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(publishActor);
|
||||
ArgumentNullException.ThrowIfNull(upstream);
|
||||
ArgumentNullException.ThrowIfNull(engine);
|
||||
ArgumentNullException.ThrowIfNull(pair.Upstream);
|
||||
ArgumentNullException.ThrowIfNull(pair.Engine);
|
||||
_publishActor = publishActor;
|
||||
_mux = mux;
|
||||
_upstream = upstream;
|
||||
_engine = engine;
|
||||
_upstream = pair.Upstream;
|
||||
_engine = pair.Engine;
|
||||
_ownsEngine = ownsEngine;
|
||||
_commandTimeout = commandTimeout is { } t && t > TimeSpan.Zero ? t : DefaultCommandTimeout;
|
||||
_localNode = localNode;
|
||||
_driverMemberCountProvider = driverMemberCountProvider;
|
||||
_telemetryHub = telemetryHub;
|
||||
@@ -294,8 +384,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
// subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the
|
||||
// engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor),
|
||||
// so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed.
|
||||
_mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self));
|
||||
_log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs", msg.DepRefs.Count);
|
||||
// ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a
|
||||
// supervision restart builds one around an EMPTY upstream) converges immediately instead of
|
||||
// waiting for every input to change again.
|
||||
_mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true));
|
||||
// Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references
|
||||
// any more will never be pushed again, so its last value is dead weight for the process lifetime.
|
||||
var dropped = _upstream.RetainOnly(new HashSet<string>(msg.DepRefs, StringComparer.Ordinal));
|
||||
_log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs (pruned {Dropped} retained path(s))",
|
||||
msg.DepRefs.Count, dropped);
|
||||
|
||||
ReassertConditionNodes();
|
||||
}
|
||||
@@ -591,18 +688,25 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
return;
|
||||
}
|
||||
|
||||
// Bounded (R02:H3): every op ends in IAlarmStateStore.SaveAsync (LocalDb/SQLite), and this is a
|
||||
// ReceiveAsync — the mailbox is suspended until the op returns, so an unbounded await on a hung
|
||||
// file lock stalled dependency-value ingestion for the duration. Linked to the actor's lifetime
|
||||
// token so a stop still cancels promptly.
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
|
||||
cts.CancelAfter(_commandTimeout);
|
||||
var ct = cts.Token;
|
||||
try
|
||||
{
|
||||
switch (cmd.Operation)
|
||||
{
|
||||
case "Acknowledge":
|
||||
await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
|
||||
await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
|
||||
break;
|
||||
case "Confirm":
|
||||
await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
|
||||
await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
|
||||
break;
|
||||
case "OneShotShelve":
|
||||
await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
|
||||
await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, ct);
|
||||
break;
|
||||
case "TimedShelve":
|
||||
// A timed shelve needs the absolute unshelve instant. T18 derives it from the OPC UA
|
||||
@@ -614,16 +718,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
cmd.AlarmId);
|
||||
return;
|
||||
}
|
||||
await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, CancellationToken.None);
|
||||
await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, ct);
|
||||
break;
|
||||
case "Unshelve":
|
||||
await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
|
||||
await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, ct);
|
||||
break;
|
||||
case "Enable":
|
||||
await _engine.EnableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
|
||||
await _engine.EnableAsync(cmd.AlarmId, cmd.User, ct);
|
||||
break;
|
||||
case "Disable":
|
||||
await _engine.DisableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
|
||||
await _engine.DisableAsync(cmd.AlarmId, cmd.User, ct);
|
||||
break;
|
||||
case "AddComment":
|
||||
if (string.IsNullOrWhiteSpace(cmd.Comment))
|
||||
@@ -631,7 +735,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
_log.Warning("ScriptedAlarmHost: rejecting AddComment for {AlarmId} — empty comment text", cmd.AlarmId);
|
||||
return;
|
||||
}
|
||||
await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
|
||||
await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
|
||||
break;
|
||||
default:
|
||||
_log.Warning("ScriptedAlarmHost: ignoring AlarmCommand with unknown operation {Op} for {AlarmId}",
|
||||
@@ -639,6 +743,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (!_cts.IsCancellationRequested)
|
||||
{
|
||||
_log.Warning("ScriptedAlarmHost: engine op {Op} for alarm {AlarmId} timed out after {Timeout} — the condition-state store is stalled",
|
||||
cmd.Operation, cmd.AlarmId, _commandTimeout);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A failing engine op must not crash the actor — mirror OnLoadFailed's log-and-stay-inert style.
|
||||
@@ -672,16 +781,28 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
|
||||
base.PreStart();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>Akka's default <c>PreRestart</c> stops the children and calls <see cref="PostStop"/>. The flag
|
||||
/// lets that <see cref="PostStop"/> keep a legacy (Props-captured) engine alive for the next incarnation
|
||||
/// instead of disposing the instance it is about to be re-created around.</remarks>
|
||||
protected override void PreRestart(Exception reason, object message)
|
||||
{
|
||||
_restarting = true;
|
||||
base.PreRestart(reason, message);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
{
|
||||
// Unregister mux interest first so no further DependencyValueChanged arrives while we tear down.
|
||||
_mux?.Tell(new DependencyMuxActor.UnregisterInterest(Self));
|
||||
// Cancel any in-flight LoadAsync, detach the OnEvent handler (so a late engine emission can't
|
||||
// Tell a stopping actor), then dispose the engine (which drains its background work + clears).
|
||||
// Tell a stopping actor), then dispose the engine (which drains its background work + clears) —
|
||||
// unless this is a RESTART around a legacy Props-captured engine, which the next incarnation will
|
||||
// re-attach to (R02:H1).
|
||||
_cts.Cancel();
|
||||
_engine.OnEvent -= _onEngineEvent;
|
||||
_engine.Dispose();
|
||||
if (_ownsEngine || !_restarting) _engine.Dispose();
|
||||
_cts.Dispose();
|
||||
base.PostStop();
|
||||
}
|
||||
|
||||
@@ -62,7 +62,10 @@ public sealed class DependencyConsumerMuxAdapter : ReceiveActor
|
||||
var refs = _consumer.DependencyRefs.ToList();
|
||||
// A RegisterInterest replaces the prior interest set for this subscriber (mux is idempotent on
|
||||
// re-register), so re-registering after an apply cleanly adopts an added/removed ref.
|
||||
_mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self));
|
||||
// ReplayRetained (R07:H6): a driver Restart is a NEW consumer behind a new adapter; the mux replays
|
||||
// the last value of every ref it retained so the calc tags compute from current inputs at once
|
||||
// instead of sitting BadWaitingForInitialData until each input happens to change.
|
||||
_mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self, ReplayRetained: true));
|
||||
_log.Debug("DependencyConsumerMuxAdapter: registered {Count} dependency ref(s)", refs.Count);
|
||||
}
|
||||
|
||||
|
||||
@@ -26,32 +26,134 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
/// <see cref="VirtualTagActor.DependencyValueChanged"/> receive as well, for direct senders and tests.
|
||||
/// The mux is deliberately not a DPS subscriber — virtual-tag evaluation is local to each node and
|
||||
/// would over-emit if it spanned the cluster.
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Last-value retention + replay (R07:H6, bounded in review fix round 1 / I2).</b> The mux
|
||||
/// retains the last <c>(value, status, timestamp)</c> of a ref <b>only while at least one subscriber
|
||||
/// registered with <see cref="RegisterInterest.ReplayRetained"/> names it</b> (the virtual-tag
|
||||
/// actors, the Calculation mux adapter, the scripted-alarm host). A registration that asks receives
|
||||
/// one batch of the retained values for its refs, in the shape a driver burst arrives in. The case
|
||||
/// this serves is a subscriber that goes away and comes back: a Calculation driver Restart is a new
|
||||
/// driver instance + a new mux adapter, and before this every calc tag sat
|
||||
/// <c>BadWaitingForInitialData</c> until each of its inputs happened to change again. So when a
|
||||
/// ref's last replay-opting subscriber leaves, its value is kept for <see cref="RetentionGrace"/>
|
||||
/// (default <see cref="DefaultRetentionGrace"/>; a returning replay registration inside it cancels
|
||||
/// the release), then dropped together with the ref's entry.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A subscriber that does NOT opt in retains nothing: the continuous-historization recorder
|
||||
/// registers every historized ref on the node (a replay there would record a stale sample per tag
|
||||
/// per deploy), and retaining on its behalf made the map grow with the historized address space and
|
||||
/// never shrink. Memory is therefore bounded by the refs replay-opting subscribers currently name,
|
||||
/// plus those released within the last grace window. A ref with no subscriber and nothing retained
|
||||
/// has no entry at all.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Source removal.</b> The mux records which driver instance last published each retained ref;
|
||||
/// <see cref="SourceRemoved"/> evicts every value that driver produced, so a removed driver's last
|
||||
/// Good is never replayed as current. A comms-loss fan (a Bad batch) already overwrites the retained
|
||||
/// value with Bad. ⚠️ As of fix round 1 no production actor sends <see cref="SourceRemoved"/> yet —
|
||||
/// <c>DriverHostActor</c> (outside the scripting lane) must Tell it when it stops / removes a driver
|
||||
/// child. Until then a removed driver's last value can still be replayed to a replay-opting
|
||||
/// subscriber that re-registers for that ref while another replay-opting subscriber holds it, or
|
||||
/// within the grace window.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class DependencyMuxActor : ReceiveActor
|
||||
public sealed class DependencyMuxActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
public const string ActorName = "dependency-mux";
|
||||
|
||||
/// <summary>How long a ref's retained value outlives its last replay-opting subscriber.</summary>
|
||||
public static readonly TimeSpan DefaultRetentionGrace = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>Register a subscriber's interest in a set of tag refs. Idempotent on re-register —
|
||||
/// the second call replaces the prior interest set for that subscriber.</summary>
|
||||
public sealed record RegisterInterest(IReadOnlyList<string> TagRefs, IActorRef Subscriber);
|
||||
/// <param name="TagRefs">The refs the subscriber wants.</param>
|
||||
/// <param name="Subscriber">The subscriber.</param>
|
||||
/// <param name="ReplayRetained">When true, the mux retains the last value of every ref in
|
||||
/// <paramref name="TagRefs"/> while this subscriber is registered, and immediately sends the subscriber one
|
||||
/// <see cref="VirtualTagActor.DependencyValueBatch"/> holding the last retained value of every ref it has
|
||||
/// one for (nothing when it has none).</param>
|
||||
public sealed record RegisterInterest(IReadOnlyList<string> TagRefs, IActorRef Subscriber, bool ReplayRetained = false);
|
||||
|
||||
/// <summary>Unregister every interest held by <see cref="Subscriber"/>. Sent on PostStop by
|
||||
/// the subscriber, or by Terminated handling when the mux watches.</summary>
|
||||
public sealed record UnregisterInterest(IActorRef Subscriber);
|
||||
|
||||
private readonly Dictionary<string, HashSet<IActorRef>> _byRef = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<IActorRef, HashSet<string>> _bySubscriber = new();
|
||||
/// <summary>A driver instance was removed / stopped: evict every retained value it produced so it is
|
||||
/// never replayed. Subscriptions are untouched.</summary>
|
||||
/// <param name="DriverInstanceId">The removed driver instance.</param>
|
||||
public sealed record SourceRemoved(string DriverInstanceId);
|
||||
|
||||
private sealed class SweepReleased
|
||||
{
|
||||
public static readonly SweepReleased Instance = new();
|
||||
}
|
||||
|
||||
private const string SweepTimerKey = "sweep-released";
|
||||
|
||||
/// <summary>Per-ref state. One lookup on the hot path answers both "who wants it" and "retain it?".</summary>
|
||||
private sealed class RefEntry
|
||||
{
|
||||
public readonly HashSet<IActorRef> Subscribers = new();
|
||||
|
||||
/// <summary>Registered subscribers of this ref that opted into replay.</summary>
|
||||
public int Replayers;
|
||||
|
||||
/// <summary>When the last replay-opting subscriber left (retention in grace), or null.</summary>
|
||||
public DateTime? ReleasedAtUtc;
|
||||
|
||||
public VirtualTagActor.DependencyValueChanged? Last;
|
||||
public string? LastSource;
|
||||
|
||||
public bool Retains => Replayers > 0 || ReleasedAtUtc is not null;
|
||||
public bool IsDead => Subscribers.Count == 0 && Replayers == 0 && Last is null;
|
||||
|
||||
public void Forget()
|
||||
{
|
||||
Last = null;
|
||||
LastSource = null;
|
||||
ReleasedAtUtc = null;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record Registration(HashSet<string> Refs, bool Replay);
|
||||
|
||||
private readonly Dictionary<string, RefEntry> _byRef = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<IActorRef, Registration> _bySubscriber = new();
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
|
||||
/// <summary>How long a released ref's value is kept.</summary>
|
||||
public TimeSpan RetentionGrace { get; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
/// <summary>Test hook — refs that currently hold a retained value.</summary>
|
||||
internal int RetainedRefCount => _byRef.Values.Count(e => e.Last is not null);
|
||||
|
||||
/// <summary>Test hook — refs the mux holds any state for (subscribers or a retained value).</summary>
|
||||
internal int TrackedRefCount => _byRef.Count;
|
||||
|
||||
/// <summary>Creates props for the DependencyMuxActor.</summary>
|
||||
/// <param name="retentionGrace">How long a ref's value outlives its last replay-opting subscriber;
|
||||
/// null ⇒ <see cref="DefaultRetentionGrace"/>.</param>
|
||||
/// <returns>The props for creating this actor.</returns>
|
||||
public static Props Props() => Akka.Actor.Props.Create<DependencyMuxActor>();
|
||||
public static Props Props(TimeSpan? retentionGrace = null)
|
||||
{
|
||||
var grace = retentionGrace ?? DefaultRetentionGrace;
|
||||
return Akka.Actor.Props.Create(() => new DependencyMuxActor(grace));
|
||||
}
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="DependencyMuxActor"/> class.</summary>
|
||||
public DependencyMuxActor()
|
||||
/// <param name="retentionGrace">See <see cref="Props"/>; non-positive ⇒ <see cref="DefaultRetentionGrace"/>.</param>
|
||||
public DependencyMuxActor(TimeSpan retentionGrace)
|
||||
{
|
||||
RetentionGrace = retentionGrace > TimeSpan.Zero ? retentionGrace : DefaultRetentionGrace;
|
||||
Receive<RegisterInterest>(OnRegister);
|
||||
Receive<UnregisterInterest>(msg => RemoveSubscriber(msg.Subscriber));
|
||||
Receive<SourceRemoved>(OnSourceRemoved);
|
||||
Receive<SweepReleased>(_ => OnSweep());
|
||||
Receive<DriverInstanceActor.AttributeValuePublished>(OnAttributeValuePublished);
|
||||
Receive<DriverInstanceActor.AttributeValueBatchPublished>(OnAttributeValueBatchPublished);
|
||||
Receive<Terminated>(msg => RemoveSubscriber(msg.ActorRef));
|
||||
@@ -59,60 +161,125 @@ public sealed class DependencyMuxActor : ReceiveActor
|
||||
|
||||
private void OnRegister(RegisterInterest msg)
|
||||
{
|
||||
// Replace any prior interest set so re-registers on actor restart don't leak old refs.
|
||||
if (_bySubscriber.TryGetValue(msg.Subscriber, out var priorRefs))
|
||||
{
|
||||
foreach (var r in priorRefs)
|
||||
{
|
||||
if (_byRef.TryGetValue(r, out var set))
|
||||
{
|
||||
set.Remove(msg.Subscriber);
|
||||
if (set.Count == 0) _byRef.Remove(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Replace any prior interest set so re-registers on actor restart don't leak old refs. Detaching first
|
||||
// may put a ref into grace; the re-attach below takes it straight back out.
|
||||
Detach(msg.Subscriber);
|
||||
|
||||
var refs = new HashSet<string>(msg.TagRefs, StringComparer.Ordinal);
|
||||
_bySubscriber[msg.Subscriber] = refs;
|
||||
_bySubscriber[msg.Subscriber] = new Registration(refs, msg.ReplayRetained);
|
||||
List<VirtualTagActor.DependencyValueChanged>? replay = null;
|
||||
foreach (var r in refs)
|
||||
{
|
||||
if (!_byRef.TryGetValue(r, out var set))
|
||||
if (!_byRef.TryGetValue(r, out var entry))
|
||||
{
|
||||
set = new HashSet<IActorRef>();
|
||||
_byRef[r] = set;
|
||||
entry = new RefEntry();
|
||||
_byRef[r] = entry;
|
||||
}
|
||||
set.Add(msg.Subscriber);
|
||||
entry.Subscribers.Add(msg.Subscriber);
|
||||
if (!msg.ReplayRetained) continue;
|
||||
entry.Replayers++;
|
||||
entry.ReleasedAtUtc = null;
|
||||
if (entry.Last is { } retained)
|
||||
(replay ??= []).Add(retained);
|
||||
}
|
||||
Context.Watch(msg.Subscriber);
|
||||
_log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count);
|
||||
_log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs (replay {Replay})",
|
||||
msg.Subscriber, refs.Count, msg.ReplayRetained);
|
||||
|
||||
// One batch, after the registration is complete, in exactly the shape a driver burst arrives in —
|
||||
// so the subscriber's hot path handles it with no special case.
|
||||
if (replay is not null)
|
||||
{
|
||||
msg.Subscriber.Tell(new VirtualTagActor.DependencyValueBatch(replay));
|
||||
_log.Debug("DependencyMux: replayed {Count} retained value(s) to {Sub}", replay.Count, msg.Subscriber);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveSubscriber(IActorRef subscriber)
|
||||
{
|
||||
if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return;
|
||||
foreach (var r in refs)
|
||||
{
|
||||
if (_byRef.TryGetValue(r, out var set))
|
||||
{
|
||||
set.Remove(subscriber);
|
||||
if (set.Count == 0) _byRef.Remove(r);
|
||||
}
|
||||
}
|
||||
_bySubscriber.Remove(subscriber);
|
||||
if (!Detach(subscriber)) return;
|
||||
Context.Unwatch(subscriber);
|
||||
_log.Debug("DependencyMux: subscriber {Sub} removed", subscriber);
|
||||
}
|
||||
|
||||
/// <summary>Drop <paramref name="subscriber"/>'s registration. A ref losing its last replay-opting
|
||||
/// subscriber goes into grace; a ref left with no state at all is removed.</summary>
|
||||
private bool Detach(IActorRef subscriber)
|
||||
{
|
||||
if (!_bySubscriber.Remove(subscriber, out var registration)) return false;
|
||||
var now = DateTime.UtcNow;
|
||||
var released = false;
|
||||
foreach (var r in registration.Refs)
|
||||
{
|
||||
if (!_byRef.TryGetValue(r, out var entry)) continue;
|
||||
entry.Subscribers.Remove(subscriber);
|
||||
if (registration.Replay && --entry.Replayers == 0 && entry.Last is not null)
|
||||
{
|
||||
entry.ReleasedAtUtc = now;
|
||||
released = true;
|
||||
}
|
||||
if (entry.IsDead) _byRef.Remove(r);
|
||||
}
|
||||
if (released && !Timers.IsTimerActive(SweepTimerKey))
|
||||
Timers.StartPeriodicTimer(SweepTimerKey, SweepReleased.Instance, RetentionGrace);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Drop retained values whose grace has run out — with a periodic sweep at the grace interval a
|
||||
/// released value lives between one and two grace periods — and stop the sweep once nothing is in grace.</summary>
|
||||
private void OnSweep()
|
||||
{
|
||||
var cutoff = DateTime.UtcNow - RetentionGrace;
|
||||
var stillInGrace = false;
|
||||
List<string>? dead = null;
|
||||
foreach (var (r, entry) in _byRef)
|
||||
{
|
||||
if (entry.ReleasedAtUtc is not { } releasedAt) continue;
|
||||
if (releasedAt > cutoff)
|
||||
{
|
||||
stillInGrace = true;
|
||||
continue;
|
||||
}
|
||||
entry.Forget();
|
||||
if (entry.IsDead) (dead ??= []).Add(r);
|
||||
}
|
||||
if (dead is not null)
|
||||
foreach (var r in dead) _byRef.Remove(r);
|
||||
if (!stillInGrace) Timers.Cancel(SweepTimerKey);
|
||||
}
|
||||
|
||||
private void OnSourceRemoved(SourceRemoved msg)
|
||||
{
|
||||
List<string>? dead = null;
|
||||
var evicted = 0;
|
||||
foreach (var (r, entry) in _byRef)
|
||||
{
|
||||
if (entry.Last is null || !string.Equals(entry.LastSource, msg.DriverInstanceId, StringComparison.Ordinal))
|
||||
continue;
|
||||
entry.Forget();
|
||||
evicted++;
|
||||
if (entry.IsDead) (dead ??= []).Add(r);
|
||||
}
|
||||
if (dead is not null)
|
||||
foreach (var r in dead) _byRef.Remove(r);
|
||||
_log.Debug("DependencyMux: source {Driver} removed — evicted {Count} retained value(s)", msg.DriverInstanceId, evicted);
|
||||
}
|
||||
|
||||
private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg)
|
||||
{
|
||||
if (!_byRef.TryGetValue(msg.FullReference, out var subscribers) || subscribers.Count == 0)
|
||||
if (!_byRef.TryGetValue(msg.FullReference, out var entry))
|
||||
{
|
||||
// No virtual tag cares about this ref — drop. Common in normal operation; the address
|
||||
// Nothing is registered for (or retaining) this ref — drop. Common in normal operation; the address
|
||||
// space carries thousands of tags and only a fraction feed virtual-tag expressions.
|
||||
return;
|
||||
}
|
||||
var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode);
|
||||
foreach (var sub in subscribers)
|
||||
if (entry.Retains)
|
||||
{
|
||||
entry.Last = dep;
|
||||
entry.LastSource = msg.DriverInstanceId;
|
||||
}
|
||||
foreach (var sub in entry.Subscribers)
|
||||
{
|
||||
// A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs
|
||||
// only the batch receive on its hot path. One list per subscriber: batch ownership transfers
|
||||
@@ -122,18 +289,9 @@ public sealed class DependencyMuxActor : ReceiveActor
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Filters one batched driver production burst per item and delivers each subscriber's matched
|
||||
/// slice as ONE <see cref="VirtualTagActor.DependencyValueBatch"/>. Items nobody registered for are
|
||||
/// dropped (the common case: only a fraction of an address space feeds virtual-tag expressions), so a
|
||||
/// burst of N values costs one inbound message plus one outbound message per genuinely interested
|
||||
/// SUBSCRIBER — not per (item, subscriber) pair as it did when this fanned out per value. That is what
|
||||
/// lets <see cref="VirtualTagActor"/> evaluate its script once per burst instead of once per input.
|
||||
/// <para>The registered set is NOT just virtual-tag inputs: <c>ContinuousHistorizationRecorder</c>
|
||||
/// registers interest in every historized ref, so on a historizing node <see cref="_byRef"/> can be
|
||||
/// large and a sizeable fraction of each burst resolves.</para></summary>
|
||||
private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg)
|
||||
{
|
||||
// Nobody has registered anything — skip the whole burst rather than probing an empty map per item.
|
||||
// Nothing registered and nothing retained — skip the whole burst rather than probing an empty map per item.
|
||||
if (_byRef.Count == 0) return;
|
||||
|
||||
// Allocated lazily on the first matched item: a burst in which nothing is registered for — still
|
||||
@@ -146,8 +304,15 @@ public sealed class DependencyMuxActor : ReceiveActor
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var item = msg.Items[i];
|
||||
if (!_byRef.TryGetValue(item.FullReference, out var subscribers) || subscribers.Count == 0) continue;
|
||||
if (!_byRef.TryGetValue(item.FullReference, out var entry)) continue;
|
||||
var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode);
|
||||
if (entry.Retains)
|
||||
{
|
||||
entry.Last = dep;
|
||||
entry.LastSource = msg.DriverInstanceId;
|
||||
}
|
||||
var subscribers = entry.Subscribers;
|
||||
if (subscribers.Count == 0) continue; // retained (in grace) for a returning subscriber; nobody to forward to now
|
||||
perSubscriber ??= new Dictionary<IActorRef, List<VirtualTagActor.DependencyValueChanged>>();
|
||||
foreach (var sub in subscribers)
|
||||
{
|
||||
|
||||
@@ -7,6 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
@@ -76,6 +77,10 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
private readonly IReadOnlyList<string> _dependencyRefs;
|
||||
private readonly IActorRef? _mux;
|
||||
private readonly ITelemetryLocalHub? _telemetryHub;
|
||||
// R02:P6 — a script failing on every trigger of a 1 Hz input published one cluster-wide DPS
|
||||
// ScriptLogEntry per second per tag; 200 such tags shed everyone's entries out of the bounded
|
||||
// telemetry channels. The per-failure entry is now rate-limited per tag (20/min + one summary).
|
||||
private readonly ScriptLogRateLimiter _failureLogLimiter;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
// Dependency SNAPSHOTS, not bare values (archreview 02/S14): the status code is what distinguishes a
|
||||
// comms-lost input from a genuine Good null, and it is what the script reads through
|
||||
@@ -104,6 +109,8 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
/// <param name="mux">Optional reference to a dependency multiplexer actor.</param>
|
||||
/// <param name="telemetryHub">Optional node-local live-telemetry hub each script-log entry is also
|
||||
/// emitted into (Phase 5); null (tests) simply skips the emit.</param>
|
||||
/// <param name="failureLogLimiter">Rate limiter for this tag's per-failure script-log entries; null ⇒
|
||||
/// the default 20 per minute + one summary.</param>
|
||||
/// <returns>A configured <see cref="Akka.Actor.Props"/> for creating the actor.</returns>
|
||||
public static Props Props(
|
||||
string virtualTagId,
|
||||
@@ -113,7 +120,8 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
Func<DPSPublisher>? publisherFactory = null,
|
||||
IReadOnlyList<string>? dependencyRefs = null,
|
||||
IActorRef? mux = null,
|
||||
ITelemetryLocalHub? telemetryHub = null) =>
|
||||
ITelemetryLocalHub? telemetryHub = null,
|
||||
ScriptLogRateLimiter? failureLogLimiter = null) =>
|
||||
Akka.Actor.Props.Create(() => new VirtualTagActor(
|
||||
virtualTagId, expression,
|
||||
evaluator ?? NullVirtualTagEvaluator.Instance,
|
||||
@@ -121,7 +129,8 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
publisherFactory,
|
||||
dependencyRefs ?? Array.Empty<string>(),
|
||||
mux,
|
||||
telemetryHub));
|
||||
telemetryHub,
|
||||
failureLogLimiter));
|
||||
|
||||
/// <summary>Initializes a virtual tag actor with the given configuration and dependencies.</summary>
|
||||
/// <param name="virtualTagId">Unique identifier for the virtual tag.</param>
|
||||
@@ -133,6 +142,8 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
/// <param name="mux">Optional reference to a dependency multiplexer actor.</param>
|
||||
/// <param name="telemetryHub">Optional node-local live-telemetry hub each script-log entry is also
|
||||
/// emitted into (Phase 5); null (tests) simply skips the emit.</param>
|
||||
/// <param name="failureLogLimiter">Rate limiter for this tag's per-failure script-log entries; null ⇒
|
||||
/// the default 20 per minute + one summary.</param>
|
||||
public VirtualTagActor(
|
||||
string virtualTagId,
|
||||
string expression,
|
||||
@@ -141,7 +152,8 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
Func<DPSPublisher>? publisherFactory,
|
||||
IReadOnlyList<string> dependencyRefs,
|
||||
IActorRef? mux,
|
||||
ITelemetryLocalHub? telemetryHub = null)
|
||||
ITelemetryLocalHub? telemetryHub = null,
|
||||
ScriptLogRateLimiter? failureLogLimiter = null)
|
||||
{
|
||||
_virtualTagId = virtualTagId;
|
||||
_scriptId = scriptId;
|
||||
@@ -151,6 +163,7 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
_dependencyRefs = dependencyRefs;
|
||||
_mux = mux;
|
||||
_telemetryHub = telemetryHub;
|
||||
_failureLogLimiter = failureLogLimiter ?? new ScriptLogRateLimiter();
|
||||
|
||||
Receive<DependencyValueChanged>(OnDependencyChanged);
|
||||
Receive<DependencyValueBatch>(OnDependencyBatch);
|
||||
@@ -162,7 +175,9 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
{
|
||||
if (_mux is not null && _dependencyRefs.Count > 0)
|
||||
{
|
||||
_mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self));
|
||||
// ReplayRetained: a respawned child (plan edit, supervision restart) evaluates from the mux's
|
||||
// last-known inputs at once instead of waiting for each to change again.
|
||||
_mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self, ReplayRetained: true));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -353,7 +368,18 @@ public sealed class VirtualTagActor : ReceiveActor
|
||||
private void OnEvaluationFailed(string reason, string level, DateTime timestampUtc)
|
||||
{
|
||||
OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair<string, object?>("outcome", "fail"));
|
||||
PublishLog(level, reason);
|
||||
|
||||
// Rate-limited per tag (R02:P6): the first 20 failures a minute publish as before; the rest are
|
||||
// counted and surfaced as ONE summary entry when the window rolls. The metric above and the Bad
|
||||
// publish below are not limited — only the cluster-wide log fan-out is.
|
||||
var admission = _failureLogLimiter.Admit(_virtualTagId);
|
||||
if (admission.SuppressedInPreviousWindow > 0)
|
||||
{
|
||||
PublishLog("Warning",
|
||||
$"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " +
|
||||
$"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {_failureLogLimiter.LimitPerWindow}/window); last: {reason}");
|
||||
}
|
||||
if (admission.Admitted) PublishLog(level, reason);
|
||||
|
||||
if ((_hasPublished && _lastPublishedStatus == OpcUaStatus.BadInternalError) || !AllDependenciesArrived())
|
||||
{
|
||||
|
||||
@@ -3,6 +3,7 @@ using Akka.Event;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.VirtualTags;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
@@ -112,6 +113,11 @@ public sealed class VirtualTagHostActor : ReceiveActor
|
||||
(_evaluator as IScriptCacheOwner)?.ClearCompiledScripts();
|
||||
}
|
||||
|
||||
// Every apply is a deploy, and a deploy lifts the run guard's quarantines unconditionally — "the
|
||||
// operator redeployed to retry" is one of the two documented ways out of a quarantine (the other,
|
||||
// an edited expression, the guard detects on its own). Cheap on a no-op redeploy (an empty map).
|
||||
(_evaluator as IScriptQuarantineOwner)?.ResetQuarantines();
|
||||
|
||||
var desired = new HashSet<string>(msg.Plans.Select(p => p.VirtualTagId), StringComparer.Ordinal);
|
||||
|
||||
// Stop + forget children whose vtagId is no longer desired. Stopping the child triggers its
|
||||
|
||||
Reference in New Issue
Block a user