fix(runtime): restart-safe alarm host, mux last-value replay, VT log rate limit, Calculation remediation (E4)

- R02:H1 ScriptedAlarmHostActor gains a factory Props (Func<(upstream, engine)>) so every incarnation builds and owns its own engine; a supervision restart no longer re-creates the actor around a disposed engine. The legacy instance-taking Props stays source-compatible and is now restart-safe in the weaker sense (PreRestart marks the stop as a restart, PostStop then detaches instead of disposing). DriverHostActor still spawns through the legacy overload (not owned by this lane — F3). - R02:H3 OnAlarmCommand runs every engine op under a 10 s token linked to the actor's lifetime (DefaultCommandTimeout / Props commandTimeout) and logs a Warning on timeout instead of suspending the mailbox on a stalled store. - R02:L3 DependencyMuxTagUpstreamSource.RetainOnly(loaded dep-ref union), called after every successful load. - R02:P6 VirtualTagActor's per-failure ScriptLogEntry is rate-limited per tag (ScriptLogRateLimiter, 20/min + one summary); the Bad publish and the metric are untouched. - R07:H6 DependencyMuxActor retains the last (value, status, ts) per ever-registered ref (through unregisters) and replays them in one batch to a RegisterInterest(ReplayRetained: true) — VirtualTagActor, the Calculation mux adapter and the alarm host opt in; the historization recorder does not (a replay there would record a stale sample per deploy). CalculationDriver records inputs regardless of _running so a replay landing before InitializeAsync still satisfies the change-gate. - R07:P4 CalculationEvaluator negative-caches compile failures by source hash (one Roslyn attempt + one Warning per source until redeploy); the driver's script-log entry fires on the Good→Bad transition or a change of reason, matching the once-per-transition Bad publish. - R07:H8 array results dedup structurally. - R07:L2 ShutdownAsync drops the compiled scripts (collectible ALCs). - R07:U5 DraftValidator rule CalculationNoEffectiveTrigger: a tag with no timer whose change trigger is off or whose script has no literal ctx.GetTag read can never evaluate. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 10:27:49 -04:00
parent 6cbf763719
commit 4d3fc9c816
15 changed files with 921 additions and 63 deletions
@@ -42,8 +42,10 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
    // publish.
    private const uint StatusNeverPushed = OpcUaStatus.BadWaitingForInitialData;

-    // Intentionally never pruned: cold-start semantics depend on retained last values so
-    // ReadTag can answer synchronously for any path that has ever been pushed.
+    // Retained last values: ReadTag answers synchronously for any path that has been pushed. Pruned
+    // ONLY through RetainOnly — the host calls it with the loaded dependency-ref union after every
+    // (re)load, so a renamed/removed raw path (a RawTree rename is a surgical remove+add) stops
+    // costing a boxed value + two DateTimes for the process lifetime (R02:L3).
    private readonly ConcurrentDictionary<string, DataValueSnapshot> _cache
        = new(StringComparer.Ordinal);
    private readonly ConcurrentDictionary<string, ImmutableList<Action<string, DataValueSnapshot>>> _observers
@@ -91,6 +93,28 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource
        }
    }

+    /// <summary>
+    ///     Drop every cached snapshot whose path is not in <paramref name="paths"/>. Observers are
+    ///     untouched. Called by the host after each successful engine (re)load with the union of the loaded
+    ///     alarms' dependency refs — exactly the set the mux will push from now on — so paths no alarm
+    ///     references any more stop being retained.
+    /// </summary>
+    /// <param name="paths">The paths to keep.</param>
+    /// <returns>The number of cached snapshots dropped.</returns>
+    public int RetainOnly(IReadOnlySet<string> paths)
+    {
+        ArgumentNullException.ThrowIfNull(paths);
+        var dropped = 0;
+        foreach (var key in _cache.Keys)
+        {
+            if (!paths.Contains(key) && _cache.TryRemove(key, out _)) dropped++;
+        }
+        return dropped;
+    }
+
+    /// <summary>Test/diagnostic: the number of paths currently retained.</summary>
+    public int RetainedPathCount => _cache.Count;
+
    /// <inheritdoc/>
    public DataValueSnapshot ReadTag(string path)
    {
@@ -23,10 +23,16 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
 ///     cluster <c>alerts</c> DistributedPubSub topic.
 ///
 ///     <para>
-///         The host is the engine's lifecycle owner: the caller (the driver-role startup) builds
-///         the <see cref="DependencyMuxTagUpstreamSource"/>, constructs a <see cref="ScriptedAlarmEngine"/>
-///         <b>around that same upstream</b>, and passes both here. The host disposes the engine in
-///         <see cref="PostStop"/>.
+///         The host is the engine's lifecycle owner. The restart-safe shape (R02:H1) is the
+///         <b>factory</b> <see cref="Props(IActorRef, IActorRef?, Func{ValueTuple{DependencyMuxTagUpstreamSource, ScriptedAlarmEngine}}, NodeId?, Func{int}?, ITelemetryLocalHub?, TimeSpan?)"/>:
+///         every incarnation builds its own <see cref="DependencyMuxTagUpstreamSource"/> + a
+///         <see cref="ScriptedAlarmEngine"/> around it, and disposes them in <see cref="PostStop"/>.
+///         Akka's default restart runs <c>PostStop</c> then re-creates the actor from the SAME Props, so the
+///         legacy instance-taking Props — a fixed engine captured in the Props — restarted around a
+///         <em>disposed</em> engine and every later <c>LoadAsync</c> threw: the node's scripted alarms
+///         silently died. The legacy overload is kept source-compatible but is restart-safe only in the
+///         weaker sense that a restart now DETACHES rather than disposes its engine (see
+///         <see cref="PreRestart"/>); production should spawn through the factory.
 ///     </para>
 ///
 ///     <para>
@@ -99,6 +105,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
    /// Warning on every clean shutdown.</summary>
    private sealed record AlarmsLoadCanceled;

+    /// <summary>Default bound on one inbound alarm command's engine op (<see cref="OnAlarmCommand"/>): 10 s.</summary>
+    public static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(10);
+
    private readonly IActorRef _publishActor;
    private readonly IActorRef? _mux;
    private readonly DependencyMuxTagUpstreamSource _upstream;
@@ -107,6 +116,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
    private readonly ILoggingAdapter _log = Context.GetLogger();
    private readonly CancellationTokenSource _cts = new();
    private readonly EventHandler<ScriptedAlarmEvent> _onEngineEvent;
+    private readonly TimeSpan _commandTimeout;
+
+    /// <summary>True when THIS incarnation built its engine (factory Props) and therefore disposes it on
+    /// every stop, restart included. False for the legacy instance-taking Props: the engine outlives the
+    /// incarnation, so a restart only detaches from it and a final stop disposes it.</summary>
+    private readonly bool _ownsEngine;
+
+    /// <summary>Set by <see cref="PreRestart"/> so the <see cref="PostStop"/> it triggers can tell a restart
+    /// (the next incarnation reuses a legacy engine) from a genuine stop.</summary>
+    private bool _restarting;

    /// <summary>Cached local <see cref="RedundancyRole"/> from the latest <see cref="RedundancyStateChanged"/>
    /// snapshot (null = unknown until the first snapshot arrives, or no <see cref="_localNode"/> wired). The
@@ -152,6 +171,8 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
    /// <param name="telemetryHub">Optional node-local live-telemetry hub (Phase 5) each cluster-wide
    /// <c>alerts</c> transition is also emitted into; null (tests) skips the emit.</param>
    /// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
+    /// <remarks>Legacy, instance-taking shape — see the class remarks: a supervision restart re-creates the
+    /// actor around these SAME instances, so prefer the factory overload in production.</remarks>
    public static Props Props(
        IActorRef publishActor,
        IActorRef? mux,
@@ -162,6 +183,60 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
        ITelemetryLocalHub? telemetryHub = null) =>
        Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub));

+    /// <summary>
+    ///     Restart-safe factory Props (R02:H1): each incarnation of the actor calls
+    ///     <paramref name="engineFactory"/> in its constructor and owns what it returns — the engine must
+    ///     be constructed <b>around</b> the returned upstream — so a supervision restart gets a fresh,
+    ///     undisposed engine and the next <see cref="ApplyScriptedAlarms"/> loads cleanly.
+    /// </summary>
+    /// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
+    /// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
+    /// <param name="engineFactory">Builds a fresh (upstream, engine) pair; called once per incarnation.</param>
+    /// <param name="localNode">The local cluster node id for the Primary alerts-emit gate; null ⇒ default-emit.</param>
+    /// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
+    /// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
+    /// <param name="commandTimeout">Bound on one inbound alarm command's engine op; null ⇒ <see cref="DefaultCommandTimeout"/>.</param>
+    /// <returns>The <see cref="Akka.Actor.Props"/> used to instantiate the actor.</returns>
+    public static Props Props(
+        IActorRef publishActor,
+        IActorRef? mux,
+        Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
+        NodeId? localNode = null,
+        Func<int>? driverMemberCountProvider = null,
+        ITelemetryLocalHub? telemetryHub = null,
+        TimeSpan? commandTimeout = null) =>
+        Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, engineFactory, localNode, driverMemberCountProvider, telemetryHub, commandTimeout));
+
+    /// <summary>Factory-built incarnation: owns (and on every stop disposes) the engine it builds.</summary>
+    /// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
+    /// <param name="mux">Optional dependency multiplexer the host registers interest with.</param>
+    /// <param name="engineFactory">Builds this incarnation's (upstream, engine) pair.</param>
+    /// <param name="localNode">The local cluster node id for the Primary alerts-emit gate.</param>
+    /// <param name="driverMemberCountProvider">Test seam for the alerts-emit gate's driver-member count.</param>
+    /// <param name="telemetryHub">Optional node-local live-telemetry hub.</param>
+    /// <param name="commandTimeout">Bound on one inbound alarm command's engine op.</param>
+    public ScriptedAlarmHostActor(
+        IActorRef publishActor,
+        IActorRef? mux,
+        Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory,
+        NodeId? localNode = null,
+        Func<int>? driverMemberCountProvider = null,
+        ITelemetryLocalHub? telemetryHub = null,
+        TimeSpan? commandTimeout = null)
+        : this(publishActor, mux, Build(engineFactory), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: true, commandTimeout)
+    {
+    }
+
+    private static (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) Build(
+        Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory)
+    {
+        ArgumentNullException.ThrowIfNull(engineFactory);
+        var pair = engineFactory();
+        ArgumentNullException.ThrowIfNull(pair.Upstream, "engineFactory().Upstream");
+        ArgumentNullException.ThrowIfNull(pair.Engine, "engineFactory().Engine");
+        return pair;
+    }
+
    /// <summary>Initializes a new instance of the <see cref="ScriptedAlarmHostActor"/> class.</summary>
    /// <param name="publishActor">The OPC UA publish actor emissions are bridged to.</param>
    /// <param name="mux">Optional dependency multiplexer the host registers dependency interest with.</param>
@@ -183,14 +258,29 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
        NodeId? localNode = null,
        Func<int>? driverMemberCountProvider = null,
        ITelemetryLocalHub? telemetryHub = null)
+        : this(publishActor, mux, (upstream, engine), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: false, commandTimeout: null)
+    {
+    }
+
+    private ScriptedAlarmHostActor(
+        IActorRef publishActor,
+        IActorRef? mux,
+        (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) pair,
+        NodeId? localNode,
+        Func<int>? driverMemberCountProvider,
+        ITelemetryLocalHub? telemetryHub,
+        bool ownsEngine,
+        TimeSpan? commandTimeout)
    {
        ArgumentNullException.ThrowIfNull(publishActor);
-        ArgumentNullException.ThrowIfNull(upstream);
-        ArgumentNullException.ThrowIfNull(engine);
+        ArgumentNullException.ThrowIfNull(pair.Upstream);
+        ArgumentNullException.ThrowIfNull(pair.Engine);
        _publishActor = publishActor;
        _mux = mux;
-        _upstream = upstream;
-        _engine = engine;
+        _upstream = pair.Upstream;
+        _engine = pair.Engine;
+        _ownsEngine = ownsEngine;
+        _commandTimeout = commandTimeout is { } t && t > TimeSpan.Zero ? t : DefaultCommandTimeout;
        _localNode = localNode;
        _driverMemberCountProvider = driverMemberCountProvider;
        _telemetryHub = telemetryHub;
@@ -294,8 +384,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
        // subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the
        // engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor),
        // so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed.
-        _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self));
-        _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs", msg.DepRefs.Count);
+        // ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a
+        // supervision restart builds one around an EMPTY upstream) converges immediately instead of
+        // waiting for every input to change again.
+        _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true));
+        // Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references
+        // any more will never be pushed again, so its last value is dead weight for the process lifetime.
+        var dropped = _upstream.RetainOnly(new HashSet<string>(msg.DepRefs, StringComparer.Ordinal));
+        _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs (pruned {Dropped} retained path(s))",
+            msg.DepRefs.Count, dropped);

        ReassertConditionNodes();
    }
@@ -591,18 +688,25 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
            return;
        }

+        // Bounded (R02:H3): every op ends in IAlarmStateStore.SaveAsync (LocalDb/SQLite), and this is a
+        // ReceiveAsync — the mailbox is suspended until the op returns, so an unbounded await on a hung
+        // file lock stalled dependency-value ingestion for the duration. Linked to the actor's lifetime
+        // token so a stop still cancels promptly.
+        using var cts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token);
+        cts.CancelAfter(_commandTimeout);
+        var ct = cts.Token;
        try
        {
            switch (cmd.Operation)
            {
                case "Acknowledge":
-                    await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
+                    await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
                    break;
                case "Confirm":
-                    await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
+                    await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
                    break;
                case "OneShotShelve":
-                    await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
+                    await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, ct);
                    break;
                case "TimedShelve":
                    // A timed shelve needs the absolute unshelve instant. T18 derives it from the OPC UA
@@ -614,16 +718,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
                            cmd.AlarmId);
                        return;
                    }
-                    await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, CancellationToken.None);
+                    await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, ct);
                    break;
                case "Unshelve":
-                    await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
+                    await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, ct);
                    break;
                case "Enable":
-                    await _engine.EnableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
+                    await _engine.EnableAsync(cmd.AlarmId, cmd.User, ct);
                    break;
                case "Disable":
-                    await _engine.DisableAsync(cmd.AlarmId, cmd.User, CancellationToken.None);
+                    await _engine.DisableAsync(cmd.AlarmId, cmd.User, ct);
                    break;
                case "AddComment":
                    if (string.IsNullOrWhiteSpace(cmd.Comment))
@@ -631,7 +735,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
                        _log.Warning("ScriptedAlarmHost: rejecting AddComment for {AlarmId} — empty comment text", cmd.AlarmId);
                        return;
                    }
-                    await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None);
+                    await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct);
                    break;
                default:
                    _log.Warning("ScriptedAlarmHost: ignoring AlarmCommand with unknown operation {Op} for {AlarmId}",
@@ -639,6 +743,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
                    break;
            }
        }
+        catch (OperationCanceledException) when (!_cts.IsCancellationRequested)
+        {
+            _log.Warning("ScriptedAlarmHost: engine op {Op} for alarm {AlarmId} timed out after {Timeout} — the condition-state store is stalled",
+                cmd.Operation, cmd.AlarmId, _commandTimeout);
+        }
        catch (Exception ex)
        {
            // A failing engine op must not crash the actor — mirror OnLoadFailed's log-and-stay-inert style.
@@ -672,16 +781,28 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor
        base.PreStart();
    }

+    /// <inheritdoc />
+    /// <remarks>Akka's default <c>PreRestart</c> stops the children and calls <see cref="PostStop"/>. The flag
+    /// lets that <see cref="PostStop"/> keep a legacy (Props-captured) engine alive for the next incarnation
+    /// instead of disposing the instance it is about to be re-created around.</remarks>
+    protected override void PreRestart(Exception reason, object message)
+    {
+        _restarting = true;
+        base.PreRestart(reason, message);
+    }
+
    /// <inheritdoc />
    protected override void PostStop()
    {
        // Unregister mux interest first so no further DependencyValueChanged arrives while we tear down.
        _mux?.Tell(new DependencyMuxActor.UnregisterInterest(Self));
        // Cancel any in-flight LoadAsync, detach the OnEvent handler (so a late engine emission can't
-        // Tell a stopping actor), then dispose the engine (which drains its background work + clears).
+        // Tell a stopping actor), then dispose the engine (which drains its background work + clears) —
+        // unless this is a RESTART around a legacy Props-captured engine, which the next incarnation will
+        // re-attach to (R02:H1).
        _cts.Cancel();
        _engine.OnEvent -= _onEngineEvent;
-        _engine.Dispose();
+        if (_ownsEngine || !_restarting) _engine.Dispose();
        _cts.Dispose();
        base.PostStop();
    }
@@ -62,7 +62,10 @@ public sealed class DependencyConsumerMuxAdapter : ReceiveActor
        var refs = _consumer.DependencyRefs.ToList();
        // A RegisterInterest replaces the prior interest set for this subscriber (mux is idempotent on
        // re-register), so re-registering after an apply cleanly adopts an added/removed ref.
-        _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self));
+        // ReplayRetained (R07:H6): a driver Restart is a NEW consumer behind a new adapter; the mux replays
+        // the last value of every ref it retained so the calc tags compute from current inputs at once
+        // instead of sitting BadWaitingForInitialData until each input happens to change.
+        _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self, ReplayRetained: true));
        _log.Debug("DependencyConsumerMuxAdapter: registered {Count} dependency ref(s)", refs.Count);
    }

@@ -26,6 +26,20 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
 ///     <see cref="VirtualTagActor.DependencyValueChanged"/> receive as well, for direct senders and tests.
 ///     The mux is deliberately not a DPS subscriber — virtual-tag evaluation is local to each node and
 ///     would over-emit if it spanned the cluster.
+///
+///     <para>
+///         <b>Last-value retention + replay (R07:H6).</b> For every ref that has EVER been registered the
+///         mux retains the last <c>(value, status, timestamp)</c> it forwarded — through unregisters too,
+///         because the case that matters is a subscriber that went away and came back: a Calculation
+///         driver Restart is a new driver instance + a new mux adapter, and before this every calc tag sat
+///         <c>BadWaitingForInitialData</c> until each of its inputs happened to change again (a
+///         scripted-alarm host restart has the same shape). A <see cref="RegisterInterest"/> with
+///         <see cref="RegisterInterest.ReplayRetained"/> receives one batch of the retained values for its
+///         refs, in the same shape a driver burst would arrive in. Opt-in per registration: the
+///         continuous-historization recorder re-registers on every deploy and a replay there would
+///         record a stale sample per tag per deploy. Memory is bounded by the number of refs scripts have
+///         ever named on this node, not by the address space.
+///     </para>
 /// </summary>
 public sealed class DependencyMuxActor : ReceiveActor
 {
@@ -33,14 +47,23 @@ public sealed class DependencyMuxActor : ReceiveActor

    /// <summary>Register a subscriber's interest in a set of tag refs. Idempotent on re-register —
    /// the second call replaces the prior interest set for that subscriber.</summary>
-    public sealed record RegisterInterest(IReadOnlyList<string> TagRefs, IActorRef Subscriber);
+    /// <param name="TagRefs">The refs the subscriber wants.</param>
+    /// <param name="Subscriber">The subscriber.</param>
+    /// <param name="ReplayRetained">When true, the mux immediately sends the subscriber one
+    /// <see cref="VirtualTagActor.DependencyValueBatch"/> holding the last retained value of every ref
+    /// in <paramref name="TagRefs"/> it has one for (nothing when it has none).</param>
+    public sealed record RegisterInterest(IReadOnlyList<string> TagRefs, IActorRef Subscriber, bool ReplayRetained = false);

    /// <summary>Unregister every interest held by <see cref="Subscriber"/>. Sent on PostStop by
    /// the subscriber, or by Terminated handling when the mux watches.</summary>
    public sealed record UnregisterInterest(IActorRef Subscriber);

+    // A ref stays in _byRef once registered — with an EMPTY set after its last subscriber leaves — so
+    // the hot path's single lookup per item also decides retention. _byRef.Count == 0 keeps meaning
+    // "nothing was ever registered", which the batch path early-outs on.
    private readonly Dictionary<string, HashSet<IActorRef>> _byRef = new(StringComparer.Ordinal);
    private readonly Dictionary<IActorRef, HashSet<string>> _bySubscriber = new();
+    private readonly Dictionary<string, VirtualTagActor.DependencyValueChanged> _last = new(StringComparer.Ordinal);
    private readonly ILoggingAdapter _log = Context.GetLogger();

    /// <summary>Creates props for the DependencyMuxActor.</summary>
@@ -64,16 +87,13 @@ public sealed class DependencyMuxActor : ReceiveActor
        {
            foreach (var r in priorRefs)
            {
-                if (_byRef.TryGetValue(r, out var set))
-                {
-                    set.Remove(msg.Subscriber);
-                    if (set.Count == 0) _byRef.Remove(r);
-                }
+                if (_byRef.TryGetValue(r, out var set)) set.Remove(msg.Subscriber);
            }
        }

        var refs = new HashSet<string>(msg.TagRefs, StringComparer.Ordinal);
        _bySubscriber[msg.Subscriber] = refs;
+        List<VirtualTagActor.DependencyValueChanged>? replay = null;
        foreach (var r in refs)
        {
            if (!_byRef.TryGetValue(r, out var set))
@@ -82,9 +102,19 @@ public sealed class DependencyMuxActor : ReceiveActor
                _byRef[r] = set;
            }
            set.Add(msg.Subscriber);
+            if (msg.ReplayRetained && _last.TryGetValue(r, out var retained))
+                (replay ??= []).Add(retained);
        }
        Context.Watch(msg.Subscriber);
        _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count);
+
+        // One batch, after the registration is complete, in exactly the shape a driver burst arrives in —
+        // so the subscriber's hot path handles it with no special case.
+        if (replay is not null)
+        {
+            msg.Subscriber.Tell(new VirtualTagActor.DependencyValueBatch(replay));
+            _log.Debug("DependencyMux: replayed {Count} retained value(s) to {Sub}", replay.Count, msg.Subscriber);
+        }
    }

    private void RemoveSubscriber(IActorRef subscriber)
@@ -92,11 +122,9 @@ public sealed class DependencyMuxActor : ReceiveActor
        if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return;
        foreach (var r in refs)
        {
-            if (_byRef.TryGetValue(r, out var set))
-            {
-                set.Remove(subscriber);
-                if (set.Count == 0) _byRef.Remove(r);
-            }
+            // The ref's entry stays (possibly empty) so its last value keeps being retained for a
+            // subscriber that comes back — see the class remarks.
+            if (_byRef.TryGetValue(r, out var set)) set.Remove(subscriber);
        }
        _bySubscriber.Remove(subscriber);
        Context.Unwatch(subscriber);
@@ -105,13 +133,14 @@ public sealed class DependencyMuxActor : ReceiveActor

    private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg)
    {
-        if (!_byRef.TryGetValue(msg.FullReference, out var subscribers) || subscribers.Count == 0)
+        if (!_byRef.TryGetValue(msg.FullReference, out var subscribers))
        {
-            // No virtual tag cares about this ref — drop. Common in normal operation; the address
+            // No virtual tag ever cared about this ref — drop. Common in normal operation; the address
            // space carries thousands of tags and only a fraction feed virtual-tag expressions.
            return;
        }
        var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode);
+        _last[msg.FullReference] = dep;
        foreach (var sub in subscribers)
        {
            // A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs
@@ -122,15 +151,6 @@ public sealed class DependencyMuxActor : ReceiveActor
        }
    }

-    /// <summary>Filters one batched driver production burst per item and delivers each subscriber's matched
-    /// slice as ONE <see cref="VirtualTagActor.DependencyValueBatch"/>. Items nobody registered for are
-    /// dropped (the common case: only a fraction of an address space feeds virtual-tag expressions), so a
-    /// burst of N values costs one inbound message plus one outbound message per genuinely interested
-    /// SUBSCRIBER — not per (item, subscriber) pair as it did when this fanned out per value. That is what
-    /// lets <see cref="VirtualTagActor"/> evaluate its script once per burst instead of once per input.
-    /// <para>The registered set is NOT just virtual-tag inputs: <c>ContinuousHistorizationRecorder</c>
-    /// registers interest in every historized ref, so on a historizing node <see cref="_byRef"/> can be
-    /// large and a sizeable fraction of each burst resolves.</para></summary>
    private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg)
    {
        // Nobody has registered anything — skip the whole burst rather than probing an empty map per item.
@@ -146,8 +166,10 @@ public sealed class DependencyMuxActor : ReceiveActor
        for (var i = 0; i < count; i++)
        {
            var item = msg.Items[i];
-            if (!_byRef.TryGetValue(item.FullReference, out var subscribers) || subscribers.Count == 0) continue;
+            if (!_byRef.TryGetValue(item.FullReference, out var subscribers)) continue;
            var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode);
+            _last[item.FullReference] = dep;
+            if (subscribers.Count == 0) continue; // retained for a returning subscriber; nobody to forward to now
            perSubscriber ??= new Dictionary<IActorRef, List<VirtualTagActor.DependencyValueChanged>>();
            foreach (var sub in subscribers)
            {
@@ -7,6 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Observability;
 using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Commons.Types;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Scripting;
 using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;

 namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
@@ -76,6 +77,10 @@ public sealed class VirtualTagActor : ReceiveActor
    private readonly IReadOnlyList<string> _dependencyRefs;
    private readonly IActorRef? _mux;
    private readonly ITelemetryLocalHub? _telemetryHub;
+    // R02:P6 — a script failing on every trigger of a 1 Hz input published one cluster-wide DPS
+    // ScriptLogEntry per second per tag; 200 such tags shed everyone's entries out of the bounded
+    // telemetry channels. The per-failure entry is now rate-limited per tag (20/min + one summary).
+    private readonly ScriptLogRateLimiter _failureLogLimiter;
    private readonly ILoggingAdapter _log = Context.GetLogger();
    // Dependency SNAPSHOTS, not bare values (archreview 02/S14): the status code is what distinguishes a
    // comms-lost input from a genuine Good null, and it is what the script reads through
@@ -104,6 +109,8 @@ public sealed class VirtualTagActor : ReceiveActor
    /// <param name="mux">Optional reference to a dependency multiplexer actor.</param>
    /// <param name="telemetryHub">Optional node-local live-telemetry hub each script-log entry is also
    /// emitted into (Phase 5); null (tests) simply skips the emit.</param>
+    /// <param name="failureLogLimiter">Rate limiter for this tag's per-failure script-log entries; null ⇒
+    /// the default 20 per minute + one summary.</param>
    /// <returns>A configured <see cref="Akka.Actor.Props"/> for creating the actor.</returns>
    public static Props Props(
        string virtualTagId,
@@ -113,7 +120,8 @@ public sealed class VirtualTagActor : ReceiveActor
        Func<DPSPublisher>? publisherFactory = null,
        IReadOnlyList<string>? dependencyRefs = null,
        IActorRef? mux = null,
-        ITelemetryLocalHub? telemetryHub = null) =>
+        ITelemetryLocalHub? telemetryHub = null,
+        ScriptLogRateLimiter? failureLogLimiter = null) =>
        Akka.Actor.Props.Create(() => new VirtualTagActor(
            virtualTagId, expression,
            evaluator ?? NullVirtualTagEvaluator.Instance,
@@ -121,7 +129,8 @@ public sealed class VirtualTagActor : ReceiveActor
            publisherFactory,
            dependencyRefs ?? Array.Empty<string>(),
            mux,
-            telemetryHub));
+            telemetryHub,
+            failureLogLimiter));

    /// <summary>Initializes a virtual tag actor with the given configuration and dependencies.</summary>
    /// <param name="virtualTagId">Unique identifier for the virtual tag.</param>
@@ -133,6 +142,8 @@ public sealed class VirtualTagActor : ReceiveActor
    /// <param name="mux">Optional reference to a dependency multiplexer actor.</param>
    /// <param name="telemetryHub">Optional node-local live-telemetry hub each script-log entry is also
    /// emitted into (Phase 5); null (tests) simply skips the emit.</param>
+    /// <param name="failureLogLimiter">Rate limiter for this tag's per-failure script-log entries; null ⇒
+    /// the default 20 per minute + one summary.</param>
    public VirtualTagActor(
        string virtualTagId,
        string expression,
@@ -141,7 +152,8 @@ public sealed class VirtualTagActor : ReceiveActor
        Func<DPSPublisher>? publisherFactory,
        IReadOnlyList<string> dependencyRefs,
        IActorRef? mux,
-        ITelemetryLocalHub? telemetryHub = null)
+        ITelemetryLocalHub? telemetryHub = null,
+        ScriptLogRateLimiter? failureLogLimiter = null)
    {
        _virtualTagId = virtualTagId;
        _scriptId = scriptId;
@@ -151,6 +163,7 @@ public sealed class VirtualTagActor : ReceiveActor
        _dependencyRefs = dependencyRefs;
        _mux = mux;
        _telemetryHub = telemetryHub;
+        _failureLogLimiter = failureLogLimiter ?? new ScriptLogRateLimiter();

        Receive<DependencyValueChanged>(OnDependencyChanged);
        Receive<DependencyValueBatch>(OnDependencyBatch);
@@ -162,7 +175,9 @@ public sealed class VirtualTagActor : ReceiveActor
    {
        if (_mux is not null && _dependencyRefs.Count > 0)
        {
-            _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self));
+            // ReplayRetained: a respawned child (plan edit, supervision restart) evaluates from the mux's
+            // last-known inputs at once instead of waiting for each to change again.
+            _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self, ReplayRetained: true));
        }
    }

@@ -353,7 +368,18 @@ public sealed class VirtualTagActor : ReceiveActor
    private void OnEvaluationFailed(string reason, string level, DateTime timestampUtc)
    {
        OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair<string, object?>("outcome", "fail"));
-        PublishLog(level, reason);
+
+        // Rate-limited per tag (R02:P6): the first 20 failures a minute publish as before; the rest are
+        // counted and surfaced as ONE summary entry when the window rolls. The metric above and the Bad
+        // publish below are not limited — only the cluster-wide log fan-out is.
+        var admission = _failureLogLimiter.Admit(_virtualTagId);
+        if (admission.SuppressedInPreviousWindow > 0)
+        {
+            PublishLog("Warning",
+                $"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " +
+                $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); last: {reason}");
+        }
+        if (admission.Admitted) PublishLog(level, reason);

        if ((_hasPublished && _lastPublishedStatus == OpcUaStatus.BadInternalError) || !AllDependenciesArrived())
        {