fix(runtime): restart-safe alarm host, mux last-value replay, VT log rate limit, Calculation remediation (E4)

- R02:H1 ScriptedAlarmHostActor gains a factory Props (Func<(upstream, engine)>) so every incarnation builds and owns its own engine; a supervision restart no longer re-creates the actor around a disposed engine. The legacy instance-taking Props stays source-compatible and is now restart-safe in the weaker sense (PreRestart marks the stop as a restart, PostStop then detaches instead of disposing). DriverHostActor still spawns through the legacy overload (not owned by this lane — F3). - R02:H3 OnAlarmCommand runs every engine op under a 10 s token linked to the actor's lifetime (DefaultCommandTimeout / Props commandTimeout) and logs a Warning on timeout instead of suspending the mailbox on a stalled store. - R02:L3 DependencyMuxTagUpstreamSource.RetainOnly(loaded dep-ref union), called after every successful load. - R02:P6 VirtualTagActor's per-failure ScriptLogEntry is rate-limited per tag (ScriptLogRateLimiter, 20/min + one summary); the Bad publish and the metric are untouched. - R07:H6 DependencyMuxActor retains the last (value, status, ts) per ever-registered ref (through unregisters) and replays them in one batch to a RegisterInterest(ReplayRetained: true) — VirtualTagActor, the Calculation mux adapter and the alarm host opt in; the historization recorder does not (a replay there would record a stale sample per deploy). CalculationDriver records inputs regardless of _running so a replay landing before InitializeAsync still satisfies the change-gate. - R07:P4 CalculationEvaluator negative-caches compile failures by source hash (one Roslyn attempt + one Warning per source until redeploy); the driver's script-log entry fires on the Good→Bad transition or a change of reason, matching the once-per-transition Bad publish. - R07:H8 array results dedup structurally. - R07:L2 ShutdownAsync drops the compiled scripts (collectible ALCs). - R07:U5 DraftValidator rule CalculationNoEffectiveTrigger: a tag with no timer whose change trigger is off or whose script has no literal ctx.GetTag read can never evaluate. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 10:27:49 -04:00
parent 6cbf763719
commit 4d3fc9c816
15 changed files with 921 additions and 63 deletions
@@ -338,6 +338,11 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
        StopTimers();
        // The pump exits at its next iteration; an evaluation already in flight is bounded by RunTimeout.
        lock (_lock) _pending.Clear();
+        // R07:L2 — nothing in the runtime disposes a driver, and every config change is a respawn, so
+        // the compiled scripts' collectible ALCs survived each generation until GC got round to the whole
+        // driver graph. Drop them here (idempotent; a later InitializeAsync recompiles on first use). An
+        // in-flight run that races this is already handled by the evaluator's retry-once path.
+        _evaluator.ClearCompiledScripts();
        return Task.CompletedTask;
    }

@@ -363,11 +368,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
    /// <inheritdoc />
    public void OnDependencyValue(string rawPath, object? value, uint statusCode, DateTime timestampUtc)
    {
-        if (!_running) return;
        lock (_lock)
        {
+            // Recorded REGARDLESS of _running (R07:H6): the mux adapter registers interest at construction,
+            // so inputs — the mux's retained-value replay in particular — can land before InitializeAsync.
+            // Dropping them left every calc tag waiting for each input to change again; keeping them means
+            // the first trigger after init evaluates from current inputs. Only the EVALUATION is gated.
            _depValues[rawPath] = new DataValueSnapshot(value, statusCode, timestampUtc, timestampUtc);
            _arrivedDeps.Add(rawPath);
+            if (!_running) return;
            if (!_changeDependents.TryGetValue(rawPath, out var dependents) || dependents.Count == 0) return;

            // QUEUE each change-triggered dependent — never evaluate here. This
@@ -600,7 +609,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
        var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token)
            .ConfigureAwait(false);

-        string? failureReason = null;
+        string? failureToLog = null;
        (object? Value, uint StatusCode)? toPublish;
        lock (_lock)
        {
@@ -610,16 +619,24 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba

            if (!result.Success)
            {
-                failureReason = result.Reason ?? "evaluator failure";
+                var reason = result.Reason ?? "evaluator failure";
                toPublish = MarkEvaluationFailed(st, pending.TimestampUtc);
+                // The script-log entry fires on the Good→Bad TRANSITION, or when the failure changes
+                // (R07:P4) — not on every trigger of a tag that is already Bad for the same reason, which
+                // flooded the script log at the input's cadence. The Bad publish was already once per
+                // transition; the log now matches it.
+                if (toPublish is not null || !string.Equals(st.LastFailureReason, reason, StringComparison.Ordinal))
+                    failureToLog = reason;
+                st.LastFailureReason = reason;
            }
            else
            {
+                st.LastFailureReason = null;
                toPublish = MarkEvaluationSucceeded(st, result.Value, pending.WorstInputStatus, pending.TimestampUtc);
            }
        }

-        if (failureReason is not null) PublishScriptLog(def, failureReason);
+        if (failureToLog is not null) PublishScriptLog(def, failureToLog);
        if (toPublish is { } p) Publish(def.RawPath, p.Value, p.StatusCode, pending.TimestampUtc);
    }

@@ -638,7 +655,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
    private (object? Value, uint StatusCode)? MarkEvaluationSucceeded(
        TagState st, object? value, uint statusCode, DateTime timestampUtc)
    {
-        if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && Equals(st.LastValue, value))
+        if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && ValuesEqual(st.LastValue, value))
            return null;

        st.HasValue = true;
@@ -650,6 +667,17 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
        return (value, statusCode);
    }

+    /// <summary>Value equality for the dedup: structural for arrays (R07:H8 — <c>object.Equals</c> is reference
+    /// equality for a <c>double[]</c>, so an array-returning timer script republished every tick through
+    /// the dual-namespace fan-out and the historian), ordinary <see cref="object.Equals(object?, object?)"/>
+    /// otherwise.</summary>
+    private static bool ValuesEqual(object? a, object? b)
+    {
+        if (a is Array x && b is Array y)
+            return System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(x, y);
+        return Equals(a, b);
+    }
+
    /// <summary>Record that this tag's INPUTS are Bad, so no evaluation ran. Publishes the
    /// worst input's status — carrying the last known value — <b>once per change of that status</b>: a
    /// repeat of the same Bad is silent, while <c>BadNoCommunication</c> arriving over an earlier
@@ -727,6 +755,12 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
    /// <summary>Test/diagnostic accessor: number of timer ticks skipped for in-flight groups.</summary>
    internal long TimerSkippedTickCount => _timers?.SkippedTickCount ?? 0;

+    /// <summary>Test/diagnostic accessor: sources currently remembered as failing to compile.</summary>
+    internal int CompileFailureCountForTest => _evaluator.CompileFailureCount;
+
+    /// <summary>Test/diagnostic accessor: compiled scripts currently held by the evaluator's cache.</summary>
+    internal int CompiledScriptCountForTest => _evaluator.CompiledScriptCount;
+
    /// <summary>Test/diagnostic accessor: <see langword="true"/> when no change-triggered evaluation is
    /// queued or in flight. A test that used to assert straight after <see cref="OnDependencyValue"/> —
    /// which evaluated inline — waits on this instead, so the assertion still runs against a settled
@@ -770,6 +804,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
        /// now carries the failing INPUT's sub-code rather than a fixed <c>BadInternalError</c>.</summary>
        public uint LastPublishedStatus;
        public DateTime LastTimestampUtc;
+
+        /// <summary>The reason of the last failed evaluation (null after a success / inputs-Bad hold) — the
+        /// gate that keeps the script-log entry to the transition and to a CHANGE of reason.</summary>
+        public string? LastFailureReason;
    }

    private sealed record CalculationSubscriptionHandle(string DiagnosticId) : ISubscriptionHandle;
@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using Microsoft.Extensions.Logging;
 using ZB.MOM.WW.OtOpcUa.Commons.Engines;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
@@ -58,6 +59,13 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
    private readonly SerilogLogger _scriptRoot;
    private readonly TimeSpan _runTimeout;
    private readonly ScriptRunGuard _guard;
+
+    // Negative compile cache (R07:P4), keyed by source hash. A compile error is a property of the SOURCE,
+    // which never changes between triggers (it is fixed in the deploy artifact) — yet every trigger of a
+    // broken tag used to re-run a full Roslyn compile under the driver-wide evaluation gate, starving the
+    // healthy tags behind it, and log the same Warning each time. The failure is now remembered until the
+    // next ClearCompiledScripts (a redeploy), and the host Warning fires once per source.
+    private readonly ConcurrentDictionary<string, string> _compileFailures = new(StringComparer.Ordinal);
    private bool _disposed;

    /// <summary>Initializes a new <see cref="CalculationEvaluator"/>.</summary>
@@ -180,6 +188,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
                    calcTagId, path),
            logger: scriptLog);

+        // A source that already failed to compile fails the same way without touching Roslyn (R07:P4).
+        var sourceKey = HashSource(scriptSource);
+        if (_compileFailures.TryGetValue(sourceKey, out var knownFailure))
+            return VirtualTagEvalResult.Failure(knownFailure);
+
        // Fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent
        // ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the run's
        // disposed guard) re-fetches and retries exactly once. The loop runs at most twice.
@@ -192,13 +205,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
            }
            catch (CompilationErrorException ex)
            {
-                _logger.LogWarning(ex, "Calculation {Id}: Roslyn compile failed", calcTagId);
-                return VirtualTagEvalResult.Failure($"compile error: {ex.Message}");
+                return RememberCompileFailure(sourceKey, calcTagId, $"compile error: {ex.Message}", ex, "Roslyn compile failed");
            }
            catch (ScriptSandboxViolationException ex)
            {
-                _logger.LogWarning(ex, "Calculation {Id}: sandbox violation", calcTagId);
-                return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}");
+                return RememberCompileFailure(sourceKey, calcTagId, $"sandbox violation: {ex.Message}", ex, "sandbox violation");
            }
            catch (ObjectDisposedException)
            {
@@ -258,10 +269,29 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
        }
    }

+    /// <summary>Record a compile failure for <paramref name="sourceKey"/> and log it ONCE — the next trigger
+    /// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning.</summary>
+    private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what)
+    {
+        if (_compileFailures.TryAdd(sourceKey, reason))
+            _logger.LogWarning(ex, "Calculation {Id}: {What} (remembered until the next redeploy)", calcTagId, what);
+        return VirtualTagEvalResult.Failure(reason);
+    }
+
+    private static string HashSource(string source)
+        => Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(source)));
+
+    /// <summary>Test/diagnostic: the number of sources currently remembered as failing to compile.</summary>
+    public int CompileFailureCount => _compileFailures.Count;
+
+    /// <summary>Test/diagnostic: the number of compiled scripts currently cached.</summary>
+    public int CompiledScriptCount => _cache.Count;
+
    /// <inheritdoc />
    public void ClearCompiledScripts()
    {
        _cache.Clear();
+        _compileFailures.Clear();
        // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine
        // (an edited source already lifts its own through the guard's source identity).
        _guard.ResetAll();
@@ -279,5 +309,6 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO
        if (_disposed) return;
        _disposed = true;
        _cache.Dispose();
+        _compileFailures.Clear();
    }
 }