using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.VirtualTags; using SerilogLogger = Serilog.ILogger; namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// /// The Calculation driver's script evaluator. A calc tag is a raw tag whose value is /// produced by a C# script over other tags' live values. This evaluator is a deliberate mirror of /// the VirtualTag RoslynVirtualTagEvaluator: it compiles each unique source once via /// (the Roslyn-backed sandbox), caches the /// resulting evaluator keyed by source in a , /// and runs steady-state evaluations as in-process method invocations against the dependency /// dictionary — fast enough to run inline on the driver's dispatch. It reuses /// verbatim so the Monaco editor, sandbox, and diagnostics pipeline /// apply to calc scripts with no divergence. /// /// /// Single-tag mode. A calc tag produces exactly one output value — its own — so cross-tag /// ctx.SetVirtualTag writes are dropped (and logged at Debug). Fan-out between tags is owned /// by the host's DependencyMuxActor (calc-of-calc chains re-enter the mux), never by the /// eval engine, exactly as in the VirtualTag adapter. /// /// /// Error surfacing . This evaluator never throws for script faults: compile error, /// sandbox violation, runtime throw, and timeout all return /// with a descriptive Reason. A successful run returns /// carrying the computed value (which may be null). The CalculationDriver maps a /// Failure to Bad quality + a ScriptLogEntry and an Ok to the published value. /// /// /// Cache management. Implements so the Calculation driver can drop compiled /// scripts at each deploy generation whose source set changed — mirroring the VirtualTag adapter's /// apply-boundary clear, which is what stops collectible AssemblyLoadContexts accreting across /// script edits. /// /// /// Run quarantine. Every compiled run goes through one keyed by /// calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive /// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — both surface as a /// the driver maps to Bad. The driver's evaluation pump awaits rather than /// blocking a pool thread on the sync , /// and the guard starts the budget clock only once the pool has picked the script up, so pool starvation /// is never reported as a script timeout. /// /// public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineOwner, IDisposable { // Source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and Clear()/Dispose() // that dispose each evaluator's collectible AssemblyLoadContext. ClearCompiledScripts() — driven by the // Calculation driver per deploy generation — is what stops ALCs accreting across script edits. private readonly CompiledScriptCache _cache = new(); private readonly ILogger _logger; private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; private readonly ScriptRunGuard _guard; private readonly IScriptRunner? _runner; private readonly IScriptShadow? _shadow; // Negative compile cache (R07:P4), keyed by source hash. A compile error is a property of the SOURCE, // which never changes between triggers (it is fixed in the deploy artifact) — yet every trigger of a // broken tag used to re-run a full Roslyn compile under the driver-wide evaluation gate, starving the // healthy tags behind it, and log the same Warning each time. The failure is now remembered until the // next ClearCompiledScripts (a redeploy), and the host Warning fires once per source. private readonly ConcurrentDictionary _compileFailures = new(StringComparer.Ordinal); private bool _disposed; /// Initializes a new . /// Logger for host-side compilation/execution diagnostics. Deliberately the /// non-generic (rather than ILogger<CalculationEvaluator>) so the /// caller can forward its OWN ILogger<T> — every ILogger<T> already /// implements this — instead of needing a second, separately-obtained logger instance. /// Root script logger; user ctx.Logger.* output flows through this to the Script-log page. /// Wall-clock budget per script run; defaults to 2 seconds (parity with the VT evaluator). /// Run-guard budget + quarantine threshold; null ⇒ defaults. /// lmxopcua#780 — the out-of-process runner when the Calculation seam runs out of /// process; null (the default) keeps the in-process guard path, unchanged. Under Scripting:Runner=Shadow it is an /// : the in-process path stays the published one and each compiled evaluation is mirrored to /// it for comparison (P2). public CalculationEvaluator( ILogger logger, ScriptRootLogger scriptRoot, TimeSpan? runTimeout = null, ScriptingOptions? scripting = null, IScriptRunner? scriptRunner = null) { _shadow = scriptRunner as IScriptShadow; _runner = _shadow is null ? scriptRunner : null; _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger; _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2); _guard = new ScriptRunGuard(scripting, _scriptRoot); } /// The run guard every compiled evaluation goes through (diagnostics / tests). public ScriptRunGuard RunGuard => _guard; /// /// Evaluate for the calc tag against /// the snapshot in (RawPath → last value). Never throws — script /// faults are reported via ; a successful run returns /// carrying the single computed value. /// /// Identity of the calc tag (bound to the script log; in the live path equals the script id). /// The C# script source producing the tag's value. /// Read-only snapshot of dependency values keyed by RawPath. /// The evaluation result, including any script fault. /// /// Value-only overload. Every dependency reaches the script stamped Good with an /// evaluation-time timestamp, because bare values carry nothing better. Prefer /// , which the /// driver uses: it carries each input's real status and source timestamp, so /// ctx.GetTag(path).StatusCode is honest. This overload remains for /// callers that genuinely hold only values — chiefly tests. /// public VirtualTagEvalResult Evaluate(string calcTagId, string scriptSource, IReadOnlyDictionary dependencies) { ArgumentNullException.ThrowIfNull(dependencies); var nowUtc = DateTime.UtcNow; var snapshots = new Dictionary(dependencies.Count, StringComparer.Ordinal); foreach (var kv in dependencies) snapshots[kv.Key] = new DataValueSnapshot(kv.Value, StatusCode: 0u, nowUtc, nowUtc); return Evaluate(calcTagId, scriptSource, snapshots); } /// /// Evaluate for the calc tag against /// the dependency snapshots in — value, status code and /// source timestamp per input. Never throws; script faults surface as /// . /// /// /// The snapshots pass through to the script's read cache rather than being re-wrapped as /// Good. The old wrapping meant a script that inspected /// ctx.GetTag(path).StatusCode was told Good for every input, including one whose driver had /// stopped communicating — so a quality-aware script could not be written at all. The driver holds /// evaluation entirely while any input is Bad, so what a script sees here is Good or Uncertain. /// /// Identity of the calc tag (bound to the script log). /// The C# script source producing the tag's value. /// Read-only snapshot of dependency values keyed by RawPath. /// The evaluation outcome. /// Blocks the calling thread for the run. The driver's pump uses ; /// this overload remains for callers with no async context — chiefly tests. public VirtualTagEvalResult Evaluate( string calcTagId, string scriptSource, IReadOnlyDictionary dependencies) => EvaluateAsync(calcTagId, scriptSource, dependencies).GetAwaiter().GetResult(); /// /// The async form of : /// the compiled script runs on the pool under the and the caller awaits /// it instead of blocking a second pool thread for the duration. Never throws for script faults. /// /// Identity of the calc tag (bound to the script log). /// The C# script source producing the tag's value. /// Read-only snapshot of dependency values keyed by RawPath. /// Caller cancellation (driver shutdown); wins over the run timeout. /// The evaluation outcome. public async Task EvaluateAsync( string calcTagId, string scriptSource, IReadOnlyDictionary dependencies, CancellationToken ct = default) { if (_disposed) return VirtualTagEvalResult.Failure("evaluator disposed"); if (string.IsNullOrWhiteSpace(scriptSource)) return VirtualTagEvalResult.Failure("empty expression"); // Passthrough fast-path: the mirror shape `return ctx.GetTag("X").Value;` needs no Roslyn. // Semantics are byte-identical to the compiled mirror: a present dependency returns its value // (Ok), an absent one makes GetTag yield a Bad snapshot whose .Value is null, so the script // returns null — Ok(null) here too. Near-misses fall through to the compiled path. if (PassthroughScript.TryMatch(scriptSource, out var passthroughRef)) { // The VALUE only — the caller stamps the output with the worst input status, which for a // single-dependency passthrough IS this input's status. Returning it here as well would put // the same decision in two places. return dependencies.TryGetValue(passthroughRef, out var ptSnapshot) ? VirtualTagEvalResult.Ok(ptSnapshot.Value) : VirtualTagEvalResult.Ok(null); } var readCache = dependencies; // Per-evaluation script logger: bind both ScriptId and CalcTagId (via the VirtualTagId slot, which // the Script-log page attributes on) from the calc-tag id so each line is attributable. var scriptLog = _scriptRoot .ForContext(ScriptLoggerFactory.ScriptIdProperty, calcTagId) .ForContext(ScriptLoggerFactory.VirtualTagIdProperty, calcTagId); var context = new VirtualTagContext( readCache, // Single-tag mode: a calc tag has exactly one output (its own), so cross-tag writes are dropped. setVirtualTag: (path, _) => _logger.LogDebug("Calculation {Id}: cross-tag write to {Path} dropped (single-tag evaluator)", calcTagId, path), logger: scriptLog); // lmxopcua#780 — OutOfProcess: the pre-emptible worker pool compiles and runs it, and owns timeouts, kills, // quarantine and the negative compile cache for this seam. Unavailable publishes BadResourceUnavailable. if (_runner is not null) { var outcome = await _runner.RunAsync(new ScriptInvocation( calcTagId, ScriptContextKind.Calculation, scriptSource, readCache, _runTimeout, scriptLog), ct).ConfigureAwait(false); return outcome.ToVirtualTagEvalResult(); } var result = await RunInProcessAsync(calcTagId, scriptSource, context, ct).ConfigureAwait(false); // lmxopcua#780 P2 — Shadow: `result` is what gets published; the same evaluation is mirrored to the worker pool // and only compared. Mirror never blocks and nothing it does reaches this result. if (_shadow is not null && result.ToMirroredOutcome() is { } mirrored) _shadow.Mirror(new ScriptInvocation(calcTagId, ScriptContextKind.Calculation, scriptSource, readCache, _runTimeout), mirrored); return result; } private async Task RunInProcessAsync( string calcTagId, string scriptSource, VirtualTagContext context, CancellationToken ct) { // A source that already failed to compile fails the same way without touching Roslyn (R07:P4). var sourceKey = HashSource(scriptSource); if (_compileFailures.TryGetValue(sourceKey, out var knownFailure)) return VirtualTagEvalResult.Failure(knownFailure); // Fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent // ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the run's // disposed guard) re-fetches and retries exactly once. The loop runs at most twice. for (var attempt = 0; ; attempt++) { ScriptEvaluator evaluator; try { evaluator = _cache.GetOrCompile(scriptSource); } catch (CompilationErrorException ex) { return RememberCompileFailure(sourceKey, calcTagId, $"compile error: {ex.Message}", ex, "Roslyn compile failed"); } catch (ScriptSandboxViolationException ex) { return RememberCompileFailure(sourceKey, calcTagId, $"sandbox violation: {ex.Message}", ex, "sandbox violation"); } catch (ObjectDisposedException) { // Fetch-time disposal is a shutdown signal (the cache is tearing down), not the // apply-boundary race — never retry. return VirtualTagEvalResult.Failure("evaluator disposed"); } catch (Exception ex) { _logger.LogWarning(ex, "Calculation {Id}: compile threw", calcTagId); return VirtualTagEvalResult.Failure($"compile failure: {ex.Message}"); } try { // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by RawPath): a raw // CancellationToken can't interrupt a CPU-bound/infinite-loop Roslyn script, so the wall-clock // budget is what bounds a runaway script — and the guard is what keeps the next trigger from // stacking another orphan behind it (one per tag, then quarantine). var raw = await _guard.RunAsync(calcTagId, scriptSource, evaluator, context, _runTimeout, ct) .ConfigureAwait(false); return VirtualTagEvalResult.Ok(raw); } catch (ScriptTimeoutException) { return VirtualTagEvalResult.Failure( $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s"); } catch (ScriptRunRefusedException ex) { return VirtualTagEvalResult.Failure(ex.Message); } catch (ScriptQuarantinedException ex) { return VirtualTagEvalResult.Failure(ex.Message); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return VirtualTagEvalResult.Failure("evaluation cancelled (driver shutting down)"); } catch (ObjectDisposedException) when (!_disposed && attempt == 0) { // A concurrent ClearCompiledScripts disposed the evaluator between our GetOrCompile and the // run's disposed-guard. Re-fetch (recompiling the same source into the CURRENT generation) // and retry once. A second disposal mid-retry falls through to the failure path below. continue; } catch (ObjectDisposedException) { return VirtualTagEvalResult.Failure(ScriptEvalFailureReasons.EvaluatorDisposedDuringEvaluation); } catch (Exception ex) { _logger.LogWarning(ex, "Calculation {Id}: script execution threw", calcTagId); return VirtualTagEvalResult.Failure($"script threw: {ex.Message}"); } } } /// /// Whether compiled runs go to an (the out-of-process seam), so the driver's pump can /// submit everything it has dequeued as one batch (lmxopcua#780, design §4.5). False on the in-process path, whose /// pump stays one evaluation at a time, unchanged. /// public bool SubmitsBatches => _runner is not null && !_disposed; /// /// Evaluates several calc tags at once (design §4.5): the passthrough fast path and the trivial failures are answered /// inline exactly as answers them, and every compiled run is handed to the runner in ONE /// . Results are per item; each item keeps 's /// outcome, budget and blame. Without a runner it is per item, in order. /// /// The evaluations: calc-tag id, script source, dependency snapshots. A calc tag must not appear twice. /// Caller cancellation (driver shutdown). /// One task per request, in order; none faults. public IReadOnlyList> EvaluateBatch( IReadOnlyList<(string CalcTagId, string ScriptSource, IReadOnlyDictionary Dependencies)> requests, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(requests); var results = new Task[requests.Count]; if (_runner is null || _disposed) { for (var i = 0; i < requests.Count; i++) results[i] = EvaluateAsync(requests[i].CalcTagId, requests[i].ScriptSource, requests[i].Dependencies, ct); return results; } List? invocations = null; List? indexes = null; for (var i = 0; i < requests.Count; i++) { var (id, source, deps) = requests[i]; // Exactly the fast answers EvaluateAsync gives before it reaches the runner. if (string.IsNullOrWhiteSpace(source) || PassthroughScript.TryMatch(source, out _)) { results[i] = EvaluateAsync(id, source, deps, ct); continue; } var scriptLog = _scriptRoot .ForContext(ScriptLoggerFactory.ScriptIdProperty, id) .ForContext(ScriptLoggerFactory.VirtualTagIdProperty, id); (invocations ??= []).Add(new ScriptInvocation(id, ScriptContextKind.Calculation, source, deps, _runTimeout, scriptLog)); (indexes ??= []).Add(i); } if (invocations is null) return results; var outcomes = _runner.RunBatch(invocations, ct); for (var j = 0; j < outcomes.Count; j++) results[indexes![j]] = ToResultAsync(outcomes[j]); return results; static async Task ToResultAsync(Task outcome) => (await outcome.ConfigureAwait(false)).ToVirtualTagEvalResult(); } /// Record a compile failure for and log it ONCE — the next trigger /// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning. private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what) { if (_compileFailures.TryAdd(sourceKey, reason)) _logger.LogWarning(ex, "Calculation {Id}: {What} (remembered until the next redeploy)", calcTagId, what); return VirtualTagEvalResult.Failure(reason); } private static string HashSource(string source) => Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(source))); /// Test/diagnostic: the number of sources currently remembered as failing to compile. public int CompileFailureCount => _compileFailures.Count; /// Test/diagnostic: the number of compiled scripts currently cached. public int CompiledScriptCount => _cache.Count; /// public void ClearCompiledScripts() { _cache.Clear(); _compileFailures.Clear(); // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine // (an edited source already lifts its own through the guard's source identity). _guard.ResetAll(); _runner?.Reset(ScriptContextKind.Calculation); _shadow?.Reset(ScriptContextKind.Calculation); } /// public void ResetQuarantines() { _guard.ResetAll(); _runner?.Reset(ScriptContextKind.Calculation); _shadow?.Reset(ScriptContextKind.Calculation); } /// public IReadOnlyCollection QuarantinedScripts => _runner is null ? _guard.QuarantinedKeys : [.. _guard.QuarantinedKeys, .. _runner.QuarantinedKeys(ScriptContextKind.Calculation)]; /// Disposes the evaluator, draining + disposing every cached compile's collectible ALC. public void Dispose() { if (_disposed) return; _disposed = true; _cache.Dispose(); _compileFailures.Clear(); } }