Merge R2-03 Virtual-Tag failure quality (arch-review round 2) [PR #430]
v2-ci / build (push) Successful in 4m2s
v2-ci / unit-tests (push) Failing after 8m20s

Findings 02/S13 (VT failure/timeout degrades node to Bad quality, no stale-Good
forever), 02/S12 (evaluator retry-once on disposed-race), 02/P7 (compile-cache
clear gated on changed expression set). T11/T12 deferred. STATUS.md conflict
resolved additively. Build clean.
This commit is contained in:
Joseph Doherty
2026-07-13 10:20:43 -04:00
9 changed files with 589 additions and 88 deletions
@@ -18,6 +18,19 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines;
/// fan-out between actors is owned by <c>DependencyMuxActor</c>, not by the eval engine.
/// Cycle detection + cascade ordering live in <see cref="VirtualTagEngine"/>; this adapter
/// stays single-tag scoped to keep <see cref="VirtualTagActor"/>'s message loop simple.
///
/// <para>
/// 02/S12 — the apply-boundary <see cref="ClearCompiledScripts"/> (driven by the host actor per
/// deploy generation) disposes cached evaluators while unchanged children may still be evaluating.
/// A clear landing between this adapter's <c>GetOrCompile</c> and the run's disposed guard would
/// otherwise surface as a spurious <c>Failure</c>. The compiled path is therefore retry-safe: an
/// <see cref="ObjectDisposedException"/> from an in-flight evaluation re-fetches (recompiling the
/// same source into the current cache generation) and retries once. A second disposal mid-retry
/// (two applies inside one evaluation — not a realistic cadence) falls through to the failure path.
/// Residual (accepted, bounded to one generation): a stopping child can recompile its stale source
/// into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no
/// longer open this window at all.
/// </para>
/// </summary>
public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable
{
@@ -63,27 +76,6 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
: VirtualTagEvalResult.Ok(null);
}
ScriptEvaluator<VirtualTagContext, object?> evaluator;
try
{
evaluator = _cache.GetOrCompile(expression);
}
catch (CompilationErrorException ex)
{
_logger.LogWarning(ex, "VirtualTag {Id}: Roslyn compile failed", virtualTagId);
return VirtualTagEvalResult.Failure($"compile error: {ex.Message}");
}
catch (ScriptSandboxViolationException ex)
{
_logger.LogWarning(ex, "VirtualTag {Id}: sandbox violation", virtualTagId);
return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "VirtualTag {Id}: compile threw", virtualTagId);
return VirtualTagEvalResult.Failure($"compile failure: {ex.Message}");
}
var readCache = BuildReadCache(dependencies);
// Per-evaluation script logger: bind both ScriptId and VirtualTagId from the virtual-tag id
// (in the live path the script id equals the virtual-tag id) so the Script-log page can
@@ -98,25 +90,71 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac
virtualTagId, path),
logger: scriptLog);
try
// 02/S12: fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent
// apply-boundary ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the
// run's disposed guard) re-fetches and retries exactly once. The loop runs at most twice — the
// retry catch only matches attempt == 0.
for (var attempt = 0; ; attempt++)
{
// Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't
// interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous
// CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when
// the wall-clock budget fires regardless of whether the inner task completes (the orphaned
// thread is the documented, accepted trade-off — TimedScriptEvaluator remarks).
var timed = new TimedScriptEvaluator<VirtualTagContext, object?>(evaluator, _runTimeout);
var raw = timed.RunAsync(context).GetAwaiter().GetResult();
return VirtualTagEvalResult.Ok(raw);
}
catch (ScriptTimeoutException)
{
return VirtualTagEvalResult.Failure($"script timed out after {_runTimeout.TotalSeconds:F1}s");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "VirtualTag {Id}: script execution threw", virtualTagId);
return VirtualTagEvalResult.Failure($"script threw: {ex.Message}");
ScriptEvaluator<VirtualTagContext, object?> evaluator;
try
{
evaluator = _cache.GetOrCompile(expression);
}
catch (CompilationErrorException ex)
{
_logger.LogWarning(ex, "VirtualTag {Id}: Roslyn compile failed", virtualTagId);
return VirtualTagEvalResult.Failure($"compile error: {ex.Message}");
}
catch (ScriptSandboxViolationException ex)
{
_logger.LogWarning(ex, "VirtualTag {Id}: sandbox violation", virtualTagId);
return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}");
}
catch (ObjectDisposedException)
{
// The cache itself is being torn down (this evaluator is disposing) — a fetch-time
// disposal is a shutdown signal, not the apply-boundary race, so never retry.
return VirtualTagEvalResult.Failure("evaluator disposed");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "VirtualTag {Id}: compile threw", virtualTagId);
return VirtualTagEvalResult.Failure($"compile failure: {ex.Message}");
}
try
{
// Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't
// interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous
// CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when
// the wall-clock budget fires regardless of whether the inner task completes (the orphaned
// thread is the documented, accepted trade-off — TimedScriptEvaluator remarks).
var timed = new TimedScriptEvaluator<VirtualTagContext, object?>(evaluator, _runTimeout);
var raw = timed.RunAsync(context).GetAwaiter().GetResult();
return VirtualTagEvalResult.Ok(raw);
}
catch (ScriptTimeoutException)
{
return VirtualTagEvalResult.Failure($"script timed out after {_runTimeout.TotalSeconds:F1}s");
}
catch (ObjectDisposedException) when (!_disposed && attempt == 0)
{
// S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our
// GetOrCompile and the run's disposed-guard. Re-fetch — GetOrCompile recompiles the same
// source into the CURRENT cache generation — and retry once. A second disposal mid-retry
// (two applies inside one evaluation) falls through to the failure path below.
continue;
}
catch (ObjectDisposedException)
{
return VirtualTagEvalResult.Failure("evaluator disposed during evaluation");
}
catch (Exception ex)
{
_logger.LogWarning(ex, "VirtualTag {Id}: script execution threw", virtualTagId);
return VirtualTagEvalResult.Failure($"script threw: {ex.Message}");
}
}
}