dohertj2/lmxopcua
1
0
Fork 0
Code Issues 36 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scripting): Scripting:Runner=Shadow — in-process results published, each compiled run mirrored to the ScriptWorker pool and compared (lmxopcua#780 P2)

Browse Source 
- ShadowScriptRunner (Core.Scripting): IScriptShadow.Mirror copies the inputs and queues the shadow run on the
  pool; compares value (structural), CLR type, status and failure class (compile / budget / threw+message /
  unsupported_type); runner-fault shadow outcomes (Unavailable, worker busy) are not compared. Bounded:
  Scripting:Shadow:MaxInFlight (256) and one per script; drops counted (behind | key_busy).
- Metrics otopcua.script.shadow.{compared,mismatch{script},dropped,inflight}; one rate-limited Warning per
  script and reason with a diff summary (Scripting:Shadow:MismatchLogsPerMinute, re-armed at the seam's deploy).
- The VirtualTag, Calculation and scripted-alarm seams recognise the shadow and keep their in-process path
  byte-for-byte (no coalescer, one-at-a-time pump, in-process engine), mirroring after each compiled run.
- ScriptRunnerSelection builds the shadow over an OutOfProcessScriptRunner whose lines carry
  ScriptRunnerRole=Shadow; Status() is null under Shadow so readiness never sees the shadow pool; the
  script-runner health entry stays Healthy and reports the shadow counters as data.
- docker-dev/docker-compose.script-runner-shadow.yml; docs/VirtualTags.md "Shadow mode".
- Tests: comparison table, per-script metric + once-per-reason log, rate limit, hung/blocking/throwing shadow
  never reaches the caller, MaxInFlight drop accounting, input copy; real-worker divergent + unsupported-type
  scripts counted against exactly those scripts, a worker-only runaway killed/quarantined there with every
  published value intact, alarm transitions identical to in process across a kill -9 of the shadow worker.
  Mutation-proved (16 mutants, 15 red; the one survivor added an assertion).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-10 07:18:40 -05:00
parent 0796928504
commit 03686eb5d7
17 changed files with 1477 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs
+29 -2
View File
@@ -87,6 +87,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// the pre-emptible worker pool, and a re-evaluation pass submits its whole dirty set as one batch.
/// </summary>
private readonly IScriptRunner? _runner;
private readonly IScriptShadow? _shadow;
// Alarms whose predicate came back Unavailable (the runner, not the script, failed) and that are waiting for the
// runner's Recovery to re-evaluate them once. Guarded by _dirtyLock.
@@ -255,7 +256,9 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// silently ran with because the host never passed a value.</param>
/// <param name="scripting">Run-guard budget + quarantine threshold; null ⇒ <see cref="ScriptingOptions"/> defaults.</param>
/// <param name="scriptRunner">lmxopcua#780 — the out-of-process runner when the ScriptedAlarm seam runs out of
/// process; null (the default) keeps the in-process guard path, unchanged.</param>
/// process; null (the default) keeps the in-process guard path, unchanged. Under <c>Scripting:Runner=Shadow</c> it is an
/// <see cref="IScriptShadow"/>: the in-process path stays the one that drives the state machine and each predicate run
/// is mirrored to it for comparison (P2).</param>
public ScriptedAlarmEngine(
ITagUpstreamSource upstream,
IAlarmStateStore store,
@@ -266,7 +269,8 @@ public sealed class ScriptedAlarmEngine : IDisposable
ScriptingOptions? scripting = null,
IScriptRunner? scriptRunner = null)
{
_runner = scriptRunner;
_shadow = scriptRunner as IScriptShadow;
_runner = _shadow is null ? scriptRunner : null;
_upstream = upstream ?? throw new ArgumentNullException(nameof(upstream));
_store = store ?? throw new ArgumentNullException(nameof(store));
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
@@ -286,6 +290,9 @@ public sealed class ScriptedAlarmEngine : IDisposable
/// <summary>The out-of-process runner predicates run through, or null in process (diagnostics / tests).</summary>
public IScriptRunner? ScriptRunner => _runner;
/// <summary>The shadow predicate runs are mirrored to (<c>Scripting:Runner=Shadow</c>), or null.</summary>
public IScriptShadow? ScriptShadow => _shadow;
/// <summary>
/// The alarm ids whose predicate is quarantined — by the in-process guard, or by the out-of-process runner when
/// one is in use (diagnostics / tests).
@@ -366,6 +373,7 @@ public sealed class ScriptedAlarmEngine : IDisposable
// its own through the guard's source identity; this covers "redeployed to retry").
_guard.ResetAll();
_runner?.Reset(ScriptContextKind.AlarmPredicate);
_shadow?.Reset(ScriptContextKind.AlarmPredicate);
lock (_dirtyLock) _recoveryArmed.Clear();
var compileFailures = new List<string>();
@@ -920,23 +928,29 @@ public sealed class ScriptedAlarmEngine : IDisposable
}
catch (ScriptTimeoutException tex)
{
MirrorPredicate(state, scratch, ScriptRunOutcome.Failed(
$"{ZB.MOM.WW.OtOpcUa.Commons.Engines.ScriptEvalFailureReasons.TimedOutPrefix}{tex.Timeout.TotalSeconds:F1}s"));
return PredicateDidNotComplete(state, seed, scratch,
$"timed out after {tex.Timeout.TotalMilliseconds:F0} ms", pendingEmissions);
}
catch (ScriptRunRefusedException ex)
{
MirrorPredicate(state, scratch, ScriptRunOutcome.Failed(ex.Message));
return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions);
}
catch (ScriptQuarantinedException ex)
{
MirrorPredicate(state, scratch, ScriptRunOutcome.Failed(ex.Message));
return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions);
}
catch (Exception ex)
{
MirrorPredicate(state, scratch, ScriptRunOutcome.Failed($"script threw: {ex.Message}"));
scratch.LastNonCompletion = null;
state.Logger.Error(ex, "Alarm predicate threw — state unchanged");
return seed;
}
MirrorPredicate(state, scratch, ScriptRunOutcome.Ok(predicateTrue));
scratch.LastNonCompletion = null;
// Track the bucket only once the predicate has COMPLETED (I3). The tracked bucket is the last
@@ -963,6 +977,19 @@ public sealed class ScriptedAlarmEngine : IDisposable
return result.State;
}
/// <summary>
/// lmxopcua#780 P2 — Shadow: mirrors one in-process predicate run, after it has driven the state machine's input,
/// to the shadow runner for comparison. Never blocks; the inputs are copied before it returns, so the alarm's
/// reused read cache can be refilled at once.
/// </summary>
private void MirrorPredicate(AlarmState state, AlarmScratch scratch, ScriptRunOutcome published)
{
if (_shadow is null) return;
_shadow.Mirror(new ScriptInvocation(
state.Definition.AlarmId, ScriptContextKind.AlarmPredicate, state.Definition.PredicateScriptSource,
scratch.ReadCache, _scriptTimeout), published);
}
/// <summary>
/// The out-of-process re-evaluation pass (lmxopcua#780, design §4.5): every dirty alarm's predicate goes to the
/// runner in ONE batch, then the results are applied — Part 9 transition, persist-before-update, emissions — in