Merge R2-03 Virtual-Tag failure quality (arch-review round 2) [PR #430]
v2-ci / build (push) Successful in 4m2s
v2-ci / unit-tests (push) Failing after 8m20s

Findings 02/S13 (VT failure/timeout degrades node to Bad quality, no stale-Good
forever), 02/S12 (evaluator retry-once on disposed-race), 02/P7 (compile-cache
clear gated on changed expression set). T11/T12 deferred. STATUS.md conflict
resolved additively. Build clean.
This commit is contained in:
Joseph Doherty
2026-07-13 10:20:43 -04:00
9 changed files with 589 additions and 88 deletions
@@ -357,6 +357,87 @@ public sealed class RoslynVirtualTagEvaluatorTests
CompiledCacheCount(sut).ShouldBe(0);
}
/// <summary>02/S12 sequential pin: evaluating a compiled expression, clearing the cache, then
/// evaluating the SAME source again succeeds both times and leaves the cache with exactly one entry
/// (the recompile). This pins the clear-then-recompile contract the T8 retry-loop restructure must
/// not regress.</summary>
[Fact]
public void Evaluate_after_ClearCompiledScripts_recompiles_and_succeeds()
{
using var sut = new RoslynVirtualTagEvaluator(NullLogger<RoslynVirtualTagEvaluator>.Instance, NoOpScriptRoot());
const string expr = "return (int)ctx.GetTag(\"a\").Value + 1;";
var first = sut.Evaluate("vt-clear", expr, new Dictionary<string, object?> { ["a"] = 1 });
first.Success.ShouldBeTrue(first.Reason);
CompiledCacheCount(sut).ShouldBe(1);
sut.ClearCompiledScripts();
CompiledCacheCount(sut).ShouldBe(0);
var second = sut.Evaluate("vt-clear", expr, new Dictionary<string, object?> { ["a"] = 41 });
second.Success.ShouldBeTrue(second.Reason);
second.Value.ShouldBe(42);
CompiledCacheCount(sut).ShouldBe(1);
}
/// <summary>02/S12 concurrent stress repro (RED on current code): while one loop runs many compiled
/// (non-passthrough) evaluations of the same expression, a second loop hammers
/// <see cref="RoslynVirtualTagEvaluator.ClearCompiledScripts"/> — the apply-boundary clear. On the
/// unfixed tree a clear landing between a child's <c>GetOrCompile</c> and the run's disposed-guard
/// surfaces an ObjectDisposedException as a <c>Failure("script threw: Cannot access a disposed
/// object…")</c>; the fix retries once and every result stays Success. The whole test is WaitAsync-
/// bounded so a regression fails fast rather than hanging.</summary>
[Fact]
public async Task Evaluate_racing_ClearCompiledScripts_never_fails_with_disposed()
{
// Generous timeout so the wall-clock budget never fires and confounds the race assertion.
using var sut = new RoslynVirtualTagEvaluator(
NullLogger<RoslynVirtualTagEvaluator>.Instance, NoOpScriptRoot(), TimeSpan.FromSeconds(5));
const string expr = "return (int)ctx.GetTag(\"a\").Value + 1;";
const int loopCount = 12;
const int iterations = 400;
var failures = new System.Collections.Concurrent.ConcurrentBag<string>();
var remaining = loopCount; // number of concurrent evaluation loops still running
// Many concurrent evaluation loops put the thread pool under enough pressure that the inner
// Task.Run hop inside TimedScriptEvaluator is measurably delayed — widening the
// fetch→run-disposed-guard window so a clear reliably lands inside SOME in-flight evaluation.
// The clear loop clears on a short spin cadence (not back-to-back) so (a) most evaluations hit
// the warm cache — keeping compile cost bounded — and (b) at most one clear lands within a
// single evaluation, so the single retry the fix adds is sufficient (a back-to-back clear loop
// would both compile-swamp the RED run into a timeout and defeat the one-shot retry). Pre-fix a
// clear disposing the just-fetched evaluator surfaces as Failure("script threw: Cannot access a
// disposed object…"); post-fix the retry re-fetches into the current generation → all Success.
var evalLoops = Enumerable.Range(0, loopCount).Select(_ => Task.Run(() =>
{
for (var i = 0; i < iterations; i++)
{
var r = sut.Evaluate("vt-race", expr, new Dictionary<string, object?> { ["a"] = i });
if (!r.Success)
{
failures.Add(r.Reason ?? "<null reason>");
}
}
Interlocked.Decrement(ref remaining);
}, TestContext.Current.CancellationToken)).ToArray();
var clearLoop = Task.Run(() =>
{
while (Volatile.Read(ref remaining) > 0)
{
sut.ClearCompiledScripts();
Thread.SpinWait(4000); // ~tens of µs between clears — frequent, but not back-to-back
}
}, TestContext.Current.CancellationToken);
await Task.WhenAll(evalLoops.Append(clearLoop))
.WaitAsync(TimeSpan.FromSeconds(45), TestContext.Current.CancellationToken);
failures.ShouldBeEmpty(
$"every evaluation must succeed despite racing clears; observed: {string.Join(" | ", failures)}");
}
/// <summary>Reads the count of the private compiled-script cache via reflection. The cache is a
/// <c>CompiledScriptCache&lt;,&gt;</c> which exposes a <c>Count</c> property (not <c>ICollection</c>).</summary>
private static int CompiledCacheCount(RoslynVirtualTagEvaluator sut)
@@ -4,6 +4,7 @@ using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
@@ -80,7 +81,138 @@ public sealed class VirtualTagActorTests : RuntimeActorTestBase
entry.VirtualTagId.ShouldBe("vt-1");
}, duration: TimeSpan.FromMilliseconds(500));
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(100));
// 02/S13: the failure now ALSO degrades the node — with no declared dependencyRefs the
// inputs-ready gate is vacuously satisfied, so a Bad EvaluationResult reaches the parent (this
// replaces the old ExpectNoMsg, which encoded the stale-Good bug).
parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
}
/// <summary>02/S13 once-per-transition: two consecutive failures publish exactly ONE Bad
/// EvaluationResult — the second failure is suppressed while already Bad.</summary>
[Fact]
public void Second_consecutive_failure_does_not_publish_a_second_Bad()
{
var parent = CreateTestProbe();
var evaluator = new QueuedEvaluator(
VirtualTagEvalResult.Failure("boom1"), VirtualTagEvalResult.Failure("boom2"));
var actor = parent.ChildActorOf(VirtualTagActor.Props("vt-1", "expr", evaluator: evaluator));
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, DateTime.UtcNow));
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
}
/// <summary>02/S13 same-value recovery: Ok(42) → fail → Ok(42) must republish Good with 42 even
/// though the value equals the pre-failure value (the dedup bypass — else the node stays Bad).</summary>
[Fact]
public void Recovery_to_the_same_value_republishes_Good_after_Bad()
{
var parent = CreateTestProbe();
var evaluator = new QueuedEvaluator(
VirtualTagEvalResult.Ok(42), VirtualTagEvalResult.Failure("boom"), VirtualTagEvalResult.Ok(42));
var actor = parent.ChildActorOf(VirtualTagActor.Props("vt-1", "expr", evaluator: evaluator));
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
var first = parent.ExpectMsg<VirtualTagActor.EvaluationResult>();
first.Quality.ShouldBe(OpcUaQuality.Good);
first.Value.ShouldBe(42);
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, DateTime.UtcNow));
parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 3, DateTime.UtcNow));
var recovered = parent.ExpectMsg<VirtualTagActor.EvaluationResult>();
recovered.Quality.ShouldBe(OpcUaQuality.Good);
recovered.Value.ShouldBe(42);
}
/// <summary>02/S13: an evaluator that THROWS (not just returns Failure) also degrades quality —
/// the try/catch path publishes a Bad EvaluationResult too.</summary>
[Fact]
public void Evaluator_throw_also_degrades_quality()
{
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(VirtualTagActor.Props(
"vt-1", "expr", evaluator: new ThrowingEvaluator()));
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
}
/// <summary>02/S13 inputs-ready gate: a multi-dep script that fails before all its declared
/// dependencies have arrived publishes NO Bad — the node stays at the materialiser's
/// BadWaitingForInitialData, avoiding a false Bad flash during deploy/respawn warm-up.</summary>
[Fact]
public void Cold_start_failure_with_missing_dependency_publishes_no_Bad()
{
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(VirtualTagActor.Props(
"vt-1", "expr",
evaluator: new FailingEvaluator("boom"),
dependencyRefs: new[] { "a", "b" }));
// Only "a" has arrived — "b" is still missing, so the gate suppresses the Bad publish.
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
}
/// <summary>02/S13 inputs-ready gate: once every declared dependency has arrived at least once, a
/// failure publishes exactly one Bad EvaluationResult.</summary>
[Fact]
public void Failure_after_all_dependencies_arrived_publishes_Bad()
{
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(VirtualTagActor.Props(
"vt-1", "expr",
evaluator: new FailingEvaluator("boom"),
dependencyRefs: new[] { "a", "b" }));
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
// Still missing "b" — no publish yet.
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
// "b" arrives — all refs now seen, so the failure degrades to Bad.
actor.Tell(new VirtualTagActor.DependencyValueChanged("b", 2, DateTime.UtcNow));
parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
}
/// <summary>Test evaluator returning a fixed sequence of results, one per Evaluate call; the last
/// result repeats once the queue is drained.</summary>
private sealed class QueuedEvaluator : IVirtualTagEvaluator
{
private readonly Queue<VirtualTagEvalResult> _results;
private VirtualTagEvalResult _last;
/// <summary>Initializes the evaluator with an ordered sequence of results.</summary>
/// <param name="results">The results returned in order, one per call.</param>
public QueuedEvaluator(params VirtualTagEvalResult[] results)
{
_results = new Queue<VirtualTagEvalResult>(results);
_last = results[^1];
}
/// <summary>Returns the next queued result (or the last one once drained).</summary>
/// <param name="id">The tag identifier.</param>
/// <param name="expr">The expression string.</param>
/// <param name="deps">The dependency values.</param>
/// <returns>The next queued result.</returns>
public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
=> _results.Count > 0 ? _last = _results.Dequeue() : _last;
}
/// <summary>Test evaluator that throws on every call — exercises the actor's try/catch degradation
/// path (distinct from a returned Failure).</summary>
private sealed class ThrowingEvaluator : IVirtualTagEvaluator
{
/// <summary>Always throws.</summary>
/// <param name="id">The tag identifier.</param>
/// <param name="expr">The expression string.</param>
/// <param name="deps">The dependency values.</param>
/// <returns>Never returns; throws.</returns>
public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
=> throw new InvalidOperationException("kaboom");
}
/// <summary>Test evaluator that sums integer dependency values.</summary>
@@ -345,24 +345,153 @@ public sealed class VirtualTagHostActorTests : RuntimeActorTestBase
mux.LastSender.ShouldNotBe(firstChild);
}
/// <summary>Wiring guard (arch-review 02/U3, theme #1): every ApplyVirtualTags calls
/// <see cref="IScriptCacheOwner.ClearCompiledScripts"/> on the evaluator at the apply boundary.
/// This is the load-bearing wire — the production evaluator roots one collectible ALC per compiled
/// script, and without this per-generation clear they accrete across deploys. A pure unit test of
/// the evaluator can't catch a missing call here; this asserts the host actor actually makes it.</summary>
/// <summary>Wiring guard (arch-review 02/U3 + 02/P7, theme #1): the host actor calls
/// <see cref="IScriptCacheOwner.ClearCompiledScripts"/> at the apply boundary — but, per P7, ONLY
/// when the deployed expression set actually changed. The clear stays the load-bearing wire (the
/// production evaluator roots one collectible ALC per compiled script), yet a byte-identical
/// redeploy — or a rename / add / remove that leaves the expression set unchanged — no longer forces
/// a full recompilation. This pins that the host makes the call, and gates it correctly.
///
/// <para>The final <c>ClearCount</c> is read at a deterministic sync point: after all applies are
/// queued we drive one EvaluationResult through the actor and wait for the bridged publish — the
/// FIFO mailbox guarantees every prior apply is processed by then, so the counter is final (a
/// polling AwaitAssert on a monotonic counter would pass transiently as it climbs, hiding the
/// unconditional-clear bug).</para></summary>
[Fact]
public void ApplyVirtualTags_clears_the_evaluator_compile_cache_each_generation()
public void ApplyVirtualTags_clears_the_compile_cache_only_when_the_expression_set_changes()
{
var publish = CreateTestProbe();
var evaluator = new CacheOwningStubEvaluator();
var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, evaluator));
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") }));
AwaitAssert(() => evaluator.ClearCount.ShouldBe(1));
// 1) First generation: previous set {} → {E1} differs → clears.
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E1", "a") }));
// 2) Byte-identical redeploy: {E1} → {E1} → NO clear (the P7 win).
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E1", "a") }));
// 3) Edited expression: {E1} → {E2} differs → clears.
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a") }));
// 4) Add a second vtag SHARING the same expression E2: {E2} → {E2} → NO clear.
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[]
{
PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a"),
PlanWithRefs("vt-2", "eq-1", "torque", "E2", "a"),
}));
// 5) Remove that second vtag — expression E2 still deployed by vt-1: {E2} → {E2} → NO clear.
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a") }));
// Sync point: a result for the still-mapped vt-1 bridges to a publish; when it lands, every
// apply above has been processed.
host.Tell(new VirtualTagActor.EvaluationResult("vt-1", 1.0, DateTime.UtcNow, CorrelationId.NewId()));
publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
// Exactly two clears: the first generation and the E1→E2 edit. The identical redeploy and the
// shared-expression add/remove did not clear.
evaluator.ClearCount.ShouldBe(2);
}
/// <summary>
/// 02/S13 stale-Good repro (RED on current code): a script that succeeds once then starts failing
/// must degrade its node's quality to Bad — carrying the last-known value — rather than freezing on
/// the last Good value forever. Drives the REAL host + real child + parent-Tell bridge: apply one
/// plan (deps ["a"]), capture the spawned child via the mux probe's RegisterInterest sender, then
/// feed it two dependency changes. The first evaluates Ok(42) → a Good AttributeValueUpdate; the
/// second fails → the fix publishes a Bad AttributeValueUpdate carrying the stale 42. On the
/// unfixed tree the second publish never arrives (the second ExpectMsg times out).
/// </summary>
[Fact]
public void Failing_script_after_success_degrades_node_quality_to_Bad()
{
var publish = CreateTestProbe();
var mux = CreateTestProbe();
var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux.Ref,
new FailAfterFirstSuccessEvaluator()));
// A second apply generation clears again — proves it's per-generation, not one-shot.
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") }));
AwaitAssert(() => evaluator.ClearCount.ShouldBe(2));
// The child self-registers with the mux in PreStart — capture its ref from the sender so we can
// drive it directly through the real evaluate → parent-Tell → bridge path.
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>();
var child = mux.LastSender;
var ts1 = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc);
var ts2 = new DateTime(2026, 7, 12, 12, 0, 1, DateTimeKind.Utc);
// First dep change → evaluator Ok(42) → Good publish.
child.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, ts1));
var good = publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
good.Quality.ShouldBe(OpcUaQuality.Good);
good.Value.ShouldBe(42.0);
// Second dep change → evaluator Failure → Bad publish carrying the last-known value 42.
child.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, ts2));
var bad = publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
bad.Quality.ShouldBe(OpcUaQuality.Bad);
bad.Value.ShouldBe(42.0);
bad.TimestampUtc.ShouldBe(ts2);
}
/// <summary>02/S13 bridge: a Bad-quality <see cref="VirtualTagActor.EvaluationResult"/> must bridge to
/// an <see cref="OpcUaPublishActor.AttributeValueUpdate"/> carrying <see cref="OpcUaQuality.Bad"/>
/// (not the old hard-coded Good).</summary>
[Fact]
public void Bad_quality_EvaluationResult_is_bridged_as_Bad()
{
var publish = CreateTestProbe();
var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, new StubEvaluator()));
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") }));
var ts = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc);
host.Tell(new VirtualTagActor.EvaluationResult(
"vt-1", 42.0, ts, CorrelationId.NewId(), OpcUaQuality.Bad));
var update = publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
update.Quality.ShouldBe(OpcUaQuality.Bad);
update.Value.ShouldBe(42.0);
}
/// <summary>02/S13 historize parity: a Bad result on a historized plan must record the dormant
/// engine's BadInternalError status (0x80020000u) in the snapshot, not the Good 0u.</summary>
[Fact]
public void Historized_Bad_result_is_recorded_with_BadInternalError_status()
{
var publish = CreateTestProbe();
var writer = new CapturingHistoryWriter();
var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, new StubEvaluator(), writer));
host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
new[] { PlanH("vt-1", "eq-1", "speed-rpm", historize: true) }));
var ts = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc);
host.Tell(new VirtualTagActor.EvaluationResult(
"vt-1", 42.0, ts, CorrelationId.NewId(), OpcUaQuality.Bad));
publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
AwaitAssert(() => writer.Calls.Count.ShouldBe(1));
writer.Calls.TryPeek(out var captured).ShouldBeTrue();
captured.Value.StatusCode.ShouldBe(0x80020000u); // BadInternalError — dormant-engine parity
}
/// <summary>Evaluator that returns <c>Ok(42.0)</c> on its first call and <c>Failure</c> thereafter —
/// the S13 "was working, now broken" shape. Interlocked counter so it is safe across the actor's
/// evaluation thread.</summary>
private sealed class FailAfterFirstSuccessEvaluator : IVirtualTagEvaluator
{
private int _calls;
/// <summary>Returns Ok(42.0) once, then Failure("boom").</summary>
/// <param name="id">The tag identifier.</param>
/// <param name="expr">The expression string.</param>
/// <param name="deps">The dependency values.</param>
/// <returns>Ok(42.0) on the first call; Failure on every subsequent call.</returns>
public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
=> Interlocked.Increment(ref _calls) == 1
? VirtualTagEvalResult.Ok(42.0)
: VirtualTagEvalResult.Failure("boom");
}
/// <summary>Capturing <see cref="IHistoryWriter"/>: records every Record call so H5c tests can