Merge R2-03 Virtual-Tag failure quality (arch-review round 2) [PR #430]

Findings 02/S13 (VT failure/timeout degrades node to Bad quality, no stale-Good
forever), 02/S12 (evaluator retry-once on disposed-race), 02/P7 (compile-cache
clear gated on changed expression set). T11/T12 deferred. STATUS.md conflict
resolved additively. Build clean.

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorTests.cs

@@ -4,6 +4,7 @@ using Shouldly;
 using Xunit;
 using ZB.MOM.WW.OtOpcUa.Commons.Engines;
 using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
+using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
 using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
 using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
 
@@ -80,7 +81,138 @@ public sealed class VirtualTagActorTests : RuntimeActorTestBase
             entry.VirtualTagId.ShouldBe("vt-1");
         }, duration: TimeSpan.FromMilliseconds(500));
 
-        parent.ExpectNoMsg(TimeSpan.FromMilliseconds(100));
+        // 02/S13: the failure now ALSO degrades the node — with no declared dependencyRefs the
+        // inputs-ready gate is vacuously satisfied, so a Bad EvaluationResult reaches the parent (this
+        // replaces the old ExpectNoMsg, which encoded the stale-Good bug).
+        parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
+    }
+
+    /// <summary>02/S13 once-per-transition: two consecutive failures publish exactly ONE Bad
+    /// EvaluationResult — the second failure is suppressed while already Bad.</summary>
+    [Fact]
+    public void Second_consecutive_failure_does_not_publish_a_second_Bad()
+    {
+        var parent = CreateTestProbe();
+        var evaluator = new QueuedEvaluator(
+            VirtualTagEvalResult.Failure("boom1"), VirtualTagEvalResult.Failure("boom2"));
+        var actor = parent.ChildActorOf(VirtualTagActor.Props("vt-1", "expr", evaluator: evaluator));
+
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
+        parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
+
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, DateTime.UtcNow));
+        parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
+    }
+
+    /// <summary>02/S13 same-value recovery: Ok(42) → fail → Ok(42) must republish Good with 42 even
+    /// though the value equals the pre-failure value (the dedup bypass — else the node stays Bad).</summary>
+    [Fact]
+    public void Recovery_to_the_same_value_republishes_Good_after_Bad()
+    {
+        var parent = CreateTestProbe();
+        var evaluator = new QueuedEvaluator(
+            VirtualTagEvalResult.Ok(42), VirtualTagEvalResult.Failure("boom"), VirtualTagEvalResult.Ok(42));
+        var actor = parent.ChildActorOf(VirtualTagActor.Props("vt-1", "expr", evaluator: evaluator));
+
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
+        var first = parent.ExpectMsg<VirtualTagActor.EvaluationResult>();
+        first.Quality.ShouldBe(OpcUaQuality.Good);
+        first.Value.ShouldBe(42);
+
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, DateTime.UtcNow));
+        parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
+
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 3, DateTime.UtcNow));
+        var recovered = parent.ExpectMsg<VirtualTagActor.EvaluationResult>();
+        recovered.Quality.ShouldBe(OpcUaQuality.Good);
+        recovered.Value.ShouldBe(42);
+    }
+
+    /// <summary>02/S13: an evaluator that THROWS (not just returns Failure) also degrades quality —
+    /// the try/catch path publishes a Bad EvaluationResult too.</summary>
+    [Fact]
+    public void Evaluator_throw_also_degrades_quality()
+    {
+        var parent = CreateTestProbe();
+        var actor = parent.ChildActorOf(VirtualTagActor.Props(
+            "vt-1", "expr", evaluator: new ThrowingEvaluator()));
+
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
+
+        parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
+    }
+
+    /// <summary>02/S13 inputs-ready gate: a multi-dep script that fails before all its declared
+    /// dependencies have arrived publishes NO Bad — the node stays at the materialiser's
+    /// BadWaitingForInitialData, avoiding a false Bad flash during deploy/respawn warm-up.</summary>
+    [Fact]
+    public void Cold_start_failure_with_missing_dependency_publishes_no_Bad()
+    {
+        var parent = CreateTestProbe();
+        var actor = parent.ChildActorOf(VirtualTagActor.Props(
+            "vt-1", "expr",
+            evaluator: new FailingEvaluator("boom"),
+            dependencyRefs: new[] { "a", "b" }));
+
+        // Only "a" has arrived — "b" is still missing, so the gate suppresses the Bad publish.
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
+
+        parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
+    }
+
+    /// <summary>02/S13 inputs-ready gate: once every declared dependency has arrived at least once, a
+    /// failure publishes exactly one Bad EvaluationResult.</summary>
+    [Fact]
+    public void Failure_after_all_dependencies_arrived_publishes_Bad()
+    {
+        var parent = CreateTestProbe();
+        var actor = parent.ChildActorOf(VirtualTagActor.Props(
+            "vt-1", "expr",
+            evaluator: new FailingEvaluator("boom"),
+            dependencyRefs: new[] { "a", "b" }));
+
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow));
+        // Still missing "b" — no publish yet.
+        parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
+
+        // "b" arrives — all refs now seen, so the failure degrades to Bad.
+        actor.Tell(new VirtualTagActor.DependencyValueChanged("b", 2, DateTime.UtcNow));
+        parent.ExpectMsg<VirtualTagActor.EvaluationResult>().Quality.ShouldBe(OpcUaQuality.Bad);
+    }
+
+    /// <summary>Test evaluator returning a fixed sequence of results, one per Evaluate call; the last
+    /// result repeats once the queue is drained.</summary>
+    private sealed class QueuedEvaluator : IVirtualTagEvaluator
+    {
+        private readonly Queue<VirtualTagEvalResult> _results;
+        private VirtualTagEvalResult _last;
+        /// <summary>Initializes the evaluator with an ordered sequence of results.</summary>
+        /// <param name="results">The results returned in order, one per call.</param>
+        public QueuedEvaluator(params VirtualTagEvalResult[] results)
+        {
+            _results = new Queue<VirtualTagEvalResult>(results);
+            _last = results[^1];
+        }
+        /// <summary>Returns the next queued result (or the last one once drained).</summary>
+        /// <param name="id">The tag identifier.</param>
+        /// <param name="expr">The expression string.</param>
+        /// <param name="deps">The dependency values.</param>
+        /// <returns>The next queued result.</returns>
+        public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
+            => _results.Count > 0 ? _last = _results.Dequeue() : _last;
+    }
+
+    /// <summary>Test evaluator that throws on every call — exercises the actor's try/catch degradation
+    /// path (distinct from a returned Failure).</summary>
+    private sealed class ThrowingEvaluator : IVirtualTagEvaluator
+    {
+        /// <summary>Always throws.</summary>
+        /// <param name="id">The tag identifier.</param>
+        /// <param name="expr">The expression string.</param>
+        /// <param name="deps">The dependency values.</param>
+        /// <returns>Never returns; throws.</returns>
+        public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
+            => throw new InvalidOperationException("kaboom");
     }
 
     /// <summary>Test evaluator that sums integer dependency values.</summary>

tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagHostActorTests.cs

@@ -345,24 +345,153 @@ public sealed class VirtualTagHostActorTests : RuntimeActorTestBase
         mux.LastSender.ShouldNotBe(firstChild);
     }
 
-    /// <summary>Wiring guard (arch-review 02/U3, theme #1): every ApplyVirtualTags calls
-    /// <see cref="IScriptCacheOwner.ClearCompiledScripts"/> on the evaluator at the apply boundary.
-    /// This is the load-bearing wire — the production evaluator roots one collectible ALC per compiled
-    /// script, and without this per-generation clear they accrete across deploys. A pure unit test of
-    /// the evaluator can't catch a missing call here; this asserts the host actor actually makes it.</summary>
+    /// <summary>Wiring guard (arch-review 02/U3 + 02/P7, theme #1): the host actor calls
+    /// <see cref="IScriptCacheOwner.ClearCompiledScripts"/> at the apply boundary — but, per P7, ONLY
+    /// when the deployed expression set actually changed. The clear stays the load-bearing wire (the
+    /// production evaluator roots one collectible ALC per compiled script), yet a byte-identical
+    /// redeploy — or a rename / add / remove that leaves the expression set unchanged — no longer forces
+    /// a full recompilation. This pins that the host makes the call, and gates it correctly.
+    ///
+    /// <para>The final <c>ClearCount</c> is read at a deterministic sync point: after all applies are
+    /// queued we drive one EvaluationResult through the actor and wait for the bridged publish — the
+    /// FIFO mailbox guarantees every prior apply is processed by then, so the counter is final (a
+    /// polling AwaitAssert on a monotonic counter would pass transiently as it climbs, hiding the
+    /// unconditional-clear bug).</para></summary>
     [Fact]
-    public void ApplyVirtualTags_clears_the_evaluator_compile_cache_each_generation()
+    public void ApplyVirtualTags_clears_the_compile_cache_only_when_the_expression_set_changes()
     {
         var publish = CreateTestProbe();
         var evaluator = new CacheOwningStubEvaluator();
         var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, evaluator));
 
-        host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") }));
-        AwaitAssert(() => evaluator.ClearCount.ShouldBe(1));
+        // 1) First generation: previous set {} → {E1} differs → clears.
+        host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
+            new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E1", "a") }));
+        // 2) Byte-identical redeploy: {E1} → {E1} → NO clear (the P7 win).
+        host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
+            new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E1", "a") }));
+        // 3) Edited expression: {E1} → {E2} differs → clears.
+        host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
+            new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a") }));
+        // 4) Add a second vtag SHARING the same expression E2: {E2} → {E2} → NO clear.
+        host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[]
+        {
+            PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a"),
+            PlanWithRefs("vt-2", "eq-1", "torque", "E2", "a"),
+        }));
+        // 5) Remove that second vtag — expression E2 still deployed by vt-1: {E2} → {E2} → NO clear.
+        host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
+            new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a") }));
+
+        // Sync point: a result for the still-mapped vt-1 bridges to a publish; when it lands, every
+        // apply above has been processed.
+        host.Tell(new VirtualTagActor.EvaluationResult("vt-1", 1.0, DateTime.UtcNow, CorrelationId.NewId()));
+        publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
+
+        // Exactly two clears: the first generation and the E1→E2 edit. The identical redeploy and the
+        // shared-expression add/remove did not clear.
+        evaluator.ClearCount.ShouldBe(2);
+    }
+
+    /// <summary>
+    /// 02/S13 stale-Good repro (RED on current code): a script that succeeds once then starts failing
+    /// must degrade its node's quality to Bad — carrying the last-known value — rather than freezing on
+    /// the last Good value forever. Drives the REAL host + real child + parent-Tell bridge: apply one
+    /// plan (deps ["a"]), capture the spawned child via the mux probe's RegisterInterest sender, then
+    /// feed it two dependency changes. The first evaluates Ok(42) → a Good AttributeValueUpdate; the
+    /// second fails → the fix publishes a Bad AttributeValueUpdate carrying the stale 42. On the
+    /// unfixed tree the second publish never arrives (the second ExpectMsg times out).
+    /// </summary>
+    [Fact]
+    public void Failing_script_after_success_degrades_node_quality_to_Bad()
+    {
+        var publish = CreateTestProbe();
+        var mux = CreateTestProbe();
+        var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux.Ref,
+            new FailAfterFirstSuccessEvaluator()));
 
-        // A second apply generation clears again — proves it's per-generation, not one-shot.
         host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") }));
-        AwaitAssert(() => evaluator.ClearCount.ShouldBe(2));
+
+        // The child self-registers with the mux in PreStart — capture its ref from the sender so we can
+        // drive it directly through the real evaluate → parent-Tell → bridge path.
+        mux.ExpectMsg<DependencyMuxActor.RegisterInterest>();
+        var child = mux.LastSender;
+
+        var ts1 = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc);
+        var ts2 = new DateTime(2026, 7, 12, 12, 0, 1, DateTimeKind.Utc);
+
+        // First dep change → evaluator Ok(42) → Good publish.
+        child.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, ts1));
+        var good = publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
+        good.Quality.ShouldBe(OpcUaQuality.Good);
+        good.Value.ShouldBe(42.0);
+
+        // Second dep change → evaluator Failure → Bad publish carrying the last-known value 42.
+        child.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, ts2));
+        var bad = publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
+        bad.Quality.ShouldBe(OpcUaQuality.Bad);
+        bad.Value.ShouldBe(42.0);
+        bad.TimestampUtc.ShouldBe(ts2);
+    }
+
+    /// <summary>02/S13 bridge: a Bad-quality <see cref="VirtualTagActor.EvaluationResult"/> must bridge to
+    /// an <see cref="OpcUaPublishActor.AttributeValueUpdate"/> carrying <see cref="OpcUaQuality.Bad"/>
+    /// (not the old hard-coded Good).</summary>
+    [Fact]
+    public void Bad_quality_EvaluationResult_is_bridged_as_Bad()
+    {
+        var publish = CreateTestProbe();
+        var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, new StubEvaluator()));
+
+        host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") }));
+
+        var ts = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc);
+        host.Tell(new VirtualTagActor.EvaluationResult(
+            "vt-1", 42.0, ts, CorrelationId.NewId(), OpcUaQuality.Bad));
+
+        var update = publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
+        update.Quality.ShouldBe(OpcUaQuality.Bad);
+        update.Value.ShouldBe(42.0);
+    }
+
+    /// <summary>02/S13 historize parity: a Bad result on a historized plan must record the dormant
+    /// engine's BadInternalError status (0x80020000u) in the snapshot, not the Good 0u.</summary>
+    [Fact]
+    public void Historized_Bad_result_is_recorded_with_BadInternalError_status()
+    {
+        var publish = CreateTestProbe();
+        var writer = new CapturingHistoryWriter();
+        var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, new StubEvaluator(), writer));
+
+        host.Tell(new VirtualTagHostActor.ApplyVirtualTags(
+            new[] { PlanH("vt-1", "eq-1", "speed-rpm", historize: true) }));
+
+        var ts = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc);
+        host.Tell(new VirtualTagActor.EvaluationResult(
+            "vt-1", 42.0, ts, CorrelationId.NewId(), OpcUaQuality.Bad));
+
+        publish.ExpectMsg<OpcUaPublishActor.AttributeValueUpdate>();
+        AwaitAssert(() => writer.Calls.Count.ShouldBe(1));
+        writer.Calls.TryPeek(out var captured).ShouldBeTrue();
+        captured.Value.StatusCode.ShouldBe(0x80020000u); // BadInternalError — dormant-engine parity
+    }
+
+    /// <summary>Evaluator that returns <c>Ok(42.0)</c> on its first call and <c>Failure</c> thereafter —
+    /// the S13 "was working, now broken" shape. Interlocked counter so it is safe across the actor's
+    /// evaluation thread.</summary>
+    private sealed class FailAfterFirstSuccessEvaluator : IVirtualTagEvaluator
+    {
+        private int _calls;
+
+        /// <summary>Returns Ok(42.0) once, then Failure("boom").</summary>
+        /// <param name="id">The tag identifier.</param>
+        /// <param name="expr">The expression string.</param>
+        /// <param name="deps">The dependency values.</param>
+        /// <returns>Ok(42.0) on the first call; Failure on every subsequent call.</returns>
+        public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
+            => Interlocked.Increment(ref _calls) == 1
+                ? VirtualTagEvalResult.Ok(42.0)
+                : VirtualTagEvalResult.Failure("boom");
     }
 
     /// <summary>Capturing <see cref="IHistoryWriter"/>: records every Record call so H5c tests can