diff --git a/archreview/plans/R2-03-vt-failure-quality-plan.md b/archreview/plans/R2-03-vt-failure-quality-plan.md index 44421932..506e74ea 100644 --- a/archreview/plans/R2-03-vt-failure-quality-plan.md +++ b/archreview/plans/R2-03-vt-failure-quality-plan.md @@ -673,3 +673,30 @@ T4, T5, T8, T10 ──► T11 (sweep + docs) ──► T12 (live /run gate) estimate for action item #3. **Overall risk:** Low-Medium, concentrated in S13's deliberate behavior change (nodes that used to freeze Good now go Bad — the point of the fix) and the one inverted legacy test expectation, both explicitly task-tracked. + +## Execution deviations (R2-03) + +Executed on branch `r2/03-vt-failure-quality` off master `1676c8f4` (not `f6eaa267` — the plan's verified +tree; `1676c8f4` is the current master and includes the round-2 plan/docs merge, so the code anchors are +unchanged). Deviations, all reversible: + +- **T7 stress-test config retuned for a reliable RED on fast hardware.** The plan's prescribed shape + (single loop × 300 sequential evaluations racing a back-to-back `Thread.Yield` clear loop) did **not** + reproduce the ObjectDisposedException on this dev machine (arm64 macOS, many idle cores → the + fetch→Task.Run-hop→disposed-guard window is sub-microsecond and the Task.Run body dispatches instantly). + Escalating to a back-to-back clear loop instead compile-swamped the run into a 60 s `WaitAsync` + timeout (every evaluation recompiles under constant clearing) — and a back-to-back clear also risks + double-disposal within one evaluation, which a single retry cannot heal (would make the fix's GREEN + flaky). Final config: **12 concurrent evaluation loops × 400 iterations** (thread-pool pressure widens + the fetch→run window) racing a clear loop that clears then `Thread.SpinWait(4000)` (~tens of µs — + frequent but not back-to-back, so at most one clear lands inside a single evaluation and the warm cache + keeps compile cost bounded). This reproduces reliably (133 disposed-reason failures observed on the + unfixed tree) and is deterministically green post-fix (verified stable across 3 repeat runs). The + assertion was **not** weakened — it still requires every evaluation to succeed with no "disposed" reason. +- **Test-run scope narrowed per controller memory constraint.** A mid-execution controller directive + flagged a ~16 GB/run memory leak in the `*.IntegrationTests` suites and forbade whole-project/whole- + solution `dotnet test`. Verified locally instead: `dotnet build ZB.MOM.WW.OtOpcUa.slnx` clean; the whole + `Runtime.Tests` unit project (372/372); and `RoslynVirtualTagEvaluatorTests` filtered (20/20). **T11** + (full cross-suite sweep) and **T12** (docker-dev live-`/run` gate) are marked `deferred-live` in + `.tasks.json` for the controller's serialized heavy pass — the build + STATUS/tasks bookkeeping portion + of T11 was completed and committed. diff --git a/archreview/plans/R2-03-vt-failure-quality-plan.md.tasks.json b/archreview/plans/R2-03-vt-failure-quality-plan.md.tasks.json index 86bfffdd..2c55fffb 100644 --- a/archreview/plans/R2-03-vt-failure-quality-plan.md.tasks.json +++ b/archreview/plans/R2-03-vt-failure-quality-plan.md.tasks.json @@ -4,75 +4,99 @@ { "id": "R2-03-T1", "subject": "S13 stale-Good repro test at host level (RED — must fail on f6eaa267): fail-after-success evaluator, publish probe sees Good then expects Bad AttributeValueUpdate", - "status": "pending", + "status": "completed", "blockedBy": [] }, { "id": "R2-03-T2", "subject": "Add OpcUaQuality Quality (default Good) to VirtualTagActor.EvaluationResult; bridge passes result.Quality into AttributeValueUpdate; historize maps Bad to 0x80020000u BadInternalError (RED bridge tests then GREEN)", - "status": "pending", - "blockedBy": ["R2-03-T1"] + "status": "completed", + "blockedBy": [ + "R2-03-T1" + ] }, { "id": "R2-03-T3", "subject": "VirtualTagActor failure/throw publishes Bad EvaluationResult once per transition, gated on all dependencyRefs arrived; recovery bypasses value-dedup; update Evaluator_failure_publishes_ScriptLogEntry_warning expectation (turns T1 green)", - "status": "pending", - "blockedBy": ["R2-03-T2"] + "status": "completed", + "blockedBy": [ + "R2-03-T2" + ] }, { "id": "R2-03-T4", "subject": "S13 transition-semantics unit guards: second consecutive failure publishes nothing; same-value recovery republishes Good; evaluator-threw path also degrades", - "status": "pending", - "blockedBy": ["R2-03-T3"] + "status": "completed", + "blockedBy": [ + "R2-03-T3" + ] }, { "id": "R2-03-T5", "subject": "S13 cold-start suppression guards: missing-dep failure publishes no Bad; failure after all deps arrived publishes Bad (inputs-ready gate)", - "status": "pending", - "blockedBy": ["R2-03-T3"] + "status": "completed", + "blockedBy": [ + "R2-03-T3" + ] }, { "id": "R2-03-T6", "subject": "S12 sequential pin: Evaluate_after_ClearCompiledScripts_recompiles_and_succeeds (green baseline the T8 retry-loop restructure must not regress)", - "status": "pending", + "status": "completed", "blockedBy": [] }, { "id": "R2-03-T7", "subject": "S12 concurrent stress repro (RED): 300 compiled evaluations racing ClearCompiledScripts, WaitAsync-bounded, assert zero disposed-reason failures", - "status": "pending", + "status": "completed", "blockedBy": [] }, { "id": "R2-03-T8", "subject": "S12 fix: retry-once on ObjectDisposedException in RoslynVirtualTagEvaluator.Evaluate (re-fetch via GetOrCompile; no retry when _disposed or second hit); update xmldoc (turns T7 green)", - "status": "pending", - "blockedBy": ["R2-03-T6", "R2-03-T7"] + "status": "completed", + "blockedBy": [ + "R2-03-T6", + "R2-03-T7" + ] }, { "id": "R2-03-T9", "subject": "P7 wiring-guard rewrite (RED): ApplyVirtualTags clears the compile cache only when the expression set changes — identical redeploy must NOT clear; expression edit must; shared-expression removal must not", - "status": "pending", - "blockedBy": ["R2-03-T3"] + "status": "completed", + "blockedBy": [ + "R2-03-T3" + ] }, { "id": "R2-03-T10", "subject": "P7 fix: gate the OnApply ClearCompiledScripts on HashSet.SetEquals of desired vs _planByVtag expression sets (ordinal), with explanatory comment (turns T9 green)", - "status": "pending", - "blockedBy": ["R2-03-T9"] + "status": "completed", + "blockedBy": [ + "R2-03-T9" + ] }, { "id": "R2-03-T11", "subject": "Full-suite sweep (Runtime.Tests whole project + RoslynVirtualTagEvaluatorTests + solution build) and STATUS.md / tasks.json bookkeeping", - "status": "pending", - "blockedBy": ["R2-03-T4", "R2-03-T5", "R2-03-T8", "R2-03-T10"] + "status": "deferred-live", + "blockedBy": [ + "R2-03-T4", + "R2-03-T5", + "R2-03-T8", + "R2-03-T10" + ], + "note": "integration/full sweep; serialized heavy pass (build clean, Runtime.Tests 372/372, RoslynVirtualTagEvaluatorTests 20/20 verified locally; whole-solution/cross-suite sweep deferred per controller memory constraint)" }, { "id": "R2-03-T12", "subject": "Live-/run gate on docker-dev (rebuild BOTH centrals): author working VT, break it, Client.CLI read shows StatusCode Bad with last value, fix and read Good; P7 no-recompile spot-check on identical redeploy", - "status": "pending", - "blockedBy": ["R2-03-T11"] + "status": "deferred-live", + "blockedBy": [ + "R2-03-T11" + ], + "note": "docker-dev live-/run; serial live pass" } ], - "lastUpdated": "2026-07-12" + "lastUpdated": "2026-07-13" } diff --git a/archreview/plans/STATUS.md b/archreview/plans/STATUS.md index f1de2b46..877ede2b 100644 --- a/archreview/plans/STATUS.md +++ b/archreview/plans/STATUS.md @@ -172,3 +172,4 @@ what the unit leg cannot; (c) migrate the 3 xunit v2 holdouts once Akka ships an | **R2-06** | 06/S-11 (rank 7), 06/U-7 (rank 8), 06/C-7 | `r2/06-serverhistorian-failfast` (off `1676c8f4`) | `3a049a13`…`4640ecb1` (10 task commits) | **Code-complete, offline-verified.** S-11: fail-fast `ServerHistorianOptionsValidator` (consumer-gated on `ServerHistorian:Enabled` OR `AlarmHistorian:Enabled`, fails empty/non-http(s) Endpoint at startup) + `AddValidatedOptions` wiring + adapter defense-in-depth (`TryCreate`→named `InvalidOperationException`, no more raw `UriFormatException` crash-loop) + `Validate()` warning parity + alarm-only-mode startup warning. U-7: pinned Boolean→Int1 explicit-presence (`HasDataType`) + `HISTGW_BOOL_SANDBOX_TAG` live Boolean EnsureTags leg + retype probe (skip-clean offline) + upsert/retype migration note (`docs/Historian.md`). C-7: keep-populating decision + dead-`Quality` doc comments (0.2.0 C-002). **Live leg operator-gated (T12) — pending a VPN run on a 0.2.0 gateway** (records the observed in-place retype policy back into `docs/Historian.md`; also discharges 06/U-2's full documented run). | | **R2-05** | 04/C-1 (High) | `r2/05-adminui-authz` (off `1676c8f4`) | `183b72b7`…`6c1f05cb` (5 commits) | **Code-complete, offline-verified.** All 38 routable AdminUI pages carry an explicit named-policy `@attribute` via new `AdminUiPolicies` constants (`Security/Auth/`). New `ConfigEditor` (Administrator+Designer) gates the 20-page config-authoring surface incl. the 8 driver pages that author live `ResilienceConfig` + `/api/script-analysis/*`; new `AuthenticatedRead` on the 16 read pages; `FleetAdmin`/`DriverOperator` unchanged (literals→constants). Anti-drift guard `PageAuthorizationGuardTests` (bans bare `[Authorize]`/`Roles=`/unknown-policy/unclassified) + `AdminUiPoliciesTests` semantics matrix. Corrections folded in: Reservations + ClusterRedundancy read-only (→`AuthenticatedRead`); Home had no attribute (now `AuthenticatedRead`). AdminUI.Tests 517/517, Security.Tests 80/80. **Live `/run` positive pass deferred (T15)** — docker-dev `DisableLogin=true` auto-admin can't observe deny; negative proof is CI-side (both guard tests). | | **R2-01** | 05/STAB-14 (High regression, rank 1), 05/STAB-15 (rank 3), 05/STAB-8 seam | `r2/01-s7-fault-hardening` (off `1676c8f4`) | `fcdcf0e2`…`bdb7ed41` (13 task commits) | **Code-complete, offline-verified.** All prod changes in `S7Driver.cs`. STAB-14: connect-timeout surfaces as `TimeoutException` (not an escaping OCE) at the `EnsureConnectedAsync`/`InitializeAsync` source + `when(token.IsCancellationRequested)` filters on the read/write ensure-wrappers and all three poll-loop OCE catches — subscription poll loops survive an unreachable-host outage and recover (T2 regression RED→GREEN). STAB-15: `IsS7ConnectionFatal` broadened to the ISO-on-TCP framing surface (TPKT/TPDU/WrongNumberOfBytes/`PlcException{WrongNumberReceivedBytes}`, NOT `InvalidDataException`); mid-PDU cancel marks handle dead; probe classifies+marks-dead under the gate. STAB-8 (S7 part): `EnsureConnectedAsync` documented as the R2-09 connect-throttle seam (note-only). Driver.S7.Tests 246/246. **Live outage gate deferred (T11)** — `S7_1500ConnectTimeoutOutageTests` (env `S7_TIMEOUT_OUTAGE_START_CMD`/`STOP_CMD`, SYN-blackhole) skips clean offline; the connect-*timeout* class the `docker restart` bounce can't produce. (Pre-existing unrelated CLI docs-scan test fails at base — out of scope.) | +| **R2-03** | 02/S13 (rank 5), 02/S12, 02/P7 | `r2/03-vt-failure-quality` (off `1676c8f4`) | `b9775485`…`6bb91e8d` (12 task commits) | **Code-complete, offline-verified.** S13: VT script failure/timeout degrades the node to `OpcUaQuality.Bad` (carrying last-known value) instead of freezing on last-Good forever — additive `EvaluationResult.Quality`, bridge historizes `0x80020000u` BadInternalError, published once per Good→Bad transition gated on all deps arrived (inputs-ready, no cold-start flash), recovery force-publishes next Good. S12: apply-boundary `ClearCompiledScripts` racing an in-flight eval into ObjectDisposedException retried once in `RoslynVirtualTagEvaluator` (re-fetch→recompile into current generation); stress guard reproduces pre-fix, deterministically green post-fix. P7: apply-boundary clear gated on a changed deployed expression set (`HashSet.SetEquals`, ordinal) — identical redeploy / rename-with-same-expressions no longer forces full VT recompile. Runtime.Tests 372/372, RoslynVirtualTagEvaluatorTests 20/20. **Deferred (T11/T12):** full cross-suite sweep + docker-dev live-`/run` (fail→Bad→recover + P7 no-recompile spot-check). | diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index 3aa47d1b..2391bd56 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -18,6 +18,19 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines; /// fan-out between actors is owned by DependencyMuxActor, not by the eval engine. /// Cycle detection + cascade ordering live in ; this adapter /// stays single-tag scoped to keep 's message loop simple. +/// +/// +/// 02/S12 — the apply-boundary (driven by the host actor per +/// deploy generation) disposes cached evaluators while unchanged children may still be evaluating. +/// A clear landing between this adapter's GetOrCompile and the run's disposed guard would +/// otherwise surface as a spurious Failure. The compiled path is therefore retry-safe: an +/// from an in-flight evaluation re-fetches (recompiling the +/// same source into the current cache generation) and retries once. A second disposal mid-retry +/// (two applies inside one evaluation — not a realistic cadence) falls through to the failure path. +/// Residual (accepted, bounded to one generation): a stopping child can recompile its stale source +/// into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no +/// longer open this window at all. +/// /// public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable { @@ -63,27 +76,6 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac : VirtualTagEvalResult.Ok(null); } - ScriptEvaluator evaluator; - try - { - evaluator = _cache.GetOrCompile(expression); - } - catch (CompilationErrorException ex) - { - _logger.LogWarning(ex, "VirtualTag {Id}: Roslyn compile failed", virtualTagId); - return VirtualTagEvalResult.Failure($"compile error: {ex.Message}"); - } - catch (ScriptSandboxViolationException ex) - { - _logger.LogWarning(ex, "VirtualTag {Id}: sandbox violation", virtualTagId); - return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}"); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "VirtualTag {Id}: compile threw", virtualTagId); - return VirtualTagEvalResult.Failure($"compile failure: {ex.Message}"); - } - var readCache = BuildReadCache(dependencies); // Per-evaluation script logger: bind both ScriptId and VirtualTagId from the virtual-tag id // (in the live path the script id equals the virtual-tag id) so the Script-log page can @@ -98,25 +90,71 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac virtualTagId, path), logger: scriptLog); - try + // 02/S12: fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent + // apply-boundary ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the + // run's disposed guard) re-fetches and retries exactly once. The loop runs at most twice — the + // retry catch only matches attempt == 0. + for (var attempt = 0; ; attempt++) { - // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't - // interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous - // CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when - // the wall-clock budget fires regardless of whether the inner task completes (the orphaned - // thread is the documented, accepted trade-off — TimedScriptEvaluator remarks). - var timed = new TimedScriptEvaluator(evaluator, _runTimeout); - var raw = timed.RunAsync(context).GetAwaiter().GetResult(); - return VirtualTagEvalResult.Ok(raw); - } - catch (ScriptTimeoutException) - { - return VirtualTagEvalResult.Failure($"script timed out after {_runTimeout.TotalSeconds:F1}s"); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "VirtualTag {Id}: script execution threw", virtualTagId); - return VirtualTagEvalResult.Failure($"script threw: {ex.Message}"); + ScriptEvaluator evaluator; + try + { + evaluator = _cache.GetOrCompile(expression); + } + catch (CompilationErrorException ex) + { + _logger.LogWarning(ex, "VirtualTag {Id}: Roslyn compile failed", virtualTagId); + return VirtualTagEvalResult.Failure($"compile error: {ex.Message}"); + } + catch (ScriptSandboxViolationException ex) + { + _logger.LogWarning(ex, "VirtualTag {Id}: sandbox violation", virtualTagId); + return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}"); + } + catch (ObjectDisposedException) + { + // The cache itself is being torn down (this evaluator is disposing) — a fetch-time + // disposal is a shutdown signal, not the apply-boundary race, so never retry. + return VirtualTagEvalResult.Failure("evaluator disposed"); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "VirtualTag {Id}: compile threw", virtualTagId); + return VirtualTagEvalResult.Failure($"compile failure: {ex.Message}"); + } + + try + { + // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't + // interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous + // CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when + // the wall-clock budget fires regardless of whether the inner task completes (the orphaned + // thread is the documented, accepted trade-off — TimedScriptEvaluator remarks). + var timed = new TimedScriptEvaluator(evaluator, _runTimeout); + var raw = timed.RunAsync(context).GetAwaiter().GetResult(); + return VirtualTagEvalResult.Ok(raw); + } + catch (ScriptTimeoutException) + { + return VirtualTagEvalResult.Failure($"script timed out after {_runTimeout.TotalSeconds:F1}s"); + } + catch (ObjectDisposedException) when (!_disposed && attempt == 0) + { + // S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our + // GetOrCompile and the run's disposed-guard. Re-fetch — GetOrCompile recompiles the same + // source into the CURRENT cache generation — and retry once. A second disposal mid-retry + // (two applies inside one evaluation) falls through to the failure path below. + continue; + } + catch (ObjectDisposedException) + { + return VirtualTagEvalResult.Failure("evaluator disposed during evaluation"); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "VirtualTag {Id}: script execution threw", virtualTagId); + return VirtualTagEvalResult.Failure($"script threw: {ex.Message}"); + } } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs index 29d1c909..c5641b81 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs @@ -4,6 +4,7 @@ using Akka.Event; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; using ZB.MOM.WW.OtOpcUa.Commons.Observability; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; @@ -12,15 +13,30 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; /// Wraps a single virtual-tag expression. Receives dependency-tag updates, recomputes the /// expression via an injected , and emits an /// to the parent (the publish actor) whenever the value -/// actually changes. Script failures publish a Warning on the +/// actually changes. Script failures publish a on the /// script-logs DPS topic so operators see the diagnostic in the live tail. +/// +/// +/// 02/S13 — a persistent script failure/timeout also degrades the node's OPC UA quality to +/// (carrying the last-known value), published once per +/// transition (Good→Bad) and gated on every declared dependency having arrived at least once +/// — so a broken script no longer serves its last Good value forever, and a fresh/respawned +/// multi-dep script does not flash Bad during deploy warm-up. Recovery to a Good value clears the +/// Bad state and is force-published even when the value equals the pre-failure value. +/// /// public sealed class VirtualTagActor : ReceiveActor { public const string ScriptLogsTopic = "script-logs"; public sealed record DependencyValueChanged(string TagId, object? Value, DateTime TimestampUtc); - public sealed record EvaluationResult(string VirtualTagId, object? Value, DateTime TimestampUtc, CorrelationId Correlation); + + /// Result emitted to the parent (the host bridge). carries the + /// OPC UA quality the node should read: for a fresh computed value, + /// when the script failed/timed out (02/S13) — in which case + /// is the last-known value (or null if none). Defaulting to Good keeps + /// every existing construction site source-compatible. + public sealed record EvaluationResult(string VirtualTagId, object? Value, DateTime TimestampUtc, CorrelationId Correlation, OpcUaQuality Quality = OpcUaQuality.Good); private readonly string _virtualTagId; private readonly string _scriptId; @@ -34,6 +50,9 @@ public sealed class VirtualTagActor : ReceiveActor private bool _hasLastValue; private object? _lastValue; + // 02/S13: true once a Bad EvaluationResult has been published and not yet cleared by a Good + // recovery. Gates Bad publishes to once-per-transition and force-publishes the recovery value. + private bool _lastPublishedBad; /// Factory method to create Props for a VirtualTagActor. /// Unique identifier for the virtual tag. @@ -115,27 +134,28 @@ public sealed class VirtualTagActor : ReceiveActor catch (Exception ex) { _log.Warning(ex, "VirtualTag {Id}: evaluator threw", _virtualTagId); - OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "fail")); - PublishLog("Error", $"evaluator threw: {ex.Message}"); + OnEvaluationFailed($"evaluator threw: {ex.Message}", "Error", msg.TimestampUtc); return; } if (!result.Success) { - OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "fail")); - PublishLog("Warning", result.Reason ?? "evaluator failure"); + OnEvaluationFailed(result.Reason ?? "evaluator failure", "Warning", msg.TimestampUtc); return; } // Skip no-change results. Real evaluator returns Ok(value); Null returns NoChange — both - // safe because Null never produces a fresh value. + // safe because Null never produces a fresh value. NoChange is "no fresh value", NOT a recovery, + // so it must stay above the dedup and must NOT clear _lastPublishedBad. if (ReferenceEquals(result, VirtualTagEvalResult.NoChange)) { OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "skip")); return; } - if (_hasLastValue && Equals(_lastValue, result.Value)) + // Value dedup — but bypass it while recovering from Bad, so a recovery whose value equals the + // pre-failure value still republishes Good (otherwise the node would stay Bad forever). + if (!_lastPublishedBad && _hasLastValue && Equals(_lastValue, result.Value)) { OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "skip")); return; @@ -143,11 +163,41 @@ public sealed class VirtualTagActor : ReceiveActor _hasLastValue = true; _lastValue = result.Value; + _lastPublishedBad = false; OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "ok")); var evalResult = new EvaluationResult(_virtualTagId, result.Value, msg.TimestampUtc, CorrelationId.NewId()); Context.Parent.Tell(evalResult); } + /// 02/S13 failure tail shared by the evaluator-threw catch and the !Success branch. + /// Always meters + publishes the per-failure script-log entry (the pre-existing operator + /// diagnostic); additionally publishes a Bad — carrying the + /// last-known value if any — once per Good→Bad transition, and only after every declared dependency + /// has arrived (the inputs-ready gate that suppresses the cold-start/respawn warm-up flash). + /// The failure reason for the script-log entry. + /// The script-log level ("Warning" for a Failure result, "Error" for a throw). + /// The triggering dependency change's timestamp, stamped on the Bad result. + private void OnEvaluationFailed(string reason, string level, DateTime timestampUtc) + { + OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "fail")); + PublishLog(level, reason); + + if (_lastPublishedBad || !AllDependenciesArrived()) + { + return; + } + + _lastPublishedBad = true; + Context.Parent.Tell(new EvaluationResult( + _virtualTagId, _hasLastValue ? _lastValue : null, timestampUtc, CorrelationId.NewId(), OpcUaQuality.Bad)); + } + + /// Inputs-ready gate (mirrors the dormant VirtualTagEngine.AreInputsReady): true once + /// every declared dependency ref has been seen at least once. Called once per failure — the ref list + /// is small, so no caching. A vacuously-true empty ref list means a no-dependency script degrades on + /// its first failure. + private bool AllDependenciesArrived() => _dependencyRefs.All(_dependencies.ContainsKey); + private void PublishLog(string level, string message) { var entry = new ScriptLogEntry( diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs index 002f1870..52f007fb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs @@ -85,11 +85,24 @@ public sealed class VirtualTagHostActor : ReceiveActor private void OnApply(ApplyVirtualTags msg) { - // Apply-boundary compile-cache drop: each deploy generation may carry edited script sources, and - // the production evaluator roots one collectible AssemblyLoadContext per compiled expression. Drop - // them here (mirroring ScriptedAlarmEngine's per-generation clear) so stale ALCs are reclaimed - // instead of accreting across edits. No-op for evaluators without a cache (NullVirtualTagEvaluator). - (_evaluator as IScriptCacheOwner)?.ClearCompiledScripts(); + // Apply-boundary compile-cache drop (gated — 02/P7): the production evaluator roots one + // collectible AssemblyLoadContext per compiled expression, so a changed script set must drop the + // stale ALCs. But clearing on EVERY apply forces a full recompilation even on a byte-identical + // redeploy (and each unnecessary clear is an unnecessary 02/S12 race window). Clear only when the + // deployed EXPRESSION set actually changed. The cache is keyed by source only, so a rename / + // FolderPath move / Historize toggle (which respawns the child but keeps the source) needs no + // recompile; set semantics also mean two vtags sharing one expression keep it alive while either + // remains. _planByVtag still holds the PREVIOUS generation here (it is rebuilt to the desired set + // at the end of this method); on the first apply after (re)start it is empty, so the sets differ + // and the clear fires (harmless — converges the singleton evaluator's cache to this generation). + // NOTE: this gate reduces clear FREQUENCY only; the S12 retry-once in RoslynVirtualTagEvaluator — + // not this gate — is the correctness fix for a clear racing an in-flight evaluation. + var newExpressions = msg.Plans.Select(p => p.Expression).ToHashSet(StringComparer.Ordinal); + var prevExpressions = _planByVtag.Values.Select(p => p.Expression).ToHashSet(StringComparer.Ordinal); + if (!newExpressions.SetEquals(prevExpressions)) + { + (_evaluator as IScriptCacheOwner)?.ClearCompiledScripts(); + } var desired = new HashSet(msg.Plans.Select(p => p.VirtualTagId), StringComparer.Ordinal); @@ -176,16 +189,22 @@ public sealed class VirtualTagHostActor : ReceiveActor return; } + // 02/S13: the child now expresses quality — a Good fresh value or a Bad degradation (script + // failure/timeout) carrying the last-known value. Bridge result.Quality through verbatim; the + // sink maps it to StatusCodes.Good/Bad on the node so a broken script no longer freezes Good. _publishActor.Tell(new OpcUaPublishActor.AttributeValueUpdate( - nodeId, result.Value, OpcUaQuality.Good, result.TimestampUtc)); + nodeId, result.Value, result.Quality, result.TimestampUtc)); // Historize iff the plan opted in. Reuses _planByVtag (kept in lock-step with _children), so // no parallel map. The historian path key is the SAME folder-scoped NodeId we just published - // to. For a computed value source == server, so both timestamps are the evaluation time. + // to. For a computed value source == server, so both timestamps are the evaluation time. Bad + // results record BadInternalError (0x80020000u) — the dormant VirtualTagEngine's Bad-snapshot + // status — so the historian trend can distinguish a degraded sample from a Good one. if (_planByVtag.TryGetValue(result.VirtualTagId, out var plan) && plan.Historize) { + var status = result.Quality == OpcUaQuality.Good ? 0u : 0x80020000u /* BadInternalError — dormant-engine parity */; _history.Record(nodeId, new DataValueSnapshot( - result.Value, 0u /* StatusCodes.Good */, result.TimestampUtc, result.TimestampUtc)); + result.Value, status, result.TimestampUtc, result.TimestampUtc)); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs index 03587563..d4edc62d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs @@ -357,6 +357,87 @@ public sealed class RoslynVirtualTagEvaluatorTests CompiledCacheCount(sut).ShouldBe(0); } + /// 02/S12 sequential pin: evaluating a compiled expression, clearing the cache, then + /// evaluating the SAME source again succeeds both times and leaves the cache with exactly one entry + /// (the recompile). This pins the clear-then-recompile contract the T8 retry-loop restructure must + /// not regress. + [Fact] + public void Evaluate_after_ClearCompiledScripts_recompiles_and_succeeds() + { + using var sut = new RoslynVirtualTagEvaluator(NullLogger.Instance, NoOpScriptRoot()); + const string expr = "return (int)ctx.GetTag(\"a\").Value + 1;"; + + var first = sut.Evaluate("vt-clear", expr, new Dictionary { ["a"] = 1 }); + first.Success.ShouldBeTrue(first.Reason); + CompiledCacheCount(sut).ShouldBe(1); + + sut.ClearCompiledScripts(); + CompiledCacheCount(sut).ShouldBe(0); + + var second = sut.Evaluate("vt-clear", expr, new Dictionary { ["a"] = 41 }); + second.Success.ShouldBeTrue(second.Reason); + second.Value.ShouldBe(42); + CompiledCacheCount(sut).ShouldBe(1); + } + + /// 02/S12 concurrent stress repro (RED on current code): while one loop runs many compiled + /// (non-passthrough) evaluations of the same expression, a second loop hammers + /// — the apply-boundary clear. On the + /// unfixed tree a clear landing between a child's GetOrCompile and the run's disposed-guard + /// surfaces an ObjectDisposedException as a Failure("script threw: Cannot access a disposed + /// object…"); the fix retries once and every result stays Success. The whole test is WaitAsync- + /// bounded so a regression fails fast rather than hanging. + [Fact] + public async Task Evaluate_racing_ClearCompiledScripts_never_fails_with_disposed() + { + // Generous timeout so the wall-clock budget never fires and confounds the race assertion. + using var sut = new RoslynVirtualTagEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromSeconds(5)); + const string expr = "return (int)ctx.GetTag(\"a\").Value + 1;"; + + const int loopCount = 12; + const int iterations = 400; + var failures = new System.Collections.Concurrent.ConcurrentBag(); + var remaining = loopCount; // number of concurrent evaluation loops still running + + // Many concurrent evaluation loops put the thread pool under enough pressure that the inner + // Task.Run hop inside TimedScriptEvaluator is measurably delayed — widening the + // fetch→run-disposed-guard window so a clear reliably lands inside SOME in-flight evaluation. + // The clear loop clears on a short spin cadence (not back-to-back) so (a) most evaluations hit + // the warm cache — keeping compile cost bounded — and (b) at most one clear lands within a + // single evaluation, so the single retry the fix adds is sufficient (a back-to-back clear loop + // would both compile-swamp the RED run into a timeout and defeat the one-shot retry). Pre-fix a + // clear disposing the just-fetched evaluator surfaces as Failure("script threw: Cannot access a + // disposed object…"); post-fix the retry re-fetches into the current generation → all Success. + var evalLoops = Enumerable.Range(0, loopCount).Select(_ => Task.Run(() => + { + for (var i = 0; i < iterations; i++) + { + var r = sut.Evaluate("vt-race", expr, new Dictionary { ["a"] = i }); + if (!r.Success) + { + failures.Add(r.Reason ?? ""); + } + } + Interlocked.Decrement(ref remaining); + }, TestContext.Current.CancellationToken)).ToArray(); + + var clearLoop = Task.Run(() => + { + while (Volatile.Read(ref remaining) > 0) + { + sut.ClearCompiledScripts(); + Thread.SpinWait(4000); // ~tens of µs between clears — frequent, but not back-to-back + } + }, TestContext.Current.CancellationToken); + + await Task.WhenAll(evalLoops.Append(clearLoop)) + .WaitAsync(TimeSpan.FromSeconds(45), TestContext.Current.CancellationToken); + + failures.ShouldBeEmpty( + $"every evaluation must succeed despite racing clears; observed: {string.Join(" | ", failures)}"); + } + /// Reads the count of the private compiled-script cache via reflection. The cache is a /// CompiledScriptCache<,> which exposes a Count property (not ICollection). private static int CompiledCacheCount(RoslynVirtualTagEvaluator sut) diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorTests.cs index af001e59..a63d9ed5 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorTests.cs @@ -4,6 +4,7 @@ using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; @@ -80,7 +81,138 @@ public sealed class VirtualTagActorTests : RuntimeActorTestBase entry.VirtualTagId.ShouldBe("vt-1"); }, duration: TimeSpan.FromMilliseconds(500)); - parent.ExpectNoMsg(TimeSpan.FromMilliseconds(100)); + // 02/S13: the failure now ALSO degrades the node — with no declared dependencyRefs the + // inputs-ready gate is vacuously satisfied, so a Bad EvaluationResult reaches the parent (this + // replaces the old ExpectNoMsg, which encoded the stale-Good bug). + parent.ExpectMsg().Quality.ShouldBe(OpcUaQuality.Bad); + } + + /// 02/S13 once-per-transition: two consecutive failures publish exactly ONE Bad + /// EvaluationResult — the second failure is suppressed while already Bad. + [Fact] + public void Second_consecutive_failure_does_not_publish_a_second_Bad() + { + var parent = CreateTestProbe(); + var evaluator = new QueuedEvaluator( + VirtualTagEvalResult.Failure("boom1"), VirtualTagEvalResult.Failure("boom2")); + var actor = parent.ChildActorOf(VirtualTagActor.Props("vt-1", "expr", evaluator: evaluator)); + + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow)); + parent.ExpectMsg().Quality.ShouldBe(OpcUaQuality.Bad); + + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, DateTime.UtcNow)); + parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + /// 02/S13 same-value recovery: Ok(42) → fail → Ok(42) must republish Good with 42 even + /// though the value equals the pre-failure value (the dedup bypass — else the node stays Bad). + [Fact] + public void Recovery_to_the_same_value_republishes_Good_after_Bad() + { + var parent = CreateTestProbe(); + var evaluator = new QueuedEvaluator( + VirtualTagEvalResult.Ok(42), VirtualTagEvalResult.Failure("boom"), VirtualTagEvalResult.Ok(42)); + var actor = parent.ChildActorOf(VirtualTagActor.Props("vt-1", "expr", evaluator: evaluator)); + + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow)); + var first = parent.ExpectMsg(); + first.Quality.ShouldBe(OpcUaQuality.Good); + first.Value.ShouldBe(42); + + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, DateTime.UtcNow)); + parent.ExpectMsg().Quality.ShouldBe(OpcUaQuality.Bad); + + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 3, DateTime.UtcNow)); + var recovered = parent.ExpectMsg(); + recovered.Quality.ShouldBe(OpcUaQuality.Good); + recovered.Value.ShouldBe(42); + } + + /// 02/S13: an evaluator that THROWS (not just returns Failure) also degrades quality — + /// the try/catch path publishes a Bad EvaluationResult too. + [Fact] + public void Evaluator_throw_also_degrades_quality() + { + var parent = CreateTestProbe(); + var actor = parent.ChildActorOf(VirtualTagActor.Props( + "vt-1", "expr", evaluator: new ThrowingEvaluator())); + + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow)); + + parent.ExpectMsg().Quality.ShouldBe(OpcUaQuality.Bad); + } + + /// 02/S13 inputs-ready gate: a multi-dep script that fails before all its declared + /// dependencies have arrived publishes NO Bad — the node stays at the materialiser's + /// BadWaitingForInitialData, avoiding a false Bad flash during deploy/respawn warm-up. + [Fact] + public void Cold_start_failure_with_missing_dependency_publishes_no_Bad() + { + var parent = CreateTestProbe(); + var actor = parent.ChildActorOf(VirtualTagActor.Props( + "vt-1", "expr", + evaluator: new FailingEvaluator("boom"), + dependencyRefs: new[] { "a", "b" })); + + // Only "a" has arrived — "b" is still missing, so the gate suppresses the Bad publish. + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow)); + + parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + /// 02/S13 inputs-ready gate: once every declared dependency has arrived at least once, a + /// failure publishes exactly one Bad EvaluationResult. + [Fact] + public void Failure_after_all_dependencies_arrived_publishes_Bad() + { + var parent = CreateTestProbe(); + var actor = parent.ChildActorOf(VirtualTagActor.Props( + "vt-1", "expr", + evaluator: new FailingEvaluator("boom"), + dependencyRefs: new[] { "a", "b" })); + + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, DateTime.UtcNow)); + // Still missing "b" — no publish yet. + parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + + // "b" arrives — all refs now seen, so the failure degrades to Bad. + actor.Tell(new VirtualTagActor.DependencyValueChanged("b", 2, DateTime.UtcNow)); + parent.ExpectMsg().Quality.ShouldBe(OpcUaQuality.Bad); + } + + /// Test evaluator returning a fixed sequence of results, one per Evaluate call; the last + /// result repeats once the queue is drained. + private sealed class QueuedEvaluator : IVirtualTagEvaluator + { + private readonly Queue _results; + private VirtualTagEvalResult _last; + /// Initializes the evaluator with an ordered sequence of results. + /// The results returned in order, one per call. + public QueuedEvaluator(params VirtualTagEvalResult[] results) + { + _results = new Queue(results); + _last = results[^1]; + } + /// Returns the next queued result (or the last one once drained). + /// The tag identifier. + /// The expression string. + /// The dependency values. + /// The next queued result. + public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary deps) + => _results.Count > 0 ? _last = _results.Dequeue() : _last; + } + + /// Test evaluator that throws on every call — exercises the actor's try/catch degradation + /// path (distinct from a returned Failure). + private sealed class ThrowingEvaluator : IVirtualTagEvaluator + { + /// Always throws. + /// The tag identifier. + /// The expression string. + /// The dependency values. + /// Never returns; throws. + public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary deps) + => throw new InvalidOperationException("kaboom"); } /// Test evaluator that sums integer dependency values. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagHostActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagHostActorTests.cs index 9b1590e8..e71a290a 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagHostActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagHostActorTests.cs @@ -345,24 +345,153 @@ public sealed class VirtualTagHostActorTests : RuntimeActorTestBase mux.LastSender.ShouldNotBe(firstChild); } - /// Wiring guard (arch-review 02/U3, theme #1): every ApplyVirtualTags calls - /// on the evaluator at the apply boundary. - /// This is the load-bearing wire — the production evaluator roots one collectible ALC per compiled - /// script, and without this per-generation clear they accrete across deploys. A pure unit test of - /// the evaluator can't catch a missing call here; this asserts the host actor actually makes it. + /// Wiring guard (arch-review 02/U3 + 02/P7, theme #1): the host actor calls + /// at the apply boundary — but, per P7, ONLY + /// when the deployed expression set actually changed. The clear stays the load-bearing wire (the + /// production evaluator roots one collectible ALC per compiled script), yet a byte-identical + /// redeploy — or a rename / add / remove that leaves the expression set unchanged — no longer forces + /// a full recompilation. This pins that the host makes the call, and gates it correctly. + /// + /// The final ClearCount is read at a deterministic sync point: after all applies are + /// queued we drive one EvaluationResult through the actor and wait for the bridged publish — the + /// FIFO mailbox guarantees every prior apply is processed by then, so the counter is final (a + /// polling AwaitAssert on a monotonic counter would pass transiently as it climbs, hiding the + /// unconditional-clear bug). [Fact] - public void ApplyVirtualTags_clears_the_evaluator_compile_cache_each_generation() + public void ApplyVirtualTags_clears_the_compile_cache_only_when_the_expression_set_changes() { var publish = CreateTestProbe(); var evaluator = new CacheOwningStubEvaluator(); var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, evaluator)); - host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") })); - AwaitAssert(() => evaluator.ClearCount.ShouldBe(1)); + // 1) First generation: previous set {} → {E1} differs → clears. + host.Tell(new VirtualTagHostActor.ApplyVirtualTags( + new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E1", "a") })); + // 2) Byte-identical redeploy: {E1} → {E1} → NO clear (the P7 win). + host.Tell(new VirtualTagHostActor.ApplyVirtualTags( + new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E1", "a") })); + // 3) Edited expression: {E1} → {E2} differs → clears. + host.Tell(new VirtualTagHostActor.ApplyVirtualTags( + new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a") })); + // 4) Add a second vtag SHARING the same expression E2: {E2} → {E2} → NO clear. + host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] + { + PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a"), + PlanWithRefs("vt-2", "eq-1", "torque", "E2", "a"), + })); + // 5) Remove that second vtag — expression E2 still deployed by vt-1: {E2} → {E2} → NO clear. + host.Tell(new VirtualTagHostActor.ApplyVirtualTags( + new[] { PlanWithRefs("vt-1", "eq-1", "speed-rpm", "E2", "a") })); + + // Sync point: a result for the still-mapped vt-1 bridges to a publish; when it lands, every + // apply above has been processed. + host.Tell(new VirtualTagActor.EvaluationResult("vt-1", 1.0, DateTime.UtcNow, CorrelationId.NewId())); + publish.ExpectMsg(); + + // Exactly two clears: the first generation and the E1→E2 edit. The identical redeploy and the + // shared-expression add/remove did not clear. + evaluator.ClearCount.ShouldBe(2); + } + + /// + /// 02/S13 stale-Good repro (RED on current code): a script that succeeds once then starts failing + /// must degrade its node's quality to Bad — carrying the last-known value — rather than freezing on + /// the last Good value forever. Drives the REAL host + real child + parent-Tell bridge: apply one + /// plan (deps ["a"]), capture the spawned child via the mux probe's RegisterInterest sender, then + /// feed it two dependency changes. The first evaluates Ok(42) → a Good AttributeValueUpdate; the + /// second fails → the fix publishes a Bad AttributeValueUpdate carrying the stale 42. On the + /// unfixed tree the second publish never arrives (the second ExpectMsg times out). + /// + [Fact] + public void Failing_script_after_success_degrades_node_quality_to_Bad() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux.Ref, + new FailAfterFirstSuccessEvaluator())); - // A second apply generation clears again — proves it's per-generation, not one-shot. host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") })); - AwaitAssert(() => evaluator.ClearCount.ShouldBe(2)); + + // The child self-registers with the mux in PreStart — capture its ref from the sender so we can + // drive it directly through the real evaluate → parent-Tell → bridge path. + mux.ExpectMsg(); + var child = mux.LastSender; + + var ts1 = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc); + var ts2 = new DateTime(2026, 7, 12, 12, 0, 1, DateTimeKind.Utc); + + // First dep change → evaluator Ok(42) → Good publish. + child.Tell(new VirtualTagActor.DependencyValueChanged("a", 1, ts1)); + var good = publish.ExpectMsg(); + good.Quality.ShouldBe(OpcUaQuality.Good); + good.Value.ShouldBe(42.0); + + // Second dep change → evaluator Failure → Bad publish carrying the last-known value 42. + child.Tell(new VirtualTagActor.DependencyValueChanged("a", 2, ts2)); + var bad = publish.ExpectMsg(); + bad.Quality.ShouldBe(OpcUaQuality.Bad); + bad.Value.ShouldBe(42.0); + bad.TimestampUtc.ShouldBe(ts2); + } + + /// 02/S13 bridge: a Bad-quality must bridge to + /// an carrying + /// (not the old hard-coded Good). + [Fact] + public void Bad_quality_EvaluationResult_is_bridged_as_Bad() + { + var publish = CreateTestProbe(); + var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, new StubEvaluator())); + + host.Tell(new VirtualTagHostActor.ApplyVirtualTags(new[] { Plan("vt-1", "eq-1", "speed-rpm") })); + + var ts = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc); + host.Tell(new VirtualTagActor.EvaluationResult( + "vt-1", 42.0, ts, CorrelationId.NewId(), OpcUaQuality.Bad)); + + var update = publish.ExpectMsg(); + update.Quality.ShouldBe(OpcUaQuality.Bad); + update.Value.ShouldBe(42.0); + } + + /// 02/S13 historize parity: a Bad result on a historized plan must record the dormant + /// engine's BadInternalError status (0x80020000u) in the snapshot, not the Good 0u. + [Fact] + public void Historized_Bad_result_is_recorded_with_BadInternalError_status() + { + var publish = CreateTestProbe(); + var writer = new CapturingHistoryWriter(); + var host = Sys.ActorOf(VirtualTagHostActor.Props(publish.Ref, mux: null, new StubEvaluator(), writer)); + + host.Tell(new VirtualTagHostActor.ApplyVirtualTags( + new[] { PlanH("vt-1", "eq-1", "speed-rpm", historize: true) })); + + var ts = new DateTime(2026, 7, 12, 12, 0, 0, DateTimeKind.Utc); + host.Tell(new VirtualTagActor.EvaluationResult( + "vt-1", 42.0, ts, CorrelationId.NewId(), OpcUaQuality.Bad)); + + publish.ExpectMsg(); + AwaitAssert(() => writer.Calls.Count.ShouldBe(1)); + writer.Calls.TryPeek(out var captured).ShouldBeTrue(); + captured.Value.StatusCode.ShouldBe(0x80020000u); // BadInternalError — dormant-engine parity + } + + /// Evaluator that returns Ok(42.0) on its first call and Failure thereafter — + /// the S13 "was working, now broken" shape. Interlocked counter so it is safe across the actor's + /// evaluation thread. + private sealed class FailAfterFirstSuccessEvaluator : IVirtualTagEvaluator + { + private int _calls; + + /// Returns Ok(42.0) once, then Failure("boom"). + /// The tag identifier. + /// The expression string. + /// The dependency values. + /// Ok(42.0) on the first call; Failure on every subsequent call. + public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary deps) + => Interlocked.Increment(ref _calls) == 1 + ? VirtualTagEvalResult.Ok(42.0) + : VirtualTagEvalResult.Failure("boom"); } /// Capturing : records every Record call so H5c tests can