fix(runtime): restart-safe alarm host, mux last-value replay, VT log rate limit, Calculation remediation (E4)
- R02:H1 ScriptedAlarmHostActor gains a factory Props (Func<(upstream, engine)>) so every incarnation builds and owns its own engine; a supervision restart no longer re-creates the actor around a disposed engine. The legacy instance-taking Props stays source-compatible and is now restart-safe in the weaker sense (PreRestart marks the stop as a restart, PostStop then detaches instead of disposing). DriverHostActor still spawns through the legacy overload (not owned by this lane — F3). - R02:H3 OnAlarmCommand runs every engine op under a 10 s token linked to the actor's lifetime (DefaultCommandTimeout / Props commandTimeout) and logs a Warning on timeout instead of suspending the mailbox on a stalled store. - R02:L3 DependencyMuxTagUpstreamSource.RetainOnly(loaded dep-ref union), called after every successful load. - R02:P6 VirtualTagActor's per-failure ScriptLogEntry is rate-limited per tag (ScriptLogRateLimiter, 20/min + one summary); the Bad publish and the metric are untouched. - R07:H6 DependencyMuxActor retains the last (value, status, ts) per ever-registered ref (through unregisters) and replays them in one batch to a RegisterInterest(ReplayRetained: true) — VirtualTagActor, the Calculation mux adapter and the alarm host opt in; the historization recorder does not (a replay there would record a stale sample per deploy). CalculationDriver records inputs regardless of _running so a replay landing before InitializeAsync still satisfies the change-gate. - R07:P4 CalculationEvaluator negative-caches compile failures by source hash (one Roslyn attempt + one Warning per source until redeploy); the driver's script-log entry fires on the Good→Bad transition or a change of reason, matching the once-per-transition Bad publish. - R07:H8 array results dedup structurally. - R07:L2 ShutdownAsync drops the compiled scripts (collectible ALCs). - R07:U5 DraftValidator rule CalculationNoEffectiveTrigger: a tag with no timer whose change trigger is off or whose script has no literal ctx.GetTag read can never evaluate. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
+22
@@ -15,6 +15,28 @@ public sealed class DependencyMuxTagUpstreamSourceTests
|
||||
private static DataValueSnapshot Good(object? value, DateTime ts)
|
||||
=> new(value, 0u, ts, ts);
|
||||
|
||||
/// <summary>R02:L3 — the host prunes the retained snapshots to the loaded dependency-ref union after each
|
||||
/// (re)load, so a renamed/removed raw path stops being retained; observers are untouched.</summary>
|
||||
[Fact]
|
||||
public void RetainOnly_drops_snapshots_outside_the_set_and_keeps_observers()
|
||||
{
|
||||
var src = new DependencyMuxTagUpstreamSource();
|
||||
var ts = DateTime.UtcNow;
|
||||
src.Push("keep", Good(1, ts));
|
||||
src.Push("old/renamed", Good(2, ts));
|
||||
var seen = new List<string>();
|
||||
using var sub = src.SubscribeTag("old/renamed", (p, _) => seen.Add(p));
|
||||
|
||||
src.RetainOnly(new HashSet<string>(StringComparer.Ordinal) { "keep" }).ShouldBe(1);
|
||||
|
||||
src.RetainedPathCount.ShouldBe(1);
|
||||
src.ReadTag("keep").Value.ShouldBe(1);
|
||||
src.ReadTag("old/renamed").StatusCode.ShouldBe(OpcUaStatus.BadWaitingForInitialData);
|
||||
src.Push("old/renamed", Good(3, ts));
|
||||
// Pruning retained values does not unsubscribe anyone.
|
||||
seen.ShouldBe(new[] { "old/renamed" });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Push_then_ReadTag_returns_the_pushed_snapshot()
|
||||
{
|
||||
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms;
|
||||
|
||||
/// <summary>
|
||||
/// R02:H1 — a supervision restart of <see cref="ScriptedAlarmHostActor"/> used to dispose the
|
||||
/// Props-captured engine and re-create the actor around the disposed instance, so every later
|
||||
/// <c>ApplyScriptedAlarms</c> failed and the node's scripted alarms silently died. With the factory
|
||||
/// Props each incarnation builds its own engine; with the legacy Props a restart only detaches. Both
|
||||
/// shapes must load cleanly after a throw inside a <see cref="VirtualTagActor.DependencyValueBatch"/>
|
||||
/// handler. Also R02:H3 — an inbound alarm command's engine op is bounded.
|
||||
/// </summary>
|
||||
public sealed class ScriptedAlarmHostActorRestartTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30);
|
||||
|
||||
private static EquipmentScriptedAlarmPlan Plan(string id = "alm-1", string depRef = "M.T") =>
|
||||
new(ScriptedAlarmId: id, EquipmentId: "Plant/Line1/M", Name: "HighTemp", AlarmType: "AlarmCondition",
|
||||
Severity: 800, MessageTemplate: "condition", PredicateScriptId: $"{id}-script",
|
||||
PredicateSource: $"return (int)ctx.GetTag(\"{depRef}\").Value > 90;",
|
||||
DependencyRefs: new[] { depRef }, HistorizeToAveva: true, Retain: true, Enabled: true);
|
||||
|
||||
private static ScriptedAlarmEngine BuildEngine(DependencyMuxTagUpstreamSource upstream, IAlarmStateStore? store = null)
|
||||
{
|
||||
var logger = new LoggerConfiguration().CreateLogger();
|
||||
return new ScriptedAlarmEngine(upstream, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
|
||||
}
|
||||
|
||||
private static (DependencyMuxTagUpstreamSource, ScriptedAlarmEngine) Pair()
|
||||
{
|
||||
var up = new DependencyMuxTagUpstreamSource();
|
||||
return (up, BuildEngine(up));
|
||||
}
|
||||
|
||||
/// <summary>Apply, crash the handler (a null TagId makes <c>Push</c> throw → restart), apply again
|
||||
/// and prove the second load registers interest and the alarm still activates.</summary>
|
||||
private void RestartThenReload(IActorRef host, TestProbe publish, TestProbe mux)
|
||||
{
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
|
||||
// The crash: a batch whose item has a null TagId makes the upstream Push throw out of the handler
|
||||
// (the documented, deliberately unguarded path) → supervision restarts the actor.
|
||||
host.Tell(new VirtualTagActor.DependencyValueBatch(new[]
|
||||
{
|
||||
new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow),
|
||||
}));
|
||||
mux.ExpectMsg<DependencyMuxActor.UnregisterInterest>(Timeout); // PostStop of the crashed incarnation
|
||||
|
||||
// A later apply must load cleanly on the restarted incarnation…
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
var reg = mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
reg.TagRefs.ShouldContain("M.T");
|
||||
|
||||
// …and the alarm must still work end to end.
|
||||
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
|
||||
var state = publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout);
|
||||
state.AlarmNodeId.ShouldBe("alm-1");
|
||||
state.State.Active.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Factory_Props_survives_a_handler_throw_and_loads_cleanly_afterwards()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var built = 0;
|
||||
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, () => { built++; return Pair(); }));
|
||||
|
||||
RestartThenReload(host, publish, mux);
|
||||
|
||||
built.ShouldBe(2, "the restarted incarnation built its own engine");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Legacy_instance_Props_survives_a_handler_throw_because_a_restart_only_detaches()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var (upstream, engine) = Pair();
|
||||
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, upstream, engine));
|
||||
|
||||
RestartThenReload(host, publish, mux);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_alarm_command_whose_store_hangs_is_bounded_and_the_mailbox_moves_on()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var store = new HangingSaveStore();
|
||||
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref,
|
||||
() => { var up = new DependencyMuxTagUpstreamSource(); return (up, BuildEngine(up, store)); },
|
||||
commandTimeout: TimeSpan.FromMilliseconds(300)));
|
||||
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).State.Active.ShouldBeTrue();
|
||||
|
||||
store.HangNext = true;
|
||||
host.Tell(new AlarmCommand("alm-1", "Acknowledge", "alice", "ack", null));
|
||||
|
||||
// The hung save would have suspended the mailbox indefinitely; the bound releases it, so the
|
||||
// next dependency value is processed and the alarm clears.
|
||||
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 10, DateTime.UtcNow));
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).State.Active.ShouldBeFalse();
|
||||
store.ObservedCancellation.ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>Store whose next SaveAsync blocks until its token is cancelled.</summary>
|
||||
private sealed class HangingSaveStore : IAlarmStateStore
|
||||
{
|
||||
private readonly InMemoryAlarmStateStore _inner = new();
|
||||
public volatile bool HangNext;
|
||||
public volatile bool ObservedCancellation;
|
||||
|
||||
public Task<AlarmConditionState?> LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct);
|
||||
public Task<IReadOnlyList<AlarmConditionState>> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct);
|
||||
public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct);
|
||||
|
||||
public async Task SaveAsync(AlarmConditionState state, CancellationToken ct)
|
||||
{
|
||||
if (HangNext)
|
||||
{
|
||||
HangNext = false;
|
||||
try { await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, ct); }
|
||||
catch (OperationCanceledException) { ObservedCancellation = true; throw; }
|
||||
}
|
||||
await _inner.SaveAsync(state, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Calculation;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags;
|
||||
|
||||
/// <summary>
|
||||
/// R07:H6 — the mux retains the last value per ever-registered ref and replays it to a registration
|
||||
/// that asks (<see cref="DependencyMuxActor.RegisterInterest.ReplayRetained"/>). A Calculation driver
|
||||
/// Restart (new driver + new adapter) therefore republishes from current inputs instead of sitting
|
||||
/// <c>BadWaitingForInitialData</c> until every input happens to change.
|
||||
/// </summary>
|
||||
public sealed class DependencyMuxReplayTests : RuntimeActorTestBase
|
||||
{
|
||||
private static DriverInstanceActor.AttributeValueBatchPublished Burst(params (string Ref, object? Value)[] items) =>
|
||||
new("driver-1", items.Select(i => new DriverInstanceActor.AttributeValueItem(i.Ref, i.Value, OpcUaStatus.Good, DateTime.UtcNow)).ToArray());
|
||||
|
||||
[Fact]
|
||||
public void RegisterInterest_with_replay_receives_the_retained_values_for_its_refs_only()
|
||||
{
|
||||
var mux = Sys.ActorOf(DependencyMuxActor.Props());
|
||||
var first = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref));
|
||||
mux.Tell(Burst(("a", 1), ("b", 2), ("c", 3)));
|
||||
first.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget).Items.Count.ShouldBe(2);
|
||||
|
||||
// The first subscriber goes away; retention must survive the unregister (that is the Restart shape).
|
||||
mux.Tell(new DependencyMuxActor.UnregisterInterest(first.Ref));
|
||||
|
||||
var second = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "c" }, second.Ref, ReplayRetained: true));
|
||||
|
||||
var replay = second.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
|
||||
replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by anyone, so it is not retained; 'b' is not in this subscriber's refs");
|
||||
replay.Items[0].Value.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RegisterInterest_without_replay_receives_nothing_and_a_fresh_ref_replays_nothing()
|
||||
{
|
||||
var mux = Sys.ActorOf(DependencyMuxActor.Props());
|
||||
var seed = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, seed.Ref));
|
||||
mux.Tell(Burst(("a", 1)));
|
||||
seed.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
|
||||
|
||||
var opted = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, opted.Ref)); // default: no replay
|
||||
opted.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
|
||||
var fresh = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "never-published" }, fresh.Ref, ReplayRetained: true));
|
||||
fresh.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
/// <summary>The user-facing scenario: a calc tag computes, the driver is restarted (new instance + new
|
||||
/// adapter) with NO input changing afterwards, and the new instance republishes the current value.</summary>
|
||||
[Fact]
|
||||
public async Task A_Calculation_driver_restart_republishes_from_the_current_inputs()
|
||||
{
|
||||
var mux = Sys.ActorOf(DependencyMuxActor.Props());
|
||||
var tag = new RawTagEntry("calc/e/sum",
|
||||
"{\"scriptId\":\"s\",\"changeTriggered\":true,\"scriptSource\":\"return (double)ctx.GetTag(\\\"src/a\\\").Value + (double)ctx.GetTag(\\\"src/b\\\").Value;\"}",
|
||||
WriteIdempotent: false);
|
||||
|
||||
// Generation 1: wire the driver through the real adapter and feed a burst.
|
||||
var gen1 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv");
|
||||
var published1 = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
gen1.OnDataChange += (_, e) => published1.Enqueue(e);
|
||||
await gen1.InitializeAsync("{}", CancellationToken.None);
|
||||
var adapter1 = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen1, mux));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(Burst(("src/a", 10.0), ("src/b", 20.0)));
|
||||
published1.Count.ShouldBeGreaterThan(0);
|
||||
}, PresenceBudget);
|
||||
published1.Last().Snapshot.Value.ShouldBe(30.0);
|
||||
|
||||
// Restart: the old instance goes away; a NEW driver + adapter come up. Nothing changes upstream.
|
||||
Sys.Stop(adapter1);
|
||||
await gen1.ShutdownAsync(CancellationToken.None);
|
||||
var gen2 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv");
|
||||
var published2 = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
gen2.OnDataChange += (_, e) => published2.Enqueue(e);
|
||||
Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen2, mux)); // registers with ReplayRetained
|
||||
await gen2.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
// The replayed inputs satisfied the change-gate; the first trigger after init is the replay itself
|
||||
// (if it lands post-init) or the next change (here: none) — so publish happens from the replay.
|
||||
AwaitAssert(() => published2.Count.ShouldBeGreaterThan(0), PresenceBudget);
|
||||
published2.Last().Snapshot.Value.ShouldBe(30.0);
|
||||
published2.Last().Snapshot.StatusCode.ShouldBe(OpcUaStatus.Good);
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags;
|
||||
|
||||
/// <summary>R02:P6 — the per-failure cluster-wide <see cref="ScriptLogEntry"/> is rate-limited per tag
|
||||
/// (first N per window, then one summary); the Bad publish and the metric are unaffected.</summary>
|
||||
public sealed class VirtualTagActorFailureLogRateLimitTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Repeated_failures_publish_at_most_the_limit_then_one_summary_when_the_window_rolls()
|
||||
{
|
||||
var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
|
||||
var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => now);
|
||||
var capture = new ConcurrentQueue<ScriptLogEntry>();
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(VirtualTagActor.Props(
|
||||
"vt-chatty", "broken",
|
||||
evaluator: new FailingEvaluator("boom"),
|
||||
publisherFactory: () => new DPSPublisher((_, p) => capture.Enqueue((ScriptLogEntry)p)),
|
||||
failureLogLimiter: limiter));
|
||||
|
||||
for (var i = 0; i < 10; i++)
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", i, DateTime.UtcNow));
|
||||
|
||||
// Positive control: the FIRST failure still degrades the node (once per transition, as before).
|
||||
parent.ExpectMsg<VirtualTagActor.EvaluationResult>(PresenceBudget).StatusCode.ShouldBe(Commons.OpcUa.OpcUaStatus.BadInternalError);
|
||||
AwaitAssert(() => capture.Count.ShouldBe(3), PresenceBudget);
|
||||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
capture.Count.ShouldBe(3, "7 of 10 failures were suppressed from the cluster-wide log");
|
||||
|
||||
now = now.AddMinutes(1);
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 99, DateTime.UtcNow));
|
||||
|
||||
AwaitAssert(() => capture.Count.ShouldBe(5), PresenceBudget);
|
||||
var entries = capture.ToArray();
|
||||
entries[3].Message.ShouldContain("7");
|
||||
entries[3].Message.ShouldContain("suppressed");
|
||||
entries[3].VirtualTagId.ShouldBe("vt-chatty");
|
||||
entries[4].Message.ShouldBe("boom");
|
||||
}
|
||||
|
||||
private sealed class FailingEvaluator(string reason) : IVirtualTagEvaluator
|
||||
{
|
||||
public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
|
||||
=> VirtualTagEvalResult.Failure(reason);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user