fix(runtime): restart-safe alarm host, mux last-value replay, VT log rate limit, Calculation remediation (E4)
- R02:H1 ScriptedAlarmHostActor gains a factory Props (Func<(upstream, engine)>) so every incarnation builds and owns its own engine; a supervision restart no longer re-creates the actor around a disposed engine. The legacy instance-taking Props stays source-compatible and is now restart-safe in the weaker sense (PreRestart marks the stop as a restart, PostStop then detaches instead of disposing). DriverHostActor still spawns through the legacy overload (not owned by this lane — F3). - R02:H3 OnAlarmCommand runs every engine op under a 10 s token linked to the actor's lifetime (DefaultCommandTimeout / Props commandTimeout) and logs a Warning on timeout instead of suspending the mailbox on a stalled store. - R02:L3 DependencyMuxTagUpstreamSource.RetainOnly(loaded dep-ref union), called after every successful load. - R02:P6 VirtualTagActor's per-failure ScriptLogEntry is rate-limited per tag (ScriptLogRateLimiter, 20/min + one summary); the Bad publish and the metric are untouched. - R07:H6 DependencyMuxActor retains the last (value, status, ts) per ever-registered ref (through unregisters) and replays them in one batch to a RegisterInterest(ReplayRetained: true) — VirtualTagActor, the Calculation mux adapter and the alarm host opt in; the historization recorder does not (a replay there would record a stale sample per deploy). CalculationDriver records inputs regardless of _running so a replay landing before InitializeAsync still satisfies the change-gate. - R07:P4 CalculationEvaluator negative-caches compile failures by source hash (one Roslyn attempt + one Warning per source until redeploy); the driver's script-log entry fires on the Good→Bad transition or a change of reason, matching the once-per-transition Bad publish. - R07:H8 array results dedup structurally. - R07:L2 ShutdownAsync drops the compiled scripts (collectible ALCs). - R07:U5 DraftValidator rule CalculationNoEffectiveTrigger: a tag with no timer whose change trigger is off or whose script has no literal ctx.GetTag read can never evaluate. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
@@ -82,7 +82,9 @@ public sealed class DraftValidatorCalculationTests
|
||||
DriverInstances = [CalcDriver()],
|
||||
Devices = [Device("dev-1", "di-calc", "engine")],
|
||||
Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))],
|
||||
Scripts = [ScriptRow("s1", "return 42.0;")],
|
||||
// A read, so the change trigger is effective (a constant change-triggered script with no timer
|
||||
// is now itself a deploy error — CalculationNoEffectiveTrigger).
|
||||
Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value + 42.0;")],
|
||||
};
|
||||
|
||||
DraftValidator.Validate(draft).ShouldNotContain(e => e.Code.StartsWith("Calculation"));
|
||||
@@ -132,6 +134,58 @@ public sealed class DraftValidatorCalculationTests
|
||||
msg.ShouldContain("calc/engine/b");
|
||||
}
|
||||
|
||||
// ── effective-trigger gate (R07:U5) ─────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Change_triggered_tag_with_no_literal_read_and_no_timer_is_rejected()
|
||||
{
|
||||
var draft = new DraftSnapshot
|
||||
{
|
||||
GenerationId = 1, ClusterId = "c",
|
||||
DriverInstances = [CalcDriver()],
|
||||
Devices = [Device("dev-1", "di-calc", "engine")],
|
||||
Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))],
|
||||
Scripts = [ScriptRow("s1", "return ctx.Now.Hour;")],
|
||||
};
|
||||
|
||||
DraftValidator.Validate(draft).ShouldContain(e =>
|
||||
e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1" && e.Message.Contains("no literal ctx.GetTag"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tag_with_change_trigger_off_and_no_timer_is_rejected()
|
||||
{
|
||||
var draft = new DraftSnapshot
|
||||
{
|
||||
GenerationId = 1, ClusterId = "c",
|
||||
DriverInstances = [CalcDriver()],
|
||||
Devices = [Device("dev-1", "di-calc", "engine")],
|
||||
Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"scriptId\":\"s1\",\"changeTriggered\":false}")],
|
||||
Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")],
|
||||
};
|
||||
|
||||
DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true,\"timerIntervalMs\":1000}", "return ctx.Now.Hour;")] // timer rescues a read-less script
|
||||
[InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":false,\"timerIntervalMs\":500}", "return 1.0;")] // timer-only tag
|
||||
[InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // change trigger with a read
|
||||
[InlineData("{\"scriptId\":\"s1\"}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // changeTriggered defaults to true
|
||||
public void A_tag_with_an_effective_trigger_is_accepted(string cfg, string source)
|
||||
{
|
||||
var draft = new DraftSnapshot
|
||||
{
|
||||
GenerationId = 1, ClusterId = "c",
|
||||
DriverInstances = [CalcDriver()],
|
||||
Devices = [Device("dev-1", "di-calc", "engine")],
|
||||
Tags = [CalcTagRow("t-1", "dev-1", "a", cfg)],
|
||||
Scripts = [ScriptRow("s1", source)],
|
||||
};
|
||||
|
||||
DraftValidator.Validate(draft).ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Cross_driver_ref_is_terminal_not_an_edge()
|
||||
{
|
||||
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Serilog;
|
||||
using Serilog.Events;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Review 07 Calculation remediation: a compile failure is negative-cached by source hash and its
|
||||
/// Warning/script-log fires on the Good→Bad transition only (P4); inputs are recorded before
|
||||
/// <c>InitializeAsync</c> so a Restart computes from current inputs (H6); array results dedup
|
||||
/// structurally (H8); <c>ShutdownAsync</c> drops the compiled scripts (L2).
|
||||
/// </summary>
|
||||
public sealed class CalculationDriverRemediationTests
|
||||
{
|
||||
private const uint StatusGood = 0u;
|
||||
private const uint StatusBadInternalError = 0x80020000u;
|
||||
|
||||
private static RawTagEntry CalcTag(string rawPath, string source, bool changeTriggered = true, int? timerMs = null)
|
||||
{
|
||||
var timer = timerMs is { } ms ? $",\"timerIntervalMs\":{ms}" : "";
|
||||
var json = $"{{\"scriptId\":\"s-{rawPath}\",\"changeTriggered\":{(changeTriggered ? "true" : "false")}{timer}," +
|
||||
$"\"scriptSource\":{System.Text.Json.JsonSerializer.Serialize(source)}}}";
|
||||
return new RawTagEntry(rawPath, json, WriteIdempotent: false);
|
||||
}
|
||||
|
||||
private static (CalculationDriver Driver, ConcurrentQueue<DataChangeEventArgs> Published, List<ScriptLogEntry> Logs, CapturingLogger HostLog)
|
||||
Build(params RawTagEntry[] tags)
|
||||
{
|
||||
var logs = new List<ScriptLogEntry>();
|
||||
var sink = new ScriptLogTopicSink(new CapturingPublisher(logs), LogEventLevel.Information);
|
||||
var scriptRoot = new ScriptRootLogger(new LoggerConfiguration().WriteTo.Sink(sink).CreateLogger());
|
||||
var hostLog = new CapturingLogger();
|
||||
var driver = new CalculationDriver(new CalculationDriverOptions { RawTags = tags }, "calc-drv", scriptRoot, hostLog);
|
||||
var published = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
driver.OnDataChange += (_, e) => published.Enqueue(e);
|
||||
return (driver, published, logs, hostLog);
|
||||
}
|
||||
|
||||
private static async Task Settle(CalculationDriver driver)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddSeconds(30);
|
||||
while (!driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(10);
|
||||
driver.IsEvaluationIdle.ShouldBeTrue("the calc driver's evaluation pump should have drained");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_compile_failure_is_remembered_and_logged_once_across_repeated_triggers()
|
||||
{
|
||||
var (driver, published, logs, hostLog) = Build(
|
||||
CalcTag("calc/e/broken", "var x = (double)ctx.GetTag(\"src/a\").Value; this is not valid C#;"));
|
||||
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
||||
|
||||
for (var i = 0; i < 5; i++)
|
||||
{
|
||||
driver.OnDependencyValue("src/a", i, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
}
|
||||
|
||||
published.Count.ShouldBe(1, "Bad is published once per transition");
|
||||
published.TryDequeue(out var bad).ShouldBeTrue();
|
||||
bad!.Snapshot.StatusCode.ShouldBe(StatusBadInternalError);
|
||||
logs.Count.ShouldBe(1, "the script-log entry fires on the Good→Bad transition only");
|
||||
hostLog.Entries.Count(e => e.Level == LogLevel.Warning && e.Message.Contains("compile failed")).ShouldBe(1,
|
||||
"Roslyn is consulted once per source; later triggers hit the negative cache");
|
||||
driver.CompileFailureCountForTest.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_changed_failure_reason_is_logged_even_while_already_Bad()
|
||||
{
|
||||
// Throws with a different message depending on the input, so the failure REASON changes while the
|
||||
// published status stays Bad — the operator must see the new reason.
|
||||
var (driver, published, logs, _) = Build(
|
||||
CalcTag("calc/e/thrower", "throw new System.InvalidOperationException(\"boom \" + (int)ctx.GetTag(\"src/a\").Value);"));
|
||||
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
||||
|
||||
driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); // same reason → no new log
|
||||
await Settle(driver);
|
||||
driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow); // new reason → logged
|
||||
await Settle(driver);
|
||||
|
||||
published.Count.ShouldBe(1);
|
||||
logs.Count.ShouldBe(2);
|
||||
logs[0].Message.ShouldContain("boom 1");
|
||||
logs[1].Message.ShouldContain("boom 2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Inputs_delivered_before_InitializeAsync_are_kept_so_the_first_trigger_computes()
|
||||
{
|
||||
// H6: a Restart is a new driver whose adapter registers interest at construction; the mux's
|
||||
// retained-value replay therefore lands BEFORE InitializeAsync. It used to be dropped.
|
||||
var (driver, published, _, _) = Build(
|
||||
CalcTag("calc/e/sum", "return (double)ctx.GetTag(\"src/a\").Value + (double)ctx.GetTag(\"src/b\").Value;"));
|
||||
|
||||
driver.OnDependencyValue("src/a", 10.0, StatusGood, DateTime.UtcNow); // replayed, pre-init
|
||||
driver.OnDependencyValue("src/b", 20.0, StatusGood, DateTime.UtcNow);
|
||||
published.ShouldBeEmpty("evaluation is gated on running");
|
||||
|
||||
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
||||
driver.OnDependencyValue("src/a", 12.0, StatusGood, DateTime.UtcNow); // only ONE input moves after init
|
||||
await Settle(driver);
|
||||
|
||||
published.Count.ShouldBe(1, "src/b's pre-init value satisfied the change-gate");
|
||||
published.TryDequeue(out var e).ShouldBeTrue();
|
||||
e!.Snapshot.Value.ShouldBe(32.0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_equal_array_result_is_deduped_structurally()
|
||||
{
|
||||
var (driver, published, _, _) = Build(
|
||||
CalcTag("calc/e/arr", "var v = (double)ctx.GetTag(\"src/a\").Value; return new double[] { v, v * 2 };"));
|
||||
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
||||
|
||||
driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); // same array contents
|
||||
await Settle(driver);
|
||||
published.Count.ShouldBe(1, "a structurally equal array is not a change");
|
||||
|
||||
driver.OnDependencyValue("src/a", 2.0, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
published.Count.ShouldBe(2, "a different array still publishes");
|
||||
published.Last().Snapshot.Value.ShouldBe(new double[] { 2.0, 4.0 });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ShutdownAsync_drops_the_compiled_scripts()
|
||||
{
|
||||
var (driver, published, _, _) = Build(CalcTag("calc/e/x", "return (double)ctx.GetTag(\"src/a\").Value * 2.0;"));
|
||||
await driver.InitializeAsync("{}", TestContext.Current.CancellationToken);
|
||||
driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
published.Count.ShouldBe(1);
|
||||
driver.CompiledScriptCountForTest.ShouldBe(1);
|
||||
|
||||
await driver.ShutdownAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
driver.CompiledScriptCountForTest.ShouldBe(0, "the collectible ALCs must not survive the generation");
|
||||
}
|
||||
|
||||
private sealed class CapturingPublisher(List<ScriptLogEntry> entries) : IScriptLogPublisher
|
||||
{
|
||||
public void Publish(ScriptLogEntry entry) { lock (entries) entries.Add(entry); }
|
||||
}
|
||||
|
||||
internal sealed class CapturingLogger : ILogger<CalculationDriver>
|
||||
{
|
||||
public ConcurrentBag<(LogLevel Level, string Message)> Entries { get; } = new();
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
||||
Func<TState, Exception?, string> formatter)
|
||||
=> Entries.Add((logLevel, formatter(state, exception)));
|
||||
}
|
||||
}
|
||||
@@ -229,11 +229,13 @@ public sealed class CalculationDriverTests
|
||||
logs[0].ScriptId.ShouldBe("s-calc/e/div");
|
||||
logs[0].VirtualTagId.ShouldBe("calc/e/div");
|
||||
|
||||
// a = 0 again → still failing, but NOT a fresh transition → no second Bad publish (script-log still emits).
|
||||
// a = 0 again → still failing for the SAME reason, NOT a fresh transition → no second Bad publish,
|
||||
// and (R07:P4) no second script-log entry either: the log fires on the transition or on a change of
|
||||
// reason, not on every trigger of a tag that is already Bad.
|
||||
driver.OnDependencyValue("src/a", 0, StatusGood, DateTime.UtcNow);
|
||||
await Settle(driver);
|
||||
published.ShouldBeEmpty();
|
||||
logs.Count.ShouldBe(2);
|
||||
logs.Count.ShouldBe(1);
|
||||
|
||||
// a = 5 → 20 → Good recovery is force-published.
|
||||
driver.OnDependencyValue("src/a", 5, StatusGood, DateTime.UtcNow);
|
||||
|
||||
+22
@@ -15,6 +15,28 @@ public sealed class DependencyMuxTagUpstreamSourceTests
|
||||
private static DataValueSnapshot Good(object? value, DateTime ts)
|
||||
=> new(value, 0u, ts, ts);
|
||||
|
||||
/// <summary>R02:L3 — the host prunes the retained snapshots to the loaded dependency-ref union after each
|
||||
/// (re)load, so a renamed/removed raw path stops being retained; observers are untouched.</summary>
|
||||
[Fact]
|
||||
public void RetainOnly_drops_snapshots_outside_the_set_and_keeps_observers()
|
||||
{
|
||||
var src = new DependencyMuxTagUpstreamSource();
|
||||
var ts = DateTime.UtcNow;
|
||||
src.Push("keep", Good(1, ts));
|
||||
src.Push("old/renamed", Good(2, ts));
|
||||
var seen = new List<string>();
|
||||
using var sub = src.SubscribeTag("old/renamed", (p, _) => seen.Add(p));
|
||||
|
||||
src.RetainOnly(new HashSet<string>(StringComparer.Ordinal) { "keep" }).ShouldBe(1);
|
||||
|
||||
src.RetainedPathCount.ShouldBe(1);
|
||||
src.ReadTag("keep").Value.ShouldBe(1);
|
||||
src.ReadTag("old/renamed").StatusCode.ShouldBe(OpcUaStatus.BadWaitingForInitialData);
|
||||
src.Push("old/renamed", Good(3, ts));
|
||||
// Pruning retained values does not unsubscribe anyone.
|
||||
seen.ShouldBe(new[] { "old/renamed" });
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Push_then_ReadTag_returns_the_pushed_snapshot()
|
||||
{
|
||||
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit;
|
||||
using Serilog;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms;
|
||||
|
||||
/// <summary>
|
||||
/// R02:H1 — a supervision restart of <see cref="ScriptedAlarmHostActor"/> used to dispose the
|
||||
/// Props-captured engine and re-create the actor around the disposed instance, so every later
|
||||
/// <c>ApplyScriptedAlarms</c> failed and the node's scripted alarms silently died. With the factory
|
||||
/// Props each incarnation builds its own engine; with the legacy Props a restart only detaches. Both
|
||||
/// shapes must load cleanly after a throw inside a <see cref="VirtualTagActor.DependencyValueBatch"/>
|
||||
/// handler. Also R02:H3 — an inbound alarm command's engine op is bounded.
|
||||
/// </summary>
|
||||
public sealed class ScriptedAlarmHostActorRestartTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30);
|
||||
|
||||
private static EquipmentScriptedAlarmPlan Plan(string id = "alm-1", string depRef = "M.T") =>
|
||||
new(ScriptedAlarmId: id, EquipmentId: "Plant/Line1/M", Name: "HighTemp", AlarmType: "AlarmCondition",
|
||||
Severity: 800, MessageTemplate: "condition", PredicateScriptId: $"{id}-script",
|
||||
PredicateSource: $"return (int)ctx.GetTag(\"{depRef}\").Value > 90;",
|
||||
DependencyRefs: new[] { depRef }, HistorizeToAveva: true, Retain: true, Enabled: true);
|
||||
|
||||
private static ScriptedAlarmEngine BuildEngine(DependencyMuxTagUpstreamSource upstream, IAlarmStateStore? store = null)
|
||||
{
|
||||
var logger = new LoggerConfiguration().CreateLogger();
|
||||
return new ScriptedAlarmEngine(upstream, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger);
|
||||
}
|
||||
|
||||
private static (DependencyMuxTagUpstreamSource, ScriptedAlarmEngine) Pair()
|
||||
{
|
||||
var up = new DependencyMuxTagUpstreamSource();
|
||||
return (up, BuildEngine(up));
|
||||
}
|
||||
|
||||
/// <summary>Apply, crash the handler (a null TagId makes <c>Push</c> throw → restart), apply again
|
||||
/// and prove the second load registers interest and the alarm still activates.</summary>
|
||||
private void RestartThenReload(IActorRef host, TestProbe publish, TestProbe mux)
|
||||
{
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
|
||||
// The crash: a batch whose item has a null TagId makes the upstream Push throw out of the handler
|
||||
// (the documented, deliberately unguarded path) → supervision restarts the actor.
|
||||
host.Tell(new VirtualTagActor.DependencyValueBatch(new[]
|
||||
{
|
||||
new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow),
|
||||
}));
|
||||
mux.ExpectMsg<DependencyMuxActor.UnregisterInterest>(Timeout); // PostStop of the crashed incarnation
|
||||
|
||||
// A later apply must load cleanly on the restarted incarnation…
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
var reg = mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
reg.TagRefs.ShouldContain("M.T");
|
||||
|
||||
// …and the alarm must still work end to end.
|
||||
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
|
||||
var state = publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout);
|
||||
state.AlarmNodeId.ShouldBe("alm-1");
|
||||
state.State.Active.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Factory_Props_survives_a_handler_throw_and_loads_cleanly_afterwards()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var built = 0;
|
||||
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, () => { built++; return Pair(); }));
|
||||
|
||||
RestartThenReload(host, publish, mux);
|
||||
|
||||
built.ShouldBe(2, "the restarted incarnation built its own engine");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Legacy_instance_Props_survives_a_handler_throw_because_a_restart_only_detaches()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var (upstream, engine) = Pair();
|
||||
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, upstream, engine));
|
||||
|
||||
RestartThenReload(host, publish, mux);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_alarm_command_whose_store_hangs_is_bounded_and_the_mailbox_moves_on()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var store = new HangingSaveStore();
|
||||
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref,
|
||||
() => { var up = new DependencyMuxTagUpstreamSource(); return (up, BuildEngine(up, store)); },
|
||||
commandTimeout: TimeSpan.FromMilliseconds(300)));
|
||||
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() }));
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).State.Active.ShouldBeTrue();
|
||||
|
||||
store.HangNext = true;
|
||||
host.Tell(new AlarmCommand("alm-1", "Acknowledge", "alice", "ack", null));
|
||||
|
||||
// The hung save would have suspended the mailbox indefinitely; the bound releases it, so the
|
||||
// next dependency value is processed and the alarm clears.
|
||||
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 10, DateTime.UtcNow));
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).State.Active.ShouldBeFalse();
|
||||
store.ObservedCancellation.ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>Store whose next SaveAsync blocks until its token is cancelled.</summary>
|
||||
private sealed class HangingSaveStore : IAlarmStateStore
|
||||
{
|
||||
private readonly InMemoryAlarmStateStore _inner = new();
|
||||
public volatile bool HangNext;
|
||||
public volatile bool ObservedCancellation;
|
||||
|
||||
public Task<AlarmConditionState?> LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct);
|
||||
public Task<IReadOnlyList<AlarmConditionState>> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct);
|
||||
public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct);
|
||||
|
||||
public async Task SaveAsync(AlarmConditionState state, CancellationToken ct)
|
||||
{
|
||||
if (HangNext)
|
||||
{
|
||||
HangNext = false;
|
||||
try { await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, ct); }
|
||||
catch (OperationCanceledException) { ObservedCancellation = true; throw; }
|
||||
}
|
||||
await _inner.SaveAsync(state, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Calculation;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags;
|
||||
|
||||
/// <summary>
|
||||
/// R07:H6 — the mux retains the last value per ever-registered ref and replays it to a registration
|
||||
/// that asks (<see cref="DependencyMuxActor.RegisterInterest.ReplayRetained"/>). A Calculation driver
|
||||
/// Restart (new driver + new adapter) therefore republishes from current inputs instead of sitting
|
||||
/// <c>BadWaitingForInitialData</c> until every input happens to change.
|
||||
/// </summary>
|
||||
public sealed class DependencyMuxReplayTests : RuntimeActorTestBase
|
||||
{
|
||||
private static DriverInstanceActor.AttributeValueBatchPublished Burst(params (string Ref, object? Value)[] items) =>
|
||||
new("driver-1", items.Select(i => new DriverInstanceActor.AttributeValueItem(i.Ref, i.Value, OpcUaStatus.Good, DateTime.UtcNow)).ToArray());
|
||||
|
||||
[Fact]
|
||||
public void RegisterInterest_with_replay_receives_the_retained_values_for_its_refs_only()
|
||||
{
|
||||
var mux = Sys.ActorOf(DependencyMuxActor.Props());
|
||||
var first = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref));
|
||||
mux.Tell(Burst(("a", 1), ("b", 2), ("c", 3)));
|
||||
first.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget).Items.Count.ShouldBe(2);
|
||||
|
||||
// The first subscriber goes away; retention must survive the unregister (that is the Restart shape).
|
||||
mux.Tell(new DependencyMuxActor.UnregisterInterest(first.Ref));
|
||||
|
||||
var second = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "c" }, second.Ref, ReplayRetained: true));
|
||||
|
||||
var replay = second.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
|
||||
replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by anyone, so it is not retained; 'b' is not in this subscriber's refs");
|
||||
replay.Items[0].Value.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RegisterInterest_without_replay_receives_nothing_and_a_fresh_ref_replays_nothing()
|
||||
{
|
||||
var mux = Sys.ActorOf(DependencyMuxActor.Props());
|
||||
var seed = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, seed.Ref));
|
||||
mux.Tell(Burst(("a", 1)));
|
||||
seed.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget);
|
||||
|
||||
var opted = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, opted.Ref)); // default: no replay
|
||||
opted.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
|
||||
var fresh = CreateTestProbe();
|
||||
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "never-published" }, fresh.Ref, ReplayRetained: true));
|
||||
fresh.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
/// <summary>The user-facing scenario: a calc tag computes, the driver is restarted (new instance + new
|
||||
/// adapter) with NO input changing afterwards, and the new instance republishes the current value.</summary>
|
||||
[Fact]
|
||||
public async Task A_Calculation_driver_restart_republishes_from_the_current_inputs()
|
||||
{
|
||||
var mux = Sys.ActorOf(DependencyMuxActor.Props());
|
||||
var tag = new RawTagEntry("calc/e/sum",
|
||||
"{\"scriptId\":\"s\",\"changeTriggered\":true,\"scriptSource\":\"return (double)ctx.GetTag(\\\"src/a\\\").Value + (double)ctx.GetTag(\\\"src/b\\\").Value;\"}",
|
||||
WriteIdempotent: false);
|
||||
|
||||
// Generation 1: wire the driver through the real adapter and feed a burst.
|
||||
var gen1 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv");
|
||||
var published1 = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
gen1.OnDataChange += (_, e) => published1.Enqueue(e);
|
||||
await gen1.InitializeAsync("{}", CancellationToken.None);
|
||||
var adapter1 = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen1, mux));
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(Burst(("src/a", 10.0), ("src/b", 20.0)));
|
||||
published1.Count.ShouldBeGreaterThan(0);
|
||||
}, PresenceBudget);
|
||||
published1.Last().Snapshot.Value.ShouldBe(30.0);
|
||||
|
||||
// Restart: the old instance goes away; a NEW driver + adapter come up. Nothing changes upstream.
|
||||
Sys.Stop(adapter1);
|
||||
await gen1.ShutdownAsync(CancellationToken.None);
|
||||
var gen2 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv");
|
||||
var published2 = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
gen2.OnDataChange += (_, e) => published2.Enqueue(e);
|
||||
Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen2, mux)); // registers with ReplayRetained
|
||||
await gen2.InitializeAsync("{}", CancellationToken.None);
|
||||
|
||||
// The replayed inputs satisfied the change-gate; the first trigger after init is the replay itself
|
||||
// (if it lands post-init) or the next change (here: none) — so publish happens from the replay.
|
||||
AwaitAssert(() => published2.Count.ShouldBeGreaterThan(0), PresenceBudget);
|
||||
published2.Last().Snapshot.Value.ShouldBe(30.0);
|
||||
published2.Last().Snapshot.StatusCode.ShouldBe(OpcUaStatus.Good);
|
||||
}
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags;
|
||||
|
||||
/// <summary>R02:P6 — the per-failure cluster-wide <see cref="ScriptLogEntry"/> is rate-limited per tag
|
||||
/// (first N per window, then one summary); the Bad publish and the metric are unaffected.</summary>
|
||||
public sealed class VirtualTagActorFailureLogRateLimitTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Repeated_failures_publish_at_most_the_limit_then_one_summary_when_the_window_rolls()
|
||||
{
|
||||
var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
|
||||
var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => now);
|
||||
var capture = new ConcurrentQueue<ScriptLogEntry>();
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(VirtualTagActor.Props(
|
||||
"vt-chatty", "broken",
|
||||
evaluator: new FailingEvaluator("boom"),
|
||||
publisherFactory: () => new DPSPublisher((_, p) => capture.Enqueue((ScriptLogEntry)p)),
|
||||
failureLogLimiter: limiter));
|
||||
|
||||
for (var i = 0; i < 10; i++)
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", i, DateTime.UtcNow));
|
||||
|
||||
// Positive control: the FIRST failure still degrades the node (once per transition, as before).
|
||||
parent.ExpectMsg<VirtualTagActor.EvaluationResult>(PresenceBudget).StatusCode.ShouldBe(Commons.OpcUa.OpcUaStatus.BadInternalError);
|
||||
AwaitAssert(() => capture.Count.ShouldBe(3), PresenceBudget);
|
||||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
capture.Count.ShouldBe(3, "7 of 10 failures were suppressed from the cluster-wide log");
|
||||
|
||||
now = now.AddMinutes(1);
|
||||
actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 99, DateTime.UtcNow));
|
||||
|
||||
AwaitAssert(() => capture.Count.ShouldBe(5), PresenceBudget);
|
||||
var entries = capture.ToArray();
|
||||
entries[3].Message.ShouldContain("7");
|
||||
entries[3].Message.ShouldContain("suppressed");
|
||||
entries[3].VirtualTagId.ShouldBe("vt-chatty");
|
||||
entries[4].Message.ShouldBe("boom");
|
||||
}
|
||||
|
||||
private sealed class FailingEvaluator(string reason) : IVirtualTagEvaluator
|
||||
{
|
||||
public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary<string, object?> deps)
|
||||
=> VirtualTagEvalResult.Failure(reason);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user