Merge Wave-1 integration: cross-lane scripting/runtime follow-ups (#731, #734)

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 15:25:43 -04:00
35 changed files with 1121 additions and 123 deletions
@@ -0,0 +1,67 @@
using Shouldly;
using Xunit;
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests;
/// <summary>
/// <c>ctx.CancellationToken</c> — the run token, visible to scripts. <see cref="ScriptRunGuard"/> already
/// cancels the token it hands <see cref="ScriptEvaluator{TContext, TResult}.RunAsync"/> when a run times out;
/// a script that polls it can now end its own abandoned run instead of burning a pool thread until it
/// returns. Cooperative only: a script that ignores it is still an orphan.
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptCancellationTokenTests
{
/// <summary>Spins until cancelled, but ends by itself after 10 s so a broken implementation cannot pin a
/// pool thread past the test.</summary>
private const string CooperativeSpin =
"""
var end = System.DateTime.UtcNow.AddSeconds(10);
while (!ctx.CancellationToken.IsCancellationRequested && System.DateTime.UtcNow < end) { }
return ctx.CancellationToken.IsCancellationRequested ? -1 : 1;
""";
[Fact]
public async Task A_script_sees_an_uncancelled_token_on_a_normal_run()
{
using var eval = ScriptEvaluator<FakeScriptContext, bool>.Compile(
"return ctx.CancellationToken.CanBeCanceled && !ctx.CancellationToken.IsCancellationRequested;");
using var cts = new CancellationTokenSource();
(await eval.RunAsync(new FakeScriptContext(), cts.Token)).ShouldBeTrue();
}
[Fact]
public async Task Outside_a_run_the_token_is_None()
{
using var eval = ScriptEvaluator<FakeScriptContext, int>.Compile("return 1;");
var ctx = new FakeScriptContext();
(await eval.RunAsync(ctx)).ShouldBe(1);
ctx.CancellationToken.ShouldBe(CancellationToken.None, "the token is scoped to the run, not left on the context");
}
[Fact]
public async Task A_timed_out_run_whose_script_polls_the_token_ends_its_own_orphan()
{
var guard = new ScriptRunGuard();
using var eval = ScriptEvaluator<FakeScriptContext, int>.Compile(CooperativeSpin);
var ct = TestContext.Current.CancellationToken;
await Should.ThrowAsync<ScriptTimeoutException>(
() => guard.RunAsync("k", CooperativeSpin, eval, new FakeScriptContext(), TimeSpan.FromMilliseconds(100), ct));
// The guard cancelled the run token on timeout; the script saw it and returned, which frees the slot well
// inside its own 10 s fallback. Without ctx.CancellationToken the orphan would hold it for the full 10 s.
var deadline = DateTime.UtcNow.AddSeconds(3);
while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline)
await Task.Delay(20, ct);
guard.InFlightCount.ShouldBe(0, "the cooperative script ended its abandoned run");
// Positive control: the slot is free, so the next run is dispatched rather than refused. Same source
// identity on purpose: a different one would reset the key and dispatch even behind a live orphan.
using var fast = ScriptEvaluator<FakeScriptContext, int>.Compile("return 5;");
(await guard.RunAsync("k", CooperativeSpin, fast, new FakeScriptContext(), TimeSpan.FromSeconds(5), ct))
.ShouldBe(5);
}
}
@@ -0,0 +1,109 @@
using Shouldly;
using Xunit;
namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests;
/// <summary>
/// <see cref="ScriptingOptions.Sanitized"/> — the bind-time clamp of the <c>Scripting</c> section's numeric
/// knobs. An absurd value must be pulled into a sane range at bind time rather than surfacing later as an
/// <see cref="OverflowException"/> inside a script run (<c>timeout * StuckOrphanQuarantineFactor</c>).
/// </summary>
[Trait("Category", "Unit")]
public sealed class ScriptingOptionsTests
{
[Fact]
public void Defaults_pass_through_unchanged_with_no_adjustments()
{
var sanitized = new ScriptingOptions().Sanitized(out var adjustments);
adjustments.ShouldBeEmpty();
sanitized.MaxAllocBytesPerRun.ShouldBe(ScriptingOptions.DefaultMaxAllocBytesPerRun);
sanitized.QuarantineAfterConsecutiveTimeouts.ShouldBe(ScriptingOptions.DefaultQuarantineAfterConsecutiveTimeouts);
sanitized.StuckOrphanQuarantineFactor.ShouldBe(ScriptingOptions.DefaultStuckOrphanQuarantineFactor);
}
[Theory]
[InlineData(double.MaxValue, ScriptingOptions.MaxStuckOrphanQuarantineFactor)]
[InlineData(double.PositiveInfinity, ScriptingOptions.MaxStuckOrphanQuarantineFactor)]
[InlineData(1e300, ScriptingOptions.MaxStuckOrphanQuarantineFactor)]
[InlineData(double.NaN, ScriptingOptions.DefaultStuckOrphanQuarantineFactor)]
[InlineData(-5, 0)]
[InlineData(double.NegativeInfinity, 0)]
[InlineData(0, 0)]
[InlineData(2.5, 2.5)]
public void StuckOrphanQuarantineFactor_is_clamped(double configured, double expected)
{
var sanitized = new ScriptingOptions { StuckOrphanQuarantineFactor = configured }.Sanitized(out var adjustments);
sanitized.StuckOrphanQuarantineFactor.ShouldBe(expected);
// Negative already means "disabled" and is documented as such; normalising it to 0 is not worth a warning.
if (!double.IsNaN(configured) && configured >= 0 && configured <= ScriptingOptions.MaxStuckOrphanQuarantineFactor)
adjustments.ShouldBeEmpty();
}
[Theory]
[InlineData(0, 1)]
[InlineData(-3, 1)]
[InlineData(int.MaxValue, ScriptingOptions.MaxQuarantineAfterConsecutiveTimeouts)]
[InlineData(5, 5)]
public void QuarantineAfterConsecutiveTimeouts_is_clamped(int configured, int expected) =>
new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = configured }.Sanitized(out _)
.QuarantineAfterConsecutiveTimeouts.ShouldBe(expected);
[Theory]
[InlineData(-1L, 0L)]
[InlineData(0L, 0L)]
[InlineData(1024L, ScriptingOptions.MinMaxAllocBytesPerRun)]
[InlineData(long.MaxValue, long.MaxValue)]
public void MaxAllocBytesPerRun_is_clamped(long configured, long expected) =>
new ScriptingOptions { MaxAllocBytesPerRun = configured }.Sanitized(out _)
.MaxAllocBytesPerRun.ShouldBe(expected);
[Fact]
public void An_out_of_range_value_is_reported_as_an_adjustment_naming_the_key()
{
_ = new ScriptingOptions { StuckOrphanQuarantineFactor = 1e300 }.Sanitized(out var adjustments);
adjustments.ShouldHaveSingleItem().ShouldContain("Scripting:StuckOrphanQuarantineFactor");
}
[Fact]
public void Sanitized_returns_a_copy_and_leaves_the_source_untouched()
{
var source = new ScriptingOptions { StuckOrphanQuarantineFactor = 1e300 };
var sanitized = source.Sanitized(out _);
sanitized.ShouldNotBeSameAs(source);
source.StuckOrphanQuarantineFactor.ShouldBe(1e300);
}
/// <summary>
/// The guard takes options from more places than the bind (the Calculation driver's per-driver DTO, tests),
/// so it must not overflow either: an absurd factor reaching the guard directly is clamped, and a refusal
/// behind a young orphan is still a plain refusal rather than an <see cref="OverflowException"/>.
/// </summary>
[Fact]
public async Task An_absurd_factor_reaching_the_guard_directly_does_not_overflow_on_a_refusal()
{
var guard = new ScriptRunGuard(new ScriptingOptions { StuckOrphanQuarantineFactor = double.MaxValue });
const string spin =
"""
var end = System.DateTime.UtcNow.AddSeconds(3);
while (System.DateTime.UtcNow < end) { }
return 1;
""";
using var eval = ScriptEvaluator<FakeScriptContext, int>.Compile(spin);
var ct = TestContext.Current.CancellationToken;
var timeout = TimeSpan.FromMilliseconds(60);
await Should.ThrowAsync<ScriptTimeoutException>(
() => guard.RunAsync("k", spin, eval, new FakeScriptContext(), timeout, ct));
// The refusal path evaluates timeout * factor. Unclamped, TimeSpan * double.MaxValue throws OverflowException.
await Should.ThrowAsync<ScriptRunRefusedException>(
() => guard.RunAsync("k", spin, eval, new FakeScriptContext(), timeout, ct));
// A huge timeout times the clamped maximum factor must not overflow either.
await Should.ThrowAsync<ScriptRunRefusedException>(
() => guard.RunAsync("k", spin, eval, new FakeScriptContext(), TimeSpan.FromDays(365 * 1000), ct));
}
}
@@ -0,0 +1,69 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests;
/// <summary>
/// The Calculation driver's run-guard options: the node-wide <c>Scripting</c> section is the base, and a key
/// authored in the driver's own config overrides only that knob. Before the section was bound, a Calculation
/// driver always ran on the compiled defaults whatever the node configured.
/// </summary>
[Trait("Category", "Unit")]
public sealed class CalculationScriptingOptionsResolutionTests
{
private static readonly ScriptingOptions Node = new()
{
MaxAllocBytesPerRun = 8L * 1024 * 1024,
QuarantineAfterConsecutiveTimeouts = 7,
StuckOrphanQuarantineFactor = 4,
};
[Fact]
public void No_driver_keys_and_no_node_options_resolves_null_so_the_guard_uses_its_defaults() =>
CalculationDriverFactoryExtensions.ResolveScripting(null, null, null, out _).ShouldBeNull();
[Fact]
public void No_driver_keys_inherits_every_node_knob()
{
var resolved = CalculationDriverFactoryExtensions.ResolveScripting(null, null, Node, out _).ShouldNotBeNull();
resolved.MaxAllocBytesPerRun.ShouldBe(Node.MaxAllocBytesPerRun);
resolved.QuarantineAfterConsecutiveTimeouts.ShouldBe(7);
resolved.StuckOrphanQuarantineFactor.ShouldBe(4);
}
[Fact]
public void A_driver_key_overrides_only_its_own_knob_and_the_node_object_is_not_mutated()
{
var resolved = CalculationDriverFactoryExtensions.ResolveScripting(null, 2, Node, out var adjustments).ShouldNotBeNull();
resolved.QuarantineAfterConsecutiveTimeouts.ShouldBe(2);
resolved.MaxAllocBytesPerRun.ShouldBe(Node.MaxAllocBytesPerRun, "the unauthored key keeps the node value");
resolved.StuckOrphanQuarantineFactor.ShouldBe(4, "the stuck factor has no driver key; it is node-wide");
resolved.ShouldNotBeSameAs(Node);
Node.QuarantineAfterConsecutiveTimeouts.ShouldBe(7);
adjustments.ShouldBeEmpty();
}
/// <summary>Review m-2: an out-of-range value authored in the driver's own config is clamped AND reported, so
/// the factory can log it rather than the guard changing it silently.</summary>
[Fact]
public void An_out_of_range_driver_key_is_clamped_and_reported()
{
var resolved = CalculationDriverFactoryExtensions.ResolveScripting(1024L, 0, Node, out var adjustments).ShouldNotBeNull();
resolved.MaxAllocBytesPerRun.ShouldBe(ScriptingOptions.MinMaxAllocBytesPerRun);
resolved.QuarantineAfterConsecutiveTimeouts.ShouldBe(1);
adjustments.Count.ShouldBe(2);
}
[Fact]
public void A_driver_key_with_no_node_options_layers_on_the_compiled_defaults()
{
var resolved = CalculationDriverFactoryExtensions.ResolveScripting(0L, null, null, out _).ShouldNotBeNull();
resolved.MaxAllocBytesPerRun.ShouldBe(0);
resolved.QuarantineAfterConsecutiveTimeouts.ShouldBe(ScriptingOptions.DefaultQuarantineAfterConsecutiveTimeouts);
}
}
@@ -57,10 +57,11 @@ public class PageAuthorizationGuardTests
// AddSecretsAuthorization(). ──
[typeof(SecretsAdmin)] = global::ZB.MOM.WW.Secrets.Ui.SecretsAuthorization.ManagePolicy,
// ── AuthenticatedRead (16): dashboards, lists, live tails (read-only) + the dev ContextMenu demo ──
// ── AuthenticatedRead (17): dashboards, lists, live tails (read-only) + the dev ContextMenu demo ──
[typeof(Home)] = AdminUiPolicies.AuthenticatedRead,
[typeof(global::ZB.MOM.WW.OtOpcUa.AdminUI.Components.Pages.Dev.ContextMenuDemo)] = AdminUiPolicies.AuthenticatedRead,
[typeof(Fleet)] = AdminUiPolicies.AuthenticatedRead,
[typeof(global::ZB.MOM.WW.OtOpcUa.AdminUI.Components.Pages.FleetDiagnostics)] = AdminUiPolicies.AuthenticatedRead,
[typeof(global::ZB.MOM.WW.OtOpcUa.AdminUI.Components.Pages.Hosts)] = AdminUiPolicies.AuthenticatedRead,
[typeof(Alerts)] = AdminUiPolicies.AuthenticatedRead,
[typeof(ScriptLog)] = AdminUiPolicies.AuthenticatedRead,
@@ -18,6 +18,7 @@ public sealed class CompletionTests
labels.ShouldContain("SetVirtualTag");
labels.ShouldContain("Now");
labels.ShouldContain("Logger");
labels.ShouldContain("CancellationToken");
}
[Fact] public async Task Scope_completion_includes_the_ctx_local()
@@ -48,6 +48,7 @@ public sealed class DiagnoseTests
[InlineData("""var x = 41; System.Threading.Interlocked.Increment(ref x); return x;""")]
[InlineData("""var x = 42; return System.Threading.Volatile.Read(ref x);""")]
[InlineData("""object o = 42; return o.GetType().IsValueType ? 1 : 0;""")]
[InlineData("""ctx.CancellationToken.ThrowIfCancellationRequested(); return ctx.CancellationToken.IsCancellationRequested ? 1 : 0;""")]
public void Allow_listed_forms_are_not_flagged_in_the_editor(string code)
=> Diag(code).ShouldBeEmpty();
@@ -0,0 +1,64 @@
using Microsoft.Extensions.Configuration;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration;
/// <summary>
/// Review m-1: the <c>Scripting</c> section binds by the documented key names and is clamped at bind. Each
/// configured value differs from the compiled default, so a misspelled key (which the binder would silently
/// ignore, falling back to the default) goes red.
/// </summary>
public sealed class ScriptingOptionsBindingTests
{
private static IConfiguration Config(Dictionary<string, string?> values) =>
new ConfigurationBuilder().AddInMemoryCollection(values).Build();
[Fact]
public void Every_documented_key_binds_to_its_property()
{
var bound = ScriptingOptionsBinding.Bind(Config(new()
{
["Scripting:MaxAllocBytesPerRun"] = "8388608",
["Scripting:QuarantineAfterConsecutiveTimeouts"] = "7",
["Scripting:StuckOrphanQuarantineFactor"] = "4.5",
// Sibling keys owned by the script-log pipeline must not disturb the bind.
["Scripting:LogFilePath"] = "logs/x-.log",
["Scripting:LogTopicMinLevel"] = "Warning",
}), out var adjustments);
adjustments.ShouldBeEmpty();
bound.MaxAllocBytesPerRun.ShouldBe(8L * 1024 * 1024);
bound.QuarantineAfterConsecutiveTimeouts.ShouldBe(7);
bound.StuckOrphanQuarantineFactor.ShouldBe(4.5);
}
[Fact]
public void Out_of_range_values_are_clamped_at_bind_and_reported()
{
var bound = ScriptingOptionsBinding.Bind(Config(new()
{
["Scripting:StuckOrphanQuarantineFactor"] = "1e300",
["Scripting:MaxAllocBytesPerRun"] = "1024",
["Scripting:QuarantineAfterConsecutiveTimeouts"] = "0",
}), out var adjustments);
bound.StuckOrphanQuarantineFactor.ShouldBe(ScriptingOptions.MaxStuckOrphanQuarantineFactor);
bound.MaxAllocBytesPerRun.ShouldBe(ScriptingOptions.MinMaxAllocBytesPerRun);
bound.QuarantineAfterConsecutiveTimeouts.ShouldBe(1);
adjustments.Count.ShouldBe(3);
}
[Fact]
public void An_absent_section_yields_the_compiled_defaults()
{
var bound = ScriptingOptionsBinding.Bind(Config(new()), out var adjustments);
adjustments.ShouldBeEmpty();
bound.MaxAllocBytesPerRun.ShouldBe(ScriptingOptions.DefaultMaxAllocBytesPerRun);
bound.QuarantineAfterConsecutiveTimeouts.ShouldBe(ScriptingOptions.DefaultQuarantineAfterConsecutiveTimeouts);
bound.StuckOrphanQuarantineFactor.ShouldBe(ScriptingOptions.DefaultStuckOrphanQuarantineFactor);
}
}
@@ -0,0 +1,103 @@
using System.Text.Json;
using Akka.Actor;
using Serilog;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Core.Scripting;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Review I-2: under the factory Props a supervision restart of the scripted-alarm host builds a fresh, EMPTY
/// engine. Nothing re-sent <c>ApplyScriptedAlarms</c>, so until the next deploy the host had no mux interest
/// (inputs stopped) and its ownership filter dropped every operator ack/shelve as "not owned". The host now
/// tells its parent <see cref="ScriptedAlarms.ScriptedAlarmHostActor.ScriptedAlarmHostRestarted"/> from <c>PostRestart</c>,
/// and <see cref="DriverHostActor"/> re-sends the last plans it applied. This test never re-applies by hand.
/// </summary>
public sealed class DriverHostActorScriptedAlarmRestartTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-alarmhost-restart-test");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('d', 64));
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30);
private static DeploymentId SeedAlarmDeployment(Microsoft.EntityFrameworkCore.IDbContextFactory<ZB.MOM.WW.OtOpcUa.Configuration.OtOpcUaConfigDbContext> db)
{
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
{
ScriptedAlarms = new[]
{
new
{
ScriptedAlarmId = "alm-1", EquipmentId = "eq-1", Name = "HighTemp", AlarmType = "AlarmCondition",
Severity = 800, MessageTemplate = "hot", PredicateScriptId = "s-1",
HistorizeToAveva = false, Retain = true, Enabled = true,
},
},
Scripts = new[] { new { ScriptId = "s-1", SourceCode = "return (int)ctx.GetTag(\"M.T\").Value > 90;" } },
});
var id = DeploymentId.NewId();
using var ctx = db.CreateDbContext();
ctx.Deployments.Add(new Deployment
{
DeploymentId = id.Value, RevisionHash = RevA.Value, Status = DeploymentStatus.Sealed,
CreatedBy = "test", SealedAtUtc = DateTime.UtcNow, ArtifactBlob = artifact,
});
ctx.SaveChanges();
return id;
}
[Fact]
public void After_a_supervision_restart_the_alarm_host_is_re_applied_and_inputs_and_acks_work_again()
{
var db = NewInMemoryDbFactory();
var deploymentId = SeedAlarmDeployment(db);
var publish = this.CreateCleanOutcomePublishProbe();
var mux = CreateTestProbe();
var coordinator = CreateTestProbe();
var host = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
localRoles: new HashSet<string> { "driver" },
dependencyMux: mux.Ref,
opcUaPublishActor: publish.Ref,
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
virtualTagHostOverride: CreateTestProbe().Ref,
scriptRootLogger: new ScriptRootLogger(new LoggerConfiguration().CreateLogger()),
alarmStateStore: new InMemoryAlarmStateStore()));
V3ArtifactSeeder.Dispatch(host, coordinator, deploymentId, RevA);
var first = mux.FishForMessage<DependencyMuxActor.RegisterInterest>(r => r.TagRefs.Contains("M.T"), Timeout);
var alarmHost = first.Subscriber;
// The crash: a batch item with a null TagId makes the upstream Push throw (the documented unguarded
// path), so supervision restarts the alarm host. NO manual re-apply follows.
alarmHost.Tell(new VirtualTagActor.DependencyValueBatch(new[]
{
new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow),
}));
mux.FishForMessage<DependencyMuxActor.UnregisterInterest>(u => u.Subscriber.Equals(alarmHost), Timeout);
// The parent re-applies on the restart signal: interest is registered again by the new incarnation.
mux.FishForMessage<DependencyMuxActor.RegisterInterest>(r => r.TagRefs.Contains("M.T"), Timeout);
// Inputs work: a dependency push activates the alarm.
alarmHost.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(
m => m.AlarmNodeId == "alm-1" && m.State.Active && !m.State.Acknowledged, Timeout);
// Acks work: the restarted engine owns alm-1 again, so the command is not filtered out as unowned.
alarmHost.Tell(new AlarmCommand(AlarmId: "alm-1", Operation: "Acknowledge", User: "alice", Comment: null, UnshelveAtUtc: null));
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(
m => m.AlarmNodeId == "alm-1" && m.State.Acknowledged, Timeout)
.State.Acknowledged.ShouldBeTrue();
}
}
@@ -0,0 +1,224 @@
using Akka.Actor;
using Akka.TestKit;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// The dependency mux retains a ref's last value for replay-opting subscribers and evicts a driver's values on
/// <see cref="DependencyMuxActor.SourceRemoved"/>. Before this, nothing in production sent that message, so a
/// driver removed by a deploy kept its last Good in the mux and replayed it to the next replay-opting
/// subscriber as if it were current. The host now sends it wherever a driver OBJECT ends (deploy removal,
/// config-change respawn, operator Restart), never on a supervision restart, and drops value bursts from the
/// child incarnations it has already stopped so a late burst cannot re-retain what was evicted. All through a
/// REAL mux and real driver children.
/// </summary>
public sealed class DriverHostActorSourceRemovedTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-sourceremoved-test");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
private const string Path1 = "Plant/Modbus/dev1/speed";
private const string Path2 = "Plant/Modbus2/dev2/flow";
private static readonly (string, string, string, string, string?)[] NoRefs = [];
private static readonly (string Driver, string DriverName, string Device, string Tag, string DataType, bool Writable)[] BothDrivers =
[
("DI-1", "Modbus", "dev1", "speed", "Double", false),
("DI-2", "Modbus2", "dev2", "flow", "Double", false),
];
private IActorRef _host = ActorRefs.Nobody;
private IActorRef _mux = ActorRefs.Nobody;
private TestProbe _coordinator = null!;
private TestProbe _holder = null!;
private RecordingSubscribableDriverFactory _factory = null!;
private IDbContextFactory<OtOpcUaConfigDbContext> _db = null!;
/// <summary>Spawns a host over a real mux with two enabled drivers, waits for both children to subscribe,
/// registers a replay-opting holder (so the mux retains both refs) and publishes one value per driver.</summary>
private void ArrangeTwoDriversWithRetainedValues()
{
_db = NewInMemoryDbFactory();
var revA = V3ArtifactSeeder.SeedV3Deployment(_db, RevA, BothDrivers, NoRefs, enabled: true);
_factory = new RecordingSubscribableDriverFactory("Modbus");
_mux = Sys.ActorOf(DependencyMuxActor.Props());
_coordinator = CreateTestProbe();
_holder = CreateTestProbe();
_host = Sys.ActorOf(DriverHostActor.Props(
_db, TestNode, _coordinator.Ref,
driverFactory: _factory,
localRoles: new HashSet<string> { "driver" },
dependencyMux: _mux,
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
virtualTagHostOverride: CreateTestProbe().Ref));
V3ArtifactSeeder.Dispatch(_host, _coordinator, revA, RevA);
AwaitAssert(() => _factory.Created.Count.ShouldBe(2), PresenceBudget);
// The spawn hands a child the PREVIOUS apply's set (empty on the first deploy) and the bulk push then
// re-subscribes with the real one; a value fired before that lands is lost, so wait for the real set.
AwaitSubscribed("DI-1", Path1);
AwaitSubscribed("DI-2", Path2);
_mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { Path1, Path2 }, _holder.Ref, ReplayRetained: true));
Driver("DI-1").FireDataChange(Path1, 11.0);
Driver("DI-2").FireDataChange(Path2, 22.0);
// One fish for both: the two drivers' bursts arrive in either order, and fishing for one at a time would
// discard the other.
var pending = new HashSet<string>(StringComparer.Ordinal) { Path1, Path2 };
_holder.FishForMessage<VirtualTagActor.DependencyValueBatch>(b =>
{
foreach (var i in b.Items)
if ((i.TagId == Path1 && Equals(i.Value, 11.0)) || (i.TagId == Path2 && Equals(i.Value, 22.0)))
pending.Remove(i.TagId);
return pending.Count == 0;
}, PresenceBudget);
}
/// <summary>Waits until the holder has been delivered <paramref name="value"/> for <paramref name="path"/>.</summary>
private void AwaitHolder(string path, object value) =>
_holder.FishForMessage<VirtualTagActor.DependencyValueBatch>(
b => b.Items.Any(i => i.TagId == path && Equals(i.Value, value)), PresenceBudget);
private void AwaitSubscribed(string driverId, string path) =>
AwaitAssert(() => (Driver(driverId).LastSubscribedRefs ?? []).ShouldContain(path), PresenceBudget);
private RecordingSubscribableDriver Driver(string id) => _factory.Created.Last(d => d.DriverInstanceId == id);
/// <summary>A fresh replay-opting subscriber for both refs; returns what it was replayed, keyed by ref.</summary>
private Dictionary<string, VirtualTagActor.DependencyValueChanged> Replayed()
{
var probe = CreateTestProbe();
_mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { Path1, Path2 }, probe.Ref, ReplayRetained: true));
return probe.ExpectMsg<VirtualTagActor.DependencyValueBatch>(PresenceBudget)
.Items.ToDictionary(i => i.TagId, StringComparer.Ordinal);
}
/// <summary>The live child actor for <paramref name="driverId"/> (named <c>drv-{id}-g{generation}</c>).</summary>
private IActorRef ChildOf(string driverId)
{
var probe = CreateTestProbe();
Sys.ActorSelection(_host.Path.ToStringWithoutAddress() + "/drv-" + driverId + "-g*").Tell(new Identify(driverId), probe.Ref);
return probe.ExpectMsg<ActorIdentity>(PresenceBudget).Subject.ShouldNotBeNull();
}
private static void ShouldNotBeTheOldGood(Dictionary<string, VirtualTagActor.DependencyValueChanged> replay, string path, object oldValue)
{
if (!replay.TryGetValue(path, out var item)) return;
(Equals(item.Value, oldValue) && OpcUaStatus.IsGood(item.StatusCode))
.ShouldBeFalse($"the stopped incarnation's last Good for {path} was replayed as current");
}
[Fact]
public void A_driver_removed_by_a_deploy_is_no_longer_replayed_while_a_still_present_driver_is()
{
ArrangeTwoDriversWithRetainedValues();
// Positive control before the removal: both values replay.
Replayed().Keys.ShouldBe(new[] { Path1, Path2 }, ignoreOrder: true);
// Deploy B drops DI-1 (StopChild).
var revB = V3ArtifactSeeder.SeedV3Deployment(_db, RevB, [("DI-2", "Modbus2", "dev2", "flow", "Double", false)], NoRefs, enabled: true);
V3ArtifactSeeder.Dispatch(_host, _coordinator, revB, RevB);
AwaitAssert(
() => Replayed().Keys.ToArray().ShouldBe(new[] { Path2 }, "DI-1 was removed: its last Good must not replay; DI-2's must"),
PresenceBudget);
}
/// <summary>Review I-1: a burst the removed child sent after the eviction (it stops asynchronously, so it keeps
/// publishing through the rest of the apply) must be dropped by the host, not forwarded to re-retain the value.</summary>
[Fact]
public void A_late_burst_from_a_removed_child_does_not_re_retain_its_value()
{
ArrangeTwoDriversWithRetainedValues();
var oldChild = ChildOf("DI-1");
// Hold the removed child in its PostStop so it has NOT terminated when the stale burst lands; a terminated
// actor cannot send, and the host forgets a retired child on Terminated.
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
Driver("DI-1").ShutdownGate = gate;
var revB = V3ArtifactSeeder.SeedV3Deployment(_db, RevB, [("DI-2", "Modbus2", "dev2", "flow", "Double", false)], NoRefs, enabled: true);
V3ArtifactSeeder.Dispatch(_host, _coordinator, revB, RevB);
// The stale burst, sent AS the stopped child.
_host.Tell(ValuePublishes.Single("DI-1", Path1, 11.0, OpcUaStatus.Good, DateTime.UtcNow), oldChild);
// Positive control / ordering fence: a live driver's value sent after it is delivered, so the stale burst
// has been processed by the host by the time we look.
Driver("DI-2").FireDataChange(Path2, 23.0);
AwaitHolder(Path2, 23.0);
Replayed().Keys.ToArray().ShouldBe(new[] { Path2 }, "the removed child's late burst re-retained its last Good");
gate.TrySetResult();
}
/// <summary>Review m-3: a config edit respawns the driver (stop + spawn); the OLD object's value is evicted, and
/// the NEW driver's value replays (positive control).</summary>
[Fact]
public void A_config_change_respawn_evicts_the_old_value_and_the_new_drivers_value_replays()
{
ArrangeTwoDriversWithRetainedValues();
// Adding a tag to DI-1 changes its merged DriverConfig, which respawns it.
var revB = V3ArtifactSeeder.SeedV3Deployment(_db, RevB,
[.. BothDrivers, ("DI-1", "Modbus", "dev1", "temp", "Double", false)], NoRefs, enabled: true);
V3ArtifactSeeder.Dispatch(_host, _coordinator, revB, RevB);
AwaitAssert(() => _factory.Created.Count.ShouldBe(3), PresenceBudget);
var respawned = Driver("DI-1");
// The spawn hands the child the PREVIOUS apply's set, then the bulk push re-subscribes with the new one
// (which names the added tag); a value fired between the two can be lost, so wait for the final set.
AwaitAssert(() => (respawned.LastSubscribedRefs ?? []).ShouldContain("Plant/Modbus/dev1/temp"), PresenceBudget);
ShouldNotBeTheOldGood(Replayed(), Path1, 11.0);
respawned.FireDataChange(Path1, 33.0);
AwaitHolder(Path1, 33.0);
var replay = Replayed();
replay[Path1].Value.ShouldBe(33.0, "the respawned driver's own value replays");
replay[Path2].Value.ShouldBe(22.0, "the untouched driver is unaffected");
}
/// <summary>Review m-3: an operator Restart builds a new driver object; the old one's value is evicted and the
/// new one's replays.</summary>
[Fact]
public void An_operator_restart_evicts_the_old_value_and_the_new_drivers_value_replays()
{
ArrangeTwoDriversWithRetainedValues();
_host.Tell(new RestartDriver("c1", "DI-1", "tester", Guid.NewGuid()));
AwaitAssert(() => _factory.Created.Count.ShouldBe(3), PresenceBudget);
var restarted = Driver("DI-1");
AwaitSubscribed("DI-1", Path1);
ShouldNotBeTheOldGood(Replayed(), Path1, 11.0);
restarted.FireDataChange(Path1, 44.0);
AwaitHolder(Path1, 44.0);
Replayed()[Path1].Value.ShouldBe(44.0);
}
[Fact]
public void A_supervision_restart_does_not_evict_the_drivers_retained_value()
{
ArrangeTwoDriversWithRetainedValues();
var di1 = Driver("DI-1");
// Crash DI-1's child: Akka restarts it around the SAME driver object, and the host reprovisions it.
di1.FireMalformedDataChange(Path1);
AwaitAssert(() => di1.InitializeCount.ShouldBe(2), PresenceBudget);
_factory.Created.Count.ShouldBe(2, "a supervision restart re-uses the driver; no new one is created");
var replay = Replayed();
replay.Keys.ShouldBe(new[] { Path1, Path2 }, ignoreOrder: true,
"a restart is the same driver instance: its last value stays replayable");
}
}
@@ -110,7 +110,7 @@ public sealed class DriverInstanceActorBatchForwardTests : RuntimeActorTestBase
batch.Items.Count.ShouldBe(2);
// The per-tag raises for the SAME burst produced nothing extra — no second batch, no
// AttributeValuePublished. Anything here means both handlers were attached.
// single-item batch. Anything here means both handlers were attached.
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(400));
}
@@ -279,11 +279,18 @@ internal sealed class RecordingSubscribableDriver : IDriver, ISubscribable, IAla
/// <inheritdoc />
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
/// <summary>
/// When set, <see cref="ShutdownAsync"/> awaits it (the actor bounds the wait), which keeps a stopping child
/// ALIVE in its PostStop for as long as the test wants — the window in which a stopped child can still
/// have messages reaching its parent.
/// </summary>
public TaskCompletionSource? ShutdownGate { get; set; }
/// <inheritdoc />
public Task ShutdownAsync(CancellationToken cancellationToken)
{
Interlocked.Increment(ref _shutdownCount);
return Task.CompletedTask;
return ShutdownGate?.Task ?? Task.CompletedTask;
}
/// <inheritdoc />
@@ -5,8 +5,8 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
/// <summary>
/// Builds the ONE message shape a driver child sends for its values — a
/// <see cref="DriverInstanceActor.AttributeValueBatchPublished"/> — for tests that push a single value into the
/// dependency mux by hand. The per-value <see cref="DriverInstanceActor.AttributeValuePublished"/> has no
/// production sender (02/U4); tests should not keep one alive either.
/// dependency mux by hand. There is no per-value message: the former per-value record had no production
/// sender and was deleted (02/U4).
/// </summary>
public static class ValuePublishes
{
@@ -19,7 +19,7 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Historian;
/// <remarks>
/// Adapted from the plan's Task 17: the recorder handles the REAL fan-out message the mux emits —
/// <see cref="VirtualTagActor.DependencyValueChanged"/> (TagId/Value/TimestampUtc/StatusCode) —
/// not <c>DriverInstanceActor.AttributeValuePublished</c>. The typed-value / real-status behaviour is
/// not the driver-side <c>DriverInstanceActor.AttributeValueBatchPublished</c>. The typed-value / real-status behaviour is
/// pinned in <see cref="ContinuousHistorizationRecorderTypedValueTests"/>.
/// </remarks>
public sealed class ContinuousHistorizationRecorderTests : TestKit
@@ -10,9 +10,9 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags;
public sealed class DependencyMuxActorTests : RuntimeActorTestBase
{
/// <summary>Verifies that AttributeValuePublished messages are forwarded only to subscribers interested in the specific reference.</summary>
/// <summary>Verifies that a published value is forwarded only to subscribers interested in the specific reference.</summary>
[Fact]
public void AttributeValuePublished_is_forwarded_only_to_subscribers_interested_in_that_ref()
public void A_published_value_is_forwarded_only_to_subscribers_interested_in_that_ref()
{
var mux = Sys.ActorOf(DependencyMuxActor.Props());
var subA = CreateTestProbe();
@@ -21,9 +21,9 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "tag-1", "tag-2" }, subA.Ref));
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "tag-2", "tag-3" }, subB.Ref));
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "tag-1", 10, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "tag-3", 30, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "tag-2", 20, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "tag-1", 10, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "tag-3", 30, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "tag-2", 20, OpcUaStatus.Good, DateTime.UtcNow));
// The mux emits ONLY batches — the per-value comms-loss ingress fans out as a single-item batch.
// subA hears tag-1 + tag-2.
@@ -54,7 +54,7 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase
var sub = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "tag-1" }, sub.Ref));
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "nobody-cares", 99, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "nobody-cares", 99, OpcUaStatus.Good, DateTime.UtcNow));
sub.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
@@ -67,11 +67,11 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase
var sub = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "tag-1" }, sub.Ref));
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "tag-1", 10, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "tag-1", 10, OpcUaStatus.Good, DateTime.UtcNow));
sub.ExpectMsg<VirtualTagActor.DependencyValueBatch>();
mux.Tell(new DependencyMuxActor.UnregisterInterest(sub.Ref));
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "tag-1", 20, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "tag-1", 20, OpcUaStatus.Good, DateTime.UtcNow));
sub.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
@@ -85,10 +85,10 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "tag-1" }, sub.Ref));
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "tag-2" }, sub.Ref)); // replaces tag-1
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "tag-1", 10, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "tag-1", 10, OpcUaStatus.Good, DateTime.UtcNow));
sub.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "tag-2", 20, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "tag-2", 20, OpcUaStatus.Good, DateTime.UtcNow));
sub.ExpectMsg<VirtualTagActor.DependencyValueBatch>().Items.ShouldHaveSingleItem().TagId.ShouldBe("tag-2");
}
@@ -110,18 +110,18 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase
// gets dropped (no subscriber for "ref-a" yet), so we re-publish each pass.
AwaitAssert(() =>
{
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
mux.Tell(ValuePublishes.Single(
"driver-1", "ref-a", 10, OpcUaStatus.Good, DateTime.UtcNow));
parent.ExpectMsg<VirtualTagActor.EvaluationResult>(TimeSpan.FromMilliseconds(200))
.Value.ShouldBe(10);
}, duration: PresenceBudget);
// From here the actor is wired — second publish drives the sum.
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "ref-b", 32, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "ref-b", 32, OpcUaStatus.Good, DateTime.UtcNow));
parent.ExpectMsg<VirtualTagActor.EvaluationResult>(PresenceBudget).Value.ShouldBe(42);
// Unrelated ref shouldn't fire eval.
mux.Tell(new DriverInstanceActor.AttributeValuePublished("driver-1", "ref-unrelated", 99, OpcUaStatus.Good, DateTime.UtcNow));
mux.Tell(ValuePublishes.Single("driver-1", "ref-unrelated", 99, OpcUaStatus.Good, DateTime.UtcNow));
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
@@ -221,7 +221,7 @@ public sealed class DependencyMuxActorTests : RuntimeActorTestBase
var sub = CreateTestProbe();
mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "tag-1" }, sub.Ref));
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
mux.Tell(ValuePublishes.Single(
"driver-1", "tag-1", 10, OpcUaStatus.Bad, DateTime.UtcNow));
sub.ExpectMsg<VirtualTagActor.DependencyValueBatch>()