using System.Text.Json;
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
///
/// Issue #485 (driver-side half) — an artifact the host could not obtain must not be mistaken for a
/// configuration that contains nothing.
///
///
///
/// ReconcileDrivers and PushDesiredSubscriptions both read the artifact as
/// …FirstOrDefault() ?? Array.Empty<byte>(). The throw path in each already
/// returns early, but a row that is absent (or carries a zero-length blob) yields empty bytes that
/// flow onwards as a real answer: zero driver specs, so DriverSpawnPlanner plans every
/// running child for StopChild, and then an empty desired-subscription set drops each
/// surviving driver's live handle. A node loses its entire field I/O and still ACKs Applied.
///
///
/// The same reasoning as the address-space half applies: nothing legitimate produces a zero-length
/// blob — an operator who really deletes every driver still deploys a JSON document with empty
/// arrays — so "no bytes" can only mean the read did not answer. Seeding a row whose
/// ArtifactBlob is empty reproduces exactly the bytes the missing-row case delivers, without
/// needing to race a delete against the two reads.
///
///
public sealed class DriverHostActorUnreadableArtifactTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
/// How long to let post-ACK subscription traffic settle before asserting it did NOT happen.
private static readonly TimeSpan SettleWindow = TimeSpan.FromSeconds(2);
private const string SpeedRef = "Plant/Modbus/dev1/speed";
/// A dispatch whose artifact reads back as no bytes must leave the running drivers — and their
/// live subscriptions — exactly as they were.
[Fact]
public void Dispatch_whose_artifact_reads_back_empty_keeps_the_drivers_and_their_subscriptions()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
// Baseline: the child is up and subscribed to its one raw tag.
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1");
// The next dispatch's artifact comes back as no bytes at all.
actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout);
// The ACK precedes the SubscribeBulk pass, and the child's unsubscribe is a further async self-tell,
// so settle before asserting an ABSENCE. SettleWindow is calibrated by
// , which observes the very same
// unsubscribe ARRIVING well inside it — without that control this assertion would pass on a race.
AskDiagnostics(actor); // ordering barrier through the host's mailbox
Thread.Sleep(SettleWindow);
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // not stopped
factory.UnsubscribeCount.ShouldBe(0); // handle not dropped
factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef);
}
///
/// Positive control for the subscription half: a READABLE artifact that drops the driver's last tag
/// genuinely does tear the live subscription down, and does so well within
/// — without this, the absence asserted above would pass on a race.
/// The observable changed with #516. Dropping a tag changes the driver's
/// DriverConfig, which used to be an in-place delta: the child received an empty desired set
/// and called UnsubscribeAsync. A config change is now a stop + respawn, so the old child is
/// STOPPED (taking its subscription with it via ShutdownAsync) and a fresh one spawns with
/// nothing to subscribe to. The teardown is therefore observed as a shutdown, not an unsubscribe —
/// the same event, reached by a different route.
///
[Fact]
public void Dropping_a_drivers_last_tag_does_clear_its_subscription()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
factory.ShutdownCount.ShouldBe(0);
actor.Tell(new DispatchDeployment(SeedTaglessDriverDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout);
AwaitAssert(() => factory.ShutdownCount.ShouldBeGreaterThanOrEqualTo(1), duration: SettleWindow);
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // respawned under the same id
}
/// Positive control: the guard keys on "the read gave us nothing", not on "the new config has
/// fewer drivers". A READABLE artifact that genuinely drops the driver still stops it — otherwise the
/// test above would pass just as happily against a host that had stopped reconciling altogether.
[Fact]
public void Dispatch_whose_readable_artifact_genuinely_drops_the_driver_still_stops_it()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
// A real artifact that simply carries no drivers — an operator deleting the last driver.
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout);
AwaitAssert(
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
duration: Timeout);
}
///
/// Issue #486 — an apply that read no artifact applied nothing, so it must NOT be reported as
/// Applied. Reporting success also advanced _currentRevision, and because
/// HandleDispatchFromSteady short-circuits on a revision match, the node could never be
/// healed by re-dispatching that revision.
///
[Fact]
public void Dispatch_whose_artifact_reads_back_empty_acks_Failed_and_leaves_the_revision_alone()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevA);
actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
var ack = coordinator.ExpectMsg(Timeout);
ack.Outcome.ShouldBe(ApplyAckOutcome.Failed);
ack.FailureReason.ShouldNotBeNullOrWhiteSpace();
var snapshot = AskDiagnostics(actor);
snapshot.CurrentRevision.ShouldBe(RevA); // never applied ⇒ never claimed
snapshot.Drivers.Select(d => d.Name).ShouldContain("drv-1"); // …and #485 still holds
}
/// Issue #486 — the point of failing the apply: the SAME revision can be dispatched again and
/// now actually applies, instead of being waved through by the "already at this rev" short-circuit. The
/// recovered artifact adds a second driver, so a short-circuited ACK (which has no side effects) cannot
/// satisfy this test — drv-2 only appears if the artifact was genuinely re-read and reconciled.
[Fact]
public void A_failed_apply_is_retried_not_short_circuited_once_the_artifact_is_readable_again()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
var deploymentId = SeedUnreadableDeployment(db, RevB);
actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Failed);
// The ConfigDb recovers: the row now carries a real artifact, under the SAME revision.
SetArtifact(db, deploymentId, TwoDriverArtifact());
actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
AwaitAssert(
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-2"),
duration: Timeout);
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevB);
}
///
/// #609 — removing a driver instance from the plan evicts its cached telemetry snapshots, so a new
/// subscriber is no longer primed with a decommissioned driver's last Health/Resilience. The
/// surviving instance's snapshot is the positive control: eviction is per-instance, not a wipe.
///
[Fact]
public void Removing_a_driver_instance_evicts_its_telemetry_snapshots_but_not_a_neighbours()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var hub = new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub();
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
driverFactory: factory,
localRoles: new HashSet { "driver" },
telemetryHub: hub));
actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
hub.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health(new(
"c1", "drv-1", "Healthy", null, null, 0, DateTime.UtcNow)));
hub.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health(new(
"c1", "other-node-driver", "Healthy", null, null, 0, DateTime.UtcNow)));
// Control: before the removal, drv-1 replays.
using (var before = hub.Subscribe(16))
{
var seen = new List();
while (before.Reader.TryRead(out var i))
if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health h) seen.Add(h.E.DriverInstanceId);
seen.ShouldContain("drv-1");
}
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
AwaitAssert(
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
duration: Timeout);
using var after = hub.Subscribe(16);
var replayed = new List();
while (after.Reader.TryRead(out var i))
if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health h) replayed.Add(h.E.DriverInstanceId);
replayed.ShouldNotContain("drv-1");
replayed.ShouldContain("other-node-driver");
}
///
/// #609 (review R1 M1) — the Resilience half of the eviction must SURVIVE the next resilience-publisher
/// tick. DriverResilienceStatusPublisherService re-emits tracker.Snapshot() into the hub
/// every interval, so evicting only the hub cache was undone within one tick: the host must also
/// release the instance's tracker state. The tick below is the publisher's own body
/// (BuildMessages → hub.Emit) minus the DPS publish, run against the REAL factory +
/// tracker the Host wires. The neighbour instance is the positive control: it is still emitted by
/// the same tick, so an absent drv-1 is not an empty tick.
///
[Fact]
public void Removing_a_driver_instance_releases_its_resilience_state_so_the_next_publisher_tick_omits_it()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var hub = new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub();
var tracker = new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResilienceStatusTracker();
// The child's subscribe is HELD across the removal so the tracker writes its capability invoker makes
// around it land AFTER StopChild's Release — the late writer that resurrected the entry (#609). Without
// the gate this test only failed when the child's in-flight SubscribeBulk happened to finish between the
// removal and the second tick (~4/6 runs); the gate forces that interleaving on every run.
var invokerFactory = new GatedSubscribeInvokerFactory(
new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverCapabilityInvokerFactory(
new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResiliencePipelineBuilder(statusTracker: tracker),
tracker,
_ => ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTier.A));
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
driverFactory: factory,
localRoles: new HashSet { "driver" },
invokerFactory: invokerFactory,
telemetryHub: hub));
actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
invokerFactory.SubscribeEntered.Wait(Timeout).ShouldBeTrue("the child never reached its subscribe");
// Resilience counters for the running driver and for an instance this host does not own.
tracker.RecordBreakerOpen("drv-1", "dev1", DateTime.UtcNow);
tracker.RecordFailure("neighbour-drv", "dev9", DateTime.UtcNow);
PublisherTick(tracker, hub);
ReplayedResilienceIds(hub).ShouldContain("drv-1"); // control: drv-1 is primed before the removal
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
AwaitAssert(
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
duration: Timeout);
// Let the removed child's held subscribe run to completion through the REAL invoker (RecordCallStart /
// RecordSuccess / RecordCallComplete). Positive control: the late write has demonstrably happened before
// the tick below, so the absence asserted after it is not "nothing has arrived yet".
invokerFactory.OpenGate();
invokerFactory.SubscribeCompleted.Wait(Timeout).ShouldBeTrue("the held subscribe never completed");
// The publisher's next tick, after the removal AND after the late writer.
PublisherTick(tracker, hub);
var replayed = ReplayedResilienceIds(hub);
replayed.ShouldNotContain("drv-1");
replayed.ShouldContain("neighbour-drv");
}
///
/// #609 — the release a removal makes must not outlive the removal: when the SAME instance id comes back
/// in a later deployment, its fresh invoker records resilience state again (the tracker un-retires the id
/// on Create).
///
[Fact]
public void A_driver_instance_re_added_after_removal_records_resilience_state_again()
{
var db = NewInMemoryDbFactory();
var factory = new SubscribingDriverFactory("Modbus");
var tracker = new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResilienceStatusTracker();
var invokerFactory = new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverCapabilityInvokerFactory(
new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResiliencePipelineBuilder(statusTracker: tracker),
tracker,
_ => ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTier.A);
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
driverFactory: factory,
localRoles: new HashSet { "driver" },
invokerFactory: invokerFactory,
telemetryHub: new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub()));
actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
AwaitAssert(
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
duration: Timeout);
var revC = RevisionHash.Parse(new string('c', 64));
actor.Tell(new DispatchDeployment(SeedV3Deployment(db, revC), revC, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
AwaitAssert(
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"),
duration: Timeout);
// An explicit counter for the re-added instance is retained, not swallowed as a late write.
tracker.RecordBreakerOpen("drv-1", "dev1", DateTime.UtcNow);
tracker.TryGet("drv-1", "dev1").ShouldNotBeNull().IsBreakerOpen.ShouldBeTrue();
}
private static void PublisherTick(
ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResilienceStatusTracker t,
ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub h)
{
var now = DateTime.UtcNow;
foreach (var e in t.Snapshot())
h.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Resilience(new(
e.DriverInstanceId, e.HostName, e.Snapshot.IsBreakerOpen, e.Snapshot.ConsecutiveFailures,
e.Snapshot.CurrentInFlight, e.Snapshot.LastBreakerOpenUtc, e.Snapshot.LastSampledUtc, now)));
}
private static List ReplayedResilienceIds(ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub h)
{
using var sub = h.Subscribe(64);
var ids = new List();
while (sub.Reader.TryRead(out var i))
if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Resilience r) ids.Add(r.E.DriverInstanceId);
return ids;
}
///
/// Wraps a real invoker factory so every call is HELD before it
/// reaches the real invoker until — forcing a removed child's subscribe (and the
/// tracker writes the real invoker makes around it) to run after the host's Release. Every other call
/// passes straight through.
///
private sealed class GatedSubscribeInvokerFactory(IDriverCapabilityInvokerFactory inner) : IDriverCapabilityInvokerFactory
{
private readonly TaskCompletionSource _gate = new(TaskCreationOptions.RunContinuationsAsynchronously);
/// Set once a subscribe has reached the gate.
public ManualResetEventSlim SubscribeEntered { get; } = new();
/// Set once a held subscribe has returned from the REAL invoker, i.e. its tracker writes are done.
public ManualResetEventSlim SubscribeCompleted { get; } = new();
public void OpenGate() => _gate.TrySetResult();
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null) =>
new Gated(this, inner.Create(driverInstanceId, driverType, resilienceConfigJson));
public void Release(string driverInstanceId) => inner.Release(driverInstanceId);
private sealed class Gated(GatedSubscribeInvokerFactory owner, IDriverCapabilityInvoker real) : IDriverCapabilityInvoker
{
public async ValueTask ExecuteAsync(
DriverCapability capability, string hostName,
Func> callSite, CancellationToken cancellationToken)
{
if (capability != DriverCapability.Subscribe)
return await real.ExecuteAsync(capability, hostName, callSite, cancellationToken);
// The gate sits INSIDE the call site, i.e. inside the real invoker's RecordCallStart … RecordSuccess /
// RecordCallComplete bracket, and every await here is ConfigureAwait(false): the real invoker resumes
// off the actor's context too, so its closing writes run on the pool after the child has stopped —
// exactly the production shape. (Gating in front of the real invoker on the actor's context does
// not work: Akka drops a stopped actor's pending continuations, so nothing late ever runs.)
try
{
return await real.ExecuteAsync(capability, hostName, async ct =>
{
owner.SubscribeEntered.Set();
await owner._gate.Task.ConfigureAwait(false);
return await callSite(CancellationToken.None).ConfigureAwait(false);
}, CancellationToken.None).ConfigureAwait(false);
}
finally { owner.SubscribeCompleted.Set(); }
}
public ValueTask ExecuteAsync(
DriverCapability capability, string hostName,
Func callSite, CancellationToken cancellationToken) =>
real.ExecuteAsync(capability, hostName, callSite, cancellationToken);
public ValueTask ExecuteWriteAsync(
string hostName, bool isIdempotent,
Func> callSite, CancellationToken cancellationToken) =>
real.ExecuteWriteAsync(hostName, isIdempotent, callSite, cancellationToken);
}
}
/// Spawns the host with the subscribing factory, dispatches and
/// waits for the Applied ACK so the child + the initial SubscribeBulk pass are in place.
private (IActorRef Actor, Akka.TestKit.TestProbe Coordinator) SpawnHostAndApply(
IDbContextFactory db, DeploymentId deploymentId, IDriverFactory factory)
{
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
driverFactory: factory,
localRoles: new HashSet { "driver" }));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
return (actor, coordinator);
}
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
{
var probe = CreateTestProbe();
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
return probe.ExpectMsg(Timeout);
}
/// Seeds a Sealed v3 deployment: RawFolder "Plant" → DriverInstance "drv-1" (Modbus, ENABLED so a
/// real child spawns) → Device "dev1" → Tag "speed" (RawPath Plant/Modbus/dev1/speed).
private static DeploymentId SeedV3Deployment(IDbContextFactory db, RevisionHash rev) =>
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
DriverInstances = new[]
{
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
},
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
TagGroups = Array.Empty