Joseph Doherty
4a10ea1e1c
DriverHostActor.StopChild released the tracker (Invalidate) and evicted the hub synchronously, but Context.Stop is asynchronous: the stopping child's in-flight SubscribeBulk completed through its capability invoker AFTER the release, and its RecordCallStart/RecordSuccess/RecordCallComplete re-created the tracker entry, which DriverResilienceStatusPublisherService then re-emitted every tick forever. Confirmed with a temporary stack capture in the tracker (removed): every post-Release write came from DriverInstanceActor.HandleSubscribeAsync -> CapabilityInvoker.ExecuteAsync; the test failed whenever it landed before the second publisher tick. - DriverResilienceStatusTracker.Retire (Release) drops the id's snapshots and ignores every later Record* for it; one guarded write path re-checks after the upsert so a write racing Retire cannot survive it. Reinstate (called by DriverCapabilityInvokerFactory.Create) lifts it, so the same id re-added in a later deployment records again. - TelemetryLocalHub.Evict now also holds the id (late Health/Resilience dropped, neither cached nor fanned out); ITelemetryLocalHub.Readmit (default no-op), called at the top of SpawnChild, lifts it. - The removal test is now deterministic: an invoker decorator holds the child's subscribe inside the real invoker's bracket across the removal and signals when its late writes are done (positive control) — red 5/5 without the fix, green 20/20 with it. New unit tests for the tracker guard (incl. a Retire race), the factory Release/Create pair, the hub hold/readmit, and a re-add-same-id host test. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
575 lines
32 KiB
C#
575 lines
32 KiB
C#
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Issue #485 (driver-side half) — an artifact the host could not obtain must not be mistaken for a
|
|
/// configuration that contains nothing.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// <c>ReconcileDrivers</c> and <c>PushDesiredSubscriptions</c> both read the artifact as
|
|
/// <c>…FirstOrDefault() ?? Array.Empty<byte>()</c>. The <b>throw</b> path in each already
|
|
/// returns early, but a row that is absent (or carries a zero-length blob) yields empty bytes that
|
|
/// flow onwards as a real answer: zero driver specs, so <c>DriverSpawnPlanner</c> plans every
|
|
/// running child for <c>StopChild</c>, and then an empty desired-subscription set drops each
|
|
/// surviving driver's live handle. A node loses its entire field I/O and still ACKs Applied.
|
|
/// </para>
|
|
/// <para>
|
|
/// The same reasoning as the address-space half applies: nothing legitimate produces a zero-length
|
|
/// blob — an operator who really deletes every driver still deploys a JSON document with empty
|
|
/// arrays — so "no bytes" can only mean the read did not answer. Seeding a row whose
|
|
/// <c>ArtifactBlob</c> is empty reproduces exactly the bytes the missing-row case delivers, without
|
|
/// needing to race a delete against the two reads.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class DriverHostActorUnreadableArtifactTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
|
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
|
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
|
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
|
|
|
|
/// <summary>How long to let post-ACK subscription traffic settle before asserting it did NOT happen.</summary>
|
|
private static readonly TimeSpan SettleWindow = TimeSpan.FromSeconds(2);
|
|
|
|
private const string SpeedRef = "Plant/Modbus/dev1/speed";
|
|
|
|
/// <summary>A dispatch whose artifact reads back as no bytes must leave the running drivers — and their
|
|
/// live subscriptions — exactly as they were.</summary>
|
|
[Fact]
|
|
public void Dispatch_whose_artifact_reads_back_empty_keeps_the_drivers_and_their_subscriptions()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
|
|
// Baseline: the child is up and subscribed to its one raw tag.
|
|
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
|
|
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1");
|
|
|
|
// The next dispatch's artifact comes back as no bytes at all.
|
|
actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout);
|
|
|
|
// The ACK precedes the SubscribeBulk pass, and the child's unsubscribe is a further async self-tell,
|
|
// so settle before asserting an ABSENCE. SettleWindow is calibrated by
|
|
// <see cref="Dropping_a_drivers_last_tag_does_clear_its_subscription"/>, which observes the very same
|
|
// unsubscribe ARRIVING well inside it — without that control this assertion would pass on a race.
|
|
AskDiagnostics(actor); // ordering barrier through the host's mailbox
|
|
Thread.Sleep(SettleWindow);
|
|
|
|
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // not stopped
|
|
factory.UnsubscribeCount.ShouldBe(0); // handle not dropped
|
|
factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Positive control for the subscription half: a READABLE artifact that drops the driver's last tag
|
|
/// genuinely does tear the live subscription down, and does so well within
|
|
/// <see cref="SettleWindow"/> — without this, the absence asserted above would pass on a race.
|
|
/// <para><b>The observable changed with #516.</b> Dropping a tag changes the driver's
|
|
/// <c>DriverConfig</c>, which used to be an in-place delta: the child received an empty desired set
|
|
/// and called <c>UnsubscribeAsync</c>. A config change is now a stop + respawn, so the old child is
|
|
/// STOPPED (taking its subscription with it via <c>ShutdownAsync</c>) and a fresh one spawns with
|
|
/// nothing to subscribe to. The teardown is therefore observed as a shutdown, not an unsubscribe —
|
|
/// the same event, reached by a different route.</para>
|
|
/// </summary>
|
|
[Fact]
|
|
public void Dropping_a_drivers_last_tag_does_clear_its_subscription()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
|
|
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
|
|
factory.ShutdownCount.ShouldBe(0);
|
|
|
|
actor.Tell(new DispatchDeployment(SeedTaglessDriverDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout);
|
|
|
|
AwaitAssert(() => factory.ShutdownCount.ShouldBeGreaterThanOrEqualTo(1), duration: SettleWindow);
|
|
AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"); // respawned under the same id
|
|
}
|
|
|
|
/// <summary>Positive control: the guard keys on "the read gave us nothing", not on "the new config has
|
|
/// fewer drivers". A READABLE artifact that genuinely drops the driver still stops it — otherwise the
|
|
/// test above would pass just as happily against a host that had stopped reconciling altogether.</summary>
|
|
[Fact]
|
|
public void Dispatch_whose_readable_artifact_genuinely_drops_the_driver_still_stops_it()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
|
|
AwaitAssert(() => factory.LastSubscribedRefs.ShouldNotBeNull()!.ShouldContain(SpeedRef), duration: Timeout);
|
|
|
|
// A real artifact that simply carries no drivers — an operator deleting the last driver.
|
|
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout);
|
|
|
|
AwaitAssert(
|
|
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
|
|
duration: Timeout);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Issue #486 — an apply that read no artifact applied nothing, so it must NOT be reported as
|
|
/// Applied. Reporting success also advanced <c>_currentRevision</c>, and because
|
|
/// <c>HandleDispatchFromSteady</c> short-circuits on a revision match, the node could never be
|
|
/// healed by re-dispatching that revision.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Dispatch_whose_artifact_reads_back_empty_acks_Failed_and_leaves_the_revision_alone()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevA);
|
|
|
|
actor.Tell(new DispatchDeployment(SeedUnreadableDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(Timeout);
|
|
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Failed);
|
|
ack.FailureReason.ShouldNotBeNullOrWhiteSpace();
|
|
|
|
var snapshot = AskDiagnostics(actor);
|
|
snapshot.CurrentRevision.ShouldBe(RevA); // never applied ⇒ never claimed
|
|
snapshot.Drivers.Select(d => d.Name).ShouldContain("drv-1"); // …and #485 still holds
|
|
}
|
|
|
|
/// <summary>Issue #486 — the point of failing the apply: the SAME revision can be dispatched again and
|
|
/// now actually applies, instead of being waved through by the "already at this rev" short-circuit. The
|
|
/// recovered artifact adds a second driver, so a short-circuited ACK (which has no side effects) cannot
|
|
/// satisfy this test — drv-2 only appears if the artifact was genuinely re-read and reconciled.</summary>
|
|
[Fact]
|
|
public void A_failed_apply_is_retried_not_short_circuited_once_the_artifact_is_readable_again()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var (actor, coordinator) = SpawnHostAndApply(db, SeedV3Deployment(db, RevA), factory);
|
|
|
|
var deploymentId = SeedUnreadableDeployment(db, RevB);
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Failed);
|
|
|
|
// The ConfigDb recovers: the row now carries a real artifact, under the SAME revision.
|
|
SetArtifact(db, deploymentId, TwoDriverArtifact());
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevB, CorrelationId.NewId()));
|
|
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
AwaitAssert(
|
|
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-2"),
|
|
duration: Timeout);
|
|
AskDiagnostics(actor).CurrentRevision.ShouldBe(RevB);
|
|
}
|
|
|
|
/// <summary>
|
|
/// #609 — removing a driver instance from the plan evicts its cached telemetry snapshots, so a new
|
|
/// subscriber is no longer primed with a decommissioned driver's last Health/Resilience. The
|
|
/// surviving instance's snapshot is the positive control: eviction is per-instance, not a wipe.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Removing_a_driver_instance_evicts_its_telemetry_snapshots_but_not_a_neighbours()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var hub = new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub();
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
driverFactory: factory,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
telemetryHub: hub));
|
|
|
|
actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
hub.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health(new(
|
|
"c1", "drv-1", "Healthy", null, null, 0, DateTime.UtcNow)));
|
|
hub.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health(new(
|
|
"c1", "other-node-driver", "Healthy", null, null, 0, DateTime.UtcNow)));
|
|
|
|
// Control: before the removal, drv-1 replays.
|
|
using (var before = hub.Subscribe(16))
|
|
{
|
|
var seen = new List<string>();
|
|
while (before.Reader.TryRead(out var i))
|
|
if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health h) seen.Add(h.E.DriverInstanceId);
|
|
seen.ShouldContain("drv-1");
|
|
}
|
|
|
|
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
AwaitAssert(
|
|
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
|
|
duration: Timeout);
|
|
|
|
using var after = hub.Subscribe(16);
|
|
var replayed = new List<string>();
|
|
while (after.Reader.TryRead(out var i))
|
|
if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health h) replayed.Add(h.E.DriverInstanceId);
|
|
replayed.ShouldNotContain("drv-1");
|
|
replayed.ShouldContain("other-node-driver");
|
|
}
|
|
|
|
/// <summary>
|
|
/// #609 (review R1 M1) — the Resilience half of the eviction must SURVIVE the next resilience-publisher
|
|
/// tick. <c>DriverResilienceStatusPublisherService</c> re-emits <c>tracker.Snapshot()</c> into the hub
|
|
/// every interval, so evicting only the hub cache was undone within one tick: the host must also
|
|
/// release the instance's tracker state. The tick below is the publisher's own body
|
|
/// (<c>BuildMessages</c> → <c>hub.Emit</c>) minus the DPS publish, run against the REAL factory +
|
|
/// tracker the Host wires. The neighbour instance is the positive control: it is still emitted by
|
|
/// the same tick, so an absent drv-1 is not an empty tick.
|
|
/// </summary>
|
|
[Fact]
|
|
public void Removing_a_driver_instance_releases_its_resilience_state_so_the_next_publisher_tick_omits_it()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var hub = new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub();
|
|
var tracker = new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResilienceStatusTracker();
|
|
// The child's subscribe is HELD across the removal so the tracker writes its capability invoker makes
|
|
// around it land AFTER StopChild's Release — the late writer that resurrected the entry (#609). Without
|
|
// the gate this test only failed when the child's in-flight SubscribeBulk happened to finish between the
|
|
// removal and the second tick (~4/6 runs); the gate forces that interleaving on every run.
|
|
var invokerFactory = new GatedSubscribeInvokerFactory(
|
|
new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverCapabilityInvokerFactory(
|
|
new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResiliencePipelineBuilder(statusTracker: tracker),
|
|
tracker,
|
|
_ => ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTier.A));
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
driverFactory: factory,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
invokerFactory: invokerFactory,
|
|
telemetryHub: hub));
|
|
|
|
actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
invokerFactory.SubscribeEntered.Wait(Timeout).ShouldBeTrue("the child never reached its subscribe");
|
|
|
|
// Resilience counters for the running driver and for an instance this host does not own.
|
|
tracker.RecordBreakerOpen("drv-1", "dev1", DateTime.UtcNow);
|
|
tracker.RecordFailure("neighbour-drv", "dev9", DateTime.UtcNow);
|
|
PublisherTick(tracker, hub);
|
|
ReplayedResilienceIds(hub).ShouldContain("drv-1"); // control: drv-1 is primed before the removal
|
|
|
|
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
AwaitAssert(
|
|
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
|
|
duration: Timeout);
|
|
|
|
// Let the removed child's held subscribe run to completion through the REAL invoker (RecordCallStart /
|
|
// RecordSuccess / RecordCallComplete). Positive control: the late write has demonstrably happened before
|
|
// the tick below, so the absence asserted after it is not "nothing has arrived yet".
|
|
invokerFactory.OpenGate();
|
|
invokerFactory.SubscribeCompleted.Wait(Timeout).ShouldBeTrue("the held subscribe never completed");
|
|
|
|
// The publisher's next tick, after the removal AND after the late writer.
|
|
PublisherTick(tracker, hub);
|
|
|
|
var replayed = ReplayedResilienceIds(hub);
|
|
replayed.ShouldNotContain("drv-1");
|
|
replayed.ShouldContain("neighbour-drv");
|
|
}
|
|
|
|
/// <summary>
|
|
/// #609 — the release a removal makes must not outlive the removal: when the SAME instance id comes back
|
|
/// in a later deployment, its fresh invoker records resilience state again (the tracker un-retires the id
|
|
/// on <c>Create</c>).
|
|
/// </summary>
|
|
[Fact]
|
|
public void A_driver_instance_re_added_after_removal_records_resilience_state_again()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var factory = new SubscribingDriverFactory("Modbus");
|
|
var tracker = new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResilienceStatusTracker();
|
|
var invokerFactory = new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverCapabilityInvokerFactory(
|
|
new ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResiliencePipelineBuilder(statusTracker: tracker),
|
|
tracker,
|
|
_ => ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTier.A);
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
driverFactory: factory,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
invokerFactory: invokerFactory,
|
|
telemetryHub: new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub()));
|
|
|
|
actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
AwaitAssert(
|
|
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
|
|
duration: Timeout);
|
|
|
|
var revC = RevisionHash.Parse(new string('c', 64));
|
|
actor.Tell(new DispatchDeployment(SeedV3Deployment(db, revC), revC, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
AwaitAssert(
|
|
() => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldContain("drv-1"),
|
|
duration: Timeout);
|
|
|
|
// An explicit counter for the re-added instance is retained, not swallowed as a late write.
|
|
tracker.RecordBreakerOpen("drv-1", "dev1", DateTime.UtcNow);
|
|
tracker.TryGet("drv-1", "dev1").ShouldNotBeNull().IsBreakerOpen.ShouldBeTrue();
|
|
}
|
|
|
|
private static void PublisherTick(
|
|
ZB.MOM.WW.OtOpcUa.Core.Resilience.DriverResilienceStatusTracker t,
|
|
ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub h)
|
|
{
|
|
var now = DateTime.UtcNow;
|
|
foreach (var e in t.Snapshot())
|
|
h.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Resilience(new(
|
|
e.DriverInstanceId, e.HostName, e.Snapshot.IsBreakerOpen, e.Snapshot.ConsecutiveFailures,
|
|
e.Snapshot.CurrentInFlight, e.Snapshot.LastBreakerOpenUtc, e.Snapshot.LastSampledUtc, now)));
|
|
}
|
|
|
|
private static List<string> ReplayedResilienceIds(ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub h)
|
|
{
|
|
using var sub = h.Subscribe(64);
|
|
var ids = new List<string>();
|
|
while (sub.Reader.TryRead(out var i))
|
|
if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Resilience r) ids.Add(r.E.DriverInstanceId);
|
|
return ids;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Wraps a real invoker factory so every <see cref="DriverCapability.Subscribe"/> call is HELD before it
|
|
/// reaches the real invoker until <see cref="OpenGate"/> — forcing a removed child's subscribe (and the
|
|
/// tracker writes the real invoker makes around it) to run after the host's Release. Every other call
|
|
/// passes straight through.
|
|
/// </summary>
|
|
private sealed class GatedSubscribeInvokerFactory(IDriverCapabilityInvokerFactory inner) : IDriverCapabilityInvokerFactory
|
|
{
|
|
private readonly TaskCompletionSource _gate = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
/// <summary>Set once a subscribe has reached the gate.</summary>
|
|
public ManualResetEventSlim SubscribeEntered { get; } = new();
|
|
|
|
/// <summary>Set once a held subscribe has returned from the REAL invoker, i.e. its tracker writes are done.</summary>
|
|
public ManualResetEventSlim SubscribeCompleted { get; } = new();
|
|
|
|
public void OpenGate() => _gate.TrySetResult();
|
|
|
|
public IDriverCapabilityInvoker Create(string driverInstanceId, string driverType, string? resilienceConfigJson = null) =>
|
|
new Gated(this, inner.Create(driverInstanceId, driverType, resilienceConfigJson));
|
|
|
|
public void Release(string driverInstanceId) => inner.Release(driverInstanceId);
|
|
|
|
private sealed class Gated(GatedSubscribeInvokerFactory owner, IDriverCapabilityInvoker real) : IDriverCapabilityInvoker
|
|
{
|
|
public async ValueTask<TResult> ExecuteAsync<TResult>(
|
|
DriverCapability capability, string hostName,
|
|
Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
|
|
{
|
|
if (capability != DriverCapability.Subscribe)
|
|
return await real.ExecuteAsync(capability, hostName, callSite, cancellationToken);
|
|
|
|
// The gate sits INSIDE the call site, i.e. inside the real invoker's RecordCallStart … RecordSuccess /
|
|
// RecordCallComplete bracket, and every await here is ConfigureAwait(false): the real invoker resumes
|
|
// off the actor's context too, so its closing writes run on the pool after the child has stopped —
|
|
// exactly the production shape. (Gating in front of the real invoker on the actor's context does
|
|
// not work: Akka drops a stopped actor's pending continuations, so nothing late ever runs.)
|
|
try
|
|
{
|
|
return await real.ExecuteAsync(capability, hostName, async ct =>
|
|
{
|
|
owner.SubscribeEntered.Set();
|
|
await owner._gate.Task.ConfigureAwait(false);
|
|
return await callSite(CancellationToken.None).ConfigureAwait(false);
|
|
}, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
finally { owner.SubscribeCompleted.Set(); }
|
|
}
|
|
|
|
public ValueTask ExecuteAsync(
|
|
DriverCapability capability, string hostName,
|
|
Func<CancellationToken, ValueTask> callSite, CancellationToken cancellationToken) =>
|
|
real.ExecuteAsync(capability, hostName, callSite, cancellationToken);
|
|
|
|
public ValueTask<TResult> ExecuteWriteAsync<TResult>(
|
|
string hostName, bool isIdempotent,
|
|
Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken) =>
|
|
real.ExecuteWriteAsync(hostName, isIdempotent, callSite, cancellationToken);
|
|
}
|
|
}
|
|
|
|
/// <summary>Spawns the host with the subscribing factory, dispatches <paramref name="deploymentId"/> and
|
|
/// waits for the Applied ACK so the child + the initial SubscribeBulk pass are in place.</summary>
|
|
private (IActorRef Actor, Akka.TestKit.TestProbe Coordinator) SpawnHostAndApply(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory factory)
|
|
{
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
driverFactory: factory,
|
|
localRoles: new HashSet<string> { "driver" }));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
return (actor, coordinator);
|
|
}
|
|
|
|
private NodeDiagnosticsSnapshot AskDiagnostics(IActorRef actor)
|
|
{
|
|
var probe = CreateTestProbe();
|
|
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), probe.Ref);
|
|
return probe.ExpectMsg<NodeDiagnosticsSnapshot>(Timeout);
|
|
}
|
|
|
|
/// <summary>Seeds a Sealed v3 deployment: RawFolder "Plant" → DriverInstance "drv-1" (Modbus, ENABLED so a
|
|
/// real child spawns) → Device "dev1" → Tag "speed" (RawPath <c>Plant/Modbus/dev1/speed</c>).</summary>
|
|
private static DeploymentId SeedV3Deployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
|
|
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
|
DriverInstances = new[]
|
|
{
|
|
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
},
|
|
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = new[]
|
|
{
|
|
new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
|
|
},
|
|
}));
|
|
|
|
/// <summary>A readable artifact carrying drv-1 AND drv-2, so an apply that genuinely happens is
|
|
/// distinguishable from a short-circuited ACK (which spawns nothing).</summary>
|
|
private static byte[] TwoDriverArtifact() => JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
|
DriverInstances = new[]
|
|
{
|
|
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
new { DriverInstanceId = "drv-2", RawFolderId = "rf-plant", Name = "Modbus2", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
},
|
|
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = new[]
|
|
{
|
|
new { TagId = "t-speed", DeviceId = "dev-1", TagGroupId = (string?)null, Name = "speed", DataType = "Double", AccessLevel = 1, TagConfig = "{}" },
|
|
},
|
|
});
|
|
|
|
/// <summary>Replaces a seeded deployment's artifact in place — the ConfigDb becoming readable again
|
|
/// without the revision changing.</summary>
|
|
private static void SetArtifact(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, byte[] artifact)
|
|
{
|
|
// ArtifactBlob is init-only, so the row is replaced rather than mutated: delete and re-add under
|
|
// the SAME id + revision. Separate SaveChanges calls so the in-memory provider does not see a
|
|
// delete and an insert of the same key in one change set.
|
|
using var ctx = db.CreateDbContext();
|
|
var row = ctx.Deployments.Single(d => d.DeploymentId == deploymentId.Value);
|
|
var rev = row.RevisionHash;
|
|
ctx.Deployments.Remove(row);
|
|
ctx.SaveChanges();
|
|
|
|
ctx.Deployments.Add(new Deployment
|
|
{
|
|
DeploymentId = deploymentId.Value,
|
|
RevisionHash = rev,
|
|
Status = DeploymentStatus.Sealed,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = DateTime.UtcNow,
|
|
ArtifactBlob = artifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
|
|
/// <summary>Seeds a Sealed deployment whose <c>ArtifactBlob</c> is zero-length — byte-for-byte what the
|
|
/// host's <c>?? Array.Empty<byte>()</c> hands downstream when the row cannot be found.</summary>
|
|
private static DeploymentId SeedUnreadableDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
|
|
SeedDeployment(db, rev, Array.Empty<byte>());
|
|
|
|
/// <summary>Seeds a Sealed deployment carrying a real, readable artifact that keeps drv-1 (so its child
|
|
/// survives the reconcile) but declares no tags for it — the driver's desired set becomes empty.</summary>
|
|
private static DeploymentId SeedTaglessDriverDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
|
|
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
|
|
DriverInstances = new[]
|
|
{
|
|
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
|
|
},
|
|
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = Array.Empty<object>(),
|
|
}));
|
|
|
|
/// <summary>Seeds a Sealed deployment carrying a real, readable artifact that declares no drivers.</summary>
|
|
private static DeploymentId SeedDriverlessDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev) =>
|
|
SeedDeployment(db, rev, JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = Array.Empty<object>(),
|
|
DriverInstances = Array.Empty<object>(),
|
|
Devices = Array.Empty<object>(),
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = Array.Empty<object>(),
|
|
}));
|
|
|
|
private static DeploymentId SeedDeployment(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev, byte[] artifact)
|
|
{
|
|
var id = DeploymentId.NewId();
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.Deployments.Add(new Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = rev.Value,
|
|
Status = DeploymentStatus.Sealed,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = DateTime.UtcNow,
|
|
ArtifactBlob = artifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
return id;
|
|
}
|
|
|
|
/// <summary>Factory producing one shared <see cref="SubscribableStubDriver"/> for the supported type, so a
|
|
/// REAL (non-stubbed) <see cref="DriverInstanceActor"/> child spawns and its subscribe/unsubscribe traffic
|
|
/// is observable.</summary>
|
|
private sealed class SubscribingDriverFactory(string supportedType) : IDriverFactory
|
|
{
|
|
private readonly SubscribableStubDriver _driver = new();
|
|
|
|
/// <summary>The reference set passed to the driver's most recent <c>SubscribeAsync</c> call.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedRefs => _driver.LastSubscribedRefs;
|
|
|
|
/// <summary>Number of <c>UnsubscribeAsync</c> calls — non-zero means a live handle was torn down.</summary>
|
|
public int UnsubscribeCount => Volatile.Read(ref _driver.UnsubscribeCount);
|
|
|
|
/// <summary>Number of <c>ShutdownAsync</c> calls — non-zero means a driver child was stopped.</summary>
|
|
public int ShutdownCount => Volatile.Read(ref _driver.ShutdownCount);
|
|
|
|
/// <inheritdoc />
|
|
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson) =>
|
|
string.Equals(driverType, supportedType, StringComparison.Ordinal) ? _driver : null;
|
|
|
|
/// <inheritdoc />
|
|
public IReadOnlyCollection<string> SupportedTypes => new[] { supportedType };
|
|
}
|
|
}
|