Merge branch 'fix/r3-03-restart-resubscribe-watchdog' — R3-03: SpawnChild always sends the desired set (operator Restart + supervision restart re-provision, stale-alarm-set re-subscribe, incarnation check, 10 s backoff); watchdog grace = max(LastSuccessfulRead, connectedSince) (archreview 03/S16, S17, U6)
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,630 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// The repo's first driver-child <b>restart</b> suite (archreview 03/S16, 03/U6): what a child obtained
|
||||
/// by each of the three incarnation routes is actually handed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>The falsification bar, and why this file exists at all.</b> An actor-restart test must assert a
|
||||
/// FUNCTION the new incarnation performs — it subscribed, it re-initialised — never that the
|
||||
/// incarnation EXISTS. Existence is exactly what
|
||||
/// <c>DriverHostActorReconcileTests.RestartDriver_respawns_the_child_without_an_actor_name_collision</c>
|
||||
/// asserts (no name collision, factory created a second driver, one child remains), and that test
|
||||
/// passes green against the 03/S16 defect: a respawned child that connects and then reads nothing
|
||||
/// forever. It is kept, because its own subject — the <c>InvalidActorNameException</c> — is real; it
|
||||
/// is simply the weaker half of a pair, and this file is the other half.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The three incarnation routes, and what re-provisions each.</b> (1) <i>Deploy reconcile</i> —
|
||||
/// <c>ApplyDriverSpecs</c> → <c>SpawnChild</c>, then the apply's bulk desired-subscription push.
|
||||
/// (2) <i>Operator Restart</i> — <c>HandleRestartDriver</c> stops the child and calls
|
||||
/// <c>SpawnChild</c> with the cached spec; before 03/S16 that was the whole story, and the new child
|
||||
/// started with an EMPTY desired set that nothing re-sent until the next deployment (the #488
|
||||
/// periodic reconcile early-returns on an empty set, so it is structurally inert here).
|
||||
/// (3) <i>Supervision restart</i> — Akka's default decider recreates the actor object around the
|
||||
/// SAME driver instance without going through <c>SpawnChild</c> at all, so nothing re-sent
|
||||
/// <c>InitializeRequested</c> either and the child sat in <c>Connecting</c> forever around a driver
|
||||
/// its own <c>PostStop</c> had already shut down. A fourth route, <i>Force Reconnect</i>, re-uses the
|
||||
/// live actor instance and needs nothing — a verified negative, pinned here so a future edit to the
|
||||
/// restart path cannot break the path that was already correct.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The supervision crash seam, and why the obvious ones do not work.</b> The seam used is the
|
||||
/// driver's own <c>OnDataChange</c> carrying a NULL <c>Snapshot</c>
|
||||
/// (<see cref="RecordingSubscribableDriver.FireMalformedDataChange"/>): <c>DataChangeEventArgs</c> is
|
||||
/// a positional record with no null validation, and <c>DriverInstanceActor.OnDataChangeForward</c>
|
||||
/// dereferences <c>msg.Snapshot</c> unguarded on the ACTOR thread, so it NREs inside the child and
|
||||
/// the default decider Restarts it. Rejected alternatives, each for a specific reason: a throwing
|
||||
/// <c>IDriverHealthPublisher</c> (<c>PublishHealthSnapshot</c> wraps the publish in a
|
||||
/// <c>catch (Exception)</c>, so it never escapes); a throwing <c>SubscribeAsync</c> (also swallowed —
|
||||
/// the stub already models it via <c>SubscribeFailuresBeforeSuccess</c>); <c>Kill.Instance</c> (the
|
||||
/// default decider <b>Stops</b> on <c>ActorKilledException</c>, a different directive and therefore a
|
||||
/// different code path); and an <c>internal</c> "throw this" test message (a production surface added
|
||||
/// purely for a test). The fault is arbitrary by design — what is under test is the RECOVERY. If
|
||||
/// <c>DataChangeEventArgs</c> ever gains null validation, the fallback is an explicit
|
||||
/// <c>OneForOneStrategy(Directive.Restart)</c> on the host plus an internal test-only throw message.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DriverHostActorRestartTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
||||
private static readonly NodeId TestNode = NodeId.Parse("node-1:4053");
|
||||
|
||||
private const string DriverId = "DI-1";
|
||||
private const string SpeedPath = "Plant/Modbus/dev1/speed";
|
||||
private const string TempPath = "Plant/Modbus/dev1/temp";
|
||||
private const string AlarmXPath = "Plant/Modbus/dev1/alarmX";
|
||||
private const string AlarmYPath = "Plant/Modbus/dev1/alarmY";
|
||||
|
||||
/// <summary>TagConfig marking a raw tag as alarm-bearing (parsed by <c>TagConfigIntent</c>).</summary>
|
||||
private const string AlarmConfig = """{"alarm":{"alarmType":"AlarmCondition","severity":500}}""";
|
||||
|
||||
/// <summary>Seeds + applies a v3 deployment whose single Modbus driver is ENABLED, so a real child is
|
||||
/// spawned around a <see cref="RecordingSubscribableDriver"/>, and waits until that child has connected
|
||||
/// and established its first subscription.</summary>
|
||||
private (IActorRef Host, RecordingSubscribableDriverFactory Factory) DeployOneDriver(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db,
|
||||
(string Driver, string DriverName, string Device, string Tag, string DataType, bool Writable)[] rawTags,
|
||||
IReadOnlyDictionary<string, string>? tagConfigByTag = null)
|
||||
{
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus");
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevA, rawTags, Array.Empty<(string, string, string, string, string?)>(),
|
||||
enabled: true, tagConfigByTag: tagConfigByTag);
|
||||
var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
|
||||
return (host, factory);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>The 03/S16 falsification.</b> An operator's Restart is supposed to recover a sick driver; before
|
||||
/// the fix it silently killed it — the respawned child came up with <c>_desiredRefs</c> empty, and the
|
||||
/// only <c>SetDesiredSubscriptions</c> sender in the repo was the apply's bulk push, so the driver
|
||||
/// subscribed to nothing until the next deployment dispatch.
|
||||
/// <para>
|
||||
/// Deliberately NOT the pass criterion: that a second driver was created, or that one child
|
||||
/// remains. Both are true against the defect.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Operator_restart_re_subscribes_the_respawned_child()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var (host, factory) = DeployOneDriver(db, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) });
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1);
|
||||
factory.Created[0].SubscribeCount.ShouldBe(1);
|
||||
factory.Created[0].LastSubscribedRefs.ShouldBe(new[] { SpeedPath });
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
host.Tell(new RestartDriver("c1", DriverId, "tester", Guid.NewGuid()));
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(2, "the restart did not respawn the child around a new driver");
|
||||
factory.Created[1].SubscribeCount.ShouldBeGreaterThanOrEqualTo(
|
||||
1,
|
||||
"the RESPAWNED child never subscribed — it is Connected and reading nothing, which is the " +
|
||||
"03/S16 defect (an operator pressed Restart to recover a driver and silently killed it)");
|
||||
factory.Created[1].LastSubscribedRefs.ShouldBe(new[] { SpeedPath });
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The value refs and the alarm refs ride in ONE <c>SetDesiredSubscriptions</c> but are applied by two
|
||||
/// different child code paths (the value set self-Tells <c>Subscribe</c>; the alarm set runs
|
||||
/// <c>SubscribeDesiredAlarms</c> against the <c>IAlarmSource</c> feed, which stays gated shut until it
|
||||
/// is called). Sending the message is therefore not evidence that both halves landed, so the alarm
|
||||
/// half is pinned separately.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Operator_restart_re_subscribes_alarm_refs_too()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var (host, factory) = DeployOneDriver(
|
||||
db,
|
||||
new[]
|
||||
{
|
||||
(DriverId, "Modbus", "dev1", "speed", "Float", false),
|
||||
(DriverId, "Modbus", "dev1", "temp", "Float", false),
|
||||
},
|
||||
// An alarm-bearing raw tag is EXCLUDED from the value set and carried in the alarm set instead.
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["temp"] = """{"alarm":{"alarmType":"AlarmCondition","severity":500}}""",
|
||||
});
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1);
|
||||
factory.Created[0].SubscribeAlarmsCount.ShouldBeGreaterThanOrEqualTo(1);
|
||||
factory.Created[0].LastSubscribedAlarmRefs.ShouldBe(new[] { TempPath });
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
host.Tell(new RestartDriver("c1", DriverId, "tester", Guid.NewGuid()));
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(2);
|
||||
factory.Created[1].SubscribeAlarmsCount.ShouldBeGreaterThanOrEqualTo(
|
||||
1,
|
||||
"the respawned child never re-subscribed its alarm refs — the IAlarmSource feed stays " +
|
||||
"gated shut, so no native alarm transition can ever reach the host again");
|
||||
factory.Created[1].LastSubscribedAlarmRefs.ShouldBe(new[] { TempPath });
|
||||
// The value half must survive the alarm half being present.
|
||||
factory.Created[1].LastSubscribedRefs.ShouldBe(new[] { SpeedPath });
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The ordering guard for the spawn-time send. <c>ApplyDriverSpecs</c> (which spawns) runs BEFORE
|
||||
/// <c>ApplyDesiredSubscriptions</c> (which rebuilds the authority), so a child spawned by a fresh
|
||||
/// deploy is momentarily handed the PREVIOUS apply's set. That transient must never be acted on: it
|
||||
/// is superseded by the bulk push on the same sender→receiver pair (Akka guarantees per-pair
|
||||
/// ordering) while the child is still <c>Connecting</c>, where <c>SetDesiredSubscriptions</c> only
|
||||
/// STORES; the subscribe happens on the Connected entry, from whatever was stored last.
|
||||
/// <para>
|
||||
/// Revision B swaps the driver's tag, so A's and B's sets are distinguishable. Note the raw tags
|
||||
/// are folded into the merged <c>DriverConfig</c>, so a tag-set change DOES change the driver
|
||||
/// spec and therefore respawns the child (#516) — which is precisely the arrangement under test:
|
||||
/// a brand-new child, spawned while the authority still holds A, must end up subscribed to B.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>What is asserted is CONVERGENCE, not inertness</b>, and the difference was measured rather
|
||||
/// than assumed. The child runs concurrently with the rest of the apply handler, so a driver
|
||||
/// whose <c>InitializeAsync</c> completes instantly (this stub does) can reach Connected and
|
||||
/// subscribe the transient stale set BEFORE the bulk push arrives — observed here as
|
||||
/// <c>SubscribeCount == 2</c>. Akka's per-pair ordering still guarantees the final state is B's,
|
||||
/// and <c>HandleSubscribeAsync</c> drops the prior subscription before establishing the new one,
|
||||
/// so the cost is one extra round-trip on a respawning deploy. Pinning the count would pin a
|
||||
/// race; the final refs are the invariant.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_child_spawned_by_a_fresh_deploy_ends_with_the_NEW_refs()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var (host, factory) = DeployOneDriver(db, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) });
|
||||
|
||||
AwaitAssert(
|
||||
() => factory.Created[0].LastSubscribedRefs.ShouldBe(new[] { SpeedPath }),
|
||||
duration: PresenceBudget);
|
||||
|
||||
var revB = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevB,
|
||||
new[] { (DriverId, "Modbus", "dev1", "temp", "Float", false) },
|
||||
Array.Empty<(string, string, string, string, string?)>(),
|
||||
enabled: true);
|
||||
host.Tell(new Commons.Messages.Deploy.DispatchDeployment(revB, RevB, CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(2, "the tag-set change re-spawns the child (#516 config respawn)");
|
||||
factory.Created[1].LastSubscribedRefs.ShouldBe(
|
||||
new[] { TempPath },
|
||||
"the newly-spawned child is on the PREVIOUS revision's refs — the spawn-time set was not superseded");
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
// A Restart on top of that must hand the new child B's set, not A's — i.e. the authority really is
|
||||
// repopulated WHOLE on every apply rather than accumulated.
|
||||
host.Tell(new RestartDriver("c1", DriverId, "tester", Guid.NewGuid()));
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(3);
|
||||
factory.Created[2].LastSubscribedRefs.ShouldBe(new[] { TempPath });
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The empty-set semantics are load-bearing and must not degrade into "skip the send": an EMPTY value
|
||||
/// set is what tells the child to <c>Unsubscribe</c> and drop a stale handle, so it must be SENT.
|
||||
/// <para>
|
||||
/// The subject is a driver whose only raw tag is alarm-bearing: alarm tags are excluded from the
|
||||
/// value set and carried in the alarm one, so its value half is genuinely empty while its alarm
|
||||
/// half is not — and both halves ride in ONE <c>SetDesiredSubscriptions</c>. That is what makes
|
||||
/// the absence assertion non-vacuous: <c>SubscribeAlarmsCount >= 1</c> proves the message
|
||||
/// ARRIVED, and <c>SubscribeCount == 0</c> then proves the empty value half was honoured rather
|
||||
/// than the whole send being skipped. Asserting only "it did not subscribe" would pass equally
|
||||
/// against a build that sent the respawned child nothing at all — the 03/S16 defect itself.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Restart_of_a_driver_with_an_empty_value_set_still_sends_the_set()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var (host, factory) = DeployOneDriver(
|
||||
db,
|
||||
new[] { (DriverId, "Modbus", "dev1", "temp", "Float", false) },
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["temp"] = """{"alarm":{"alarmType":"AlarmCondition","severity":500}}""",
|
||||
});
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1);
|
||||
factory.Created[0].SubscribeAlarmsCount.ShouldBeGreaterThanOrEqualTo(1);
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
host.Tell(new RestartDriver("c1", DriverId, "tester", Guid.NewGuid()));
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(2);
|
||||
var respawned = factory.Created[1];
|
||||
respawned.InitializeCount.ShouldBe(1, "the respawned child was never initialised");
|
||||
// Positive control: the SetDesiredSubscriptions genuinely reached the respawned child.
|
||||
respawned.SubscribeAlarmsCount.ShouldBeGreaterThanOrEqualTo(
|
||||
1, "the respawned child received no desired set at all — the absence below would be vacuous");
|
||||
respawned.LastSubscribedAlarmRefs.ShouldBe(new[] { TempPath });
|
||||
respawned.SubscribeCount.ShouldBe(
|
||||
0, "an EMPTY value set must route to Unsubscribe / no-op, never to a spurious subscribe");
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>Review finding F1 — the alarm subscription must FOLLOW a changed desired set, not latch.</b>
|
||||
/// The value side converges by construction (<c>HandleSubscribeAsync</c> drops the prior subscription
|
||||
/// before establishing the new one, and <c>StoreDesiredSubscriptions</c> is a full replace), but the
|
||||
/// alarm side is guarded by <c>_alarmSubscriptionHandle is not null</c> in BOTH
|
||||
/// <c>SubscribeDesiredAlarms</c> and <c>HandleSubscribeAlarmsAsync</c> — so once a handle exists it
|
||||
/// was never re-established for the life of the connection.
|
||||
/// <para>
|
||||
/// That became reachable when every spawn started carrying a desired set: a tag-set edit forces a
|
||||
/// respawn (raw tags fold into the merged <c>DriverConfig</c>, #516), and a fast-initialising
|
||||
/// driver connects and subscribes the transient spawn-time set — the PREVIOUS revision's alarm
|
||||
/// refs — before the bulk push lands. The bulk push then early-returns, latching the old set for
|
||||
/// the connection's life. Drivers that FILTER on <c>sourceNodeIds</c> (OPC UA Client, AB CIP,
|
||||
/// FOCAS) would silently never deliver the newly-added alarm.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_deploy_that_widens_the_alarm_set_re_subscribes_the_new_refs()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var (host, factory) = DeployOneDriver(
|
||||
db,
|
||||
new[] { (DriverId, "Modbus", "dev1", "alarmX", "Float", false) },
|
||||
new Dictionary<string, string> { ["alarmX"] = AlarmConfig });
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1);
|
||||
factory.Created[0].LastSubscribedAlarmRefs.ShouldBe(new[] { AlarmXPath }, ignoreOrder: true);
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
// Revision B WIDENS the alarm set: X stays, Y is added.
|
||||
var revB = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevB,
|
||||
new[]
|
||||
{
|
||||
(DriverId, "Modbus", "dev1", "alarmX", "Float", false),
|
||||
(DriverId, "Modbus", "dev1", "alarmY", "Float", false),
|
||||
},
|
||||
Array.Empty<(string, string, string, string, string?)>(),
|
||||
enabled: true,
|
||||
tagConfigByTag: new Dictionary<string, string>
|
||||
{
|
||||
["alarmX"] = AlarmConfig,
|
||||
["alarmY"] = AlarmConfig,
|
||||
});
|
||||
host.Tell(new Commons.Messages.Deploy.DispatchDeployment(revB, RevB, CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
var current = factory.Created[^1];
|
||||
current.LastSubscribedAlarmRefs.ShouldBe(
|
||||
new[] { AlarmXPath, AlarmYPath },
|
||||
ignoreOrder: true,
|
||||
"the driver is still subscribed to the PREVIOUS revision's alarm refs — the alarm " +
|
||||
"subscription latched on the transient spawn-time set and never followed the new one");
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>The repo's first genuine Akka SUPERVISION-restart test (03/U6).</b> Not a stop, not a stub: the
|
||||
/// driver raises a malformed <c>OnDataChange</c>, the actor NREs on its own thread, and Akka's default
|
||||
/// decider Restarts the child in place.
|
||||
/// <para>
|
||||
/// A restart is strictly worse than the operator-Restart case 03/S16 names. It recreates the actor
|
||||
/// object around the SAME driver instance without going through <c>SpawnChild</c>, so nothing
|
||||
/// re-sends <c>InitializeRequested</c> — while <c>PreRestart</c> has already run <c>PostStop</c>,
|
||||
/// which called <c>ShutdownAsync</c> on that very driver. The child therefore sat in
|
||||
/// <c>Connecting</c> forever around a shut-down driver, with its desired set gone too, and
|
||||
/// silently: Akka logs the fault, nothing logs the non-recovery.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The assertions are FUNCTIONS of the new incarnation — it re-initialised, it re-subscribed the
|
||||
/// deployed refs — never that it exists. <c>Created.Count == 1</c> is what distinguishes this from
|
||||
/// the operator path: a restart must NOT produce a second driver object.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Supervision_restart_reinitialises_and_re_subscribes_the_child()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var (_, factory) = DeployOneDriver(db, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) });
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1);
|
||||
factory.Created[0].SubscribeCount.ShouldBe(1);
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
var driver = factory.Created[0];
|
||||
driver.InitializeCount.ShouldBe(1);
|
||||
|
||||
// The crash. Subscribing is what attaches the actor's OnDataChange handler, so this only reaches the
|
||||
// actor thread because the arrangement above got as far as a live subscription.
|
||||
driver.FireMalformedDataChange(SpeedPath);
|
||||
|
||||
// FIRST: prove the crash seam actually tripped supervision, before asserting anything about the
|
||||
// recovery. PreRestart runs PostStop, which calls ShutdownAsync on the driver — so a raised
|
||||
// ShutdownCount is the evidence that a restart happened at all. Without this the assertions below
|
||||
// would be ambiguous: "the child was not re-provisioned" and "the child never crashed" look
|
||||
// identical from the outside, and a seam that silently stopped working would read as a passing
|
||||
// regression test forever.
|
||||
AwaitAssert(
|
||||
() => driver.ShutdownCount.ShouldBeGreaterThanOrEqualTo(
|
||||
1, "the crash seam did not trip supervision — nothing below is testing a restart"),
|
||||
duration: PresenceBudget);
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(
|
||||
1, "a supervision restart must re-use the same driver object, not respawn the child");
|
||||
driver.InitializeCount.ShouldBe(
|
||||
2,
|
||||
"the restarted child was never re-initialised — it is sitting in Connecting around a " +
|
||||
"driver its own PostStop already shut down (03/S6, the worse half of 03/S16)");
|
||||
driver.SubscribeCount.ShouldBeGreaterThanOrEqualTo(
|
||||
2, "the restarted child never re-subscribed — its desired set died with the old instance");
|
||||
driver.LastSubscribedRefs.ShouldBe(new[] { SpeedPath });
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>Review finding F2 — a reprovision must be routed by INCARNATION, not just by driver id.</b>
|
||||
/// <c>ReprovisionRequested</c> is delivered asynchronously, so one can still be in flight from an old
|
||||
/// incarnation when <c>HandleRestartDriver</c> has already stopped that child and swapped in a new
|
||||
/// one under the same <c>DriverInstanceId</c>. Routing on the id alone delivers it to the NEW child,
|
||||
/// which is in <c>Connecting</c> around a healthy driver: it triggers a second, concurrent
|
||||
/// <c>InitializeAsync</c> that races the very dispose-before-overwrite the #527 teardown exists to
|
||||
/// serialise (<c>ModbusDriver._transport</c>, <c>MqttDriver._connection</c>).
|
||||
/// <para>
|
||||
/// A stale sender is simulated directly — a probe standing in for the dead incarnation — because
|
||||
/// provoking the real race would be timing-dependent. The assertion is an ABSENCE, so it carries
|
||||
/// a positive control: the current child is proven alive and provisioned first, and its
|
||||
/// <c>InitializeCount</c> must then not move.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_reprovision_from_a_stale_incarnation_is_ignored()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
// The child must be in CONNECTING for this defect to bite: InitializeRequested is unhandled in the
|
||||
// Connected behaviour, so an already-connected child would swallow the stale reprovision for the
|
||||
// wrong reason and the test would pass vacuously. The gated init holds it in Connecting.
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus", gateInitialize: true);
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevA,
|
||||
new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) },
|
||||
Array.Empty<(string, string, string, string, string?)>(),
|
||||
enabled: true);
|
||||
var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
|
||||
|
||||
// Positive control: the current child is alive and has begun its one and only initialise.
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1);
|
||||
factory.Created[0].InitializeCount.ShouldBe(1);
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
var driver = factory.Created[0];
|
||||
|
||||
// A ReprovisionRequested that did NOT come from the child the host currently holds.
|
||||
var staleIncarnation = CreateTestProbe();
|
||||
host.Tell(
|
||||
new DriverInstanceActor.ReprovisionRequested(DriverId, "NullReferenceException"),
|
||||
staleIncarnation.Ref);
|
||||
|
||||
// Round-trip a message the host must answer, so the reprovision above is provably processed before
|
||||
// the absence is asserted — otherwise "nothing happened yet" would pass for "it was ignored".
|
||||
var coordinator = CreateTestProbe();
|
||||
host.Tell(new Commons.Messages.Fleet.GetDiagnostics(CorrelationId.NewId()), coordinator.Ref);
|
||||
coordinator.ExpectMsg<Commons.Interfaces.NodeDiagnosticsSnapshot>(PresenceBudget);
|
||||
|
||||
driver.InitializeCount.ShouldBe(
|
||||
1,
|
||||
"a reprovision from a dead incarnation reached the CURRENT child and started a second, " +
|
||||
"concurrent InitializeAsync on a driver that was already connected");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>Review finding F3 - a crash-looping driver must not become a reconnect storm.</b> There is no
|
||||
/// <c>SupervisorStrategy</c> anywhere under <c>src/Server</c>, so the default decider restarts a
|
||||
/// faulting child <b>immediately and without limit</b>. That was survivable while a restart did
|
||||
/// nothing; the reprovision hook now makes every one of them do
|
||||
/// <c>ShutdownAsync</c> + <c>InitializeAsync</c> + <c>SubscribeAsync</c>, so a driver that faults on
|
||||
/// every value would hammer its device as fast as it can crash.
|
||||
/// <para>
|
||||
/// The reprovision is therefore rate-limited per driver. Crucially it is <b>deferred, not
|
||||
/// dropped</b> - the last request must still be honoured, or a child whose final crash fell
|
||||
/// inside the window would be left un-provisioned forever, which is the very failure this whole
|
||||
/// plan exists to close. Two genuine back-to-back supervision restarts: the first reprovision
|
||||
/// applies immediately, the second is held and then lands.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_second_reprovision_inside_the_backoff_window_is_deferred_not_dropped()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
// Generous relative to the in-memory step time below, so "inside the window" is a decision rather
|
||||
// than a race, while still far under the presence budget the deferred landing waits on.
|
||||
var backoff = TimeSpan.FromSeconds(3);
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus");
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevA,
|
||||
new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) },
|
||||
Array.Empty<(string, string, string, string, string?)>(),
|
||||
enabled: true);
|
||||
V3ArtifactSeeder.SpawnHostAndApply(
|
||||
this, db, TestNode, deploymentId, RevA, factory, reprovisionBackoff: backoff);
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1);
|
||||
factory.Created[0].SubscribeCount.ShouldBe(1);
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
var driver = factory.Created[0];
|
||||
|
||||
// Crash 1: the first reprovision has no prior stamp, so it applies immediately.
|
||||
driver.FireMalformedDataChange(SpeedPath);
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
driver.InitializeCount.ShouldBe(2);
|
||||
driver.SubscribeCount.ShouldBe(2, "the re-subscribe must land before the second crash is armed");
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
// Crash 2, immediately after - well inside the window.
|
||||
driver.FireMalformedDataChange(SpeedPath);
|
||||
AwaitAssert(
|
||||
() => driver.ShutdownCount.ShouldBeGreaterThanOrEqualTo(
|
||||
2, "the second crash seam did not trip supervision - nothing below is testing a backoff"),
|
||||
duration: PresenceBudget);
|
||||
|
||||
driver.InitializeCount.ShouldBe(
|
||||
2, "the second reprovision was applied immediately - the backoff is not rate-limiting anything");
|
||||
|
||||
// ...but it must still land once the window elapses.
|
||||
AwaitAssert(
|
||||
() => driver.InitializeCount.ShouldBe(
|
||||
3, "the deferred reprovision was DROPPED, not deferred"),
|
||||
duration: PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The <c>_children.Keys</c> arm of the desired-set authority's union. A driver with NO authored tags
|
||||
/// is absent from both the value- and alarm-ref maps, so keying the authority off those maps alone
|
||||
/// would leave it with no entry at all — and "absent" must not be allowed to differ from "empty",
|
||||
/// because an empty set is what tells a child to <c>Unsubscribe</c> and drop a stale handle.
|
||||
/// <para>
|
||||
/// Asserted the only way it is observable: the tagless child is initialised (so it is provably
|
||||
/// alive and was provisioned) and never subscribes, while its tag-owning sibling in the same
|
||||
/// deployment does — the sibling is the positive control proving the apply's push pass actually
|
||||
/// ran, so "it did not subscribe" cannot pass for "nothing happened".
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_running_driver_with_no_authored_tags_is_handed_an_empty_set()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus");
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevA,
|
||||
new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) },
|
||||
Array.Empty<(string, string, string, string, string?)>(),
|
||||
enabled: true,
|
||||
taglessDrivers: new[] { "DI-TAGLESS" });
|
||||
V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(2);
|
||||
// Positive control: the sibling that DOES have a tag subscribed, so the push pass ran.
|
||||
factory.Created.Single(d => d.DriverInstanceId == DriverId)
|
||||
.SubscribeCount.ShouldBe(1);
|
||||
factory.Created.Single(d => d.DriverInstanceId == "DI-TAGLESS")
|
||||
.InitializeCount.ShouldBe(1);
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
factory.Created.Single(d => d.DriverInstanceId == "DI-TAGLESS")
|
||||
.SubscribeCount.ShouldBe(
|
||||
0, "a driver with no authored tags must be handed an EMPTY set, never a spurious subscribe");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The verified negative, pinned so this plan's edits to the restart paths cannot break the one that
|
||||
/// was already correct. Force Reconnect (<c>ReconnectDriver</c>) Tells <c>ForceReconnect</c> to the
|
||||
/// SAME child actor: the instance survives, so <c>_desiredRefs</c>/<c>_desiredAlarmRefs</c> are intact
|
||||
/// and the reconnect's Connected entry re-applies them by itself. Nothing needs re-provisioning, and
|
||||
/// no new driver object is created.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The budget here is deliberately wider than <see cref="RuntimeActorTestBase.PresenceBudget"/>:
|
||||
/// <c>SpawnChild</c> gives the child the production <c>DefaultReconnectInterval</c> (10 s) and the
|
||||
/// retry is a periodic timer, so the reconnect cannot land sooner than that. A budget is spent only
|
||||
/// when the assertion fails, so widening it costs nothing on the happy path.
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public void Force_reconnect_keeps_the_desired_set_and_re_subscribes()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var (host, factory) = DeployOneDriver(db, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) });
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1);
|
||||
factory.Created[0].SubscribeCount.ShouldBe(1);
|
||||
},
|
||||
duration: PresenceBudget);
|
||||
|
||||
var driver = factory.Created[0];
|
||||
host.Tell(new ReconnectDriver("c1", DriverId, "tester", Guid.NewGuid()));
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
factory.Created.Count.ShouldBe(1, "Force Reconnect must NOT respawn the child");
|
||||
driver.SubscribeCount.ShouldBeGreaterThanOrEqualTo(
|
||||
2, "the reconnected driver never re-subscribed its (surviving) desired set");
|
||||
driver.LastSubscribedRefs.ShouldBe(new[] { SpeedPath });
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(30));
|
||||
}
|
||||
}
|
||||
+83
@@ -40,6 +40,10 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest
|
||||
private static readonly TimeSpan FastPoll = TimeSpan.FromMilliseconds(50);
|
||||
private static readonly TimeSpan FastTimeout = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>The threshold used by the per-connection grace pair (03/S17). Deliberately an order of
|
||||
/// magnitude above <see cref="FastPoll"/> so the absence window below is a decision, not a race.</summary>
|
||||
private static readonly TimeSpan Grace = TimeSpan.FromMilliseconds(1500);
|
||||
|
||||
/// <summary>Drives the actor to Connected and returns once the connect has been announced.</summary>
|
||||
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) ConnectDriver(
|
||||
HealthControlledStubDriver driver,
|
||||
@@ -87,6 +91,14 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest
|
||||
/// <c>ModbusDriver.ReadAsync</c> leaves latched whenever the bad tag is read last — but it is still
|
||||
/// reading successfully, so <c>LastSuccessfulRead</c> keeps advancing. It must be left alone.
|
||||
/// A state-only watchdog would tear this driver down on every threshold, forever.
|
||||
/// <para>
|
||||
/// <b>Falsification partner (03/S17):</b>
|
||||
/// <see cref="A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again"/>.
|
||||
/// The silence baseline is <c>max(LastSuccessfulRead, _connectedSinceUtc)</c>, and each test kills
|
||||
/// one term of that max — this one goes red if the <c>LastSuccessfulRead</c> term is dropped (its
|
||||
/// connect ages far past the threshold while its reads keep succeeding), that one goes red if the
|
||||
/// <c>_connectedSinceUtc</c> term is dropped (i.e. if the original <c>??</c> is restored).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_degraded_driver_that_is_still_reading_successfully_is_left_alone()
|
||||
@@ -258,6 +270,77 @@ public sealed class DriverInstanceActorCommsLossWatchdogTests : RuntimeActorTest
|
||||
|
||||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 03/S17 — <b>the grace is per connection.</b> Drivers deliberately <em>preserve</em>
|
||||
/// <see cref="DriverHealth.LastSuccessfulRead"/> across Degraded/Faulted health writes (that
|
||||
/// preservation is the #527 design), so for a device that accepts a TCP connection but never serves
|
||||
/// data the stamp stays frozen from before the outage — already older than the threshold. Measuring
|
||||
/// silence from that stamp alone (<c>??</c>) makes the <b>first</b> health tick of every new
|
||||
/// connection re-trip, so the reconnect cycle runs at health-tick cadence (~30 s in production)
|
||||
/// instead of the designed grace. Measuring from <c>max(LastSuccessfulRead, _connectedSinceUtc)</c>
|
||||
/// gives each connection its own full window, which is exactly what the <c>_connectedSinceUtc</c>
|
||||
/// re-stamp on every Connected entry already promises in its comment.
|
||||
/// <para>
|
||||
/// <b>Falsification partner:</b>
|
||||
/// <see cref="A_degraded_driver_that_is_still_reading_successfully_is_left_alone"/> — see its
|
||||
/// remarks for how the pair covers both terms of the max.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The absence window carries the house positive control: a proven number of health-poll ticks
|
||||
/// processed <em>after</em> the reconnect, so "the watchdog did not re-trip" cannot pass because
|
||||
/// the actor stopped ticking. It is also deliberately far shorter than <see cref="Grace"/>, so
|
||||
/// it asserts a delay, not a disablement — <see cref="A_half_broken_device_is_still_declared_lost_once_the_grace_elapses"/>
|
||||
/// is the anti-vacuity half.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again()
|
||||
{
|
||||
var driver = new HealthControlledStubDriver
|
||||
{
|
||||
// The half-broken device: the socket connects, every read fails, and the last successful read is
|
||||
// frozen well before the outage — preserved across the Degraded writes, as every driver does.
|
||||
Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "reads time out"),
|
||||
};
|
||||
var (_, parent) = ConnectDriver(driver, commsLossTimeout: Grace, reconnectInterval: TimeSpan.FromMilliseconds(100));
|
||||
|
||||
// The first trip is correct and expected — the device really has been silent for five minutes.
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget)
|
||||
.Connected.ShouldBeFalse();
|
||||
// The retry re-initialises successfully (the socket still accepts), so a NEW connection starts here.
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget)
|
||||
.Connected.ShouldBeTrue();
|
||||
|
||||
var ticksAtReconnect = driver.HealthReadCount;
|
||||
parent.ExpectNoMsg(TimeSpan.FromMilliseconds(400));
|
||||
(driver.HealthReadCount - ticksAtReconnect).ShouldBeGreaterThanOrEqualTo(
|
||||
4,
|
||||
"health-poll ticks did not run after the reconnect — the absence assertion above would be vacuous");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The anti-vacuity half of the 03/S17 pair: the per-connection grace <b>delays</b> the watchdog, it
|
||||
/// does not disable it. The same half-broken device is still declared lost once the grace elapses —
|
||||
/// without this, a "fix" that simply stopped evaluating the watchdog after the first connection would
|
||||
/// pass <see cref="A_reconnected_driver_gets_a_full_grace_window_before_the_watchdog_trips_again"/>
|
||||
/// and look correct.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_half_broken_device_is_still_declared_lost_once_the_grace_elapses()
|
||||
{
|
||||
var driver = new HealthControlledStubDriver
|
||||
{
|
||||
Health = new DriverHealth(DriverState.Degraded, DateTime.UtcNow.AddMinutes(-5), "reads time out"),
|
||||
};
|
||||
var (_, parent) = ConnectDriver(driver, commsLossTimeout: Grace, reconnectInterval: TimeSpan.FromMilliseconds(100));
|
||||
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget).Connected.ShouldBeFalse();
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget).Connected.ShouldBeTrue();
|
||||
|
||||
// The second connection serves no data either, so once ITS grace elapses the watchdog trips again.
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(PresenceBudget).Connected.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -188,3 +188,215 @@ internal sealed class StubHandle : ISubscriptionHandle
|
||||
/// <summary>Gets the diagnostic ID of the subscription.</summary>
|
||||
public string DiagnosticId => "stub-sub";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A per-instance recording driver for the restart / re-subscribe suite (03/S16, U6): unlike
|
||||
/// <see cref="SubscribableStubDriver"/> it carries the <b>spec's</b> <see cref="DriverInstanceId"/>
|
||||
/// (<see cref="StubDriver"/> hard-codes a constant on a non-virtual member), and it counts and records the
|
||||
/// alarm subscription alongside the value one — the two travel in a single
|
||||
/// <c>SetDesiredSubscriptions</c> but through two different child code paths, so both need observing.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>The falsification bar these counters exist for:</b> a restart test must assert a FUNCTION the new
|
||||
/// incarnation performs — it subscribed, it re-initialised — never that the incarnation exists.
|
||||
/// Existence is what
|
||||
/// <c>DriverHostActorReconcileTests.RestartDriver_respawns_the_child_without_an_actor_name_collision</c>
|
||||
/// asserts, and that test passes against the very defect 03/S16 names.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b><see cref="FireMalformedDataChange"/> is the supervision crash seam</b> — see
|
||||
/// <c>DriverHostActorRestartTests</c>'s file remarks for why the obvious alternatives do not work.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class RecordingSubscribableDriver : IDriver, ISubscribable, IAlarmSource
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
private readonly StubHandle _handle = new();
|
||||
private readonly RecordingAlarmHandle _alarmHandle = new();
|
||||
private int _initializeCount;
|
||||
private int _shutdownCount;
|
||||
private int _subscribeCount;
|
||||
private int _subscribeAlarmsCount;
|
||||
private IReadOnlyList<string>? _lastSubscribedRefs;
|
||||
private IReadOnlyList<string>? _lastSubscribedAlarmRefs;
|
||||
|
||||
/// <summary>Initializes a new recording driver stamped with the spec's identity.</summary>
|
||||
/// <param name="driverInstanceId">The driver instance id from the spec that created it.</param>
|
||||
/// <param name="driverType">The driver type from the spec that created it.</param>
|
||||
public RecordingSubscribableDriver(string driverInstanceId, string driverType)
|
||||
{
|
||||
DriverInstanceId = driverInstanceId;
|
||||
DriverType = driverType;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string DriverInstanceId { get; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public string DriverType { get; }
|
||||
|
||||
/// <summary>Number of <see cref="InitializeAsync"/> calls. A supervision restart re-uses the SAME driver
|
||||
/// object (the Props closure captured it), so this going 1 to 2 is how the reprovision is observed.</summary>
|
||||
public int InitializeCount => Volatile.Read(ref _initializeCount);
|
||||
|
||||
/// <summary>Number of <see cref="ShutdownAsync"/> calls — a restart's <c>PostStop</c> raises it.</summary>
|
||||
public int ShutdownCount => Volatile.Read(ref _shutdownCount);
|
||||
|
||||
/// <summary>Number of <see cref="SubscribeAsync"/> calls.</summary>
|
||||
public int SubscribeCount => Volatile.Read(ref _subscribeCount);
|
||||
|
||||
/// <summary>Number of <see cref="SubscribeAlarmsAsync"/> calls.</summary>
|
||||
public int SubscribeAlarmsCount => Volatile.Read(ref _subscribeAlarmsCount);
|
||||
|
||||
/// <summary>The refs passed to the most recent <see cref="SubscribeAsync"/>, or null if never called.</summary>
|
||||
public IReadOnlyList<string>? LastSubscribedRefs
|
||||
{
|
||||
get { lock (_gate) return _lastSubscribedRefs; }
|
||||
}
|
||||
|
||||
/// <summary>The refs passed to the most recent <see cref="SubscribeAlarmsAsync"/>, or null if never called.</summary>
|
||||
public IReadOnlyList<string>? LastSubscribedAlarmRefs
|
||||
{
|
||||
get { lock (_gate) return _lastSubscribedAlarmRefs; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// When set, <see cref="InitializeAsync"/> awaits it instead of completing inline — so the actor
|
||||
/// stays in <c>Connecting</c> for as long as the test wants. Needed because several defects only
|
||||
/// bite a child that has NOT yet connected (<c>InitializeRequested</c> is unhandled in
|
||||
/// <c>Connected</c>, so a test built on an already-connected child passes vacuously).
|
||||
/// </summary>
|
||||
public TaskCompletionSource? InitializeGate { get; set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _initializeCount);
|
||||
if (InitializeGate is { } gate) await gate.Task.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _shutdownCount);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
||||
|
||||
/// <inheritdoc />
|
||||
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<ISubscriptionHandle> SubscribeAsync(
|
||||
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _subscribeCount);
|
||||
lock (_gate) _lastSubscribedRefs = fullReferences.ToArray();
|
||||
return Task.FromResult<ISubscriptionHandle>(_handle);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public event EventHandler<AlarmEventArgs>? OnAlarmEvent;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IAlarmSubscriptionHandle> SubscribeAlarmsAsync(
|
||||
IReadOnlyList<string> sourceNodeIds, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _subscribeAlarmsCount);
|
||||
lock (_gate) _lastSubscribedAlarmRefs = sourceNodeIds.ToArray();
|
||||
return Task.FromResult<IAlarmSubscriptionHandle>(_alarmHandle);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task AcknowledgeAsync(
|
||||
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <summary>Fires a well-formed data change (the ordinary production path).</summary>
|
||||
/// <param name="fullRef">The reference whose value changed.</param>
|
||||
/// <param name="value">The new value.</param>
|
||||
public void FireDataChange(string fullRef, object? value) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(
|
||||
_handle, fullRef, new DataValueSnapshot(value, 0u, DateTime.UtcNow, DateTime.UtcNow)));
|
||||
|
||||
/// <summary>
|
||||
/// Raises <see cref="OnDataChange"/> with a NULL <c>Snapshot</c> — the supervision crash seam.
|
||||
/// <c>DataChangeEventArgs</c> is a positional record with no null validation, and the actor's
|
||||
/// <c>OnDataChangeForward</c> dereferences <c>msg.Snapshot</c> unguarded ON THE ACTOR THREAD, so this
|
||||
/// produces a genuine unhandled exception inside the child, which Akka's default decider turns into a
|
||||
/// Restart. The fault itself is arbitrary; what is under test is the RECOVERY.
|
||||
/// </summary>
|
||||
/// <param name="fullRef">The reference to name in the malformed event.</param>
|
||||
public void FireMalformedDataChange(string fullRef) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, null!));
|
||||
|
||||
/// <summary>Raises <see cref="OnAlarmEvent"/>; present so the event is not merely written-to-never-read.</summary>
|
||||
/// <param name="conditionId">The condition id to raise.</param>
|
||||
public void FireAlarm(string conditionId) =>
|
||||
OnAlarmEvent?.Invoke(this, new AlarmEventArgs(
|
||||
_alarmHandle, conditionId, conditionId, "AlarmCondition", "test", AlarmSeverity.High, DateTime.UtcNow));
|
||||
|
||||
private sealed class RecordingAlarmHandle : IAlarmSubscriptionHandle
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public string DiagnosticId => "recording-alarm-sub";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="IDriverFactory"/> that hands out <see cref="RecordingSubscribableDriver"/>s and KEEPS
|
||||
/// every one it created, in creation order. The list — not a single "the driver" reference — is the point:
|
||||
/// an operator Restart tears the child down and spawns a new one around a NEW driver object, so the
|
||||
/// assertion subject is <c>Created[1]</c>, while a SUPERVISION restart re-uses <c>Created[0]</c> (the Props
|
||||
/// closure captured it) and part of the assertion is that <c>Created.Count</c> stayed 1.
|
||||
/// </summary>
|
||||
internal sealed class RecordingSubscribableDriverFactory : IDriverFactory
|
||||
{
|
||||
private readonly string _supportedType;
|
||||
private readonly Lock _gate = new();
|
||||
private readonly List<RecordingSubscribableDriver> _created = new();
|
||||
|
||||
private readonly bool _gateInitialize;
|
||||
|
||||
/// <summary>Initializes a factory that materialises exactly one driver type.</summary>
|
||||
/// <param name="supportedType">The driver type this factory answers for.</param>
|
||||
/// <param name="gateInitialize">When true, every created driver gets an un-completed
|
||||
/// <see cref="RecordingSubscribableDriver.InitializeGate"/>, so its child actor stays in
|
||||
/// <c>Connecting</c> until the test completes it.</param>
|
||||
public RecordingSubscribableDriverFactory(string supportedType, bool gateInitialize = false)
|
||||
{
|
||||
_supportedType = supportedType;
|
||||
_gateInitialize = gateInitialize;
|
||||
}
|
||||
|
||||
/// <summary>Every driver this factory created, in creation order.</summary>
|
||||
public IReadOnlyList<RecordingSubscribableDriver> Created
|
||||
{
|
||||
get { lock (_gate) return _created.ToArray(); }
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
|
||||
{
|
||||
if (!string.Equals(driverType, _supportedType, StringComparison.Ordinal)) return null;
|
||||
var driver = new RecordingSubscribableDriver(driverInstanceId, driverType);
|
||||
if (_gateInitialize)
|
||||
driver.InitializeGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
lock (_gate) _created.Add(driver);
|
||||
return driver;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyCollection<string> SupportedTypes => new[] { _supportedType };
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
|
||||
@@ -36,16 +37,35 @@ public static class V3ArtifactSeeder
|
||||
/// <param name="rev">The revision hash the deployment is sealed under.</param>
|
||||
/// <param name="rawTags">The raw tags to materialise, one per (driver, device, tag).</param>
|
||||
/// <param name="unsRefs">UNS references projecting a raw tag (by <c>BackingTag</c>) into an equipment.</param>
|
||||
/// <param name="enabled">Whether the seeded <c>DriverInstances</c> are Enabled — i.e. whether applying this
|
||||
/// artifact actually SPAWNS driver children. Defaults to <c>false</c>, the shape every pre-existing caller
|
||||
/// relies on: the value-routing suites inject values straight into the host and want no children at all.
|
||||
/// The restart/re-subscribe suite (03/S16) passes <c>true</c>, because its whole subject is what a spawned
|
||||
/// child is handed.</param>
|
||||
/// <param name="tagConfigByTag">Optional per-tag <c>TagConfig</c> JSON, keyed by tag name; tags not named
|
||||
/// here keep the default <c>"{}"</c>. The only way to seed an <b>alarm-bearing</b> raw tag (the alarm intent
|
||||
/// is parsed out of <c>TagConfig.alarm</c> by <c>TagConfigIntent</c>), which the alarm-ref half of the
|
||||
/// restart suite needs — an alarm-bearing raw tag is excluded from the VALUE subscription set and carried
|
||||
/// in the alarm one instead, so the two travel through different child code paths.</param>
|
||||
/// <param name="taglessDrivers">Extra <c>DriverInstance</c> ids carrying NO devices and NO tags. The
|
||||
/// only way to express a driver that is a running child yet absent from both the value- and alarm-ref
|
||||
/// maps — the <c>_children.Keys</c> arm of <c>ApplyDesiredSubscriptions</c>'s union, which must still
|
||||
/// hand that child an explicit EMPTY set rather than skipping it.</param>
|
||||
/// <returns>The seeded deployment's id.</returns>
|
||||
public static DeploymentId SeedV3Deployment(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev,
|
||||
(string Driver, string DriverName, string Device, string Tag, string DataType, bool Writable)[] rawTags,
|
||||
(string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs)
|
||||
(string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs,
|
||||
bool enabled = false,
|
||||
IReadOnlyDictionary<string, string>? tagConfigByTag = null,
|
||||
string[]? taglessDrivers = null)
|
||||
{
|
||||
var drivers = rawTags
|
||||
.Select(t => (t.Driver, t.DriverName))
|
||||
.Distinct()
|
||||
.Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = false })
|
||||
.Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })
|
||||
.Concat((taglessDrivers ?? Array.Empty<string>())
|
||||
.Select(id => new { DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }))
|
||||
.ToArray();
|
||||
var devices = rawTags
|
||||
.Select(t => (t.Driver, t.Device))
|
||||
@@ -60,7 +80,7 @@ public static class V3ArtifactSeeder
|
||||
Name = t.Tag,
|
||||
DataType = t.DataType,
|
||||
AccessLevel = t.Writable ? 1 : 0, // TagAccessLevel.ReadWrite = 1
|
||||
TagConfig = "{}",
|
||||
TagConfig = tagConfigByTag is not null && tagConfigByTag.TryGetValue(t.Tag, out var tc) ? tc : "{}",
|
||||
}).ToArray();
|
||||
|
||||
// UNS topology: one area/line per distinct (Area, Line); one equipment per (Area, Line, Equip).
|
||||
@@ -114,13 +134,21 @@ public static class V3ArtifactSeeder
|
||||
/// <param name="localNode">The host's local node id.</param>
|
||||
/// <param name="deploymentId">The deployment to dispatch.</param>
|
||||
/// <param name="rev">The revision hash the deployment was sealed under.</param>
|
||||
/// <param name="driverFactory">Optional driver factory forwarded to <c>DriverHostActor.Props</c>. Null (the
|
||||
/// default, preserving every existing call site) leaves the host on its own default, which materialises no
|
||||
/// real drivers. Supply one — with <c>enabled: true</c> on the seed — to actually spawn driver children.</param>
|
||||
/// <param name="reprovisionBackoff">Optional per-driver supervision-reprovision rate limit forwarded to
|
||||
/// <c>DriverHostActor.Props</c>; null keeps the production default. Exists so a test can observe the
|
||||
/// deferral without waiting the production window.</param>
|
||||
/// <returns>The host actor plus the publish and mux probes wired into it.</returns>
|
||||
public static (IActorRef Actor, TestProbe Publish, TestProbe Mux) SpawnHostAndApply(
|
||||
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
|
||||
NodeId localNode, DeploymentId deploymentId, RevisionHash rev)
|
||||
NodeId localNode, DeploymentId deploymentId, RevisionHash rev,
|
||||
IDriverFactory? driverFactory = null,
|
||||
TimeSpan? reprovisionBackoff = null)
|
||||
{
|
||||
var publish = kit.CreateTestProbe();
|
||||
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref);
|
||||
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory, reprovisionBackoff);
|
||||
|
||||
Dispatch(actor, coordinator, deploymentId, rev);
|
||||
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(RuntimeActorTestBase.PresenceBudget);
|
||||
@@ -140,14 +168,14 @@ public static class V3ArtifactSeeder
|
||||
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
|
||||
NodeId localNode, DeploymentId deploymentId, RevisionHash rev)
|
||||
{
|
||||
var (actor, mux, coordinator) = Spawn(kit, db, localNode, opcUaPublishActor: null);
|
||||
var (actor, mux, coordinator) = Spawn(kit, db, localNode, opcUaPublishActor: null, driverFactory: null, reprovisionBackoff: null);
|
||||
Dispatch(actor, coordinator, deploymentId, rev);
|
||||
return (actor, mux);
|
||||
}
|
||||
|
||||
private static (IActorRef Actor, TestProbe Mux, TestProbe Coordinator) Spawn(
|
||||
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db, NodeId localNode,
|
||||
IActorRef? opcUaPublishActor)
|
||||
IActorRef? opcUaPublishActor, IDriverFactory? driverFactory, TimeSpan? reprovisionBackoff)
|
||||
{
|
||||
var coordinator = kit.CreateTestProbe();
|
||||
var mux = kit.CreateTestProbe();
|
||||
@@ -155,11 +183,13 @@ public static class V3ArtifactSeeder
|
||||
|
||||
var actor = kit.Sys.ActorOf(DriverHostActor.Props(
|
||||
db, localNode, coordinator.Ref,
|
||||
driverFactory: driverFactory,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
dependencyMux: mux.Ref,
|
||||
opcUaPublishActor: opcUaPublishActor,
|
||||
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
|
||||
virtualTagHostOverride: vtHost.Ref));
|
||||
virtualTagHostOverride: vtHost.Ref,
|
||||
reprovisionBackoff: reprovisionBackoff));
|
||||
|
||||
return (actor, mux, coordinator);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user