dohertj2/lmxopcua
1
0
Fork 0
Code Issues 32 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(runtime): rate-limit the supervision reprovision — deferred, not dropped (review F3)

Browse Source 
The default decider restarts a faulting child immediately and without limit, and
the hook makes every restart do ShutdownAsync+InitializeAsync+SubscribeAsync, so a
driver faulting on every value became a reconnect storm. Per-driver last-reprovision
stamp + DefaultReprovisionBackoff (10s, matched to DefaultReconnectInterval);
a held request is re-delivered by a single per-driver timer carrying the original
incarnation ref, so the F2 identity check still applies.

Seen RED with the backoff disabled: InitializeCount == 3 (second applied
immediately) where the test requires 2-then-3.

Also: the _children.Keys union-arm test (seeder gained taglessDrivers) and the
_desiredByDriver xmldoc note that it lingers on last-known-good after a failed apply.
This commit is contained in:
Joseph Doherty
2026-08-20 10:39:13 -04:00
parent 2b7100cf19
commit 0793b0cba8
4 changed files with 212 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorRestartTests.cs
+109
View File
@@ -478,6 +478,115 @@ public sealed class DriverHostActorRestartTests : RuntimeActorTestBase
"concurrent InitializeAsync on a driver that was already connected");
}
/// <summary>
/// <b>Review finding F3 - a crash-looping driver must not become a reconnect storm.</b> There is no
/// <c>SupervisorStrategy</c> anywhere under <c>src/Server</c>, so the default decider restarts a
/// faulting child <b>immediately and without limit</b>. That was survivable while a restart did
/// nothing; the reprovision hook now makes every one of them do
/// <c>ShutdownAsync</c> + <c>InitializeAsync</c> + <c>SubscribeAsync</c>, so a driver that faults on
/// every value would hammer its device as fast as it can crash.
/// <para>
/// The reprovision is therefore rate-limited per driver. Crucially it is <b>deferred, not
/// dropped</b> - the last request must still be honoured, or a child whose final crash fell
/// inside the window would be left un-provisioned forever, which is the very failure this whole
/// plan exists to close. Two genuine back-to-back supervision restarts: the first reprovision
/// applies immediately, the second is held and then lands.
/// </para>
/// </summary>
[Fact]
public void A_second_reprovision_inside_the_backoff_window_is_deferred_not_dropped()
{
var db = NewInMemoryDbFactory();
// Generous relative to the in-memory step time below, so "inside the window" is a decision rather
// than a race, while still far under the presence budget the deferred landing waits on.
var backoff = TimeSpan.FromSeconds(3);
var factory = new RecordingSubscribableDriverFactory("Modbus");
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
db, RevA,
new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) },
Array.Empty<(string, string, string, string, string?)>(),
enabled: true);
V3ArtifactSeeder.SpawnHostAndApply(
this, db, TestNode, deploymentId, RevA, factory, reprovisionBackoff: backoff);
AwaitAssert(
() =>
{
factory.Created.Count.ShouldBe(1);
factory.Created[0].SubscribeCount.ShouldBe(1);
},
duration: PresenceBudget);
var driver = factory.Created[0];
// Crash 1: the first reprovision has no prior stamp, so it applies immediately.
driver.FireMalformedDataChange(SpeedPath);
AwaitAssert(
() =>
{
driver.InitializeCount.ShouldBe(2);
driver.SubscribeCount.ShouldBe(2, "the re-subscribe must land before the second crash is armed");
},
duration: PresenceBudget);
// Crash 2, immediately after - well inside the window.
driver.FireMalformedDataChange(SpeedPath);
AwaitAssert(
() => driver.ShutdownCount.ShouldBeGreaterThanOrEqualTo(
2, "the second crash seam did not trip supervision - nothing below is testing a backoff"),
duration: PresenceBudget);
driver.InitializeCount.ShouldBe(
2, "the second reprovision was applied immediately - the backoff is not rate-limiting anything");
// ...but it must still land once the window elapses.
AwaitAssert(
() => driver.InitializeCount.ShouldBe(
3, "the deferred reprovision was DROPPED, not deferred"),
duration: PresenceBudget);
}
/// <summary>
/// The <c>_children.Keys</c> arm of the desired-set authority's union. A driver with NO authored tags
/// is absent from both the value- and alarm-ref maps, so keying the authority off those maps alone
/// would leave it with no entry at all — and "absent" must not be allowed to differ from "empty",
/// because an empty set is what tells a child to <c>Unsubscribe</c> and drop a stale handle.
/// <para>
/// Asserted the only way it is observable: the tagless child is initialised (so it is provably
/// alive and was provisioned) and never subscribes, while its tag-owning sibling in the same
/// deployment does — the sibling is the positive control proving the apply's push pass actually
/// ran, so "it did not subscribe" cannot pass for "nothing happened".
/// </para>
/// </summary>
[Fact]
public void A_running_driver_with_no_authored_tags_is_handed_an_empty_set()
{
var db = NewInMemoryDbFactory();
var factory = new RecordingSubscribableDriverFactory("Modbus");
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
db, RevA,
new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) },
Array.Empty<(string, string, string, string, string?)>(),
enabled: true,
taglessDrivers: new[] { "DI-TAGLESS" });
V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
AwaitAssert(
() =>
{
factory.Created.Count.ShouldBe(2);
// Positive control: the sibling that DOES have a tag subscribed, so the push pass ran.
factory.Created.Single(d => d.DriverInstanceId == DriverId)
.SubscribeCount.ShouldBe(1);
factory.Created.Single(d => d.DriverInstanceId == "DI-TAGLESS")
.InitializeCount.ShouldBe(1);
},
duration: PresenceBudget);
factory.Created.Single(d => d.DriverInstanceId == "DI-TAGLESS")
.SubscribeCount.ShouldBe(
0, "a driver with no authored tags must be handed an EMPTY set, never a spurious subscribe");
}
/// <summary>
/// The verified negative, pinned so this plan's edits to the restart paths cannot break the one that
/// was already correct. Force Reconnect (<c>ReconnectDriver</c>) Tells <c>ForceReconnect</c> to the
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Harness/V3ArtifactSeeder.cs
+18 -6
View File
@@ -47,18 +47,25 @@ public static class V3ArtifactSeeder
/// is parsed out of <c>TagConfig.alarm</c> by <c>TagConfigIntent</c>), which the alarm-ref half of the
/// restart suite needs — an alarm-bearing raw tag is excluded from the VALUE subscription set and carried
/// in the alarm one instead, so the two travel through different child code paths.</param>
/// <param name="taglessDrivers">Extra <c>DriverInstance</c> ids carrying NO devices and NO tags. The
/// only way to express a driver that is a running child yet absent from both the value- and alarm-ref
/// maps — the <c>_children.Keys</c> arm of <c>ApplyDesiredSubscriptions</c>'s union, which must still
/// hand that child an explicit EMPTY set rather than skipping it.</param>
/// <returns>The seeded deployment's id.</returns>
public static DeploymentId SeedV3Deployment(
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev,
(string Driver, string DriverName, string Device, string Tag, string DataType, bool Writable)[] rawTags,
(string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs,
bool enabled = false,
IReadOnlyDictionary<string, string>? tagConfigByTag = null)
IReadOnlyDictionary<string, string>? tagConfigByTag = null,
string[]? taglessDrivers = null)
{
var drivers = rawTags
.Select(t => (t.Driver, t.DriverName))
.Distinct()
.Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })
.Concat((taglessDrivers ?? Array.Empty<string>())
.Select(id => new { DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }))
.ToArray();
var devices = rawTags
.Select(t => (t.Driver, t.Device))
@@ -130,14 +137,18 @@ public static class V3ArtifactSeeder
/// <param name="driverFactory">Optional driver factory forwarded to <c>DriverHostActor.Props</c>. Null (the
/// default, preserving every existing call site) leaves the host on its own default, which materialises no
/// real drivers. Supply one — with <c>enabled: true</c> on the seed — to actually spawn driver children.</param>
/// <param name="reprovisionBackoff">Optional per-driver supervision-reprovision rate limit forwarded to
/// <c>DriverHostActor.Props</c>; null keeps the production default. Exists so a test can observe the
/// deferral without waiting the production window.</param>
/// <returns>The host actor plus the publish and mux probes wired into it.</returns>
public static (IActorRef Actor, TestProbe Publish, TestProbe Mux) SpawnHostAndApply(
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
NodeId localNode, DeploymentId deploymentId, RevisionHash rev,
IDriverFactory? driverFactory = null)
IDriverFactory? driverFactory = null,
TimeSpan? reprovisionBackoff = null)
{
var publish = kit.CreateTestProbe();
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory);
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory, reprovisionBackoff);
Dispatch(actor, coordinator, deploymentId, rev);
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(RuntimeActorTestBase.PresenceBudget);
@@ -157,14 +168,14 @@ public static class V3ArtifactSeeder
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
NodeId localNode, DeploymentId deploymentId, RevisionHash rev)
{
var (actor, mux, coordinator) = Spawn(kit, db, localNode, opcUaPublishActor: null, driverFactory: null);
var (actor, mux, coordinator) = Spawn(kit, db, localNode, opcUaPublishActor: null, driverFactory: null, reprovisionBackoff: null);
Dispatch(actor, coordinator, deploymentId, rev);
return (actor, mux);
}
private static (IActorRef Actor, TestProbe Mux, TestProbe Coordinator) Spawn(
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db, NodeId localNode,
IActorRef? opcUaPublishActor, IDriverFactory? driverFactory)
IActorRef? opcUaPublishActor, IDriverFactory? driverFactory, TimeSpan? reprovisionBackoff)
{
var coordinator = kit.CreateTestProbe();
var mux = kit.CreateTestProbe();
@@ -177,7 +188,8 @@ public static class V3ArtifactSeeder
dependencyMux: mux.Ref,
opcUaPublishActor: opcUaPublishActor,
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
virtualTagHostOverride: vtHost.Ref));
virtualTagHostOverride: vtHost.Ref,
reprovisionBackoff: reprovisionBackoff));
return (actor, mux, coordinator);
}