dohertj2/lmxopcua
1
0
Fork 0
Code Issues 43 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(runtime,adminui): per-tag write ordering, pessimistic init bound, unique generations, PostStop budget (fix round 1)

Browse Source 
- Writes: a second write to a tag already inside the driver waits in the
  queue behind it (_tagsInFlight) while other tags still run under the
  cap of 4; queued writes are stamped and failed BadTimeout at dequeue
  once older than the 8 s host Ask budget (Props queuedWriteBudget)
  instead of being written late.
- Initialise attempts are bounded pessimistically (initializeTimeout,
  default 30 s = 2x Tier C) on top of the invoker's token-only timeout:
  an overrun is logged as abandoned, reported InitializeFailed so the
  retry loop continues, and its late result is dropped by generation.
  Generations come from a process-wide Interlocked counter so a prior
  incarnation's late result cannot collide (F6).
- PostStop shares one shutdownTimeout budget across shutdown + dispose.
- TelemetryDialSupervisor cancels the establishment deadline on
  StreamStopped; FleetDiagnosticsClient builds the ActorSelection inside
  the try (a malformed node degrades to "unavailable"); the diagnostics
  page labels the column "Last health publish".
- Tests: same-tag serialisation with a distinct-tag positive control;
  stale queued write BadTimeout; token-ignoring hung init -> later
  attempt connects; stopwatch assertion demoted to a ceiling.

Refs #731

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 10:36:30 -04:00
parent 59b72fe385
commit 953bd27533
6 changed files with 257 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorConcurrentWriteTests.cs
+68 -9
View File
@@ -22,11 +22,13 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
{
private static readonly TimeSpan WriteLatency = TimeSpan.FromMilliseconds(1200);
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(IDriver driver, int? maxWritesInFlight = null)
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(
IDriver driver, int? maxWritesInFlight = null, TimeSpan? queuedWriteBudget = null)
{
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, commsLossTimeout: TimeSpan.Zero, maxWritesInFlight: maxWritesInFlight));
driver, commsLossTimeout: TimeSpan.Zero, maxWritesInFlight: maxWritesInFlight,
queuedWriteBudget: queuedWriteBudget));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => m.Connected, PresenceBudget);
return (actor, parent);
@@ -49,10 +51,10 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
results.ShouldAllBe(r => r.Success);
driver.Writes.Select(w => w.FullReference).ShouldBe(new[] { "tag-1", "tag-2" }, ignoreOrder: true);
// Serialised writes take two latencies; concurrent ones take one. The margin covers a loaded machine.
sw.Elapsed.ShouldBeLessThan(WriteLatency + WriteLatency / 2,
"the second write waited for the first — writes are still serialised on the mailbox");
driver.MaxConcurrentWrites.ShouldBe(2, "positive control: both writes were inside the driver at once");
// Both were inside the driver at once — that is the proof; the wall clock is only a sanity ceiling
// (serialised writes would take two latencies, and the host's Ask is 8 s).
driver.MaxConcurrentWrites.ShouldBe(2, "the second write waited for the first — writes are still serialised on the mailbox");
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(8));
}
[Fact]
@@ -96,12 +98,69 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
driver.ReleaseWrites();
var results = await Task.WhenAll(asks);
results.ShouldAllBe(r => r.Success);
// Entry order: the first two slots went to the first two writes (in either order — they are
// concurrent), the queued pair entered only after a slot freed, in queue order.
// Entry order: the first two slots went to the first two writes, the queued pair entered only after a
// slot freed. Within each pair the order is unspecified — distinct tags run concurrently on the pool;
// only SAME-tag order is guaranteed (pinned by the same-tag test below).
var entered = driver.EnteredRefs;
entered.Count.ShouldBe(4);
entered.Take(2).ShouldBe(new[] { "tag-0", "tag-1" }, ignoreOrder: true);
entered.Skip(2).ShouldBe(new[] { "tag-2", "tag-3" });
entered.Skip(2).ShouldBe(new[] { "tag-2", "tag-3" }, ignoreOrder: true);
}
/// <summary>
/// Fix round 1: writes to the SAME tag must keep their order — two concurrent writes v1, v2 could land as
/// v1. A same-tag write waits behind its tag while other tags still run concurrently.
/// </summary>
[Fact]
public async Task Writes_to_the_same_tag_are_serialised_in_order_while_other_tags_run_concurrently()
{
var driver = new SlowWritableDriver { HoldWrites = true };
var (actor, _) = Connect(driver, maxWritesInFlight: 4);
var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
// Positive control: a DIFFERENT tag is not held behind tag-a.
var other = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-b", 9), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-b" });
await Task.Delay(300);
driver.WritesEntered.ShouldBe(2, "the second tag-a write entered the driver while the first was still inside it");
second.IsCompleted.ShouldBeFalse();
driver.ReleaseWrites();
(await Task.WhenAll(first, second, other)).ShouldAllBe(r => r.Success);
driver.Writes.Where(w => w.FullReference == "tag-a").Select(w => w.Value).ShouldBe(new object[] { 1, 2 });
}
/// <summary>
/// Fix round 1: a write that sat in the queue longer than the host's Ask budget has already been reported
/// as a timeout (and reverted) upstream — writing it late would push a stale value to the device.
/// </summary>
[Fact]
public async Task A_queued_write_older_than_the_budget_is_failed_BadTimeout_instead_of_written_late()
{
var driver = new SlowWritableDriver { HoldWrites = true };
var (actor, _) = Connect(driver, maxWritesInFlight: 1, queuedWriteBudget: TimeSpan.FromMilliseconds(300));
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
var stale = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
await Task.Delay(600);
driver.ReleaseWrites();
(await inFlight).Success.ShouldBeTrue();
var staleResult = await stale;
staleResult.Success.ShouldBeFalse();
staleResult.Reason!.ShouldContain("800A0000", customMessage: "BadTimeout");
driver.EnteredRefs.ShouldBe(new[] { "tag-0" }, "the stale write must never reach the device");
}
[Fact]