fix(runtime,adminui): per-tag write ordering, pessimistic init bound, unique generations, PostStop budget (fix round 1)

- Writes: a second write to a tag already inside the driver waits in the queue behind it (_tagsInFlight) while other tags still run under the cap of 4; queued writes are stamped and failed BadTimeout at dequeue once older than the 8 s host Ask budget (Props queuedWriteBudget) instead of being written late. - Initialise attempts are bounded pessimistically (initializeTimeout, default 30 s = 2x Tier C) on top of the invoker's token-only timeout: an overrun is logged as abandoned, reported InitializeFailed so the retry loop continues, and its late result is dropped by generation. Generations come from a process-wide Interlocked counter so a prior incarnation's late result cannot collide (F6). - PostStop shares one shutdownTimeout budget across shutdown + dispose. - TelemetryDialSupervisor cancels the establishment deadline on StreamStopped; FleetDiagnosticsClient builds the ActorSelection inside the try (a malformed node degrades to "unavailable"); the diagnostics page labels the column "Last health publish". - Tests: same-tag serialisation with a distinct-tag positive control; stale queued write BadTimeout; token-ignoring hung init -> later attempt connects; stopwatch assertion demoted to a ceiling. Refs #731 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 10:36:30 -04:00
parent 59b72fe385
commit 953bd27533
6 changed files with 257 additions and 49 deletions
@@ -22,11 +22,13 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
 {
    private static readonly TimeSpan WriteLatency = TimeSpan.FromMilliseconds(1200);

-    private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(IDriver driver, int? maxWritesInFlight = null)
+    private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(
+        IDriver driver, int? maxWritesInFlight = null, TimeSpan? queuedWriteBudget = null)
    {
        var parent = CreateTestProbe();
        var actor = parent.ChildActorOf(DriverInstanceActor.Props(
-            driver, commsLossTimeout: TimeSpan.Zero, maxWritesInFlight: maxWritesInFlight));
+            driver, commsLossTimeout: TimeSpan.Zero, maxWritesInFlight: maxWritesInFlight,
+            queuedWriteBudget: queuedWriteBudget));
        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
        parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => m.Connected, PresenceBudget);
        return (actor, parent);
@@ -49,10 +51,10 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa

        results.ShouldAllBe(r => r.Success);
        driver.Writes.Select(w => w.FullReference).ShouldBe(new[] { "tag-1", "tag-2" }, ignoreOrder: true);
-        // Serialised writes take two latencies; concurrent ones take one. The margin covers a loaded machine.
-        sw.Elapsed.ShouldBeLessThan(WriteLatency + WriteLatency / 2,
-            "the second write waited for the first — writes are still serialised on the mailbox");
-        driver.MaxConcurrentWrites.ShouldBe(2, "positive control: both writes were inside the driver at once");
+        // Both were inside the driver at once — that is the proof; the wall clock is only a sanity ceiling
+        // (serialised writes would take two latencies, and the host's Ask is 8 s).
+        driver.MaxConcurrentWrites.ShouldBe(2, "the second write waited for the first — writes are still serialised on the mailbox");
+        sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(8));
    }

    [Fact]
@@ -96,12 +98,69 @@ public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBa
        driver.ReleaseWrites();
        var results = await Task.WhenAll(asks);
        results.ShouldAllBe(r => r.Success);
-        // Entry order: the first two slots went to the first two writes (in either order — they are
-        // concurrent), the queued pair entered only after a slot freed, in queue order.
+        // Entry order: the first two slots went to the first two writes, the queued pair entered only after a
+        // slot freed. Within each pair the order is unspecified — distinct tags run concurrently on the pool;
+        // only SAME-tag order is guaranteed (pinned by the same-tag test below).
        var entered = driver.EnteredRefs;
        entered.Count.ShouldBe(4);
        entered.Take(2).ShouldBe(new[] { "tag-0", "tag-1" }, ignoreOrder: true);
-        entered.Skip(2).ShouldBe(new[] { "tag-2", "tag-3" });
+        entered.Skip(2).ShouldBe(new[] { "tag-2", "tag-3" }, ignoreOrder: true);
+    }
+
+    /// <summary>
+    ///     Fix round 1: writes to the SAME tag must keep their order — two concurrent writes v1, v2 could land as
+    ///     v1. A same-tag write waits behind its tag while other tags still run concurrently.
+    /// </summary>
+    [Fact]
+    public async Task Writes_to_the_same_tag_are_serialised_in_order_while_other_tags_run_concurrently()
+    {
+        var driver = new SlowWritableDriver { HoldWrites = true };
+        var (actor, _) = Connect(driver, maxWritesInFlight: 4);
+
+        var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
+            new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
+        AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
+        var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
+            new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
+        // Positive control: a DIFFERENT tag is not held behind tag-a.
+        var other = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
+            new DriverInstanceActor.WriteAttribute("tag-b", 9), PresenceBudget);
+        AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
+        driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-b" });
+
+        await Task.Delay(300);
+        driver.WritesEntered.ShouldBe(2, "the second tag-a write entered the driver while the first was still inside it");
+        second.IsCompleted.ShouldBeFalse();
+
+        driver.ReleaseWrites();
+        (await Task.WhenAll(first, second, other)).ShouldAllBe(r => r.Success);
+        driver.Writes.Where(w => w.FullReference == "tag-a").Select(w => w.Value).ShouldBe(new object[] { 1, 2 });
+    }
+
+    /// <summary>
+    ///     Fix round 1: a write that sat in the queue longer than the host's Ask budget has already been reported
+    ///     as a timeout (and reverted) upstream — writing it late would push a stale value to the device.
+    /// </summary>
+    [Fact]
+    public async Task A_queued_write_older_than_the_budget_is_failed_BadTimeout_instead_of_written_late()
+    {
+        var driver = new SlowWritableDriver { HoldWrites = true };
+        var (actor, _) = Connect(driver, maxWritesInFlight: 1, queuedWriteBudget: TimeSpan.FromMilliseconds(300));
+
+        var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
+            new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
+        AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
+        var stale = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
+            new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
+
+        await Task.Delay(600);
+        driver.ReleaseWrites();
+
+        (await inFlight).Success.ShouldBeTrue();
+        var staleResult = await stale;
+        staleResult.Success.ShouldBeFalse();
+        staleResult.Reason!.ShouldContain("800A0000", customMessage: "BadTimeout");
+        driver.EnteredRefs.ShouldBe(new[] { "tag-0" }, "the stale write must never reach the device");
    }

    [Fact]
@@ -60,6 +60,32 @@ public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActor
        driver.Attempts.ShouldBe(2);
    }

+    /// <summary>
+    ///     Fix round 1: the single-flight gate only cleared on the attempt's RESULT, and the invoker's timeout is
+    ///     token-only — a driver whose InitializeAsync ignores its token wedged the reconnect loop forever. The
+    ///     attempt is now bounded pessimistically (<c>initializeTimeout</c>); an overrun is abandoned, reported
+    ///     as a failure, and the next tick retries.
+    /// </summary>
+    [Fact]
+    public void A_token_ignoring_never_completing_init_is_abandoned_and_a_later_attempt_connects()
+    {
+        var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 1)
+        {
+            // Attempt 1 hangs forever and ignores cancellation; attempt 2 completes.
+            HangOnAttempt = 1,
+        };
+        var parent = CreateTestProbe();
+        var actor = parent.ChildActorOf(DriverInstanceActor.Props(
+            driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
+            initializeTimeout: TimeSpan.FromMilliseconds(300)));
+
+        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
+
+        parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
+        driver.Attempts.ShouldBe(2);
+        driver.HungAttemptStillRunning.ShouldBeTrue("positive control: the first attempt really was wedged");
+    }
+
    /// <summary>Records every Shutdown/Initialize entry and exit so an overlap is a fact, not an inference.</summary>
    private sealed class InterleavingDriver : IDriver
    {
@@ -79,6 +105,13 @@ public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActor

        public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }

+        /// <summary>The 1-based attempt that never completes and ignores its token (0 = none).</summary>
+        public int HangOnAttempt { get; init; }
+
+        private readonly TaskCompletionSource _never = new(TaskCreationOptions.RunContinuationsAsynchronously);
+
+        public bool HungAttemptStillRunning => !_never.Task.IsCompleted && Volatile.Read(ref _initsInProgress) > 0;
+
        public int Attempts => Volatile.Read(ref _attempts);
        public int Overlaps => Volatile.Read(ref _overlaps);
        public string Interleaving { get { lock (_gate) return string.Join(" ", _events); } }
@@ -93,6 +126,7 @@ public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActor
            Record($"init{attempt}>");
            try
            {
+                if (attempt == HangOnAttempt) await _never.Task;
                if (_initDuration > TimeSpan.Zero) await Task.Delay(_initDuration, CancellationToken.None);
                if (attempt < _failUntilAttempt) throw new InvalidOperationException($"init attempt {attempt} refused");
            }