fix(runtime): malformed artifact bytes on boot/restore are no answer (#601); evict telemetry snapshots on driver removal (#609)

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
2026-10-08 06:53:45 -04:00
parent 3baa332d6b
commit 53247a056a
10 changed files with 162 additions and 6 deletions
@@ -133,6 +133,8 @@ public sealed class DriverResilienceStatusPublisherServiceTests
            lock (Items) Items.Add(item);
        }

+        public void Evict(string driverInstanceId) { }
+
        public ITelemetrySubscription Subscribe(int boundedCapacity) => throw new NotSupportedException();
    }
 }
@@ -80,6 +80,32 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
        rebuild.Artifact.ShouldBe(blob);
    }

+    [Theory]
+    [InlineData("not json at all")]
+    [InlineData("{\"DriverInstances\": [")]               // truncated JSON
+    [InlineData("\u001f\u008b\u0008garbage-after-gzip-magic")] // gzip magic, undecompressible body
+    public void BootFromCache_withMalformedNonEmptyBytes_isNoAnswer_notAnEmptyConfiguration(string text)
+    {
+        // #601 (S-10): the lenient parse turned malformed bytes into "zero driver specs", which plans every
+        // running child for StopChild and then rebuilds the address space from the garbage. Malformed bytes
+        // must be treated like the zero-length case: nothing reconciled, nothing rebuilt.
+        var cached = new CachedDeploymentArtifact(
+            DeploymentId.NewId().ToString(), CachedRev.Value,
+            System.Text.Encoding.Latin1.GetBytes(text), DateTimeOffset.UtcNow.AddHours(-1));
+        var publishProbe = CreateTestProbe();
+
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            new ThrowingDbFactory(), TestNode, ackRouter: null,
+            localRoles: new HashSet<string> { "driver" },
+            opcUaPublishActor: publishProbe.Ref,
+            deploymentArtifactCache: new StubArtifactCache(cached)));
+
+        // Barrier: diagnostics are served only after PreStart's boot work ran, so the absence below is
+        // asserted after the boot path had its chance to Tell (positive control is the valid-bytes test).
+        AskDiagnostics(actor);
+        publishProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
+    }
+
    [Fact]
    public void CentralUnreachableAndCacheEmpty_StaysStaleWithNoConfiguration()
    {
@@ -178,6 +178,55 @@ public sealed class DriverHostActorUnreadableArtifactTests : RuntimeActorTestBas
        AskDiagnostics(actor).CurrentRevision.ShouldBe(RevB);
    }

+    /// <summary>
+    ///     #609 — removing a driver instance from the plan evicts its cached telemetry snapshots, so a new
+    ///     subscriber is no longer primed with a decommissioned driver's last Health/Resilience. The
+    ///     surviving instance's snapshot is the positive control: eviction is per-instance, not a wipe.
+    /// </summary>
+    [Fact]
+    public void Removing_a_driver_instance_evicts_its_telemetry_snapshots_but_not_a_neighbours()
+    {
+        var db = NewInMemoryDbFactory();
+        var factory = new SubscribingDriverFactory("Modbus");
+        var hub = new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryLocalHub();
+        var coordinator = CreateTestProbe();
+        var actor = Sys.ActorOf(DriverHostActor.Props(
+            db, TestNode, coordinator.Ref,
+            driverFactory: factory,
+            localRoles: new HashSet<string> { "driver" },
+            telemetryHub: hub));
+
+        actor.Tell(new DispatchDeployment(SeedV3Deployment(db, RevA), RevA, CorrelationId.NewId()));
+        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
+
+        hub.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health(new(
+            "c1", "drv-1", "Healthy", null, null, 0, DateTime.UtcNow)));
+        hub.Emit(new ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health(new(
+            "c1", "other-node-driver", "Healthy", null, null, 0, DateTime.UtcNow)));
+
+        // Control: before the removal, drv-1 replays.
+        using (var before = hub.Subscribe(16))
+        {
+            var seen = new List<string>();
+            while (before.Reader.TryRead(out var i))
+                if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health h) seen.Add(h.E.DriverInstanceId);
+            seen.ShouldContain("drv-1");
+        }
+
+        actor.Tell(new DispatchDeployment(SeedDriverlessDeployment(db, RevB), RevB, CorrelationId.NewId()));
+        coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
+        AwaitAssert(
+            () => AskDiagnostics(actor).Drivers.Select(d => d.Name).ShouldNotContain("drv-1"),
+            duration: Timeout);
+
+        using var after = hub.Subscribe(16);
+        var replayed = new List<string>();
+        while (after.Reader.TryRead(out var i))
+            if (i is ZB.MOM.WW.OtOpcUa.Runtime.Telemetry.TelemetryItem.Health h) replayed.Add(h.E.DriverInstanceId);
+        replayed.ShouldNotContain("drv-1");
+        replayed.ShouldContain("other-node-driver");
+    }
+
    /// <summary>Spawns the host with the subscribing factory, dispatches <paramref name="deploymentId"/> and
    /// waits for the Applied ACK so the child + the initial SubscribeBulk pass are in place.</summary>
    private (IActorRef Actor, Akka.TestKit.TestProbe Coordinator) SpawnHostAndApply(
@@ -80,6 +80,8 @@ public sealed class DpsScriptLogPublisherTests : RuntimeActorTestBase
    {
        public void Emit(TelemetryItem item) { }

+        public void Evict(string driverInstanceId) { }
+
        public ITelemetrySubscription Subscribe(int boundedCapacity) => throw new NotSupportedException();
    }
 }
@@ -312,6 +312,8 @@ public sealed class PublishSeamEmitsToHubTests : RuntimeActorTestBase

        public void Emit(TelemetryItem item) => Items.Add(item);

+        public void Evict(string driverInstanceId) { }
+
        public ITelemetrySubscription Subscribe(int boundedCapacity) => throw new NotSupportedException();
    }
 }
@@ -36,6 +36,28 @@ public sealed class TelemetryLocalHubTests
        return items;
    }

+    [Fact]
+    public void Evict_drops_the_instances_Health_and_every_Resilience_snapshot_but_keeps_others()
+    {
+        var hub = new TelemetryLocalHub();
+        hub.Emit(new TelemetryItem.Health(Health("drv-a")));
+        hub.Emit(new TelemetryItem.Resilience(Resilience("drv-a", "host-1")));
+        hub.Emit(new TelemetryItem.Resilience(Resilience("drv-a", "host-2")));
+        hub.Emit(new TelemetryItem.Health(Health("drv-b")));
+        hub.Emit(new TelemetryItem.Resilience(Resilience("drv-b", "host-1")));
+
+        hub.Evict("drv-a");
+        hub.Evict("never-existed"); // unknown id is a no-op
+
+        using var sub = hub.Subscribe(boundedCapacity: 16);
+        var items = Drain(sub);
+
+        // Positive control: drv-b still replays; drv-a does not.
+        items.Count.ShouldBe(2);
+        items.OfType<TelemetryItem.Health>().Single().E.DriverInstanceId.ShouldBe("drv-b");
+        items.OfType<TelemetryItem.Resilience>().Single().E.DriverInstanceId.ShouldBe("drv-b");
+    }
+
    [Fact]
    public void New_subscriber_is_primed_with_cached_Health_snapshot()
    {