fix(adminui): evict unconfigured drivers from status stores; isolate throwing subscribers (R04 L1, L5)

The driver-status and resilience stores had no removal path, so a deleted driver stayed on /hosts as a frozen row until restart. Both stores gain RetainOnly, which /hosts calls after each successful config load (never on a failed one). The stores and InProcessBroadcaster now raise their events one subscriber at a time, so a throwing handler cannot starve later subscribers or escape onto the bridge actor. The alert/script-log bridges publish inside their try, and the resilience bridge catches a store failure instead of restarting. Refs #745 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 17:24:04 -04:00
parent 82380d2bee
commit a04196d725
14 changed files with 314 additions and 13 deletions
@@ -0,0 +1,25 @@
+using Shouldly;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests;
+
+/// <summary>
+///     Reads an AdminUI source file for a source-scan pin. The repo has no bUnit, so Razor wiring that a
+///     unit test cannot render is pinned by scanning the markup; it proves the wiring exists, not that it
+///     renders correctly (the docker-dev live pass covers that).
+/// </summary>
+internal static class AdminUiSource
+{
+    /// <summary>Reads a file under <c>src/Server/ZB.MOM.WW.OtOpcUa.AdminUI</c>.</summary>
+    /// <param name="relativeSegments">Path segments below the AdminUI project directory.</param>
+    /// <returns>The file text.</returns>
+    public static string Read(params string[] relativeSegments)
+    {
+        var dir = new DirectoryInfo(AppContext.BaseDirectory);
+        while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "ZB.MOM.WW.OtOpcUa.slnx")))
+            dir = dir.Parent;
+        dir.ShouldNotBeNull("could not locate the repository root (no ZB.MOM.WW.OtOpcUa.slnx above the test binary)");
+
+        var parts = new[] { dir!.FullName, "src", "Server", "ZB.MOM.WW.OtOpcUa.AdminUI" }.Concat(relativeSegments).ToArray();
+        return File.ReadAllText(Path.Combine(parts));
+    }
+}
@@ -0,0 +1,139 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
+
+namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Hubs;
+
+/// <summary>
+///     Review R04 L1 + L5: the in-process stores evict driver instances that are no longer configured
+///     (so <c>/hosts</c> stops rendering ghost rows), and a throwing subscriber no longer aborts the
+///     multicast for the subscribers after it (nor escapes onto the bridge actor).
+/// </summary>
+public sealed class StoreEvictionAndIsolationTests
+{
+    private static DriverHealthChanged Health(string instance) =>
+        new("MAIN", instance, "Healthy", null, null, 0, new DateTime(2026, 10, 22, 0, 0, 0, DateTimeKind.Utc));
+
+    private static DriverResilienceStatusChanged Res(string instance, string host) =>
+        new(instance, host, false, 0, 0, null,
+            new DateTime(2026, 10, 22, 0, 0, 0, DateTimeKind.Utc),
+            new DateTime(2026, 10, 22, 0, 0, 0, DateTimeKind.Utc));
+
+    [Fact]
+    public void HealthStore_RetainOnly_evicts_unconfigured_instances_and_keeps_configured_ones()
+    {
+        var store = new InMemoryDriverStatusSnapshotStore();
+        store.Upsert(Health("drv-live"));
+        store.Upsert(Health("drv-deleted"));
+
+        var removed = store.RetainOnly(new HashSet<string>(["drv-live"], StringComparer.OrdinalIgnoreCase));
+
+        removed.ShouldBe(1);
+        store.TryGet("drv-deleted", out _).ShouldBeFalse();
+        // Positive control: the configured instance survives the sweep.
+        store.TryGet("drv-live", out _).ShouldBeTrue();
+        store.GetAll().Select(s => s.DriverInstanceId).ShouldBe(["drv-live"]);
+    }
+
+    [Fact]
+    public void HealthStore_RetainOnly_matches_ids_case_insensitively()
+    {
+        var store = new InMemoryDriverStatusSnapshotStore();
+        store.Upsert(Health("DRV-Live"));
+
+        store.RetainOnly(new HashSet<string>(["drv-live"], StringComparer.OrdinalIgnoreCase)).ShouldBe(0);
+        store.TryGet("DRV-Live", out _).ShouldBeTrue();
+    }
+
+    [Fact]
+    public void ResilienceStore_RetainOnly_evicts_every_host_of_an_unconfigured_instance()
+    {
+        var store = new InMemoryDriverResilienceStatusStore();
+        store.Upsert(Res("drv-live", "h1"));
+        store.Upsert(Res("drv-gone", "h1"));
+        store.Upsert(Res("drv-gone", "h2"));
+
+        var removed = store.RetainOnly(new HashSet<string>(["drv-live"], StringComparer.OrdinalIgnoreCase));
+
+        removed.ShouldBe(2);
+        store.GetForInstance("drv-gone").ShouldBeEmpty();
+        store.GetForInstance("drv-live").Count.ShouldBe(1);
+    }
+
+    [Fact]
+    public void HealthStore_throwing_subscriber_does_not_stop_later_subscribers()
+    {
+        var store = new InMemoryDriverStatusSnapshotStore();
+        var received = new List<string>();
+        store.SnapshotChanged += _ => throw new InvalidOperationException("renderer torn down");
+        store.SnapshotChanged += s => received.Add(s.DriverInstanceId);
+
+        Should.NotThrow(() => store.Upsert(Health("drv-1")));
+
+        received.ShouldBe(["drv-1"]);
+        store.TryGet("drv-1", out _).ShouldBeTrue();
+    }
+
+    [Fact]
+    public void ResilienceStore_throwing_subscriber_does_not_stop_later_subscribers()
+    {
+        var store = new InMemoryDriverResilienceStatusStore();
+        var received = 0;
+        store.SnapshotChanged += _ => throw new InvalidOperationException("boom");
+        store.SnapshotChanged += _ => received++;
+
+        Should.NotThrow(() => store.Upsert(Res("drv-1", "h1")));
+
+        received.ShouldBe(1);
+    }
+
+    [Fact]
+    public void Broadcaster_throwing_subscriber_does_not_stop_later_subscribers()
+    {
+        var broadcaster = new InProcessBroadcaster<string>();
+        var received = new List<string>();
+        broadcaster.Received += _ => throw new InvalidOperationException("boom");
+        broadcaster.Received += received.Add;
+
+        Should.NotThrow(() => broadcaster.Publish("evt"));
+
+        received.ShouldBe(["evt"]);
+    }
+
+    [Fact]
+    public void Broadcaster_throwing_connection_subscriber_does_not_stop_later_subscribers()
+    {
+        var broadcaster = new InProcessBroadcaster<string>();
+        var raised = new List<bool>();
+        broadcaster.ConnectionStateChanged += _ => throw new InvalidOperationException("boom");
+        broadcaster.ConnectionStateChanged += raised.Add;
+
+        Should.NotThrow(() => broadcaster.SetConnected(true));
+
+        raised.ShouldBe([true]);
+        broadcaster.IsConnected.ShouldBeTrue();
+    }
+}
+
+/// <summary>
+///     Source pin (no bUnit): <c>/hosts</c> sweeps both stores after a successful config load, and the
+///     sweep sits before the failure catch, so a failed load (which blanks the config) never evicts.
+/// </summary>
+[Trait("Category", "Unit")]
+public sealed class HostsEvictionWiringTests
+{
+    [Fact]
+    public void Hosts_page_evicts_both_stores_inside_the_successful_load_only()
+    {
+        var src = AdminUiSource.Read("Components", "Pages", "Hosts.razor");
+
+        var load = src.IndexOf("private async Task LoadConfigAsync()", StringComparison.Ordinal);
+        load.ShouldBeGreaterThan(0);
+        var firstCatch = src.IndexOf("catch (Exception ex) when (ex is OperationCanceledException", load, StringComparison.Ordinal);
+        firstCatch.ShouldBeGreaterThan(load);
+
+        src.IndexOf("DriverStore.RetainOnly(", load, StringComparison.Ordinal).ShouldBeInRange(load, firstCatch);
+        src.IndexOf("ResilienceStore.RetainOnly(", load, StringComparison.Ordinal).ShouldBeInRange(load, firstCatch);
+    }
+}
@@ -141,6 +141,7 @@ public sealed class TelemetryDialDeadlineTests : TestKit
        public void Upsert(DriverHealthChanged snapshot) { Upserts.Enqueue(snapshot); SnapshotChanged?.Invoke(snapshot); }
        public bool TryGet(string driverInstanceId, out DriverHealthChanged snapshot) { snapshot = null!; return false; }
        public IReadOnlyCollection<DriverHealthChanged> GetAll() => Upserts.ToArray();
+        public int RetainOnly(IReadOnlySet<string> configuredInstanceIds) => 0;
    }

    private sealed class NullResilienceStore : IDriverResilienceStatusStore
@@ -149,5 +150,6 @@ public sealed class TelemetryDialDeadlineTests : TestKit
        public void Upsert(DriverResilienceStatusChanged snapshot) => SnapshotChanged?.Invoke(snapshot);
        public IReadOnlyList<DriverResilienceStatusChanged> GetForInstance(string driverInstanceId) => Array.Empty<DriverResilienceStatusChanged>();
        public IReadOnlyCollection<DriverResilienceStatusChanged> GetAll() => Array.Empty<DriverResilienceStatusChanged>();
+        public int RetainOnly(IReadOnlySet<string> configuredInstanceIds) => 0;
    }
 }
@@ -438,6 +438,7 @@ public sealed class TelemetryDialSupervisorTests : TestKit
        }

        public IReadOnlyCollection<DriverHealthChanged> GetAll() => Upserts.ToArray();
+        public int RetainOnly(IReadOnlySet<string> configuredInstanceIds) => 0;
    }

    private sealed class RecordingResilienceStore : IDriverResilienceStatusStore
@@ -456,6 +457,7 @@ public sealed class TelemetryDialSupervisorTests : TestKit
            Upserts.ToArray();

        public IReadOnlyCollection<DriverResilienceStatusChanged> GetAll() => Upserts.ToArray();
+        public int RetainOnly(IReadOnlySet<string> configuredInstanceIds) => 0;
    }

    private sealed class InMemoryFactory(string name) : IDbContextFactory<OtOpcUaConfigDbContext>