feat(hosts): surface per-host connectivity on /hosts; drop the unwritable table (#521)
`IHostConnectivityProbe` was a dead surface: eleven drivers implement it, `GetHostStatuses()` had ZERO production call sites, and `OnHostStatusChanged` had no subscriber outside the Galaxy driver's own aggregator. Per-host connectivity was computed by every driver and read by nobody. The issue offered "build the publisher or delete it". The publisher as its entity doc described it — driver nodes upserting `DriverHostStatus` rows — is not buildable: per-cluster mesh Phase 4 gates `AddOtOpcUaConfigDb` on the `admin` role, so a driver-only node has no ConfigDb connection to write rows with. So the capability is kept and the transport changed. `DriverHealthChanged.HostStatuses` now carries the probe result to `/hosts` as a Hosts column. That channel already reached the page, already survives the mesh split via the Phase 5 gRPC telemetry stream, and already replays a last-value snapshot on re-subscribe — so per-host state re-primes after a reconnect without a durable store. Both halves of the interface finally do what they are for: the event triggers a prompt publish, the pull is the source of truth. The point of the column is the case the driver-level state chip structurally cannot express: a multi-device driver stays aggregate-Healthy while ONE of its devices is unreachable. Two traps, both pinned by tests that were falsified against the prod code: - The host digest MUST be in the publish fingerprint. On a single-host-down transition every other fingerprint component is unchanged, so the dedup would swallow exactly the publish carrying the news — the trap that already bit the rediscovery signal. Removing it turns the guard test red, verified. - null (no probe) must stay distinct from empty (probe with no hosts). proto3 cannot tell an absent repeated field from an empty one, hence the explicit `has_host_statuses` flag; collapsing them would render every probe-less driver as one whose devices are all fine. Dropped: the DriverHostStatus entity, enum, DbSet, model config and table (migration DropDriverHostStatusTable — empty on every deployment, so the scaffolder's data-loss warning is moot, and Down() recreates it exactly). Found en route, NOT fixed here: `DriverInstanceResilienceStatus` is the identical defect — no writer, no reader, only a DbSet declaration, while the live data rides the `driver-resilience-status` telemetry channel. Its doc-comment now states that rather than describing the sampler and AdminUI join that were never built. Filed as #524 rather than widening this schema change beyond what was asked. Claude-Session: https://claude.ai/code/session_015p7wGqy3YpZNCpDzTpGMKo
This commit is contained in:
Joseph Doherty
@@ -1,131 +0,0 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Configuration.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end round-trip through the DB for the <see cref="DriverHostStatus"/> entity
|
||||
/// added in PR 33 — exercises the composite primary key (NodeId, DriverInstanceId,
|
||||
/// HostName), string-backed <c>DriverHostState</c> conversion, and the two indexes the
|
||||
/// Admin UI's drill-down queries will scan (NodeId, LastSeenUtc).
|
||||
/// </summary>
|
||||
[Trait("Category", "SchemaCompliance")]
|
||||
[Collection(nameof(SchemaComplianceCollection))]
|
||||
public sealed class DriverHostStatusTests(SchemaComplianceFixture fixture)
|
||||
{
|
||||
/// <summary>Verifies that the composite key allows the same host across different nodes or drivers.</summary>
|
||||
[Fact]
|
||||
public async Task Composite_key_allows_same_host_across_different_nodes_or_drivers()
|
||||
{
|
||||
await using var ctx = NewContext();
|
||||
|
||||
// Same HostName + DriverInstanceId across two different server nodes — classic 2-node
|
||||
// redundancy case. Both rows must be insertable because each server node owns its own
|
||||
// runtime view of the shared host.
|
||||
var now = DateTime.UtcNow;
|
||||
ctx.DriverHostStatuses.Add(new DriverHostStatus
|
||||
{
|
||||
NodeId = "node-a", DriverInstanceId = "galaxy-1", HostName = "GRPlatform",
|
||||
State = DriverHostState.Running,
|
||||
StateChangedUtc = now, LastSeenUtc = now,
|
||||
});
|
||||
ctx.DriverHostStatuses.Add(new DriverHostStatus
|
||||
{
|
||||
NodeId = "node-b", DriverInstanceId = "galaxy-1", HostName = "GRPlatform",
|
||||
State = DriverHostState.Stopped,
|
||||
StateChangedUtc = now, LastSeenUtc = now,
|
||||
Detail = "secondary hasn't taken over yet",
|
||||
});
|
||||
// Same server node + host, different driver instance — second driver doesn't clobber.
|
||||
ctx.DriverHostStatuses.Add(new DriverHostStatus
|
||||
{
|
||||
NodeId = "node-a", DriverInstanceId = "modbus-plc1", HostName = "GRPlatform",
|
||||
State = DriverHostState.Running,
|
||||
StateChangedUtc = now, LastSeenUtc = now,
|
||||
});
|
||||
await ctx.SaveChangesAsync();
|
||||
|
||||
var rows = await ctx.DriverHostStatuses.AsNoTracking()
|
||||
.Where(r => r.HostName == "GRPlatform").ToListAsync();
|
||||
|
||||
rows.Count.ShouldBe(3);
|
||||
rows.ShouldContain(r => r.NodeId == "node-a" && r.DriverInstanceId == "galaxy-1");
|
||||
rows.ShouldContain(r => r.NodeId == "node-b" && r.State == DriverHostState.Stopped && r.Detail == "secondary hasn't taken over yet");
|
||||
rows.ShouldContain(r => r.NodeId == "node-a" && r.DriverInstanceId == "modbus-plc1");
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the upsert pattern updates existing records in place.</summary>
|
||||
[Fact]
|
||||
public async Task Upsert_pattern_for_same_key_updates_in_place()
|
||||
{
|
||||
// The publisher hosted service (follow-up PR) upserts on every transition +
|
||||
// heartbeat. This test pins the two-step pattern it will use: check-then-add-or-update
|
||||
// keyed on the composite PK. If the composite key ever changes, this test breaks
|
||||
// loudly so the publisher gets a synchronized update.
|
||||
await using var ctx = NewContext();
|
||||
var t0 = DateTime.UtcNow;
|
||||
ctx.DriverHostStatuses.Add(new DriverHostStatus
|
||||
{
|
||||
NodeId = "upsert-node", DriverInstanceId = "upsert-driver", HostName = "upsert-host",
|
||||
State = DriverHostState.Running,
|
||||
StateChangedUtc = t0, LastSeenUtc = t0,
|
||||
});
|
||||
await ctx.SaveChangesAsync();
|
||||
|
||||
var t1 = t0.AddSeconds(30);
|
||||
await using (var ctx2 = NewContext())
|
||||
{
|
||||
var existing = await ctx2.DriverHostStatuses.SingleAsync(r =>
|
||||
r.NodeId == "upsert-node" && r.DriverInstanceId == "upsert-driver" && r.HostName == "upsert-host");
|
||||
existing.State = DriverHostState.Faulted;
|
||||
existing.StateChangedUtc = t1;
|
||||
existing.LastSeenUtc = t1;
|
||||
existing.Detail = "transport reset by peer";
|
||||
await ctx2.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await using var ctx3 = NewContext();
|
||||
var final = await ctx3.DriverHostStatuses.AsNoTracking().SingleAsync(r =>
|
||||
r.NodeId == "upsert-node" && r.HostName == "upsert-host");
|
||||
final.State.ShouldBe(DriverHostState.Faulted);
|
||||
final.Detail.ShouldBe("transport reset by peer");
|
||||
// Only one row — a naive "always insert" would have created a duplicate PK and thrown.
|
||||
(await ctx3.DriverHostStatuses.CountAsync(r => r.NodeId == "upsert-node")).ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the State enum is persisted as a string, not an integer.</summary>
|
||||
[Fact]
|
||||
public async Task Enum_persists_as_string_not_int()
|
||||
{
|
||||
// Fluent config sets HasConversion<string>() on State — the DB stores 'Running' /
|
||||
// 'Stopped' / 'Faulted' / 'Unknown' as nvarchar(16). Verify by reading the raw
|
||||
// string back via ADO; if someone drops the conversion the column will contain '1'
|
||||
// / '2' / '3' and this assertion fails. Matters because DBAs inspecting the table
|
||||
// directly should see readable state names, not enum ordinals.
|
||||
await using var ctx = NewContext();
|
||||
ctx.DriverHostStatuses.Add(new DriverHostStatus
|
||||
{
|
||||
NodeId = "enum-node", DriverInstanceId = "enum-driver", HostName = "enum-host",
|
||||
State = DriverHostState.Faulted,
|
||||
StateChangedUtc = DateTime.UtcNow, LastSeenUtc = DateTime.UtcNow,
|
||||
});
|
||||
await ctx.SaveChangesAsync();
|
||||
|
||||
await using var conn = fixture.OpenConnection();
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT [State] FROM DriverHostStatus WHERE NodeId = 'enum-node'";
|
||||
var rawValue = (string?)await cmd.ExecuteScalarAsync();
|
||||
rawValue.ShouldBe("Faulted");
|
||||
}
|
||||
|
||||
private OtOpcUaConfigDbContext NewContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseSqlServer(fixture.ConnectionString)
|
||||
.Options;
|
||||
return new OtOpcUaConfigDbContext(options);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Hosts;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Hosts;
|
||||
|
||||
@@ -154,4 +155,62 @@ public sealed class HostsDriverViewTests
|
||||
|
||||
groups.Select(g => g.ClusterId).ShouldBe(new[] { "Alpha", "Beta", "zeta" });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-host connectivity flows through to the row (Gitea #521), and <c>DegradedHosts</c> picks out
|
||||
/// exactly the hosts an operator needs to look at. This is the case the driver-level Status chip
|
||||
/// cannot express: the driver is Healthy and one of its devices is not.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Build_carries_host_statuses_and_flags_only_the_degraded_ones()
|
||||
{
|
||||
var snapshot = Snap("MAIN", "drv-a") with
|
||||
{
|
||||
HostStatuses =
|
||||
[
|
||||
new HostConnectivityStatus("plc-a", HostState.Running, When),
|
||||
new HostConnectivityStatus("plc-b", HostState.Stopped, When),
|
||||
new HostConnectivityStatus("plc-c", HostState.Faulted, When),
|
||||
],
|
||||
};
|
||||
|
||||
var row = HostsDriverView.Build([snapshot], nodes: null, instances: null).Single().Drivers.Single();
|
||||
|
||||
row.State.ShouldBe("Healthy");
|
||||
row.HostStatuses!.Count.ShouldBe(3);
|
||||
row.DegradedHosts.Select(h => h.HostName).ShouldBe(["plc-b", "plc-c"]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="HostState.Unknown"/> counts as degraded. A probe that has not completed its first tick
|
||||
/// — or one a driver failed to start at all, which AbCip logs explicitly — reports Unknown, and
|
||||
/// rendering that as healthy is how an unstarted probe stays invisible.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Unknown_host_state_counts_as_degraded()
|
||||
{
|
||||
var snapshot = Snap("MAIN", "drv-a") with
|
||||
{
|
||||
HostStatuses = [new HostConnectivityStatus("plc-a", HostState.Unknown, When)],
|
||||
};
|
||||
|
||||
var row = HostsDriverView.Build([snapshot], nodes: null, instances: null).Single().Drivers.Single();
|
||||
|
||||
row.DegradedHosts.ShouldHaveSingleItem().HostName.ShouldBe("plc-a");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A driver with no probe keeps a null host list — distinct from a probe reporting zero hosts. The
|
||||
/// /hosts column renders "—" for the former and "0 hosts" for the latter, and collapsing them would
|
||||
/// claim every probe-less driver's devices are fine.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_driver_without_host_statuses_keeps_null_and_reports_no_degraded_hosts()
|
||||
{
|
||||
var row = HostsDriverView.Build([Snap("MAIN", "drv-a")], nodes: null, instances: null)
|
||||
.Single().Drivers.Single();
|
||||
|
||||
row.HostStatuses.ShouldBeNull();
|
||||
row.DegradedHosts.ShouldBeEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
+74
@@ -3,6 +3,9 @@ using Shouldly;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane.Telemetry;
|
||||
// Aliased, not imported wholesale: Core.Abstractions also declares a DriverHealth, which collides with
|
||||
// the proto DriverHealth these tests construct.
|
||||
using HostState = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HostState;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Telemetry;
|
||||
@@ -168,6 +171,77 @@ public sealed class TelemetryProtoMapCentralTests
|
||||
e.PublishedUtc.Kind.ShouldBe(DateTimeKind.Utc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-host connectivity (Gitea #521) survives the wire with its tri-state intact:
|
||||
/// <b>null</b> (driver has no probe) must stay distinguishable from an <b>empty list</b> (it has one
|
||||
/// that knows no hosts). proto3 cannot tell an absent repeated field from an empty one, which is why
|
||||
/// <c>has_host_statuses</c> exists — without it a driver with no probe would arrive looking like one
|
||||
/// whose devices are all fine, and the /hosts column would render "0 hosts" for every driver in the
|
||||
/// fleet.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ToHealth_host_statuses_round_trip_with_the_null_vs_empty_distinction_intact()
|
||||
{
|
||||
var populated = new DriverHealth
|
||||
{
|
||||
ClusterId = "c1", DriverInstanceId = "d1", State = "Healthy",
|
||||
PublishedUtc = Timestamp.FromDateTime(SampleUtc),
|
||||
HasHostStatuses = true,
|
||||
HostStatuses =
|
||||
{
|
||||
new HostConnectivity { HostName = "plc-a", State = "Running", LastChangedUtc = Timestamp.FromDateTime(OtherUtc) },
|
||||
new HostConnectivity { HostName = "plc-b", State = "Stopped", LastChangedUtc = Timestamp.FromDateTime(SampleUtc) },
|
||||
},
|
||||
};
|
||||
|
||||
var mapped = TelemetryProtoMapCentral.ToHealth(populated).HostStatuses;
|
||||
mapped.ShouldNotBeNull();
|
||||
mapped!.Count.ShouldBe(2);
|
||||
mapped[0].HostName.ShouldBe("plc-a");
|
||||
mapped[0].State.ShouldBe(HostState.Running);
|
||||
mapped[0].LastChangedUtc.ShouldBe(OtherUtc);
|
||||
mapped[1].State.ShouldBe(HostState.Stopped);
|
||||
|
||||
// A probe that currently knows no hosts: empty, NOT null.
|
||||
var empty = new DriverHealth
|
||||
{
|
||||
ClusterId = "c1", DriverInstanceId = "d1", State = "Healthy",
|
||||
PublishedUtc = Timestamp.FromDateTime(SampleUtc),
|
||||
HasHostStatuses = true,
|
||||
};
|
||||
TelemetryProtoMapCentral.ToHealth(empty).HostStatuses.ShouldBeEmpty();
|
||||
|
||||
// No probe at all: null, NOT empty.
|
||||
var absent = new DriverHealth
|
||||
{
|
||||
ClusterId = "c1", DriverInstanceId = "d1", State = "Healthy",
|
||||
PublishedUtc = Timestamp.FromDateTime(SampleUtc),
|
||||
};
|
||||
TelemetryProtoMapCentral.ToHealth(absent).HostStatuses.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An unparseable host state degrades to <see cref="HostState.Unknown"/> rather than throwing. A node
|
||||
/// running a newer build that added an enum member must not be able to kill central's telemetry
|
||||
/// stream — this is observability, and it has no business failing closed.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ToHealth_unknown_host_state_string_degrades_instead_of_throwing()
|
||||
{
|
||||
var proto = new DriverHealth
|
||||
{
|
||||
ClusterId = "c1", DriverInstanceId = "d1", State = "Healthy",
|
||||
PublishedUtc = Timestamp.FromDateTime(SampleUtc),
|
||||
HasHostStatuses = true,
|
||||
HostStatuses = { new HostConnectivity { HostName = "plc-a", State = "Quiescing" } },
|
||||
};
|
||||
|
||||
var mapped = TelemetryProtoMapCentral.ToHealth(proto).HostStatuses;
|
||||
|
||||
mapped.ShouldNotBeNull();
|
||||
mapped![0].State.ShouldBe(HostState.Unknown);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ToHealth_absent_nullable_timestamp_and_optional_string_map_to_null()
|
||||
{
|
||||
|
||||
@@ -8,6 +8,10 @@ using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.Telemetry.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
// Aliased, not imported wholesale: Core.Abstractions also declares a DriverHealth, which would collide
|
||||
// with the proto DriverHealth these tests assert on.
|
||||
using HostConnectivityStatus = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HostConnectivityStatus;
|
||||
using HostState = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HostState;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Grpc;
|
||||
|
||||
@@ -210,6 +214,40 @@ public sealed class TelemetryStreamGrpcServiceTests
|
||||
evt.DriverHealth.LastSuccessfulReadUtc.ToDateTime().ShouldBe(expectedUtc);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Node side of the #521 host-status carry: the presence flag must be written, so the tri-state
|
||||
/// (no probe / probe with no hosts / probe with hosts) survives a wire proto3 cannot express on the
|
||||
/// repeated field alone. Paired with the central-side decode in <c>TelemetryProtoMapCentralTests</c>.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ToProto_health_writes_the_host_status_presence_flag_and_entries()
|
||||
{
|
||||
var changedAt = new DateTime(2026, 7, 30, 11, 0, 0, DateTimeKind.Utc);
|
||||
var withHosts = new DriverHealthChanged(
|
||||
"cluster-a", "drv-1", "Healthy", null, null, 0, DateTime.UtcNow,
|
||||
HostStatuses: [new HostConnectivityStatus("plc-a", HostState.Stopped, changedAt)]);
|
||||
|
||||
var evt = TelemetryProtoMapNode.ToProto(new TelemetryItem.Health(withHosts), "c");
|
||||
|
||||
evt.DriverHealth.HasHostStatuses.ShouldBeTrue();
|
||||
evt.DriverHealth.HostStatuses.Count.ShouldBe(1);
|
||||
evt.DriverHealth.HostStatuses[0].HostName.ShouldBe("plc-a");
|
||||
evt.DriverHealth.HostStatuses[0].State.ShouldBe("Stopped");
|
||||
evt.DriverHealth.HostStatuses[0].LastChangedUtc.ToDateTime().ShouldBe(changedAt);
|
||||
|
||||
// A probe reporting zero hosts still sets the flag — that is the whole point of having one.
|
||||
var emptyProbe = new DriverHealthChanged(
|
||||
"cluster-a", "drv-1", "Healthy", null, null, 0, DateTime.UtcNow, HostStatuses: []);
|
||||
var emptyEvt = TelemetryProtoMapNode.ToProto(new TelemetryItem.Health(emptyProbe), "c");
|
||||
emptyEvt.DriverHealth.HasHostStatuses.ShouldBeTrue();
|
||||
emptyEvt.DriverHealth.HostStatuses.ShouldBeEmpty();
|
||||
|
||||
// No probe: flag clear.
|
||||
var noProbe = new DriverHealthChanged("cluster-a", "drv-1", "Healthy", null, null, 0, DateTime.UtcNow);
|
||||
TelemetryProtoMapNode.ToProto(new TelemetryItem.Health(noProbe), "c")
|
||||
.DriverHealth.HasHostStatuses.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Client_disconnect_mid_stream_ends_cleanly_without_leaking_a_slot()
|
||||
{
|
||||
|
||||
+298
@@ -0,0 +1,298 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Covers the <see cref="IHostConnectivityProbe"/> consumer (Gitea #521). Eleven drivers implement the
|
||||
/// capability; before this wiring <c>GetHostStatuses()</c> had ZERO production call sites and
|
||||
/// <c>OnHostStatusChanged</c> had no subscriber outside the Galaxy driver's own aggregator, so per-host
|
||||
/// connectivity was computed by every driver and read by nobody.
|
||||
/// <para><b>Why it does not go to the DB.</b> The <c>DriverHostStatus</c> table's doc-comment described a
|
||||
/// publisher hosted service that upserted rows from each driver node. It was never built, and
|
||||
/// per-cluster mesh Phase 4 made it unbuildable as described — <c>AddOtOpcUaConfigDb</c> is gated on the
|
||||
/// <c>admin</c> role, so a driver-only node has no ConfigDb connection. The table was dropped; the data
|
||||
/// rides the driver-health snapshot instead.</para>
|
||||
/// </summary>
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class DriverInstanceActorHostStatusTests : RuntimeActorTestBase
|
||||
{
|
||||
/// <summary>The base case: a probe driver's hosts reach the health publisher.</summary>
|
||||
[Fact]
|
||||
public void Host_statuses_reach_the_health_publisher()
|
||||
{
|
||||
var driver = new ProbeStubDriver();
|
||||
driver.SetHost("plc-a", HostState.Running);
|
||||
driver.SetHost("plc-b", HostState.Running);
|
||||
var publisher = new RecordingHealthPublisher();
|
||||
var actor = SpawnDriverActor(driver, publisher);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
var latest = publisher.Published.LastOrDefault();
|
||||
latest.ShouldNotBeNull();
|
||||
latest!.HostStatuses.ShouldNotBeNull();
|
||||
latest.HostStatuses!.Select(h => h.HostName).OrderBy(n => n, StringComparer.Ordinal)
|
||||
.ShouldBe(["plc-a", "plc-b"]);
|
||||
},
|
||||
TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>The load-bearing case, and the entire reason this channel exists.</b> A multi-device driver
|
||||
/// stays aggregate-<c>Healthy</c> when ONE of its devices drops — the driver-level state chip cannot
|
||||
/// express it. So the transition must reach the operator through the per-host detail.
|
||||
/// <para>This is also the dedup trap that already bit the rediscovery signal once.
|
||||
/// <c>PublishHealthSnapshot</c> suppresses a publish whose fingerprint repeats, and on this
|
||||
/// transition (state, lastSuccessfulRead, lastError, errorCount) are ALL unchanged. Unless the
|
||||
/// host-status digest is part of the fingerprint, the dedup swallows exactly the publish that
|
||||
/// carries the news.</para>
|
||||
/// <para><b>Falsifiability:</b> the assertion is that the publish count STRICTLY INCREASES across the
|
||||
/// transition — an "eventually shows Stopped" assertion would be satisfied by the warm-up publish
|
||||
/// plus a later 30 s heartbeat and would prove nothing. Drop <c>HostStatusDigest</c> from the
|
||||
/// fingerprint tuple in <c>DriverInstanceActor</c> and this test must go red. Verified by doing so.</para>
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_single_host_going_down_is_not_swallowed_by_the_unchanged_health_dedup()
|
||||
{
|
||||
var driver = new ProbeStubDriver();
|
||||
driver.SetHost("plc-a", HostState.Running);
|
||||
driver.SetHost("plc-b", HostState.Running);
|
||||
var publisher = new RecordingHealthPublisher();
|
||||
var actor = SpawnDriverActor(driver, publisher);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
AwaitAssert(() => publisher.Published.Count.ShouldBeGreaterThan(0), TimeSpan.FromSeconds(3));
|
||||
|
||||
// Settle, so the baseline is a quiet actor: from here only the host state changes. The driver's
|
||||
// OWN health stays Healthy throughout — that is the point.
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
var before = publisher.Published.Count;
|
||||
|
||||
driver.SetHost("plc-b", HostState.Stopped, raise: true);
|
||||
|
||||
AwaitAssert(
|
||||
() => publisher.Published.Count.ShouldBeGreaterThan(before),
|
||||
TimeSpan.FromSeconds(3));
|
||||
|
||||
var latest = publisher.Published[^1];
|
||||
latest.Health.State.ShouldBe(DriverState.Healthy, "the driver itself never faulted — only one of its devices did");
|
||||
latest.HostStatuses.ShouldNotBeNull();
|
||||
latest.HostStatuses!.Single(h => h.HostName == "plc-b").State.ShouldBe(HostState.Stopped);
|
||||
latest.HostStatuses.Single(h => h.HostName == "plc-a").State.ShouldBe(HostState.Running);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The other half of the dedup contract: when nothing changes, the digest must NOT churn. A digest
|
||||
/// built over an <c>IReadOnlyList</c> by reference, or one sensitive to the order a driver happens to
|
||||
/// enumerate its hosts in, would differ on every call and re-publish on every 30 s heartbeat forever
|
||||
/// — turning the dedup off without anyone noticing.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Unchanged_hosts_do_not_defeat_the_dedup()
|
||||
{
|
||||
var driver = new ProbeStubDriver();
|
||||
driver.SetHost("plc-a", HostState.Running);
|
||||
driver.SetHost("plc-b", HostState.Running);
|
||||
var publisher = new RecordingHealthPublisher();
|
||||
var actor = SpawnDriverActor(driver, publisher);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
AwaitAssert(() => publisher.Published.Count.ShouldBeGreaterThan(0), TimeSpan.FromSeconds(3));
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
var before = publisher.Published.Count;
|
||||
|
||||
// The driver re-shuffles its host order without changing any state. A real driver builds this list
|
||||
// from a Dictionary, so enumeration order is not guaranteed stable between calls.
|
||||
driver.ReverseHostOrder();
|
||||
// Poke the actor into re-publishing without changing anything material.
|
||||
driver.RaiseHostStatusChanged();
|
||||
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
publisher.Published.Count.ShouldBe(before, "an order flip with no state change must be deduped, not re-published");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <b>Leak guard.</b> The <see cref="IDriver"/> can OUTLIVE the actor — the host respawns a child
|
||||
/// around the same driver object — so a missing <c>-=</c> in <c>PostStop</c> accumulates one handler
|
||||
/// per respawn, each holding a dead <c>Self</c>.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Stopping_the_actor_detaches_the_host_status_handler()
|
||||
{
|
||||
var driver = new ProbeStubDriver();
|
||||
var parent = CreateTestProbe();
|
||||
parent.IgnoreMessages(_ => true);
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver));
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
AwaitAssert(() => driver.SubscriberCount.ShouldBe(1), TimeSpan.FromSeconds(3));
|
||||
|
||||
Watch(actor);
|
||||
actor.Tell(PoisonPill.Instance);
|
||||
ExpectTerminated(actor, TimeSpan.FromSeconds(3));
|
||||
|
||||
driver.SubscriberCount.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A driver with no probe publishes null host statuses — NOT an empty list. The two mean different
|
||||
/// things at the UI ("no per-host detail available" vs "a probe that currently knows no hosts") and
|
||||
/// collapsing them would render a driver with no probe as one whose devices are all fine.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_driver_without_a_probe_publishes_null_host_statuses()
|
||||
{
|
||||
var publisher = new RecordingHealthPublisher();
|
||||
var actor = SpawnDriverActor(new StubDriver(), publisher);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
AwaitAssert(() => publisher.Published.Count.ShouldBeGreaterThan(0), TimeSpan.FromSeconds(3));
|
||||
|
||||
publisher.Published.ShouldAllBe(p => p.HostStatuses == null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A probe that throws must not take the health publish down with it. <c>GetHostStatuses()</c> is
|
||||
/// documented as a pure in-memory snapshot (which is why the capability analyzer exempts it from
|
||||
/// the guarded-call rule), but a driver is free to violate that, and losing the whole health channel
|
||||
/// for one misbehaving probe would be a much worse outcome than losing the host detail.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_throwing_probe_degrades_to_null_without_killing_the_health_publish()
|
||||
{
|
||||
var driver = new ProbeStubDriver { ThrowOnGetHostStatuses = true };
|
||||
var publisher = new RecordingHealthPublisher();
|
||||
var actor = SpawnDriverActor(driver, publisher);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
|
||||
AwaitAssert(() => publisher.Published.Count.ShouldBeGreaterThan(0), TimeSpan.FromSeconds(3));
|
||||
publisher.Published[^1].Health.State.ShouldBe(DriverState.Healthy);
|
||||
publisher.Published[^1].HostStatuses.ShouldBeNull();
|
||||
}
|
||||
|
||||
private IActorRef SpawnDriverActor(IDriver driver, IDriverHealthPublisher publisher)
|
||||
{
|
||||
var parent = CreateTestProbe();
|
||||
parent.IgnoreMessages(_ => true);
|
||||
return parent.ChildActorOf(DriverInstanceActor.Props(driver, healthPublisher: publisher));
|
||||
}
|
||||
|
||||
/// <summary>Captures every health publish so a test can assert on the host-status field.</summary>
|
||||
private sealed record HealthPublish(
|
||||
DriverHealth Health,
|
||||
IReadOnlyList<HostConnectivityStatus>? HostStatuses);
|
||||
|
||||
private sealed class RecordingHealthPublisher : IDriverHealthPublisher
|
||||
{
|
||||
private readonly List<HealthPublish> _published = [];
|
||||
|
||||
/// <summary>Thread-safe snapshot — <c>Publish</c> runs on the actor thread while the test asserts
|
||||
/// from its own.</summary>
|
||||
public IReadOnlyList<HealthPublish> Published
|
||||
{
|
||||
get { lock (_published) return _published.ToArray(); }
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Publish(
|
||||
string clusterId,
|
||||
string driverInstanceId,
|
||||
DriverHealth health,
|
||||
int errorCount5Min,
|
||||
DateTime? rediscoveryNeededUtc = null,
|
||||
string? rediscoveryReason = null,
|
||||
IReadOnlyList<HostConnectivityStatus>? hostStatuses = null)
|
||||
{
|
||||
lock (_published) _published.Add(new HealthPublish(health, hostStatuses));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A stub driver exposing <see cref="IHostConnectivityProbe"/>.
|
||||
/// <para><b>Its own health is deliberately STABLE</b> (a fixed last-read timestamp), for the same
|
||||
/// reason <c>RediscoverableStubDriver</c> is: the shared <c>StubDriver</c> returns
|
||||
/// <c>DateTime.UtcNow</c> from <c>GetHealth()</c>, so its fingerprint differs on every call, the
|
||||
/// dedup never engages, and any test built on it passes vacuously whether or not the fix is
|
||||
/// present.</para>
|
||||
/// </summary>
|
||||
private sealed class ProbeStubDriver : IDriver, IHostConnectivityProbe
|
||||
{
|
||||
private static readonly DateTime FixedLastRead = new(2026, 7, 30, 12, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly DateTime FixedChangedAt = new(2026, 7, 30, 11, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private readonly List<HostConnectivityStatus> _hosts = [];
|
||||
|
||||
/// <summary>When set, <see cref="GetHostStatuses"/> throws — the misbehaving-probe case.</summary>
|
||||
public bool ThrowOnGetHostStatuses { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public event EventHandler<HostStatusChangedEventArgs>? OnHostStatusChanged;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string DriverInstanceId => "probe-stub-1";
|
||||
|
||||
/// <inheritdoc />
|
||||
public string DriverType => "Stub";
|
||||
|
||||
/// <summary>Number of live subscribers on <see cref="OnHostStatusChanged"/>.</summary>
|
||||
public int SubscriberCount => OnHostStatusChanged?.GetInvocationList().Length ?? 0;
|
||||
|
||||
/// <summary>Adds or updates a host, optionally raising the transition event as a real probe loop does.</summary>
|
||||
public void SetHost(string hostName, HostState state, bool raise = false)
|
||||
{
|
||||
lock (_hosts)
|
||||
{
|
||||
var index = _hosts.FindIndex(h => h.HostName == hostName);
|
||||
var entry = new HostConnectivityStatus(hostName, state, FixedChangedAt);
|
||||
if (index >= 0) _hosts[index] = entry;
|
||||
else _hosts.Add(entry);
|
||||
}
|
||||
|
||||
if (raise) RaiseHostStatusChanged();
|
||||
}
|
||||
|
||||
/// <summary>Flips enumeration order without changing any host's state.</summary>
|
||||
public void ReverseHostOrder()
|
||||
{
|
||||
lock (_hosts) _hosts.Reverse();
|
||||
}
|
||||
|
||||
/// <summary>Raises the event exactly as a real probe loop does.</summary>
|
||||
public void RaiseHostStatusChanged()
|
||||
=> OnHostStatusChanged?.Invoke(this, new HostStatusChangedEventArgs("plc-b", HostState.Running, HostState.Stopped));
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyList<HostConnectivityStatus> GetHostStatuses()
|
||||
{
|
||||
if (ThrowOnGetHostStatuses) throw new InvalidOperationException("probe is broken");
|
||||
lock (_hosts) return _hosts.ToArray();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, FixedLastRead, null);
|
||||
|
||||
/// <inheritdoc />
|
||||
public long GetMemoryFootprint() => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task FlushOptionalCachesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -185,7 +185,8 @@ public sealed class DriverInstanceActorRediscoverySignalTests : RuntimeActorTest
|
||||
DriverHealth health,
|
||||
int errorCount5Min,
|
||||
DateTime? rediscoveryNeededUtc = null,
|
||||
string? rediscoveryReason = null)
|
||||
string? rediscoveryReason = null,
|
||||
IReadOnlyList<HostConnectivityStatus>? hostStatuses = null)
|
||||
{
|
||||
lock (_published)
|
||||
{
|
||||
|
||||
+2
-1
@@ -122,7 +122,8 @@ public sealed class DriverInstanceActorSubscriptionReconcileTests : RuntimeActor
|
||||
DriverHealth health,
|
||||
int errorCount5Min,
|
||||
DateTime? rediscoveryNeededUtc = null,
|
||||
string? rediscoveryReason = null)
|
||||
string? rediscoveryReason = null,
|
||||
IReadOnlyList<HostConnectivityStatus>? hostStatuses = null)
|
||||
=> Interlocked.Increment(ref _count);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user