feat(hosts): surface per-host connectivity on /hosts; drop the unwritable table (#521)

`IHostConnectivityProbe` was a dead surface: eleven drivers implement it, `GetHostStatuses()`
had ZERO production call sites, and `OnHostStatusChanged` had no subscriber outside the Galaxy
driver's own aggregator. Per-host connectivity was computed by every driver and read by nobody.

The issue offered "build the publisher or delete it". The publisher as its entity doc described
it — driver nodes upserting `DriverHostStatus` rows — is not buildable: per-cluster mesh Phase 4
gates `AddOtOpcUaConfigDb` on the `admin` role, so a driver-only node has no ConfigDb connection
to write rows with. So the capability is kept and the transport changed.

`DriverHealthChanged.HostStatuses` now carries the probe result to `/hosts` as a Hosts column.
That channel already reached the page, already survives the mesh split via the Phase 5 gRPC
telemetry stream, and already replays a last-value snapshot on re-subscribe — so per-host state
re-primes after a reconnect without a durable store. Both halves of the interface finally do
what they are for: the event triggers a prompt publish, the pull is the source of truth.

The point of the column is the case the driver-level state chip structurally cannot express: a
multi-device driver stays aggregate-Healthy while ONE of its devices is unreachable.

Two traps, both pinned by tests that were falsified against the prod code:

- The host digest MUST be in the publish fingerprint. On a single-host-down transition every
  other fingerprint component is unchanged, so the dedup would swallow exactly the publish
  carrying the news — the trap that already bit the rediscovery signal. Removing it turns the
  guard test red, verified.
- null (no probe) must stay distinct from empty (probe with no hosts). proto3 cannot tell an
  absent repeated field from an empty one, hence the explicit `has_host_statuses` flag;
  collapsing them would render every probe-less driver as one whose devices are all fine.

Dropped: the DriverHostStatus entity, enum, DbSet, model config and table (migration
DropDriverHostStatusTable — empty on every deployment, so the scaffolder's data-loss warning is
moot, and Down() recreates it exactly).

Found en route, NOT fixed here: `DriverInstanceResilienceStatus` is the identical defect — no
writer, no reader, only a DbSet declaration, while the live data rides the
`driver-resilience-status` telemetry channel. Its doc-comment now states that rather than
describing the sampler and AdminUI join that were never built. Filed as #524 rather than widening
this schema change beyond what was asked.

Claude-Session: https://claude.ai/code/session_015p7wGqy3YpZNCpDzTpGMKo
This commit is contained in:
Joseph Doherty
2026-07-30 04:21:08 -04:00
parent dc9d947bca
commit 30d0697c28
24 changed files with 2411 additions and 305 deletions
@@ -1,3 +1,5 @@
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
/// <summary>
@@ -26,6 +28,22 @@ namespace ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
/// The driver-supplied reason string from the same event (e.g. <c>"deploy-time-changed"</c>), shown
/// to the operator alongside the prompt. Null when <paramref name="RediscoveryNeededUtc"/> is null.
/// </param>
/// <param name="HostStatuses">
/// Per-host connectivity as reported by <c>IHostConnectivityProbe.GetHostStatuses()</c>, or null when
/// the driver does not implement that capability. Empty (not null) when it does but knows no hosts yet.
/// <para><b>Why this rides the health snapshot rather than a table.</b> The <c>DriverHostStatus</c>
/// entity used to own this, with a doc-comment describing a publisher hosted service that upserted rows
/// from each driver node. That publisher was never built, and since per-cluster mesh Phase 4 it is
/// unbuildable as described: <c>Program.cs</c> gates <c>AddOtOpcUaConfigDb</c> on the <c>admin</c> role,
/// so a driver-only node has no ConfigDb connection to write rows to. This channel already reaches
/// <c>/hosts</c>, already survives the mesh split via the Phase 5 gRPC telemetry stream, and already
/// replays a last-value snapshot on every (re)subscribe — so per-host state re-primes after a
/// reconnect without a durable store. The table was dropped; see Gitea #521.</para>
/// <para><b>Value-equality caveat.</b> A record's generated <c>Equals</c> compares this list by
/// REFERENCE, so two <c>DriverHealthChanged</c> carrying equal-but-distinct lists are not equal.
/// Nothing dedups on record equality — <c>DriverInstanceActor</c> keeps its own flattened fingerprint
/// precisely because of this — but do not introduce such a comparison without fixing it here first.</para>
/// </param>
public sealed record DriverHealthChanged(
string ClusterId,
string DriverInstanceId,
@@ -35,7 +53,8 @@ public sealed record DriverHealthChanged(
int ErrorCount5Min,
DateTime PublishedUtc,
DateTime? RediscoveryNeededUtc = null,
string? RediscoveryReason = null)
string? RediscoveryReason = null,
IReadOnlyList<HostConnectivityStatus>? HostStatuses = null)
{
/// <summary>
/// DPS topic name. Both the runtime <c>AkkaDriverHealthPublisher</c> and the AdminUI
@@ -70,6 +70,19 @@ message DriverHealth {
google.protobuf.Timestamp published_utc = 7;
google.protobuf.Timestamp rediscovery_needed_utc = 8; // DateTime? — absent Timestamp encodes null
optional string rediscovery_reason = 9; // nullable in the record
// Per-host connectivity (Gitea #521). proto3 cannot distinguish an absent repeated field from an
// empty one, and the two mean different things here — "driver is not an IHostConnectivityProbe" vs
// "it is one and currently knows no hosts" — so the presence flag carries that bit explicitly rather
// than letting an empty list silently claim the driver has no probe.
bool has_host_statuses = 10;
repeated HostConnectivity host_statuses = 11;
}
// Mirrors ZB.MOM.WW.OtOpcUa.Core.Abstractions.HostConnectivityStatus.
message HostConnectivity {
string host_name = 1;
string state = 2; // HostState-as-string, matching DriverHealth.state
google.protobuf.Timestamp last_changed_utc = 3;
}
// Mirrors ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers.DriverResilienceStatusChanged.
@@ -1,62 +0,0 @@
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
namespace ZB.MOM.WW.OtOpcUa.Configuration.Entities;
/// <summary>
/// Per-host connectivity snapshot the Server publishes for each driver's
/// <c>IHostConnectivityProbe.GetHostStatuses</c> entry. One row per
/// (<see cref="NodeId"/>, <see cref="DriverInstanceId"/>, <see cref="HostName"/>) triple —
/// a redundant 2-node cluster with one Galaxy driver reporting 3 platforms produces 6
/// rows, not 3, because each server node owns its own runtime view.
/// </summary>
/// <remarks>
/// <para>
/// Supports the per-AppEngine Admin dashboard drill-down. The publisher hosted
/// service on the Server side subscribes to every
/// registered driver's <c>OnHostStatusChanged</c> and upserts rows on transitions +
/// periodic liveness heartbeats. <see cref="LastSeenUtc"/> advances on every
/// heartbeat so the Admin UI can flag stale rows from a crashed Server.
/// </para>
/// <para>
/// No foreign-key to <see cref="ClusterNode"/> — a Server may start reporting host
/// status before its ClusterNode row exists (e.g. first-boot bootstrap), and we'd
/// rather keep the status row than drop it. The Admin-side service left-joins on
/// NodeId when presenting rows.
/// </para>
/// </remarks>
public sealed class DriverHostStatus
{
/// <summary>Server node that's running the driver.</summary>
public required string NodeId { get; set; }
/// <summary>Driver instance's stable id (matches <c>IDriver.DriverInstanceId</c>).</summary>
public required string DriverInstanceId { get; set; }
/// <summary>
/// Driver-side host identifier — Galaxy Platform / AppEngine name, Modbus
/// <c>host:port</c>, whatever the probe returns. Opaque to the Admin UI except as
/// a display string.
/// </summary>
public required string HostName { get; set; }
/// <summary>Gets or sets the current connectivity state of the host.</summary>
public DriverHostState State { get; set; } = DriverHostState.Unknown;
/// <summary>Timestamp of the last state transition (not of the most recent heartbeat).</summary>
public DateTime StateChangedUtc { get; set; }
/// <summary>
/// Advances on every publisher heartbeat — the Admin UI uses
/// <c>now - LastSeenUtc &gt; threshold</c> to flag rows whose owning Server has
/// stopped reporting (crashed, network-partitioned, etc.), independent of
/// <see cref="State"/>.
/// </summary>
public DateTime LastSeenUtc { get; set; }
/// <summary>
/// Optional human-readable detail populated when <see cref="State"/> is
/// <see cref="DriverHostState.Faulted"/> — e.g. the exception message from the
/// driver's probe. Null for Running / Stopped / Unknown transitions.
/// </summary>
public string? Detail { get; set; }
}
@@ -1,17 +1,26 @@
namespace ZB.MOM.WW.OtOpcUa.Configuration.Entities;
/// <summary>
/// Runtime resilience counters the CapabilityInvoker + MemoryTracking + MemoryRecycle
/// surfaces for each <c>(DriverInstanceId, HostName)</c> pair. Separate from
/// <see cref="DriverHostStatus"/> (which owns per-host <i>connectivity</i> state) so a
/// host that's Running but has tripped its breaker or is approaching its memory ceiling
/// shows up distinctly on Admin <c>/hosts</c>.
/// Runtime resilience counters per <c>(DriverInstanceId, HostName)</c> pair.
/// <para><b>⚠️ This table is DEAD: nothing writes it and nothing reads it.</b> The only reference in
/// the repo is the <c>DriverInstanceResilienceStatuses</c> DbSet declaration. Do not treat a query
/// against it as a source of runtime state — it returns empty on every deployment.</para>
/// </summary>
/// <remarks>
/// Per <c>docs/v2/implementation/phase-6-1-resilience-and-observability.md</c> §Stream E.1.
/// The Admin UI left-joins this table on DriverHostStatus for display; rows are written
/// by the runtime via a HostedService that samples the tracker at a configurable
/// interval (default 5 s) — writes are non-critical, a missed sample is tolerated.
/// <para>
/// The original design (<c>docs/v2/implementation/phase-6-1-resilience-and-observability.md</c>
/// §Stream E.1) called for a HostedService sampling the tracker every ~5 s into this table, and
/// an Admin UI that left-joined it on <c>DriverHostStatus</c> for display. Neither was built:
/// there is no sampler, the join exists in no razor file, and <c>DriverHostStatus</c> itself was
/// removed in Gitea #521.
/// </para>
/// <para>
/// The live data does exist — it just never goes through SQL. Resilience state reaches the
/// AdminUI as <c>DriverResilienceStatusChanged</c> over the Phase 5 telemetry stream into the
/// in-memory <c>IDriverResilienceStatusStore</c>, the same shape #521 adopted for per-host
/// connectivity, and for the same reason: per-cluster mesh Phase 4 leaves a driver-only node with
/// no ConfigDb connection to write rows with. Keep-or-delete is tracked as Gitea #524.
/// </para>
/// </remarks>
public sealed class DriverInstanceResilienceStatus
{
@@ -1,21 +0,0 @@
namespace ZB.MOM.WW.OtOpcUa.Configuration.Enums;
/// <summary>
/// Persisted mirror of <c>Core.Abstractions.HostState</c> — the lifecycle state each
/// <c>IHostConnectivityProbe</c>-capable driver reports for its per-host topology
/// (Galaxy Platforms / AppEngines, Modbus PLC endpoints, future OPC UA gateway upstreams).
/// Defined here instead of re-using <c>Core.Abstractions.HostState</c> so the
/// Configuration project stays free of driver-runtime dependencies.
/// </summary>
/// <remarks>
/// The server-side publisher (follow-up PR) translates
/// <c>HostStatusChangedEventArgs.NewState</c> to this enum on every transition and
/// upserts into <see cref="Entities.DriverHostStatus"/>. Admin UI reads from the DB.
/// </remarks>
public enum DriverHostState
{
Unknown,
Running,
Stopped,
Faulted,
}
@@ -0,0 +1,56 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations
{
/// <summary>
/// Drops the DriverHostStatus table (Gitea #521). The scaffolder warns this "may result in the loss
/// of data"; here it cannot. No code ever inserted a row — the publisher hosted service its entity
/// doc described was never written, and per-cluster mesh Phase 4 made it unbuildable as described,
/// since a driver-only node has no ConfigDb connection. Every deployment's copy of this table is
/// empty. Per-host connectivity now rides DriverHealthChanged.HostStatuses over the Phase 5
/// telemetry stream to /hosts. Down() recreates the table exactly, so the rollback is lossless too.
/// </summary>
public partial class DropDriverHostStatusTable : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "DriverHostStatus");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "DriverHostStatus",
columns: table => new
{
NodeId = table.Column<string>(type: "nvarchar(64)", maxLength: 64, nullable: false),
DriverInstanceId = table.Column<string>(type: "nvarchar(64)", maxLength: 64, nullable: false),
HostName = table.Column<string>(type: "nvarchar(256)", maxLength: 256, nullable: false),
Detail = table.Column<string>(type: "nvarchar(1024)", maxLength: 1024, nullable: true),
LastSeenUtc = table.Column<DateTime>(type: "datetime2(3)", nullable: false),
State = table.Column<string>(type: "nvarchar(16)", maxLength: 16, nullable: false),
StateChangedUtc = table.Column<DateTime>(type: "datetime2(3)", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_DriverHostStatus", x => new { x.NodeId, x.DriverInstanceId, x.HostName });
});
migrationBuilder.CreateIndex(
name: "IX_DriverHostStatus_LastSeen",
table: "DriverHostStatus",
column: "LastSeenUtc");
migrationBuilder.CreateIndex(
name: "IX_DriverHostStatus_Node",
table: "DriverHostStatus",
column: "NodeId");
}
}
}
@@ -394,46 +394,6 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations
});
});
modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverHostStatus", b =>
{
b.Property<string>("NodeId")
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<string>("DriverInstanceId")
.HasMaxLength(64)
.HasColumnType("nvarchar(64)");
b.Property<string>("HostName")
.HasMaxLength(256)
.HasColumnType("nvarchar(256)");
b.Property<string>("Detail")
.HasMaxLength(1024)
.HasColumnType("nvarchar(1024)");
b.Property<DateTime>("LastSeenUtc")
.HasColumnType("datetime2(3)");
b.Property<string>("State")
.IsRequired()
.HasMaxLength(16)
.HasColumnType("nvarchar(16)");
b.Property<DateTime>("StateChangedUtc")
.HasColumnType("datetime2(3)");
b.HasKey("NodeId", "DriverInstanceId", "HostName");
b.HasIndex("LastSeenUtc")
.HasDatabaseName("IX_DriverHostStatus_LastSeen");
b.HasIndex("NodeId")
.HasDatabaseName("IX_DriverHostStatus_Node");
b.ToTable("DriverHostStatus", (string)null);
});
modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b =>
{
b.Property<Guid>("DriverInstanceRowId")
@@ -44,8 +44,6 @@ public sealed class OtOpcUaConfigDbContext(DbContextOptions<OtOpcUaConfigDbConte
public DbSet<ConfigAuditLog> ConfigAuditLogs => Set<ConfigAuditLog>();
/// <summary>Gets the DbSet of external ID reservations.</summary>
public DbSet<ExternalIdReservation> ExternalIdReservations => Set<ExternalIdReservation>();
/// <summary>Gets the DbSet of driver host statuses.</summary>
public DbSet<DriverHostStatus> DriverHostStatuses => Set<DriverHostStatus>();
/// <summary>Gets the DbSet of driver instance resilience statuses.</summary>
public DbSet<DriverInstanceResilienceStatus> DriverInstanceResilienceStatuses => Set<DriverInstanceResilienceStatus>();
/// <summary>Gets the DbSet of LDAP group role mappings.</summary>
@@ -90,7 +88,6 @@ public sealed class OtOpcUaConfigDbContext(DbContextOptions<OtOpcUaConfigDbConte
ConfigureNodeAcl(modelBuilder);
ConfigureConfigAuditLog(modelBuilder);
ConfigureExternalIdReservation(modelBuilder);
ConfigureDriverHostStatus(modelBuilder);
ConfigureDriverInstanceResilienceStatus(modelBuilder);
ConfigureLdapGroupRoleMapping(modelBuilder);
ConfigureScript(modelBuilder);
@@ -542,31 +539,12 @@ public sealed class OtOpcUaConfigDbContext(DbContextOptions<OtOpcUaConfigDbConte
});
}
private static void ConfigureDriverHostStatus(ModelBuilder modelBuilder)
{
modelBuilder.Entity<DriverHostStatus>(e =>
{
e.ToTable("DriverHostStatus");
// Composite key — one row per (server node, driver instance, probe-reported host).
// A redundant 2-node cluster with one Galaxy driver reporting 3 platforms produces
// 6 rows because each server node owns its own runtime view; the composite key is
// what lets both views coexist without shadowing each other.
e.HasKey(x => new { x.NodeId, x.DriverInstanceId, x.HostName });
e.Property(x => x.NodeId).HasMaxLength(64);
e.Property(x => x.DriverInstanceId).HasMaxLength(64);
e.Property(x => x.HostName).HasMaxLength(256);
e.Property(x => x.State).HasConversion<string>().HasMaxLength(16);
e.Property(x => x.StateChangedUtc).HasColumnType("datetime2(3)");
e.Property(x => x.LastSeenUtc).HasColumnType("datetime2(3)");
e.Property(x => x.Detail).HasMaxLength(1024);
// NodeId-only index drives the Admin UI's per-cluster drill-down (select all host
// statuses for the nodes of a specific cluster via join on ClusterNode.ClusterId).
e.HasIndex(x => x.NodeId).HasDatabaseName("IX_DriverHostStatus_Node");
// LastSeenUtc index powers the Admin UI's stale-row query (now - LastSeen > N).
e.HasIndex(x => x.LastSeenUtc).HasDatabaseName("IX_DriverHostStatus_LastSeen");
});
}
// ConfigureDriverHostStatus is GONE (Gitea #521). The DriverHostStatus table held per-host
// connectivity that a publisher hosted service was supposed to upsert from each driver node. That
// publisher was never written, and per-cluster mesh Phase 4 made it unwritable as designed: ConfigDb
// is registered only on the admin role, so a driver-only node has no connection to write rows with.
// Per-host connectivity now rides DriverHealthChanged.HostStatuses to /hosts over the Phase 5
// telemetry stream, which needs no DB and survives the mesh split.
private static void ConfigureDriverInstanceResilienceStatus(ModelBuilder modelBuilder)
{
@@ -22,13 +22,20 @@ public interface IDriverHealthPublisher
/// </param>
/// <param name="rediscoveryReason">The driver-supplied reason from that event; null when
/// <paramref name="rediscoveryNeededUtc"/> is null.</param>
/// <param name="hostStatuses">
/// Per-host connectivity from <see cref="IHostConnectivityProbe.GetHostStatuses"/>, or null when the
/// driver is not an <see cref="IHostConnectivityProbe"/>. Lets a multi-device driver (a FOCAS or
/// AbLegacy instance owning several PLCs) surface ONE unreachable device that would otherwise be
/// invisible behind an aggregate-Healthy driver row.
/// </param>
void Publish(
string clusterId,
string driverInstanceId,
DriverHealth health,
int errorCount5Min,
DateTime? rediscoveryNeededUtc = null,
string? rediscoveryReason = null);
string? rediscoveryReason = null,
IReadOnlyList<HostConnectivityStatus>? hostStatuses = null);
}
/// <summary>
@@ -49,6 +56,7 @@ public sealed class NullDriverHealthPublisher : IDriverHealthPublisher
DriverHealth health,
int errorCount5Min,
DateTime? rediscoveryNeededUtc = null,
string? rediscoveryReason = null)
string? rediscoveryReason = null,
IReadOnlyList<HostConnectivityStatus>? hostStatuses = null)
{ /* no-op */ }
}