feat(hosts): surface per-host connectivity on /hosts; drop the unwritable table (#521)
`IHostConnectivityProbe` was a dead surface: eleven drivers implement it, `GetHostStatuses()` had ZERO production call sites, and `OnHostStatusChanged` had no subscriber outside the Galaxy driver's own aggregator. Per-host connectivity was computed by every driver and read by nobody. The issue offered "build the publisher or delete it". The publisher as its entity doc described it — driver nodes upserting `DriverHostStatus` rows — is not buildable: per-cluster mesh Phase 4 gates `AddOtOpcUaConfigDb` on the `admin` role, so a driver-only node has no ConfigDb connection to write rows with. So the capability is kept and the transport changed. `DriverHealthChanged.HostStatuses` now carries the probe result to `/hosts` as a Hosts column. That channel already reached the page, already survives the mesh split via the Phase 5 gRPC telemetry stream, and already replays a last-value snapshot on re-subscribe — so per-host state re-primes after a reconnect without a durable store. Both halves of the interface finally do what they are for: the event triggers a prompt publish, the pull is the source of truth. The point of the column is the case the driver-level state chip structurally cannot express: a multi-device driver stays aggregate-Healthy while ONE of its devices is unreachable. Two traps, both pinned by tests that were falsified against the prod code: - The host digest MUST be in the publish fingerprint. On a single-host-down transition every other fingerprint component is unchanged, so the dedup would swallow exactly the publish carrying the news — the trap that already bit the rediscovery signal. Removing it turns the guard test red, verified. - null (no probe) must stay distinct from empty (probe with no hosts). proto3 cannot tell an absent repeated field from an empty one, hence the explicit `has_host_statuses` flag; collapsing them would render every probe-less driver as one whose devices are all fine. Dropped: the DriverHostStatus entity, enum, DbSet, model config and table (migration DropDriverHostStatusTable — empty on every deployment, so the scaffolder's data-loss warning is moot, and Down() recreates it exactly). Found en route, NOT fixed here: `DriverInstanceResilienceStatus` is the identical defect — no writer, no reader, only a DbSet declaration, while the live data rides the `driver-resilience-status` telemetry channel. Its doc-comment now states that rather than describing the sampler and AdminUI join that were never built. Filed as #524 rather than widening this schema change beyond what was asked. Claude-Session: https://claude.ai/code/session_015p7wGqy3YpZNCpDzTpGMKo
This commit is contained in:
@@ -120,6 +120,14 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// connection affinity (a Galaxy redeploy or a TwinCAT symbol-version bump can land while the driver is
|
||||
/// between connects), and dropping it in one state would lose the signal silently.</summary>
|
||||
private sealed record RediscoveryRaised(RediscoveryEventArgs Args);
|
||||
/// <summary>Self-sent when the wrapped driver raises <see cref="IHostConnectivityProbe.OnHostStatusChanged"/>
|
||||
/// — one of its hosts went Running ↔ Stopped ↔ Faulted. Marshals the event off the driver's probe thread
|
||||
/// onto the actor thread. Carries NO payload: the handler re-pulls
|
||||
/// <see cref="IHostConnectivityProbe.GetHostStatuses"/>, so the driver stays the single source of truth
|
||||
/// for the host set and a host that appeared since the last publish is picked up too. Handled in every
|
||||
/// behaviour for the same reason as <see cref="RediscoveryRaised"/> — a probe tick can land while the
|
||||
/// driver is between connects.</summary>
|
||||
private sealed record HostStatusRaised;
|
||||
|
||||
public sealed class RetryConnect
|
||||
{
|
||||
@@ -171,6 +179,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
private EventHandler<DataChangeEventArgs>? _dataChangeHandler;
|
||||
private EventHandler<AlarmEventArgs>? _alarmEventHandler;
|
||||
private EventHandler<RediscoveryEventArgs>? _rediscoveryHandler;
|
||||
private EventHandler<HostStatusChangedEventArgs>? _hostStatusHandler;
|
||||
|
||||
/// <summary>When the driver last raised <see cref="IRediscoverable.OnRediscoveryNeeded"/>, and the reason
|
||||
/// it gave. Null until the first raise. Carried on every subsequent health snapshot so the AdminUI can
|
||||
@@ -306,6 +315,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// Attach the rediscovery signal before the first publish. Not per-connect: an IRediscoverable raise
|
||||
// has no connection affinity, and a driver can observe a remote change while disconnected.
|
||||
AttachRediscoverySource();
|
||||
AttachHostStatusSource();
|
||||
PublishHealthSnapshot();
|
||||
Timers.StartPeriodicTimer("health-poll", HealthPollTick.Instance, _healthPollInterval);
|
||||
}
|
||||
@@ -325,6 +335,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// Stubbed drivers never enter Connected, so they never kick discovery; swallow defensively in case a
|
||||
// re-discovery self-tick is ever routed here so it doesn't surface as an Akka Unhandled message.
|
||||
Receive<RediscoveryRaised>(HandleRediscoveryRaised);
|
||||
Receive<HostStatusRaised>(_ => PublishHealthSnapshot());
|
||||
Receive<HealthPollTick>(_ => PublishHealthSnapshot());
|
||||
}
|
||||
|
||||
@@ -378,6 +389,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// this state; swallow it so it doesn't dead-letter — the next Connected entry re-subscribes.
|
||||
Receive<SubscribeAlarms>(_ => { });
|
||||
Receive<RediscoveryRaised>(HandleRediscoveryRaised);
|
||||
Receive<HostStatusRaised>(_ => PublishHealthSnapshot());
|
||||
Receive<HealthPollTick>(_ => PublishHealthSnapshot());
|
||||
}
|
||||
|
||||
@@ -438,6 +450,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Receive<SubscriptionFailed>(msg =>
|
||||
_log.Debug("DriverInstance {Id}: resubscribe reported failure: {Reason}", _driverInstanceId, msg.Reason));
|
||||
Receive<RediscoveryRaised>(HandleRediscoveryRaised);
|
||||
Receive<HostStatusRaised>(_ => PublishHealthSnapshot());
|
||||
Receive<HealthPollTick>(_ =>
|
||||
{
|
||||
PublishHealthSnapshot();
|
||||
@@ -541,6 +554,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// this state; swallow it so it doesn't dead-letter — the next Connected entry re-subscribes.
|
||||
Receive<SubscribeAlarms>(_ => { });
|
||||
Receive<RediscoveryRaised>(HandleRediscoveryRaised);
|
||||
Receive<HostStatusRaised>(_ => PublishHealthSnapshot());
|
||||
Receive<HealthPollTick>(_ => PublishHealthSnapshot());
|
||||
Timers.StartPeriodicTimer("retry-connect", RetryConnect.Instance, _reconnectInterval);
|
||||
}
|
||||
@@ -809,6 +823,48 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
_rediscoveryHandler = null;
|
||||
}
|
||||
|
||||
/// <summary>Subscribe the driver's <see cref="IHostConnectivityProbe.OnHostStatusChanged"/> (if it is
|
||||
/// one), marshaling each transition to the actor thread. Idempotent; mirrors
|
||||
/// <see cref="AttachRediscoverySource"/>, including the PreStart-not-per-connect placement — a probe
|
||||
/// loop can report a host down while the driver itself is between connects.</summary>
|
||||
private void AttachHostStatusSource()
|
||||
{
|
||||
if (_driver is not IHostConnectivityProbe src || _hostStatusHandler is not null) return;
|
||||
var self = Self;
|
||||
_hostStatusHandler = (_, _) => self.Tell(new HostStatusRaised());
|
||||
src.OnHostStatusChanged += _hostStatusHandler;
|
||||
}
|
||||
|
||||
/// <summary>Symmetric teardown, called from PostStop — same leak argument as
|
||||
/// <see cref="DetachRediscoverySource"/>.</summary>
|
||||
private void DetachHostStatusSource()
|
||||
{
|
||||
if (_driver is IHostConnectivityProbe src && _hostStatusHandler is not null)
|
||||
src.OnHostStatusChanged -= _hostStatusHandler;
|
||||
_hostStatusHandler = null;
|
||||
}
|
||||
|
||||
/// <summary>Current per-host connectivity, or null when the driver is not an
|
||||
/// <see cref="IHostConnectivityProbe"/>. Pulled fresh rather than cached: the driver owns the host set,
|
||||
/// and a stale local copy would be a second source of truth to keep in sync.
|
||||
/// <para><c>GetHostStatuses()</c> is called UNGUARDED (not through <see cref="_invoker"/>) on purpose —
|
||||
/// it is a pure in-memory snapshot with no I/O, which is exactly why
|
||||
/// <c>UnwrappedCapabilityCallAnalyzer</c> exempts it. A driver that makes it do I/O breaks that
|
||||
/// contract; the try/catch below keeps a misbehaving one from killing the health publish.</para></summary>
|
||||
private IReadOnlyList<HostConnectivityStatus>? CurrentHostStatuses()
|
||||
{
|
||||
if (_driver is not IHostConnectivityProbe probe) return null;
|
||||
try
|
||||
{
|
||||
return probe.GetHostStatuses();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex, "DriverInstance {Id}: GetHostStatuses threw during health publish; omitting host detail", _driverInstanceId);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Records the driver's rediscovery raise and re-publishes health so the signal reaches the
|
||||
/// AdminUI promptly rather than waiting for the next 30 s heartbeat.
|
||||
/// <para><b>Advisory only.</b> The served address space is deliberately NOT rebuilt: v3 authors raw tags
|
||||
@@ -961,16 +1017,26 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
var health = _driver.GetHealth();
|
||||
var errorCount = ErrorCount5Min();
|
||||
var hostStatuses = CurrentHostStatuses();
|
||||
// _rediscoveryNeededUtc is PART OF THE FINGERPRINT on purpose. A rediscovery raise on an
|
||||
// otherwise-unchanged Healthy driver leaves (state, lastSuccess, lastError, errorCount)
|
||||
// identical, so without it the dedup below would swallow the very publish that carries the
|
||||
// signal and the operator would never see the prompt.
|
||||
var fingerprint = (health.State, health.LastSuccessfulRead, health.LastError, errorCount, _rediscoveryNeededUtc);
|
||||
//
|
||||
// The host-status digest is in the fingerprint for EXACTLY the same reason, and the failure
|
||||
// mode is the more likely one of the two: a multi-device driver stays aggregate-Healthy when a
|
||||
// single device drops, so every other fingerprint component is unchanged and the transition —
|
||||
// the whole point of this channel — would be deduped away. It is a flattened STRING because a
|
||||
// tuple holding IReadOnlyList compares by reference, which would never match and so would
|
||||
// defeat the dedup in the opposite direction (re-publishing every 30 s heartbeat).
|
||||
var fingerprint = (health.State, health.LastSuccessfulRead, health.LastError, errorCount,
|
||||
_rediscoveryNeededUtc, HostStatusDigest(hostStatuses));
|
||||
if (_lastPublishedFingerprint is { } prev && prev.Equals(fingerprint))
|
||||
return;
|
||||
_lastPublishedFingerprint = fingerprint;
|
||||
_healthPublisher.Publish(
|
||||
_clusterId, _driverInstanceId, health, errorCount, _rediscoveryNeededUtc, _rediscoveryReason);
|
||||
_clusterId, _driverInstanceId, health, errorCount, _rediscoveryNeededUtc, _rediscoveryReason,
|
||||
hostStatuses);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -978,8 +1044,22 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Order-insensitive value digest of a host-status list for the publish fingerprint. Null in,
|
||||
/// null out — so "driver is not a probe" and "probe reports no hosts" stay distinguishable rather than
|
||||
/// both collapsing to the empty string.
|
||||
/// <para>Ordered by host name because a driver is free to return its hosts in any order (several build
|
||||
/// the list from a <c>Dictionary</c>), and an order flip would otherwise read as a real transition and
|
||||
/// re-publish forever.</para></summary>
|
||||
private static string? HostStatusDigest(IReadOnlyList<HostConnectivityStatus>? statuses) =>
|
||||
statuses is null
|
||||
? null
|
||||
: string.Join('|', statuses
|
||||
.OrderBy(s => s.HostName, StringComparer.Ordinal)
|
||||
.Select(s => $"{s.HostName}={s.State}@{s.LastChangedUtc:O}"));
|
||||
|
||||
/// <summary>Fingerprint of the last <see cref="PublishHealthSnapshot"/> call; null until first publish.</summary>
|
||||
private (DriverState State, DateTime? LastSuccess, string? LastError, int ErrorCount, DateTime? RediscoveryNeededUtc)? _lastPublishedFingerprint;
|
||||
private (DriverState State, DateTime? LastSuccess, string? LastError, int ErrorCount,
|
||||
DateTime? RediscoveryNeededUtc, string? HostStatusDigest)? _lastPublishedFingerprint;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PostStop()
|
||||
@@ -988,6 +1068,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// MUST happen: the IDriver instance can outlive this actor (the host respawns a child around the
|
||||
// same driver object), so a missing unsubscribe accumulates a handler per respawn holding a dead Self.
|
||||
DetachRediscoverySource();
|
||||
DetachHostStatusSource();
|
||||
try { _driver.ShutdownAsync(CancellationToken.None).GetAwaiter().GetResult(); }
|
||||
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
|
||||
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
|
||||
|
||||
Reference in New Issue
Block a user