feat(runtime,adminui): real GetDiagnostics from child health + /fleet/diagnostics consumer (A6)

DriverHostActor wraps the health publisher it hands its children in a
ChildHealthLedger (last snapshot per driver id, forwarded to the real
publisher), so GetDiagnostics now answers with each child's row id,
authored name, last-published state, device counts (host statuses when
the driver probes hosts, else one device) and last-change time — instead
of Guid.Empty / "Spawned" / 0 / now. Entries are forgotten on StopChild
and operator Restart.

IFleetDiagnosticsClient gains its first consumer: the new
/fleet/diagnostics page (node selector + ?node= query, revision,
running-from-cache notice, per-driver table). FleetDiagnosticsClient
logs the degrade-to-empty path at Warning.

Review ref: 02/U1.

Refs #731

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
Joseph Doherty
2026-10-22 06:44:38 -04:00
parent c6d4b14c93
commit d1cc9e2cfc
6 changed files with 459 additions and 15 deletions
@@ -195,7 +195,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
private readonly IReadOnlySet<string> _localRoles;
private readonly IActorRef? _dependencyMux;
private readonly IActorRef? _opcUaPublishActor;
private readonly IDriverHealthPublisher _healthPublisher;
/// <summary>The publisher handed to every child — a <see cref="ChildHealthLedger"/> over the configured one, so
/// <see cref="HandleGetDiagnostics"/> answers from the children's own last-published health.</summary>
private readonly ChildHealthLedger _healthPublisher;
private readonly IVirtualTagEvaluator _virtualTagEvaluator;
private readonly IHistoryWriter _historyWriter;
private readonly IActorRef? _virtualTagHostOverride;
@@ -594,6 +596,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
private sealed record ChildEntry(IActorRef Actor, DriverInstanceSpec Spec, bool Stubbed)
{
/// <summary>When this child was spawned — the diagnostics' last-change time until it publishes health.</summary>
public DateTime SpawnedUtc { get; } = DateTime.UtcNow;
// Convenience accessors for sites that don't need the full spec.
/// <summary>Gets the driver type name from the child's spec.</summary>
public string DriverType => Spec.DriverType;
@@ -886,7 +891,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
_localRoles = localRoles ?? new HashSet<string>(StringComparer.Ordinal);
_dependencyMux = dependencyMux;
_opcUaPublishActor = opcUaPublishActor;
_healthPublisher = healthPublisher ?? NullDriverHealthPublisher.Instance;
_healthPublisher = new ChildHealthLedger(healthPublisher ?? NullDriverHealthPublisher.Instance);
_virtualTagEvaluator = virtualTagEvaluator ?? NullVirtualTagEvaluator.Instance;
_historyWriter = historyWriter ?? NullHistoryWriter.Instance;
_virtualTagHostOverride = virtualTagHostOverride;
@@ -2514,16 +2519,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Timers.StartPeriodicTimer("retry-db", RetryConfigDbConnection.Instance, ReconnectInterval);
}
/// <summary>
/// Answers <see cref="GetDiagnostics"/> from the live child set and each child's last-published health
/// (<see cref="ChildHealthLedger"/>). Used to be a placeholder — <c>Guid.Empty</c>, "Spawned", zero
/// devices, "now" — that looked live in every behaviour and was read by nobody (02/U1); the AdminUI
/// fleet-diagnostics page is its consumer.
/// </summary>
private void HandleGetDiagnostics(GetDiagnostics msg)
{
var drivers = _children
.Select(kv => new DriverInstanceDiagnostics(
DriverInstanceId: Guid.Empty,
Name: kv.Key,
State: kv.Value.Stubbed ? "Stubbed" : "Spawned",
ConnectedDevices: 0,
FaultedDevices: 0,
LastChangeUtc: DateTime.UtcNow))
.Select(kv => ToDiagnostics(kv.Key, kv.Value, _healthPublisher.TryGet(kv.Key)))
.ToArray();
var snapshot = new NodeDiagnosticsSnapshot(
NodeId: _localNode,
@@ -2534,6 +2539,37 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Sender.Tell(snapshot);
}
/// <summary>
/// One child's diagnostics row. Device counts come from the driver's host statuses when it is an
/// <c>IHostConnectivityProbe</c> (Running = connected; Stopped/Faulted = faulted; Unknown = neither);
/// a single-host driver counts as one device, connected while it reports Healthy or Degraded and
/// faulted while it reports Faulted. A child that has not published yet reports "Connecting" (or
/// "Stubbed") with its spawn time as the last change.
/// </summary>
private static DriverInstanceDiagnostics ToDiagnostics(string driverInstanceId, ChildEntry entry, ChildHealthRecord? health)
{
var name = string.IsNullOrEmpty(entry.Spec.Name) ? driverInstanceId : entry.Spec.Name;
if (entry.Stubbed)
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Stubbed", 0, 0, entry.SpawnedUtc);
if (health is null)
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Connecting", 0, 0, entry.SpawnedUtc);
int connected, faulted;
if (health.HostStatuses is { Count: > 0 } hosts)
{
connected = hosts.Count(h => h.State == HostState.Running);
faulted = hosts.Count(h => h.State is HostState.Stopped or HostState.Faulted);
}
else
{
connected = health.State is DriverState.Healthy or DriverState.Degraded ? 1 : 0;
faulted = health.State == DriverState.Faulted ? 1 : 0;
}
return new DriverInstanceDiagnostics(
entry.Spec.DriverInstanceRowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc);
}
private void HandleDispatchFromSteady(DispatchDeployment msg)
{
// The short-circuit claims a CLEAN Applied, so it is only honest while the current revision's
@@ -3794,6 +3830,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// backoff it never earned (its first crash was deferred as a "crash loop").
_lastReprovisionUtc.Remove(driverInstanceId);
Timers.Cancel(ReprovisionTimerKey(driverInstanceId));
_healthPublisher.Forget(driverInstanceId);
_log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
}
@@ -3844,6 +3881,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
StopDependencyAdapter(msg.DriverInstanceId);
Context.Stop(entry.Actor);
_children.Remove(msg.DriverInstanceId);
// The old incarnation's health must not read as the new one's until it publishes its own.
_healthPublisher.Forget(msg.DriverInstanceId);
// Respawn from the same spec the last reconcile used — preserves RowId, Name, ClusterId.
SpawnChild(entry.Spec);