Merge branch 'master' into worktree-agent-a720278a57ae41792

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:25:58 -05:00
69 changed files with 6709 additions and 915 deletions
@@ -295,7 +295,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// idempotent "already at this revision" short-circuit: the short-circuit would ack a clean Applied
/// for an address space known to be partial, and the re-apply is what repairs it (a degraded
/// non-rebuild apply leaves the publish actor's composition un-advanced, so it re-diffs the missing
/// nodes). Reset by every clean outcome and by every boot path that restores a revision.
/// nodes). Reset by every clean outcome and by the ConfigDb boot path (an Applied row is a final outcome);
/// SET by the cache boot paths, whose cached revision may never have reported an outcome (lmxopcua#868).
/// </summary>
private bool _currentRevisionDegraded;
@@ -1434,6 +1435,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
}
_currentRevision = revision;
// Unconfirmed (lmxopcua#868 third review #2): the cache is written BEFORE the address-space outcome, so a
// restart in that window boots a revision central never got an ack for. Marking it unconfirmed makes a
// re-dispatch of it RE-APPLY (and ack the real outcome) instead of short-circuiting a clean Applied.
_currentRevisionDegraded = true;
BecomeSteady();
_log.Info(
@@ -1514,6 +1519,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// Stale would have started does not run here, so recovery rides on the next dispatch —
// matching how a normally-booted node behaves.
_currentRevision = revision;
// Unconfirmed, as in BootstrapFromCache (third review #2): a re-dispatch of it re-applies.
_currentRevisionDegraded = true;
BecomeSteady();
_isRunningFromCache = true;
@@ -2759,9 +2766,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
/// One child's diagnostics row. <c>Name</c> carries the string driver-instance id — the identity every
/// other surface (/hosts, telemetry, the artifact) keys on — and <c>DriverInstanceId</c> the DB row Guid;
/// the wire record has no slot for the authored display name. Device counts come from the driver's host
/// statuses when it is an <c>IHostConnectivityProbe</c> (Running = connected; Stopped/Faulted = faulted;
/// Unknown = neither); a single-host driver counts as one device, connected while it reports Healthy or
/// Degraded and faulted while it reports Faulted. A child that has not published yet reports
/// statuses when it is an <c>IHostConnectivityProbe</c> (Running = connected; Faulted = faulted;
/// Stopped = stopped, counted separately since #786; Unknown = none) and the faulted hosts' names ride
/// along so the AdminUI can de-duplicate a device across a redundant pair; a single-host driver counts as
/// one device, connected while it reports Healthy or Degraded and faulted while it reports Faulted. A child that has not published yet reports
/// "Connecting" (or "Stubbed") with its spawn time as the last change.
/// </summary>
private static DriverInstanceDiagnostics ToDiagnostics(string driverInstanceId, ChildEntry entry, ChildHealthRecord? health)
@@ -2772,20 +2780,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
if (health is null)
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Connecting", 0, 0, entry.SpawnedUtc);
int connected, faulted;
return DiagnosticsFromHealth(entry.Spec.DriverInstanceRowId, name, health);
}
/// <summary>The device counts of a child that has published health (#786: Faulted and Stopped split).</summary>
/// <param name="rowId">The driver instance's ConfigDb row id.</param>
/// <param name="name">The string driver-instance id.</param>
/// <param name="health">The child's last-published health.</param>
/// <returns>The diagnostics row.</returns>
internal static DriverInstanceDiagnostics DiagnosticsFromHealth(Guid rowId, string name, ChildHealthRecord health)
{
int connected, faulted, stopped;
IReadOnlyList<string> faultedHostNames;
if (health.HostStatuses is { Count: > 0 } hosts)
{
connected = hosts.Count(h => h.State == HostState.Running);
faulted = hosts.Count(h => h.State is HostState.Stopped or HostState.Faulted);
// #786: Faulted only. A Stopped host (e.g. a Galaxy engine an operator took off scan) is a
// deliberate, separately-reported state, not a fault.
var faultedHosts = hosts.Where(h => h.State == HostState.Faulted).ToArray();
faulted = faultedHosts.Length;
stopped = hosts.Count(h => h.State == HostState.Stopped);
faultedHostNames = faultedHosts.Select(h => h.HostName).ToArray();
}
else
{
connected = health.State is DriverState.Healthy or DriverState.Degraded ? 1 : 0;
faulted = health.State == DriverState.Faulted ? 1 : 0;
stopped = 0;
// A single-host driver is its own device: no host name; the AdminUI keys it on the instance id.
faultedHostNames = Array.Empty<string>();
}
return new DriverInstanceDiagnostics(
entry.Spec.DriverInstanceRowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc);
rowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc,
StoppedDevices: stopped, FaultedHostNames: faultedHostNames);
}
private void HandleDispatchFromSteady(DispatchDeployment msg)
@@ -2967,7 +2995,13 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
}
_currentRevision = msg.Revision;
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Applied, failureReason: null);
// lmxopcua#868 review M1: the own row stays APPLYING (written by BeginDirectApply) until the
// address-space outcome is known — AwaitAddressSpaceOutcomeThenAck / ReleasePendingAck write the
// final Applied (clean, or with the degraded reason). Writing Applied/null here, up to ~105 s before
// that outcome, made the row read as a CLEAN ack: a coordinator restart in that window recovered it
// as complete and sealed the deployment clean while this node was about to report degraded (#601).
// A restart of THIS node in the window now replays the orphan Applying row (re-applies), which is
// the honest answer to "outcome unknown".
// The Applied ack is NOT sent here any more. It used to go out right now, before the
// address-space apply below had even been dequeued, so a degraded apply (swallowed sink faults,
// _lastApplied not advanced) still sealed the deployment green fleet-wide. The ack is now held
@@ -3052,6 +3086,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
if (_opcUaPublishActor is null)
{
_currentRevisionDegraded = false;
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applied, failureReason: null);
SendAck(deploymentId, ApplyAckOutcome.Applied, failureReason: null, correlation);
return;
}
@@ -3109,13 +3144,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
if (_currentRevision is { } cur && cur == pending.Revision)
_currentRevisionDegraded = degraded;
// The own row has been Applying since the apply began (lmxopcua#868 review M1); the FINAL outcome is
// written here, clean or degraded. Central adds the reason from the ack only when it ACCEPTS the ack (a
// timed-out or superseded deployment's ack is discarded), so a node with a ConfigDb records it itself; a
// driver-only node no-ops here.
UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
degraded ? ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded") : null);
if (degraded)
{
// The apply wrote this node's own row Applied/null before the outcome was known. Central
// adds the reason from the ack only when it ACCEPTS the ack (a timed-out or superseded deployment's ack
// is discarded), so a node with a ConfigDb records it itself; a driver-only node no-ops here.
UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded"));
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "degraded"));
_log.Warning(
"DriverHost {Node}: deployment {Id} (rev {Rev}) applied DEGRADED — {Reason}. Acking Applied+degraded; re-dispatching the same revision re-applies it",
@@ -4311,6 +4347,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
private void UpsertNodeDeploymentState(DeploymentId deploymentId, NodeDeploymentStatus status, string? failureReason)
{
// nvarchar(2048): an exception message or a degraded reason can exceed it, and SQL Server then refuses the
// write — the row would stay Applying forever (lmxopcua#868 third review #1).
failureReason = FailureReasons.Truncate(failureReason);
// Per-cluster mesh Phase 4: a driver-only node has NO ConfigDb. Central is the ack system of
// record — ConfigPublishCoordinator.PersistNodeAck already writes NodeDeploymentState from every
// ApplyAck it receives (and seeds the Applying rows at dispatch) — so this node's own upsert is