Merge branch 'master' into worktree-agent-a720278a57ae41792
Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -295,7 +295,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// idempotent "already at this revision" short-circuit: the short-circuit would ack a clean Applied
|
||||
/// for an address space known to be partial, and the re-apply is what repairs it (a degraded
|
||||
/// non-rebuild apply leaves the publish actor's composition un-advanced, so it re-diffs the missing
|
||||
/// nodes). Reset by every clean outcome and by every boot path that restores a revision.
|
||||
/// nodes). Reset by every clean outcome and by the ConfigDb boot path (an Applied row is a final outcome);
|
||||
/// SET by the cache boot paths, whose cached revision may never have reported an outcome (lmxopcua#868).
|
||||
/// </summary>
|
||||
private bool _currentRevisionDegraded;
|
||||
|
||||
@@ -1434,6 +1435,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
}
|
||||
|
||||
_currentRevision = revision;
|
||||
// Unconfirmed (lmxopcua#868 third review #2): the cache is written BEFORE the address-space outcome, so a
|
||||
// restart in that window boots a revision central never got an ack for. Marking it unconfirmed makes a
|
||||
// re-dispatch of it RE-APPLY (and ack the real outcome) instead of short-circuiting a clean Applied.
|
||||
_currentRevisionDegraded = true;
|
||||
BecomeSteady();
|
||||
|
||||
_log.Info(
|
||||
@@ -1514,6 +1519,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// Stale would have started does not run here, so recovery rides on the next dispatch —
|
||||
// matching how a normally-booted node behaves.
|
||||
_currentRevision = revision;
|
||||
// Unconfirmed, as in BootstrapFromCache (third review #2): a re-dispatch of it re-applies.
|
||||
_currentRevisionDegraded = true;
|
||||
BecomeSteady();
|
||||
|
||||
_isRunningFromCache = true;
|
||||
@@ -2759,9 +2766,10 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
/// One child's diagnostics row. <c>Name</c> carries the string driver-instance id — the identity every
|
||||
/// other surface (/hosts, telemetry, the artifact) keys on — and <c>DriverInstanceId</c> the DB row Guid;
|
||||
/// the wire record has no slot for the authored display name. Device counts come from the driver's host
|
||||
/// statuses when it is an <c>IHostConnectivityProbe</c> (Running = connected; Stopped/Faulted = faulted;
|
||||
/// Unknown = neither); a single-host driver counts as one device, connected while it reports Healthy or
|
||||
/// Degraded and faulted while it reports Faulted. A child that has not published yet reports
|
||||
/// statuses when it is an <c>IHostConnectivityProbe</c> (Running = connected; Faulted = faulted;
|
||||
/// Stopped = stopped, counted separately since #786; Unknown = none) and the faulted hosts' names ride
|
||||
/// along so the AdminUI can de-duplicate a device across a redundant pair; a single-host driver counts as
|
||||
/// one device, connected while it reports Healthy or Degraded and faulted while it reports Faulted. A child that has not published yet reports
|
||||
/// "Connecting" (or "Stubbed") with its spawn time as the last change.
|
||||
/// </summary>
|
||||
private static DriverInstanceDiagnostics ToDiagnostics(string driverInstanceId, ChildEntry entry, ChildHealthRecord? health)
|
||||
@@ -2772,20 +2780,40 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
if (health is null)
|
||||
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Connecting", 0, 0, entry.SpawnedUtc);
|
||||
|
||||
int connected, faulted;
|
||||
return DiagnosticsFromHealth(entry.Spec.DriverInstanceRowId, name, health);
|
||||
}
|
||||
|
||||
/// <summary>The device counts of a child that has published health (#786: Faulted and Stopped split).</summary>
|
||||
/// <param name="rowId">The driver instance's ConfigDb row id.</param>
|
||||
/// <param name="name">The string driver-instance id.</param>
|
||||
/// <param name="health">The child's last-published health.</param>
|
||||
/// <returns>The diagnostics row.</returns>
|
||||
internal static DriverInstanceDiagnostics DiagnosticsFromHealth(Guid rowId, string name, ChildHealthRecord health)
|
||||
{
|
||||
int connected, faulted, stopped;
|
||||
IReadOnlyList<string> faultedHostNames;
|
||||
if (health.HostStatuses is { Count: > 0 } hosts)
|
||||
{
|
||||
connected = hosts.Count(h => h.State == HostState.Running);
|
||||
faulted = hosts.Count(h => h.State is HostState.Stopped or HostState.Faulted);
|
||||
// #786: Faulted only. A Stopped host (e.g. a Galaxy engine an operator took off scan) is a
|
||||
// deliberate, separately-reported state, not a fault.
|
||||
var faultedHosts = hosts.Where(h => h.State == HostState.Faulted).ToArray();
|
||||
faulted = faultedHosts.Length;
|
||||
stopped = hosts.Count(h => h.State == HostState.Stopped);
|
||||
faultedHostNames = faultedHosts.Select(h => h.HostName).ToArray();
|
||||
}
|
||||
else
|
||||
{
|
||||
connected = health.State is DriverState.Healthy or DriverState.Degraded ? 1 : 0;
|
||||
faulted = health.State == DriverState.Faulted ? 1 : 0;
|
||||
stopped = 0;
|
||||
// A single-host driver is its own device: no host name; the AdminUI keys it on the instance id.
|
||||
faultedHostNames = Array.Empty<string>();
|
||||
}
|
||||
|
||||
return new DriverInstanceDiagnostics(
|
||||
entry.Spec.DriverInstanceRowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc);
|
||||
rowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc,
|
||||
StoppedDevices: stopped, FaultedHostNames: faultedHostNames);
|
||||
}
|
||||
|
||||
private void HandleDispatchFromSteady(DispatchDeployment msg)
|
||||
@@ -2967,7 +2995,13 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
}
|
||||
|
||||
_currentRevision = msg.Revision;
|
||||
UpsertNodeDeploymentState(msg.DeploymentId, NodeDeploymentStatus.Applied, failureReason: null);
|
||||
// lmxopcua#868 review M1: the own row stays APPLYING (written by BeginDirectApply) until the
|
||||
// address-space outcome is known — AwaitAddressSpaceOutcomeThenAck / ReleasePendingAck write the
|
||||
// final Applied (clean, or with the degraded reason). Writing Applied/null here, up to ~105 s before
|
||||
// that outcome, made the row read as a CLEAN ack: a coordinator restart in that window recovered it
|
||||
// as complete and sealed the deployment clean while this node was about to report degraded (#601).
|
||||
// A restart of THIS node in the window now replays the orphan Applying row (re-applies), which is
|
||||
// the honest answer to "outcome unknown".
|
||||
// The Applied ack is NOT sent here any more. It used to go out right now, before the
|
||||
// address-space apply below had even been dequeued, so a degraded apply (swallowed sink faults,
|
||||
// _lastApplied not advanced) still sealed the deployment green fleet-wide. The ack is now held
|
||||
@@ -3052,6 +3086,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
if (_opcUaPublishActor is null)
|
||||
{
|
||||
_currentRevisionDegraded = false;
|
||||
UpsertNodeDeploymentState(deploymentId, NodeDeploymentStatus.Applied, failureReason: null);
|
||||
SendAck(deploymentId, ApplyAckOutcome.Applied, failureReason: null, correlation);
|
||||
return;
|
||||
}
|
||||
@@ -3109,13 +3144,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
if (_currentRevision is { } cur && cur == pending.Revision)
|
||||
_currentRevisionDegraded = degraded;
|
||||
|
||||
// The own row has been Applying since the apply began (lmxopcua#868 review M1); the FINAL outcome is
|
||||
// written here, clean or degraded. Central adds the reason from the ack only when it ACCEPTS the ack (a
|
||||
// timed-out or superseded deployment's ack is discarded), so a node with a ConfigDb records it itself; a
|
||||
// driver-only node no-ops here.
|
||||
UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
|
||||
degraded ? ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded") : null);
|
||||
if (degraded)
|
||||
{
|
||||
// The apply wrote this node's own row Applied/null before the outcome was known. Central
|
||||
// adds the reason from the ack only when it ACCEPTS the ack (a timed-out or superseded deployment's ack
|
||||
// is discarded), so a node with a ConfigDb records it itself; a driver-only node no-ops here.
|
||||
UpsertNodeDeploymentState(pending.DeploymentId, NodeDeploymentStatus.Applied,
|
||||
ApplyAck.DegradedReasonPrefix + (reason ?? "address-space apply degraded"));
|
||||
OtOpcUaTelemetry.DeploymentApplied.Add(1, new KeyValuePair<string, object?>("outcome", "degraded"));
|
||||
_log.Warning(
|
||||
"DriverHost {Node}: deployment {Id} (rev {Rev}) applied DEGRADED — {Reason}. Acking Applied+degraded; re-dispatching the same revision re-applies it",
|
||||
@@ -4311,6 +4347,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
|
||||
private void UpsertNodeDeploymentState(DeploymentId deploymentId, NodeDeploymentStatus status, string? failureReason)
|
||||
{
|
||||
// nvarchar(2048): an exception message or a degraded reason can exceed it, and SQL Server then refuses the
|
||||
// write — the row would stay Applying forever (lmxopcua#868 third review #1).
|
||||
failureReason = FailureReasons.Truncate(failureReason);
|
||||
// Per-cluster mesh Phase 4: a driver-only node has NO ConfigDb. Central is the ack system of
|
||||
// record — ConfigPublishCoordinator.PersistNodeAck already writes NodeDeploymentState from every
|
||||
// ApplyAck it receives (and seeds the Applying rows at dispatch) — so this node's own upsert is
|
||||
|
||||
Reference in New Issue
Block a user