feat(mtconnect): refused Agent connections mark values BadNoCommunication after a short confirmation; clamp the raised sample count to the Agent's bufferSize (#846)

- A /sample or re-baseline failure whose chain holds SocketError.ConnectionRefused, unbroken for
  CommsLossConfirmMs (new, default 2000; 0 = first refusal; negative = off), publishes every subscribed
  value BadNoCommunication once per outage; the first delivery afterwards republishes all of them from
  the index (unchanged values included). Timeouts/resets/DNS/pause keep the #527 watchdog contract.
- SampleCountCeiling clamps the adaptive raise to the /probe bufferSize (never below SampleCount).
- Docs: MTConnect.md comms-loss section + config key; drivers.md rule.

Refs #846
This commit is contained in:
Joseph Doherty
2026-11-11 03:48:05 -05:00
parent ddc1f29e3a
commit 19ffb3c902
7 changed files with 522 additions and 10 deletions
@@ -63,6 +63,15 @@ public sealed class MTConnectDriverOptions
/// </summary>
public int HeartbeatMs { get; init; } = 10000;
/// <summary>
/// Gets how long, in milliseconds, the Agent must keep REFUSING connections (a definite loss — nothing is
/// listening at <see cref="AgentUri"/>) before every subscribed value is marked <c>BadNoCommunication</c>
/// (lmxopcua#846). Default <c>2000</c>; <c>0</c> marks on the first refusal; a negative value disables the fast
/// path, leaving the comms-loss watchdog (~90 s) as the only signal. Ambiguous failures (timeouts, a stalled or
/// reset stream, DNS) never use it — they keep the watchdog contract.
/// </summary>
public int CommsLossConfirmMs { get; init; } = 2000;
/// <summary>
/// The authored tags this driver instance serves — one <see cref="MTConnectTagDefinition"/>
/// per tag, keyed by the DataItem <c>id</c> it binds
@@ -0,0 +1,201 @@
using System.Net.Sockets;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
/// <summary>
/// lmxopcua#846 — the fast comms-loss path for a DEFINITE Agent loss. Before it, an Agent outage left every value
/// Good-but-stale for ~94–118 s, until the runtime's comms-loss watchdog (Degraded + a frozen
/// <c>LastSuccessfulRead</c> for 90 s) re-initialized the driver and the host fanned <c>BadNoCommunication</c>.
/// </summary>
/// <remarks>
/// <para>
/// <b>Only a refusal is definite.</b> A connection REFUSED (<see cref="SocketError.ConnectionRefused"/> anywhere
/// in the exception chain — what an <c>HttpClient</c> connect to a port nobody listens on produces) proves
/// nothing is serving the Agent URI. Everything else — a request or stream-idle timeout, a reset or half-closed
/// stream, a DNS failure, a paused Agent — is ambiguous and keeps the watchdog contract
/// (<c>docs/DriverLifecycle.md</c>, <c>.claude/rules/drivers.md</c> #527/#676): those must never mark early.
/// </para>
/// <para>
/// <b>Confirmed, not instant.</b> Refusals must continue, unbroken, for
/// <see cref="MTConnectDriverOptions.CommsLossConfirmMs"/> (default 2 s; 0 = first refusal; negative = off)
/// before the values are marked, so a single refused connect during an Agent restart does not flap every value.
/// While confirmation is pending the reconnect delay is capped at the time remaining, so the confirming
/// attempt is not pushed out by the backoff ladder. Anything that reaches the Agent ends the run.
/// </para>
/// <para>
/// <b>What marking does — the driver half of the quality contract.</b> Every subscribed value is published
/// <c>BadNoCommunication</c> (value null) ONCE per outage; the observation index keeps the last values. Health
/// is already Degraded with <c>LastSuccessfulRead</c> frozen (the stream failure degraded it), so the watchdog
/// still re-initializes at 90 s exactly as before — this path only stops serving stale values as Good in the
/// meantime. Reads already fall back to <c>/current</c> on a degraded stream and fail Bad on their own.
/// Native CONDITIONS are not touched: the host annotates them on the connectivity edge, as before.
/// </para>
/// <para>
/// <b>Recovery republishes everything.</b> The first delivered chunk or re-baseline after a marked outage
/// republishes every subscribed value from the index — a resumed <c>/sample</c> only carries what CHANGED, so
/// an unchanged data item would otherwise stay Bad until it next moved (the (value, status) recovery rule).
/// </para>
/// </remarks>
public sealed partial class MTConnectDriver
{
/// <summary>Canonical <c>Opc.Ua.StatusCodes.BadNoCommunication</c>.</summary>
private const uint BadNoCommunicationStatus = 0x80310000u;
/// <summary>1 while a confirmed refused-connection outage has marked the subscribed values Bad.</summary>
private int _commsLostMarked;
/// <summary>Whether a refused-connection outage currently has the values marked Bad (test seam / diagnostics).</summary>
internal bool CommsLossMarked => Volatile.Read(ref _commsLostMarked) == 1;
/// <summary>One pump's run of consecutive refused connections. Only that pump's task touches it.</summary>
private sealed class RefusalState
{
/// <summary>When the current unbroken run of refusals started, or null when there is none.</summary>
public DateTime? Since { get; set; }
/// <summary>Ends the run (anything reached the Agent, or an ambiguous failure broke it).</summary>
public void Reset() => Since = null;
}
/// <summary>
/// Whether <paramref name="error"/> proves nothing is listening at the Agent URI — a connection refused anywhere
/// in its chain. Every other failure is ambiguous (see the class remarks).
/// </summary>
/// <param name="error">The stream / re-baseline failure.</param>
/// <returns>True for a definite loss.</returns>
internal static bool IsDefiniteConnectionLoss(Exception? error)
{
for (var e = error; e is not null; e = e.InnerException)
{
if (e is SocketException { SocketErrorCode: SocketError.ConnectionRefused })
{
return true;
}
if (e is AggregateException agg && agg.InnerExceptions.Any(IsDefiniteConnectionLoss))
{
return true;
}
}
return false;
}
/// <summary>
/// Records one transient stream failure and, once refusals have persisted for the confirmation window, marks
/// every subscribed value <c>BadNoCommunication</c> (once per outage).
/// </summary>
/// <returns><see langword="false"/> when the pump's session has been replaced (nothing marked).</returns>
private bool NoteStreamFailure(RefusalState refusals, StreamOutcome outcome, MTConnectDriverOptions options, long generation)
{
if (options.CommsLossConfirmMs < 0 || !IsDefiniteConnectionLoss(outcome.Error))
{
// An ambiguous failure breaks the run: only an UNBROKEN series of refusals is a confirmed loss.
refusals.Reset();
return true;
}
var now = DateTime.UtcNow;
refusals.Since ??= now;
if (now - refusals.Since.Value < TimeSpan.FromMilliseconds(options.CommsLossConfirmMs))
{
return true;
}
return TryMarkCommsLost(generation, options, now - refusals.Since.Value);
}
/// <summary>
/// While a refusal run is waiting for confirmation, the next attempt is made no later than the moment the window
/// closes — so the confirming refusal is not deferred by a long backoff step.
/// </summary>
private static TimeSpan CapForCommsLossConfirmation(TimeSpan delay, RefusalState refusals, MTConnectDriverOptions options)
{
if (refusals.Since is not { } since || options.CommsLossConfirmMs < 0)
{
return delay;
}
var remaining = since.AddMilliseconds(options.CommsLossConfirmMs) - DateTime.UtcNow;
if (remaining <= TimeSpan.Zero)
{
return delay; // already confirmed (or about to be): the ordinary ladder paces the outage
}
return remaining < delay ? remaining : delay;
}
private bool TryMarkCommsLost(long generation, MTConnectDriverOptions options, TimeSpan refusedFor)
{
lock (_subscriptionLock)
{
if (_sessionGeneration != generation)
{
return false;
}
}
if (Interlocked.Exchange(ref _commsLostMarked, 1) == 1)
{
return true; // already marked for this outage
}
var references = SubscribedReferences();
_logger.LogWarning(
"MTConnect driver {DriverInstanceId}: {AgentUri} has refused every connection for {RefusedMs} ms (definite loss); marking {Count} subscribed value(s) BadNoCommunication until it answers again.",
_driverInstanceId, AgentDisplay(options), (long)refusedFor.TotalMilliseconds, references.Count);
FanOutStatus(references, BadNoCommunicationStatus);
return true;
}
/// <summary>
/// The Agent answered again after a marked outage: republish every subscribed value from the index (once).
/// Generation-checked so a superseded pump cannot republish into the session that replaced it.
/// </summary>
private void RepublishAfterCommsLoss(long generation)
{
if (Volatile.Read(ref _commsLostMarked) == 0 || !IsCurrentSessionGeneration(generation))
{
return;
}
if (Interlocked.Exchange(ref _commsLostMarked, 0) == 0)
{
return;
}
_logger.LogInformation(
"MTConnect driver {DriverInstanceId}: the Agent answered again; republishing every subscribed value.",
_driverInstanceId);
FanOut(SubscribedReferences(), ObservationIndex);
}
/// <summary>Publishes a value-less <paramref name="status"/> to every handle subscribing each reference.</summary>
private void FanOutStatus(IReadOnlyList<string> references, uint status)
{
if (references.Count == 0)
{
return;
}
SubscriptionState[] states;
lock (_subscriptionLock)
{
states = [.. _subscriptions.Values];
}
var snapshot = new DataValueSnapshot(null, status, null, DateTime.UtcNow);
foreach (var reference in references)
{
foreach (var state in states)
{
if (state.References.Contains(reference))
{
RaiseDataChange(state.Handle, reference, snapshot);
}
}
}
}
}
@@ -79,9 +79,23 @@ public sealed partial class MTConnectDriver : IDriverDiagnostics
/// <summary>The ceiling a pump started with <paramref name="configured"/> may raise its count to.</summary>
/// <param name="configured">The configured <see cref="MTConnectDriverOptions.SampleCount"/>.</param>
/// <returns>The ceiling — never below <paramref name="configured"/>.</returns>
internal static int SampleCountCeiling(int configured) =>
Math.Max(configured, (int)Math.Min((long)configured * SampleCountMaxGrowth, SampleCountAbsoluteCeiling));
/// <param name="agentBufferSize">
/// The Agent's <c>/probe</c> <c>bufferSize</c>, or <c>0</c> when unknown. lmxopcua#846: an Agent refuses a
/// <c>count</c> above it (<c>OUT_OF_RANGE</c>; live, <c>count == bufferSize</c> is accepted), so a known buffer
/// caps the raised count — a raise past it would only buy a refused stream and a step-back.
/// </param>
/// <returns>The ceiling — never below <paramref name="configured"/> (a configured count above the buffer is the
/// #825 configuration warning's business, not something to raise further).</returns>
internal static int SampleCountCeiling(int configured, long agentBufferSize = 0)
{
long ceiling = Math.Min((long)configured * SampleCountMaxGrowth, SampleCountAbsoluteCeiling);
if (agentBufferSize > 0)
{
ceiling = Math.Min(ceiling, agentBufferSize);
}
return (int)Math.Max(configured, ceiling);
}
/// <inheritdoc />
public IReadOnlyDictionary<string, string> GetDiagnostics()
@@ -132,7 +146,7 @@ public sealed partial class MTConnectDriver : IDriverDiagnostics
}
/// <summary>One pump's adaptive-count state. Only that pump's task touches it.</summary>
private sealed class SampleCountState(int configured)
private sealed class SampleCountState(int configured, long agentBufferSize)
{
/// <summary>The count the next stream opens with.</summary>
public int Count { get; set; } = configured;
@@ -141,7 +155,7 @@ public sealed partial class MTConnectDriver : IDriverDiagnostics
public int LastDelivered { get; set; } = configured;
/// <summary>The most <see cref="Count"/> may be raised to; lowered to the last good count on a refusal.</summary>
public int Ceiling { get; set; } = SampleCountCeiling(configured);
public int Ceiling { get; set; } = SampleCountCeiling(configured, agentBufferSize);
/// <summary>Whether the at-the-ceiling Warning has been logged for this pump.</summary>
public bool CeilingWarned { get; set; }
@@ -2229,8 +2229,9 @@ public sealed partial class MTConnectDriver
var cursor = from;
var attempt = 0;
_reconnectAttempts = 0;
var counts = new SampleCountState(options.SampleCount);
var counts = new SampleCountState(options.SampleCount, Interlocked.Read(ref _agentBufferSize));
Volatile.Write(ref _effectiveSampleCount, counts.Count);
var refusals = new RefusalState();
// #804 (Java-driver parity) — consecutive re-baselines with no /sample chunk delivered in between.
var rebaselinesWithoutProgress = 0;
@@ -2250,7 +2251,7 @@ public sealed partial class MTConnectDriver
{
if (attempt > 0)
{
var delay = BackoffFor(attempt, options.Reconnect);
var delay = CapForCommsLossConfirmation(BackoffFor(attempt, options.Reconnect), refusals, options);
if (delay > TimeSpan.Zero)
{
try
@@ -2273,6 +2274,12 @@ public sealed partial class MTConnectDriver
{
rebaselinesWithoutProgress = 0;
}
// #846 — only an unbroken run of refusals confirms a loss: anything that reached the Agent ends it.
if (outcome.DeliveredAny || outcome.Verdict is not StreamVerdict.Transient)
{
refusals.Reset();
}
cursor = outcome.Cursor;
instanceId = outcome.InstanceId;
@@ -2375,6 +2382,13 @@ public sealed partial class MTConnectDriver
return;
}
// #846 — a REFUSED connection is a definite loss; after CommsLossConfirmMs of nothing but
// refusals the values go BadNoCommunication now instead of at the ~90 s watchdog.
if (!NoteStreamFailure(refusals, outcome, options, generation))
{
return;
}
// A stream that DELIVERED before it dropped proves the endpoint works, so
// its failure starts a fresh backoff ladder. Without this the counter is
// monotonic for the life of the process: a healthy agent that drops a
@@ -2519,6 +2533,8 @@ public sealed partial class MTConnectDriver
return new StreamOutcome(StreamVerdict.Cancelled, expected, instanceId, delivered, null);
}
RepublishAfterCommsLoss(generation);
// R06:T-5 — a stream persistently behind the Agent reopens at the cursor it reached with a larger
// count, rather than falling out of the ring buffer and re-baselining every buffer-length.
if (!MeasureSampleLag(chunk, counts.Count))
@@ -2643,6 +2659,9 @@ public sealed partial class MTConnectDriver
StreamVerdict.Cancelled, fallbackCursor, fallbackInstanceId, deliveredAny, null);
}
// #846 — the Agent answered: if a refused-connection outage marked the values Bad, put them all back.
RepublishAfterCommsLoss(generation);
_logger.LogInformation(
"MTConnect driver {DriverInstanceId} re-baselined from {AgentUri}'s /current ({ObservationCount} observation(s), agent instanceId {InstanceId}) and resumes /sample from sequence {NextSequence}.",
_driverInstanceId,
@@ -3247,6 +3266,7 @@ public sealed partial class MTConnectDriver
SampleIntervalMs = dto.SampleIntervalMs ?? 1_000,
SampleCount = dto.SampleCount ?? 1_000,
HeartbeatMs = dto.HeartbeatMs ?? 10_000,
CommsLossConfirmMs = dto.CommsLossConfirmMs ?? 2_000,
Tags = dto.Tags is { Count: > 0 }
? [.. dto.Tags.Select(t => BuildTag(t, driverInstanceId))]
: [],
@@ -4447,6 +4467,8 @@ public sealed partial class MTConnectDriver
public int? SampleCount { get; init; }
/// <summary>The /sample heartbeat in milliseconds.</summary>
public int? HeartbeatMs { get; init; }
/// <summary>How long refused connections must persist before values go BadNoCommunication (#846).</summary>
public int? CommsLossConfirmMs { get; init; }
/// <summary>The authored tags.</summary>
public List<MTConnectTagDto>? Tags { get; init; }