Files
lmxopcua/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectDriver.CommsLoss.cs
T
Joseph Doherty 19ffb3c902 feat(mtconnect): refused Agent connections mark values BadNoCommunication after a short confirmation; clamp the raised sample count to the Agent's bufferSize (#846)
- A /sample or re-baseline failure whose chain holds SocketError.ConnectionRefused, unbroken for
  CommsLossConfirmMs (new, default 2000; 0 = first refusal; negative = off), publishes every subscribed
  value BadNoCommunication once per outage; the first delivery afterwards republishes all of them from
  the index (unchanged values included). Timeouts/resets/DNS/pause keep the #527 watchdog contract.
- SampleCountCeiling clamps the adaptive raise to the /probe bufferSize (never below SampleCount).
- Docs: MTConnect.md comms-loss section + config key; drivers.md rule.

Refs #846
2026-11-11 03:48:05 -05:00

202 lines
8.9 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Net.Sockets;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
/// <summary>
/// lmxopcua#846 — the fast comms-loss path for a DEFINITE Agent loss. Before it, an Agent outage left every value
/// Good-but-stale for ~94–118 s, until the runtime's comms-loss watchdog (Degraded + a frozen
/// <c>LastSuccessfulRead</c> for 90 s) re-initialized the driver and the host fanned <c>BadNoCommunication</c>.
/// </summary>
/// <remarks>
/// <para>
/// <b>Only a refusal is definite.</b> A connection REFUSED (<see cref="SocketError.ConnectionRefused"/> anywhere
/// in the exception chain — what an <c>HttpClient</c> connect to a port nobody listens on produces) proves
/// nothing is serving the Agent URI. Everything else — a request or stream-idle timeout, a reset or half-closed
/// stream, a DNS failure, a paused Agent — is ambiguous and keeps the watchdog contract
/// (<c>docs/DriverLifecycle.md</c>, <c>.claude/rules/drivers.md</c> #527/#676): those must never mark early.
/// </para>
/// <para>
/// <b>Confirmed, not instant.</b> Refusals must continue, unbroken, for
/// <see cref="MTConnectDriverOptions.CommsLossConfirmMs"/> (default 2 s; 0 = first refusal; negative = off)
/// before the values are marked, so a single refused connect during an Agent restart does not flap every value.
/// While confirmation is pending the reconnect delay is capped at the time remaining, so the confirming
/// attempt is not pushed out by the backoff ladder. Anything that reaches the Agent ends the run.
/// </para>
/// <para>
/// <b>What marking does — the driver half of the quality contract.</b> Every subscribed value is published
/// <c>BadNoCommunication</c> (value null) ONCE per outage; the observation index keeps the last values. Health
/// is already Degraded with <c>LastSuccessfulRead</c> frozen (the stream failure degraded it), so the watchdog
/// still re-initializes at 90 s exactly as before — this path only stops serving stale values as Good in the
/// meantime. Reads already fall back to <c>/current</c> on a degraded stream and fail Bad on their own.
/// Native CONDITIONS are not touched: the host annotates them on the connectivity edge, as before.
/// </para>
/// <para>
/// <b>Recovery republishes everything.</b> The first delivered chunk or re-baseline after a marked outage
/// republishes every subscribed value from the index — a resumed <c>/sample</c> only carries what CHANGED, so
/// an unchanged data item would otherwise stay Bad until it next moved (the (value, status) recovery rule).
/// </para>
/// </remarks>
public sealed partial class MTConnectDriver
{
/// <summary>Canonical <c>Opc.Ua.StatusCodes.BadNoCommunication</c>.</summary>
private const uint BadNoCommunicationStatus = 0x80310000u;
/// <summary>1 while a confirmed refused-connection outage has marked the subscribed values Bad.</summary>
private int _commsLostMarked;
/// <summary>Whether a refused-connection outage currently has the values marked Bad (test seam / diagnostics).</summary>
internal bool CommsLossMarked => Volatile.Read(ref _commsLostMarked) == 1;
/// <summary>One pump's run of consecutive refused connections. Only that pump's task touches it.</summary>
private sealed class RefusalState
{
/// <summary>When the current unbroken run of refusals started, or null when there is none.</summary>
public DateTime? Since { get; set; }
/// <summary>Ends the run (anything reached the Agent, or an ambiguous failure broke it).</summary>
public void Reset() => Since = null;
}
/// <summary>
/// Whether <paramref name="error"/> proves nothing is listening at the Agent URI — a connection refused anywhere
/// in its chain. Every other failure is ambiguous (see the class remarks).
/// </summary>
/// <param name="error">The stream / re-baseline failure.</param>
/// <returns>True for a definite loss.</returns>
internal static bool IsDefiniteConnectionLoss(Exception? error)
{
for (var e = error; e is not null; e = e.InnerException)
{
if (e is SocketException { SocketErrorCode: SocketError.ConnectionRefused })
{
return true;
}
if (e is AggregateException agg && agg.InnerExceptions.Any(IsDefiniteConnectionLoss))
{
return true;
}
}
return false;
}
/// <summary>
/// Records one transient stream failure and, once refusals have persisted for the confirmation window, marks
/// every subscribed value <c>BadNoCommunication</c> (once per outage).
/// </summary>
/// <returns><see langword="false"/> when the pump's session has been replaced (nothing marked).</returns>
private bool NoteStreamFailure(RefusalState refusals, StreamOutcome outcome, MTConnectDriverOptions options, long generation)
{
if (options.CommsLossConfirmMs < 0 || !IsDefiniteConnectionLoss(outcome.Error))
{
// An ambiguous failure breaks the run: only an UNBROKEN series of refusals is a confirmed loss.
refusals.Reset();
return true;
}
var now = DateTime.UtcNow;
refusals.Since ??= now;
if (now - refusals.Since.Value < TimeSpan.FromMilliseconds(options.CommsLossConfirmMs))
{
return true;
}
return TryMarkCommsLost(generation, options, now - refusals.Since.Value);
}
/// <summary>
/// While a refusal run is waiting for confirmation, the next attempt is made no later than the moment the window
/// closes — so the confirming refusal is not deferred by a long backoff step.
/// </summary>
private static TimeSpan CapForCommsLossConfirmation(TimeSpan delay, RefusalState refusals, MTConnectDriverOptions options)
{
if (refusals.Since is not { } since || options.CommsLossConfirmMs < 0)
{
return delay;
}
var remaining = since.AddMilliseconds(options.CommsLossConfirmMs) - DateTime.UtcNow;
if (remaining <= TimeSpan.Zero)
{
return delay; // already confirmed (or about to be): the ordinary ladder paces the outage
}
return remaining < delay ? remaining : delay;
}
private bool TryMarkCommsLost(long generation, MTConnectDriverOptions options, TimeSpan refusedFor)
{
lock (_subscriptionLock)
{
if (_sessionGeneration != generation)
{
return false;
}
}
if (Interlocked.Exchange(ref _commsLostMarked, 1) == 1)
{
return true; // already marked for this outage
}
var references = SubscribedReferences();
_logger.LogWarning(
"MTConnect driver {DriverInstanceId}: {AgentUri} has refused every connection for {RefusedMs} ms (definite loss); marking {Count} subscribed value(s) BadNoCommunication until it answers again.",
_driverInstanceId, AgentDisplay(options), (long)refusedFor.TotalMilliseconds, references.Count);
FanOutStatus(references, BadNoCommunicationStatus);
return true;
}
/// <summary>
/// The Agent answered again after a marked outage: republish every subscribed value from the index (once).
/// Generation-checked so a superseded pump cannot republish into the session that replaced it.
/// </summary>
private void RepublishAfterCommsLoss(long generation)
{
if (Volatile.Read(ref _commsLostMarked) == 0 || !IsCurrentSessionGeneration(generation))
{
return;
}
if (Interlocked.Exchange(ref _commsLostMarked, 0) == 0)
{
return;
}
_logger.LogInformation(
"MTConnect driver {DriverInstanceId}: the Agent answered again; republishing every subscribed value.",
_driverInstanceId);
FanOut(SubscribedReferences(), ObservationIndex);
}
/// <summary>Publishes a value-less <paramref name="status"/> to every handle subscribing each reference.</summary>
private void FanOutStatus(IReadOnlyList<string> references, uint status)
{
if (references.Count == 0)
{
return;
}
SubscriptionState[] states;
lock (_subscriptionLock)
{
states = [.. _subscriptions.Values];
}
var snapshot = new DataValueSnapshot(null, status, null, DateTime.UtcNow);
foreach (var reference in references)
{
foreach (var state in states)
{
if (state.References.Contains(reference))
{
RaiseDataChange(state.Handle, reference, snapshot);
}
}
}
}
}