340d9d26b7
- AdminUI MTConnect driver form: "Comms-loss confirm (ms)" in the Reconnect section, always written (0 = first refusal, negative = off is never dropped), case-insensitive load, unknown keys preserved, advisory at >= 90000 ms (the watchdog fires first). Form output byte-pinned and parsed through MTConnectDriver.ParseOptions on the driver side. - Driver: a confirmed refused-connection outage now annotates every told CONDITION quality-only BadNoCommunication (state/ack kept, once per outage, UNAVAILABLE-Bad skipped); the first delivery afterwards re-diffs all subscribed conditions so unchanged ones get a quality-only Good. - docker-dev/docker-compose.mtconnect-gate.yml: throwaway Agent in its adapter's network namespace so a stop is a genuine ConnectionRefused (a plain stopped container is a DNS failure = ambiguous). - Docs: MTConnect.md comms-loss section + live-gate recipe, docker-dev README, drivers.md rule. Refs #852
218 lines
9.9 KiB
C#
218 lines
9.9 KiB
C#
using System.Net.Sockets;
|
||
using Microsoft.Extensions.Logging;
|
||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
|
||
|
||
/// <summary>
|
||
/// lmxopcua#846 — the fast comms-loss path for a DEFINITE Agent loss. Before it, an Agent outage left every value
|
||
/// Good-but-stale for ~94–118 s, until the runtime's comms-loss watchdog (Degraded + a frozen
|
||
/// <c>LastSuccessfulRead</c> for 90 s) re-initialized the driver and the host fanned <c>BadNoCommunication</c>.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// <para>
|
||
/// <b>Only a refusal is definite.</b> A connection REFUSED (<see cref="SocketError.ConnectionRefused"/> anywhere
|
||
/// in the exception chain — what an <c>HttpClient</c> connect to a port nobody listens on produces) proves
|
||
/// nothing is serving the Agent URI. Everything else — a request or stream-idle timeout, a reset or half-closed
|
||
/// stream, a DNS failure, a paused Agent — is ambiguous and keeps the watchdog contract
|
||
/// (<c>docs/DriverLifecycle.md</c>, <c>.claude/rules/drivers.md</c> #527/#676): those must never mark early.
|
||
/// </para>
|
||
/// <para>
|
||
/// <b>Confirmed, not instant.</b> Refusals must continue, unbroken, for
|
||
/// <see cref="MTConnectDriverOptions.CommsLossConfirmMs"/> (default 2 s; 0 = first refusal; negative = off)
|
||
/// before the values are marked, so a single refused connect during an Agent restart does not flap every value.
|
||
/// While confirmation is pending the reconnect delay is capped at the time remaining, so the confirming
|
||
/// attempt is not pushed out by the backoff ladder. Anything that reaches the Agent ends the run.
|
||
/// </para>
|
||
/// <para>
|
||
/// <b>What marking does — the driver half of the quality contract.</b> Every subscribed value is published
|
||
/// <c>BadNoCommunication</c> (value null) ONCE per outage; the observation index keeps the last values. Health
|
||
/// is already Degraded with <c>LastSuccessfulRead</c> frozen (the stream failure degraded it), so the watchdog
|
||
/// still re-initializes at 90 s exactly as before — this path only stops serving stale values as Good in the
|
||
/// meantime. Reads already fall back to <c>/current</c> on a degraded stream and fail Bad on their own.
|
||
/// Native CONDITIONS (#852) get what the host's connectivity edge gives them: a quality-only
|
||
/// <c>BadNoCommunication</c> annotation, Active state and ack untouched (<c>MarkConditionsCommsLost</c>).
|
||
/// </para>
|
||
/// <para>
|
||
/// <b>Recovery republishes everything.</b> The first delivered chunk or re-baseline after a marked outage
|
||
/// republishes every subscribed value from the index — a resumed <c>/sample</c> only carries what CHANGED, so
|
||
/// an unchanged data item would otherwise stay Bad until it next moved (the (value, status) recovery rule).
|
||
/// Conditions are re-diffed in full for the same reason, which lifts each annotated one's quality.
|
||
/// </para>
|
||
/// </remarks>
|
||
public sealed partial class MTConnectDriver
|
||
{
|
||
/// <summary>Canonical <c>Opc.Ua.StatusCodes.BadNoCommunication</c>.</summary>
|
||
private const uint StatusBadNoCommunication = 0x80310000u;
|
||
|
||
/// <summary>1 while a confirmed refused-connection outage has marked the subscribed values Bad.</summary>
|
||
private int _commsLostMarked;
|
||
|
||
/// <summary>Whether a refused-connection outage currently has the values marked Bad (test seam / diagnostics).</summary>
|
||
internal bool CommsLossMarked => Volatile.Read(ref _commsLostMarked) == 1;
|
||
|
||
/// <summary>One pump's run of consecutive refused connections. Only that pump's task touches it.</summary>
|
||
private sealed class RefusalState
|
||
{
|
||
/// <summary>When the current unbroken run of refusals started, or null when there is none.</summary>
|
||
public DateTime? Since { get; set; }
|
||
|
||
/// <summary>Ends the run (anything reached the Agent, or an ambiguous failure broke it).</summary>
|
||
public void Reset() => Since = null;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Whether <paramref name="error"/> proves nothing is listening at the Agent URI — a connection refused anywhere
|
||
/// in its chain. Every other failure is ambiguous (see the class remarks).
|
||
/// </summary>
|
||
/// <param name="error">The stream / re-baseline failure.</param>
|
||
/// <returns>True for a definite loss.</returns>
|
||
internal static bool IsDefiniteConnectionLoss(Exception? error)
|
||
{
|
||
for (var e = error; e is not null; e = e.InnerException)
|
||
{
|
||
if (e is SocketException { SocketErrorCode: SocketError.ConnectionRefused })
|
||
{
|
||
return true;
|
||
}
|
||
|
||
if (e is AggregateException agg && agg.InnerExceptions.Any(IsDefiniteConnectionLoss))
|
||
{
|
||
return true;
|
||
}
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Records one transient stream failure and, once refusals have persisted for the confirmation window, marks
|
||
/// every subscribed value <c>BadNoCommunication</c> (once per outage).
|
||
/// </summary>
|
||
/// <returns><see langword="false"/> when the pump's session has been replaced (nothing marked).</returns>
|
||
private bool NoteStreamFailure(RefusalState refusals, StreamOutcome outcome, MTConnectDriverOptions options, long generation)
|
||
{
|
||
if (options.CommsLossConfirmMs < 0 || !IsDefiniteConnectionLoss(outcome.Error))
|
||
{
|
||
// An ambiguous failure breaks the run: only an UNBROKEN series of refusals is a confirmed loss.
|
||
refusals.Reset();
|
||
return true;
|
||
}
|
||
|
||
var now = DateTime.UtcNow;
|
||
refusals.Since ??= now;
|
||
if (now - refusals.Since.Value < TimeSpan.FromMilliseconds(options.CommsLossConfirmMs))
|
||
{
|
||
return true;
|
||
}
|
||
|
||
return TryMarkCommsLost(generation, options, now - refusals.Since.Value);
|
||
}
|
||
|
||
/// <summary>
|
||
/// While a refusal run is waiting for confirmation, the next attempt is made no later than the moment the window
|
||
/// closes — so the confirming refusal is not deferred by a long backoff step.
|
||
/// </summary>
|
||
private static TimeSpan CapForCommsLossConfirmation(TimeSpan delay, RefusalState refusals, MTConnectDriverOptions options)
|
||
{
|
||
if (refusals.Since is not { } since || options.CommsLossConfirmMs < 0)
|
||
{
|
||
return delay;
|
||
}
|
||
|
||
var remaining = since.AddMilliseconds(options.CommsLossConfirmMs) - DateTime.UtcNow;
|
||
if (remaining <= TimeSpan.Zero)
|
||
{
|
||
return delay; // already confirmed (or about to be): the ordinary ladder paces the outage
|
||
}
|
||
|
||
return remaining < delay ? remaining : delay;
|
||
}
|
||
|
||
private bool TryMarkCommsLost(long generation, MTConnectDriverOptions options, TimeSpan refusedFor)
|
||
{
|
||
lock (_subscriptionLock)
|
||
{
|
||
if (_sessionGeneration != generation)
|
||
{
|
||
return false;
|
||
}
|
||
}
|
||
|
||
if (Interlocked.Exchange(ref _commsLostMarked, 1) == 1)
|
||
{
|
||
return true; // already marked for this outage
|
||
}
|
||
|
||
var references = SubscribedReferences();
|
||
_logger.LogWarning(
|
||
"MTConnect driver {DriverInstanceId}: {AgentUri} has refused every connection for {RefusedMs} ms (definite loss); marking {Count} subscribed value(s) BadNoCommunication until it answers again.",
|
||
_driverInstanceId, AgentDisplay(options), (long)refusedFor.TotalMilliseconds, references.Count);
|
||
FanOutStatus(references, StatusBadNoCommunication);
|
||
|
||
// #852 — and the native conditions, quality-only (state + ack kept), as the host's connectivity edge does.
|
||
var conditions = MarkConditionsCommsLost();
|
||
if (conditions > 0)
|
||
{
|
||
_logger.LogWarning(
|
||
"MTConnect driver {DriverInstanceId}: annotated {Count} native condition(s) BadNoCommunication (quality only; state and acknowledgement kept).",
|
||
_driverInstanceId, conditions);
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
/// <summary>
|
||
/// The Agent answered again after a marked outage: republish every subscribed value from the index (once).
|
||
/// Generation-checked so a superseded pump cannot republish into the session that replaced it.
|
||
/// </summary>
|
||
private void RepublishAfterCommsLoss(long generation)
|
||
{
|
||
if (Volatile.Read(ref _commsLostMarked) == 0 || !IsCurrentSessionGeneration(generation))
|
||
{
|
||
return;
|
||
}
|
||
|
||
if (Interlocked.Exchange(ref _commsLostMarked, 0) == 0)
|
||
{
|
||
return;
|
||
}
|
||
|
||
_logger.LogInformation(
|
||
"MTConnect driver {DriverInstanceId}: the Agent answered again; republishing every subscribed value.",
|
||
_driverInstanceId);
|
||
FanOut(SubscribedReferences(), ObservationIndex);
|
||
|
||
// #852 — every subscribed condition, not just the ones this delivery touched: an unchanged condition the
|
||
// outage annotated Bad gets a quality-only Good; one still UNAVAILABLE at the Agent stays Bad.
|
||
EmitConditionTransitions(touchedDataItemIds: null, resync: false);
|
||
}
|
||
|
||
/// <summary>Publishes a value-less <paramref name="status"/> to every handle subscribing each reference.</summary>
|
||
private void FanOutStatus(IReadOnlyList<string> references, uint status)
|
||
{
|
||
if (references.Count == 0)
|
||
{
|
||
return;
|
||
}
|
||
|
||
SubscriptionState[] states;
|
||
lock (_subscriptionLock)
|
||
{
|
||
states = [.. _subscriptions.Values];
|
||
}
|
||
|
||
var snapshot = new DataValueSnapshot(null, status, null, DateTime.UtcNow);
|
||
foreach (var reference in references)
|
||
{
|
||
foreach (var state in states)
|
||
{
|
||
if (state.References.Contains(reference))
|
||
{
|
||
RaiseDataChange(state.Handle, reference, snapshot);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|