feat(mtconnect): CommsLossConfirmMs AdminUI field, refused-connection fast path reaches native conditions, dedicated live-gate Agent (#852)

- AdminUI MTConnect driver form: "Comms-loss confirm (ms)" in the Reconnect section, always written
  (0 = first refusal, negative = off is never dropped), case-insensitive load, unknown keys preserved,
  advisory at >= 90000 ms (the watchdog fires first). Form output byte-pinned and parsed through
  MTConnectDriver.ParseOptions on the driver side.
- Driver: a confirmed refused-connection outage now annotates every told CONDITION quality-only
  BadNoCommunication (state/ack kept, once per outage, UNAVAILABLE-Bad skipped); the first delivery
  afterwards re-diffs all subscribed conditions so unchanged ones get a quality-only Good.
- docker-dev/docker-compose.mtconnect-gate.yml: throwaway Agent in its adapter's network namespace so a
  stop is a genuine ConnectionRefused (a plain stopped container is a DNS failure = ambiguous).
- Docs: MTConnect.md comms-loss section + live-gate recipe, docker-dev README, drivers.md rule.

Refs #852
This commit is contained in:
Joseph Doherty
2026-11-11 06:23:18 -05:00
parent d1b6cad299
commit 340d9d26b7
11 changed files with 635 additions and 35 deletions
@@ -92,7 +92,7 @@ internal static class MTConnectConditionAlarmMapper
// State untouched; only the quality moves (or is re-asserted on a resync).
var next = new MTConnectToldCondition(statement, QualityBad: true);
return !bad || resync
? (QualityOnly(statement ?? current, current, conditionId, dataItemId, handle, BadNoCommunication), next)
? (QualityOnly(statement ?? current, current.TimestampUtc, conditionId, dataItemId, handle, BadNoCommunication), next)
: (null, next);
}
@@ -103,7 +103,7 @@ internal static class MTConnectConditionAlarmMapper
// Unchanged statement: keep the FIRST text the server was shown. Only a Bad quality is lifted.
var kept = told2 with { Statement = statement };
return bad
? (QualityOnly(statement, current, conditionId, dataItemId, handle, Good), kept)
? (QualityOnly(statement, current.TimestampUtc, conditionId, dataItemId, handle, Good), kept)
: (null, kept);
}
@@ -123,10 +123,41 @@ internal static class MTConnectConditionAlarmMapper
// Normal after nothing / after Normal: nothing to say about the state — the host's connectivity
// annotation confirms a materialised condition Good — except lifting a Bad this driver put there.
return bad
? (QualityOnly(current, current, conditionId, dataItemId, handle, Good), told2)
? (QualityOnly(current, current.TimestampUtc, conditionId, dataItemId, handle, Good), told2)
: (null, told2);
}
/// <summary>
/// lmxopcua#852 — the quality-only <c>BadNoCommunication</c> annotation for a condition whose Agent has been
/// confirmed gone (the #846 refused-connection fast path), or <c>null</c> when the server already holds a Bad
/// quality for it (or was never told a statement). The told STATEMENT is kept, so the recovery diff that
/// follows sees "same statement, quality Bad" and lifts the quality without re-raising — an acked alarm stays
/// acked across the outage, exactly as for the host's own connectivity annotation.
/// </summary>
/// <param name="told">What the server was last told for this condition.</param>
/// <param name="conditionId">The condition's ConditionId (RawPath).</param>
/// <param name="dataItemId">The Agent DataItem id.</param>
/// <param name="handle">The alarm subscription the event belongs to.</param>
/// <param name="timestampUtc">When the loss was confirmed.</param>
/// <returns>The annotation to raise (or <c>null</c>) and what the server has been told afterwards.</returns>
public static (AlarmEventArgs? Event, MTConnectToldCondition Told) CommsLost(
MTConnectToldCondition told,
string conditionId,
string dataItemId,
IAlarmSubscriptionHandle handle,
DateTime timestampUtc)
{
ArgumentNullException.ThrowIfNull(told);
if (told.QualityBad || told.Statement is not { } statement)
{
return (null, told);
}
return (QualityOnly(statement, timestampUtc, conditionId, dataItemId, handle, BadNoCommunication),
told with { QualityBad = true });
}
/// <summary>The Part 9 severity of an active level.</summary>
/// <param name="level">An active level.</param>
/// <returns>The Fault or Warning severity.</returns>
@@ -163,7 +194,7 @@ internal static class MTConnectConditionAlarmMapper
/// </summary>
private static AlarmEventArgs QualityOnly(
MTConnectConditionState shown,
MTConnectConditionState current,
DateTime sourceTimestampUtc,
string conditionId,
string dataItemId,
IAlarmSubscriptionHandle handle,
@@ -178,7 +209,7 @@ internal static class MTConnectConditionAlarmMapper
AlarmType: shown.Detail.ConditionType ?? "CONDITION",
Message: MessageOf(shown),
Severity: severity >= FaultSeverity ? AlarmSeverity.High : AlarmSeverity.Medium,
SourceTimestampUtc: current.TimestampUtc,
SourceTimestampUtc: sourceTimestampUtc,
Kind: AlarmTransitionKind.Unspecified,
OpcUaSeverity: severity,
QualityStatusCode: quality);
@@ -36,10 +36,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
/// <see cref="_subscriptionLock"/> or the lifecycle semaphore, and never takes either.
/// </para>
/// <para>
/// <b>Comms loss emits nothing here</b>, matching every other native source: a driver that cannot reach
/// its Agent has nothing new to say, and the host annotates every condition it owns
/// <c>BadNoCommunication</c> on the connectivity edge, keeping each one's Active state. On recovery
/// the re-baselined <c>/current</c> is diffed like any other document.
/// <b>Comms loss.</b> An AMBIGUOUS loss emits nothing here: the host annotates every condition it owns
/// <c>BadNoCommunication</c> on the connectivity edge (the ~90 s watchdog), keeping each one's Active
/// state. A CONFIRMED refused-connection loss (#846/#852, <c>MarkConditionsCommsLost</c>) annotates
/// every told condition the same way — quality-only, state and ack untouched — at the same moment the data
/// values go Bad, and the recovery re-diffs every subscribed condition so the unchanged ones get their
/// quality lifted (a resumed <c>/sample</c> carries only what changed).
/// </para>
/// </remarks>
public sealed partial class MTConnectDriver : IAlarmSource
@@ -201,20 +203,67 @@ public sealed partial class MTConnectDriver : IAlarmSource
if (evt is null) continue;
foreach (var registration in handlers.GetInvocationList())
{
try
{
((EventHandler<AlarmEventArgs>)registration).Invoke(this, evt);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"MTConnect driver {DriverInstanceId} caught a fault from an alarm subscriber for {ConditionId}; delivery continues.",
_driverInstanceId, conditionId);
}
}
RaiseAlarmEvent(handlers, evt, conditionId);
}
}
}
/// <summary>
/// lmxopcua#852 — the alarm half of the #846 refused-connection fast path: annotates every subscribed
/// condition the server holds a Good-quality statement for <c>BadNoCommunication</c> (quality-only — no state,
/// severity, message or ack change; no <c>/alerts</c> row) and records that it did, so
/// <see cref="EmitConditionTransitions"/> lifts it on recovery. Once per outage per condition: a condition
/// already annotated Bad (by this or by an Agent UNAVAILABLE) is skipped. Like every other emit, it records
/// nothing when no handler is attached — the host has then detached because it is reconnecting, and its own
/// connectivity annotation covers the conditions.
/// </summary>
/// <returns>How many conditions were annotated.</returns>
private int MarkConditionsCommsLost()
{
var subscription = Volatile.Read(ref _alarmSubscription);
if (subscription is not { References.Count: > 0 }) return 0;
var annotated = 0;
lock (_alarmLock)
{
var handlers = OnAlarmEvent;
if (handlers is null) return 0;
var binding = Volatile.Read(ref _binding);
var now = DateTime.UtcNow;
foreach (var conditionId in subscription.References)
{
if (!_deliveredConditions.TryGetValue(conditionId, out var told)) continue;
var (evt, nowTold) = MTConnectConditionAlarmMapper.CommsLost(
told, conditionId, binding.ToDataItemId(conditionId), subscription.Handle, now);
if (evt is null) continue;
_deliveredConditions[conditionId] = nowTold;
annotated++;
RaiseAlarmEvent(handlers, evt, conditionId);
}
}
return annotated;
}
/// <summary>Invokes every alarm handler; a faulting one is logged and the next still runs. Caller holds <see cref="_alarmLock"/>.</summary>
private void RaiseAlarmEvent(EventHandler<AlarmEventArgs> handlers, AlarmEventArgs evt, string conditionId)
{
foreach (var registration in handlers.GetInvocationList())
{
try
{
((EventHandler<AlarmEventArgs>)registration).Invoke(this, evt);
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"MTConnect driver {DriverInstanceId} caught a fault from an alarm subscriber for {ConditionId}; delivery continues.",
_driverInstanceId, conditionId);
}
}
}
@@ -30,12 +30,14 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
/// is already Degraded with <c>LastSuccessfulRead</c> frozen (the stream failure degraded it), so the watchdog
/// still re-initializes at 90 s exactly as before — this path only stops serving stale values as Good in the
/// meantime. Reads already fall back to <c>/current</c> on a degraded stream and fail Bad on their own.
/// Native CONDITIONS are not touched: the host annotates them on the connectivity edge, as before.
/// Native CONDITIONS (#852) get what the host's connectivity edge gives them: a quality-only
/// <c>BadNoCommunication</c> annotation, Active state and ack untouched (<c>MarkConditionsCommsLost</c>).
/// </para>
/// <para>
/// <b>Recovery republishes everything.</b> The first delivered chunk or re-baseline after a marked outage
/// republishes every subscribed value from the index — a resumed <c>/sample</c> only carries what CHANGED, so
/// an unchanged data item would otherwise stay Bad until it next moved (the (value, status) recovery rule).
/// Conditions are re-diffed in full for the same reason, which lifts each annotated one's quality.
/// </para>
/// </remarks>
public sealed partial class MTConnectDriver
@@ -147,6 +149,16 @@ public sealed partial class MTConnectDriver
"MTConnect driver {DriverInstanceId}: {AgentUri} has refused every connection for {RefusedMs} ms (definite loss); marking {Count} subscribed value(s) BadNoCommunication until it answers again.",
_driverInstanceId, AgentDisplay(options), (long)refusedFor.TotalMilliseconds, references.Count);
FanOutStatus(references, StatusBadNoCommunication);
// #852 — and the native conditions, quality-only (state + ack kept), as the host's connectivity edge does.
var conditions = MarkConditionsCommsLost();
if (conditions > 0)
{
_logger.LogWarning(
"MTConnect driver {DriverInstanceId}: annotated {Count} native condition(s) BadNoCommunication (quality only; state and acknowledgement kept).",
_driverInstanceId, conditions);
}
return true;
}
@@ -170,6 +182,10 @@ public sealed partial class MTConnectDriver
"MTConnect driver {DriverInstanceId}: the Agent answered again; republishing every subscribed value.",
_driverInstanceId);
FanOut(SubscribedReferences(), ObservationIndex);
// #852 — every subscribed condition, not just the ones this delivery touched: an unchanged condition the
// outage annotated Bad gets a quality-only Good; one still UNAVAILABLE at the Agent stays Bad.
EmitConditionTransitions(touchedDataItemIds: null, resync: false);
}
/// <summary>Publishes a value-less <paramref name="status"/> to every handle subscribing each reference.</summary>
@@ -107,6 +107,16 @@
<InputNumber @bind-Value="_form.ReconnectBackoffMultiplier" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
<div class="form-text">Default 2.0 (doubles each retry).</div>
</div>
<div class="col-md-3">
<label class="form-label">Comms-loss confirm (ms)</label>
<InputNumber @bind-Value="_form.CommsLossConfirmMs" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
<div class="form-text">
Default 2000. How long the Agent must keep <strong>refusing</strong> connections before every
subscribed value and native condition goes <span class="mono">BadNoCommunication</span>.
<span class="mono">0</span> = on the first refusal; negative = fast path off (only the ~90 s
comms-loss watchdog). Timeouts and a paused Agent always wait for the watchdog.
</div>
</div>
</div>
</div>
</section>
@@ -299,6 +309,12 @@
/// <summary>Multiplier applied to the reconnect backoff each retry.</summary>
public double ReconnectBackoffMultiplier { get; set; } = 2.0;
/// <summary>
/// #846/#852 — how long refused connections must persist before values and native conditions go
/// <c>BadNoCommunication</c>. Every integer is meaningful to the driver: <c>0</c> = the first refusal,
/// negative = the fast path is off (watchdog only). So, unlike the timing knobs, it is always written.
/// </summary>
public int CommsLossConfirmMs { get; set; } = DefaultCommsLossConfirmMs;
/// <summary>
/// Whether the driver's half of the write-back gate is armed. Default <c>false</c> — the
/// driver shipped read-only, and the Agent's own <c>AllowPut</c> is an independent second
/// gate that this cannot arm.
@@ -329,6 +345,7 @@
ReconnectMinBackoffMs = dto.Reconnect?.MinBackoffMs ?? defaults.ReconnectMinBackoffMs,
ReconnectMaxBackoffMs = dto.Reconnect?.MaxBackoffMs ?? defaults.ReconnectMaxBackoffMs,
ReconnectBackoffMultiplier = dto.Reconnect?.BackoffMultiplier ?? defaults.ReconnectBackoffMultiplier,
CommsLossConfirmMs = dto.CommsLossConfirmMs ?? defaults.CommsLossConfirmMs,
WriteEnabled = dto.Write?.Enabled ?? defaults.WriteEnabled,
};
}
@@ -344,6 +361,9 @@
SampleIntervalMs = Positive(SampleIntervalMs),
SampleCount = Positive(SampleCount),
HeartbeatMs = Positive(HeartbeatMs),
// Written as-is, negative included: a negative is "off", not an unusable value to drop — dropping it
// would silently re-enable the fast path at the driver's 2000 ms default.
CommsLossConfirmMs = CommsLossConfirmMs,
Probe = new MTConnectProbeConfigDto
{
Enabled = ProbeEnabled,
@@ -383,11 +403,24 @@
if (ProbeEnabled && ProbeTimeoutMs <= 0) { bad.Add("Probe timeout"); }
if (ReconnectMaxBackoffMs <= 0) { bad.Add("Max backoff"); }
return bad.Count == 0
? null
: $"Must be greater than zero: {string.Join(", ", bad)}. A non-positive value is dropped on save and the driver's default applies.";
if (bad.Count > 0)
{
return $"Must be greater than zero: {string.Join(", ", bad)}. A non-positive value is dropped on save and the driver's default applies.";
}
// Advisory only (the value is still saved): the runtime's comms-loss watchdog re-initializes a driver
// whose reads have been frozen for 90 s, so a window that long can never fire first.
return CommsLossConfirmMs >= CommsLossWatchdogMs
? $"Comms-loss confirm of {CommsLossConfirmMs} ms is at or beyond the ~{CommsLossWatchdogMs / 1000} s comms-loss watchdog, so the fast path never fires before it. Use a negative value to turn the fast path off explicitly."
: null;
}
/// <summary>The driver's own default for <see cref="CommsLossConfirmMs"/> (<c>MTConnectDriverOptions</c>).</summary>
public const int DefaultCommsLossConfirmMs = 2_000;
/// <summary>The runtime's hardcoded comms-loss watchdog threshold (<c>DefaultCommsLossTimeout</c>, 90 s).</summary>
public const int CommsLossWatchdogMs = 90_000;
private static int? Positive(int value) => value > 0 ? value : null;
private static string? Trimmed(string? value)
@@ -418,6 +451,8 @@
public int? SampleCount { get; init; }
/// <summary>The <c>/sample</c> <c>heartbeat</c> query parameter in ms.</summary>
public int? HeartbeatMs { get; init; }
/// <summary>Refused-connection confirmation window in ms (#846; 0 = first refusal, negative = off).</summary>
public int? CommsLossConfirmMs { get; init; }
/// <summary>Background connectivity-probe knobs.</summary>
public MTConnectProbeConfigDto? Probe { get; init; }
/// <summary>Reconnect backoff knobs.</summary>