Files
lmxopcua/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect/MTConnectDriver.SampleLag.cs
T
Joseph Doherty 19ffb3c902 feat(mtconnect): refused Agent connections mark values BadNoCommunication after a short confirmation; clamp the raised sample count to the Agent's bufferSize (#846)
- A /sample or re-baseline failure whose chain holds SocketError.ConnectionRefused, unbroken for
  CommsLossConfirmMs (new, default 2000; 0 = first refusal; negative = off), publishes every subscribed
  value BadNoCommunication once per outage; the first delivery afterwards republishes all of them from
  the index (unchanged values included). Timeouts/resets/DNS/pause keep the #527 watchdog contract.
- SampleCountCeiling clamps the adaptive raise to the /probe bufferSize (never below SampleCount).
- Docs: MTConnect.md comms-loss section + config key; drivers.md rule.

Refs #846
2026-11-11 03:48:05 -05:00

199 lines
10 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Globalization;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
/// <summary>
/// Review 2026-10-21 R06:T-5 — <c>/sample</c> throughput is capped at
/// <see cref="MTConnectDriverOptions.SampleCount"/> observations per chunk, one chunk per
/// <see cref="MTConnectDriverOptions.SampleIntervalMs"/>. A machine producing more than that falls steadily behind
/// the Agent's ring buffer: every buffer-length the driver re-baselines (losing every intermediate transition) and
/// every delivered value is backlog-stale. This half measures the lag, surfaces it, and raises the per-stream
/// <c>count</c> so the pump can catch up.
/// </summary>
/// <remarks>
/// <para>
/// <b>Measured per chunk.</b> <c>lag = lastSequence + 1 − nextSequence</c> — the observations the Agent holds
/// that this stream has not been sent yet (unknown when the Agent omits <c>lastSequence</c>). A chunk is
/// <b>saturated</b> when it carries the full <c>count</c>: the Agent had more and stopped. A stream is behind
/// when <see cref="BehindChunksBeforeRaise"/> consecutive chunks are saturated with a positive (or unknown) lag.
/// </para>
/// <para>
/// <b>Adapted per pump.</b> A behind stream is reopened at the cursor it reached with <c>count</c> doubled, up
/// to <see cref="SampleCountCeiling"/>. A raised stream that then delivers nothing (an Agent that refuses the
/// larger count) steps back to the last count that delivered and stops raising for that pump. At the ceiling,
/// a stream still behind logs one Warning per pump. A new pump (a re-initialize, a resubscribe) starts again
/// from the configured <see cref="MTConnectDriverOptions.SampleCount"/> — the count is an upper bound, so a
/// raised one costs nothing while the Agent has little to send.
/// </para>
/// <para>
/// <b>Surfaced</b> on the <see cref="IDriverDiagnostics"/> bag the <c>/hosts</c> page shows: the effective
/// count, the last measured lag, saturated chunks and count raises.
/// </para>
/// </remarks>
public sealed partial class MTConnectDriver : IDriverDiagnostics
{
/// <summary>Consecutive behind chunks before the count is raised — one saturated chunk is just a burst.</summary>
internal const int BehindChunksBeforeRaise = 3;
/// <summary>The most the per-stream count is ever raised to, relative to the configured one.</summary>
internal const int SampleCountMaxGrowth = 16;
/// <summary>Absolute ceiling on a raised count (a stock cppagent buffer holds 131072 observations).</summary>
internal const int SampleCountAbsoluteCeiling = 65_536;
/// <summary>The count the current pump is streaming with. Diagnostics only.</summary>
private int _effectiveSampleCount;
/// <summary>The last measured lag (observations), or -1 when unknown. Diagnostics only.</summary>
private long _sampleLag = -1;
/// <summary>Chunks that carried the full count. Diagnostics only.</summary>
private long _saturatedChunks;
/// <summary>Times a pump raised its count. Diagnostics only.</summary>
private long _sampleCountRaises;
/// <summary>The Agent's <c>/probe</c> <c>bufferSize</c> as of the last initialize, or 0 when unknown. Diagnostics only.</summary>
private long _agentBufferSize;
/// <summary>
/// lmxopcua#825 — the configuration warning for a <see cref="MTConnectDriverOptions.SampleCount"/> the Agent
/// will refuse. An Agent answers <c>/sample?count=N</c> with <c>OUT_OF_RANGE</c> ("'count' must be less than
/// bufferSize + 1") when <c>N &gt; bufferSize</c> — live-measured on cppagent 2.7 (bufferSize 16384: 16384 is
/// accepted, 16385 refused) — so such a driver never streams: it re-baselines from
/// <c>/current</c> and reports Degraded after three of them. Shared by <see cref="MTConnectDriverProbe"/>
/// (Test Connect) and the driver's initialize (deploy), so both say the same thing.
/// </summary>
/// <param name="sampleCount">The configured count.</param>
/// <param name="bufferSize">The Agent's buffer size, or null when its header did not say.</param>
/// <returns>The warning text, or <see langword="null"/> when the count fits (or the buffer size is unknown).</returns>
internal static string? SampleCountBufferWarning(int sampleCount, long? bufferSize) =>
bufferSize is { } size && sampleCount > size
? $"SampleCount {sampleCount.ToString(CultureInfo.InvariantCulture)} is above the Agent's bufferSize "
+ $"{size.ToString(CultureInfo.InvariantCulture)}: the Agent refuses every /sample stream (OUT_OF_RANGE), so "
+ "values will only re-baseline from /current and the driver reports Degraded. Set SampleCount to at most "
+ $"{size.ToString(CultureInfo.InvariantCulture)}."
: null;
/// <summary>The ceiling a pump started with <paramref name="configured"/> may raise its count to.</summary>
/// <param name="configured">The configured <see cref="MTConnectDriverOptions.SampleCount"/>.</param>
/// <param name="agentBufferSize">
/// The Agent's <c>/probe</c> <c>bufferSize</c>, or <c>0</c> when unknown. lmxopcua#846: an Agent refuses a
/// <c>count</c> above it (<c>OUT_OF_RANGE</c>; live, <c>count == bufferSize</c> is accepted), so a known buffer
/// caps the raised count — a raise past it would only buy a refused stream and a step-back.
/// </param>
/// <returns>The ceiling — never below <paramref name="configured"/> (a configured count above the buffer is the
/// #825 configuration warning's business, not something to raise further).</returns>
internal static int SampleCountCeiling(int configured, long agentBufferSize = 0)
{
long ceiling = Math.Min((long)configured * SampleCountMaxGrowth, SampleCountAbsoluteCeiling);
if (agentBufferSize > 0)
{
ceiling = Math.Min(ceiling, agentBufferSize);
}
return (int)Math.Max(configured, ceiling);
}
/// <inheritdoc />
public IReadOnlyDictionary<string, string> GetDiagnostics()
{
var bag = new Dictionary<string, string>(StringComparer.Ordinal);
var count = Volatile.Read(ref _effectiveSampleCount);
if (count > 0)
{
bag["sampleCount"] = count.ToString(CultureInfo.InvariantCulture);
}
var lag = Interlocked.Read(ref _sampleLag);
if (lag >= 0)
{
bag["sampleLag"] = lag.ToString(CultureInfo.InvariantCulture);
}
var bufferSize = Interlocked.Read(ref _agentBufferSize);
if (bufferSize > 0)
{
bag["agentBufferSize"] = bufferSize.ToString(CultureInfo.InvariantCulture);
}
bag["sampleSaturatedChunks"] = Interlocked.Read(ref _saturatedChunks).ToString(CultureInfo.InvariantCulture);
bag["sampleCountRaises"] = Interlocked.Read(ref _sampleCountRaises).ToString(CultureInfo.InvariantCulture);
return bag;
}
/// <summary>
/// Measures one delivered chunk against the count its stream was opened with and records the result.
/// </summary>
/// <param name="chunk">The chunk just applied.</param>
/// <param name="sampleCount">The count the stream was opened with.</param>
/// <returns>Whether this chunk says the stream is behind the Agent.</returns>
private bool MeasureSampleLag(MTConnectStreamsResult chunk, int sampleCount)
{
long? lag = chunk.LastSequence is { } last ? Math.Max(0, last + 1 - chunk.NextSequence) : null;
Interlocked.Exchange(ref _sampleLag, lag ?? -1);
var saturated = chunk.Observations.Count >= sampleCount;
if (saturated)
{
Interlocked.Increment(ref _saturatedChunks);
}
return saturated && lag is not 0;
}
/// <summary>One pump's adaptive-count state. Only that pump's task touches it.</summary>
private sealed class SampleCountState(int configured, long agentBufferSize)
{
/// <summary>The count the next stream opens with.</summary>
public int Count { get; set; } = configured;
/// <summary>The last count a stream actually delivered at.</summary>
public int LastDelivered { get; set; } = configured;
/// <summary>The most <see cref="Count"/> may be raised to; lowered to the last good count on a refusal.</summary>
public int Ceiling { get; set; } = SampleCountCeiling(configured, agentBufferSize);
/// <summary>Whether the at-the-ceiling Warning has been logged for this pump.</summary>
public bool CeilingWarned { get; set; }
}
/// <summary>
/// The count a behind stream should reopen with, or <see langword="null"/> when it is already at its ceiling
/// (logged once per pump).
/// </summary>
private int? NextSampleCount(SampleCountState state, MTConnectDriverOptions options, long cursor)
{
var sampleCount = state.Count;
var next = (int)Math.Min((long)sampleCount * 2, state.Ceiling);
if (next > sampleCount)
{
Interlocked.Increment(ref _sampleCountRaises);
_logger.LogInformation(
"MTConnect driver {DriverInstanceId} is falling behind {AgentUri}'s /sample stream (chunks full at count {Count}, lag {Lag} observation(s)); reopening at sequence {Cursor} with count {NextCount}.",
_driverInstanceId, AgentDisplay(options), sampleCount, DescribeLag(), cursor, next);
return next;
}
if (!state.CeilingWarned)
{
state.CeilingWarned = true;
_logger.LogWarning(
"MTConnect driver {DriverInstanceId} is still behind {AgentUri}'s /sample stream at its count ceiling {Count} (lag {Lag} observation(s)): the machine produces more than count / SampleIntervalMs observations a second, so values are backlog-stale and the stream will re-baseline (dropping intermediate transitions) whenever it falls out of the Agent's buffer. Lower SampleIntervalMs or raise SampleCount.",
_driverInstanceId, AgentDisplay(options), sampleCount, DescribeLag());
}
return null;
}
private string DescribeLag()
{
var lag = Interlocked.Read(ref _sampleLag);
return lag >= 0 ? lag.ToString(CultureInfo.InvariantCulture) : "unknown";
}
}