19ffb3c902
- A /sample or re-baseline failure whose chain holds SocketError.ConnectionRefused, unbroken for CommsLossConfirmMs (new, default 2000; 0 = first refusal; negative = off), publishes every subscribed value BadNoCommunication once per outage; the first delivery afterwards republishes all of them from the index (unchanged values included). Timeouts/resets/DNS/pause keep the #527 watchdog contract. - SampleCountCeiling clamps the adaptive raise to the /probe bufferSize (never below SampleCount). - Docs: MTConnect.md comms-loss section + config key; drivers.md rule. Refs #846
199 lines
10 KiB
C#
199 lines
10 KiB
C#
using System.Globalization;
|
||
using Microsoft.Extensions.Logging;
|
||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
|
||
|
||
/// <summary>
|
||
/// Review 2026-10-21 R06:T-5 — <c>/sample</c> throughput is capped at
|
||
/// <see cref="MTConnectDriverOptions.SampleCount"/> observations per chunk, one chunk per
|
||
/// <see cref="MTConnectDriverOptions.SampleIntervalMs"/>. A machine producing more than that falls steadily behind
|
||
/// the Agent's ring buffer: every buffer-length the driver re-baselines (losing every intermediate transition) and
|
||
/// every delivered value is backlog-stale. This half measures the lag, surfaces it, and raises the per-stream
|
||
/// <c>count</c> so the pump can catch up.
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// <para>
|
||
/// <b>Measured per chunk.</b> <c>lag = lastSequence + 1 − nextSequence</c> — the observations the Agent holds
|
||
/// that this stream has not been sent yet (unknown when the Agent omits <c>lastSequence</c>). A chunk is
|
||
/// <b>saturated</b> when it carries the full <c>count</c>: the Agent had more and stopped. A stream is behind
|
||
/// when <see cref="BehindChunksBeforeRaise"/> consecutive chunks are saturated with a positive (or unknown) lag.
|
||
/// </para>
|
||
/// <para>
|
||
/// <b>Adapted per pump.</b> A behind stream is reopened at the cursor it reached with <c>count</c> doubled, up
|
||
/// to <see cref="SampleCountCeiling"/>. A raised stream that then delivers nothing (an Agent that refuses the
|
||
/// larger count) steps back to the last count that delivered and stops raising for that pump. At the ceiling,
|
||
/// a stream still behind logs one Warning per pump. A new pump (a re-initialize, a resubscribe) starts again
|
||
/// from the configured <see cref="MTConnectDriverOptions.SampleCount"/> — the count is an upper bound, so a
|
||
/// raised one costs nothing while the Agent has little to send.
|
||
/// </para>
|
||
/// <para>
|
||
/// <b>Surfaced</b> on the <see cref="IDriverDiagnostics"/> bag the <c>/hosts</c> page shows: the effective
|
||
/// count, the last measured lag, saturated chunks and count raises.
|
||
/// </para>
|
||
/// </remarks>
|
||
public sealed partial class MTConnectDriver : IDriverDiagnostics
|
||
{
|
||
/// <summary>Consecutive behind chunks before the count is raised — one saturated chunk is just a burst.</summary>
|
||
internal const int BehindChunksBeforeRaise = 3;
|
||
|
||
/// <summary>The most the per-stream count is ever raised to, relative to the configured one.</summary>
|
||
internal const int SampleCountMaxGrowth = 16;
|
||
|
||
/// <summary>Absolute ceiling on a raised count (a stock cppagent buffer holds 131072 observations).</summary>
|
||
internal const int SampleCountAbsoluteCeiling = 65_536;
|
||
|
||
/// <summary>The count the current pump is streaming with. Diagnostics only.</summary>
|
||
private int _effectiveSampleCount;
|
||
|
||
/// <summary>The last measured lag (observations), or -1 when unknown. Diagnostics only.</summary>
|
||
private long _sampleLag = -1;
|
||
|
||
/// <summary>Chunks that carried the full count. Diagnostics only.</summary>
|
||
private long _saturatedChunks;
|
||
|
||
/// <summary>Times a pump raised its count. Diagnostics only.</summary>
|
||
private long _sampleCountRaises;
|
||
|
||
/// <summary>The Agent's <c>/probe</c> <c>bufferSize</c> as of the last initialize, or 0 when unknown. Diagnostics only.</summary>
|
||
private long _agentBufferSize;
|
||
|
||
/// <summary>
|
||
/// lmxopcua#825 — the configuration warning for a <see cref="MTConnectDriverOptions.SampleCount"/> the Agent
|
||
/// will refuse. An Agent answers <c>/sample?count=N</c> with <c>OUT_OF_RANGE</c> ("'count' must be less than
|
||
/// bufferSize + 1") when <c>N > bufferSize</c> — live-measured on cppagent 2.7 (bufferSize 16384: 16384 is
|
||
/// accepted, 16385 refused) — so such a driver never streams: it re-baselines from
|
||
/// <c>/current</c> and reports Degraded after three of them. Shared by <see cref="MTConnectDriverProbe"/>
|
||
/// (Test Connect) and the driver's initialize (deploy), so both say the same thing.
|
||
/// </summary>
|
||
/// <param name="sampleCount">The configured count.</param>
|
||
/// <param name="bufferSize">The Agent's buffer size, or null when its header did not say.</param>
|
||
/// <returns>The warning text, or <see langword="null"/> when the count fits (or the buffer size is unknown).</returns>
|
||
internal static string? SampleCountBufferWarning(int sampleCount, long? bufferSize) =>
|
||
bufferSize is { } size && sampleCount > size
|
||
? $"SampleCount {sampleCount.ToString(CultureInfo.InvariantCulture)} is above the Agent's bufferSize "
|
||
+ $"{size.ToString(CultureInfo.InvariantCulture)}: the Agent refuses every /sample stream (OUT_OF_RANGE), so "
|
||
+ "values will only re-baseline from /current and the driver reports Degraded. Set SampleCount to at most "
|
||
+ $"{size.ToString(CultureInfo.InvariantCulture)}."
|
||
: null;
|
||
|
||
/// <summary>The ceiling a pump started with <paramref name="configured"/> may raise its count to.</summary>
|
||
/// <param name="configured">The configured <see cref="MTConnectDriverOptions.SampleCount"/>.</param>
|
||
/// <param name="agentBufferSize">
|
||
/// The Agent's <c>/probe</c> <c>bufferSize</c>, or <c>0</c> when unknown. lmxopcua#846: an Agent refuses a
|
||
/// <c>count</c> above it (<c>OUT_OF_RANGE</c>; live, <c>count == bufferSize</c> is accepted), so a known buffer
|
||
/// caps the raised count — a raise past it would only buy a refused stream and a step-back.
|
||
/// </param>
|
||
/// <returns>The ceiling — never below <paramref name="configured"/> (a configured count above the buffer is the
|
||
/// #825 configuration warning's business, not something to raise further).</returns>
|
||
internal static int SampleCountCeiling(int configured, long agentBufferSize = 0)
|
||
{
|
||
long ceiling = Math.Min((long)configured * SampleCountMaxGrowth, SampleCountAbsoluteCeiling);
|
||
if (agentBufferSize > 0)
|
||
{
|
||
ceiling = Math.Min(ceiling, agentBufferSize);
|
||
}
|
||
|
||
return (int)Math.Max(configured, ceiling);
|
||
}
|
||
|
||
/// <inheritdoc />
|
||
public IReadOnlyDictionary<string, string> GetDiagnostics()
|
||
{
|
||
var bag = new Dictionary<string, string>(StringComparer.Ordinal);
|
||
var count = Volatile.Read(ref _effectiveSampleCount);
|
||
if (count > 0)
|
||
{
|
||
bag["sampleCount"] = count.ToString(CultureInfo.InvariantCulture);
|
||
}
|
||
|
||
var lag = Interlocked.Read(ref _sampleLag);
|
||
if (lag >= 0)
|
||
{
|
||
bag["sampleLag"] = lag.ToString(CultureInfo.InvariantCulture);
|
||
}
|
||
|
||
var bufferSize = Interlocked.Read(ref _agentBufferSize);
|
||
if (bufferSize > 0)
|
||
{
|
||
bag["agentBufferSize"] = bufferSize.ToString(CultureInfo.InvariantCulture);
|
||
}
|
||
|
||
bag["sampleSaturatedChunks"] = Interlocked.Read(ref _saturatedChunks).ToString(CultureInfo.InvariantCulture);
|
||
bag["sampleCountRaises"] = Interlocked.Read(ref _sampleCountRaises).ToString(CultureInfo.InvariantCulture);
|
||
|
||
return bag;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Measures one delivered chunk against the count its stream was opened with and records the result.
|
||
/// </summary>
|
||
/// <param name="chunk">The chunk just applied.</param>
|
||
/// <param name="sampleCount">The count the stream was opened with.</param>
|
||
/// <returns>Whether this chunk says the stream is behind the Agent.</returns>
|
||
private bool MeasureSampleLag(MTConnectStreamsResult chunk, int sampleCount)
|
||
{
|
||
long? lag = chunk.LastSequence is { } last ? Math.Max(0, last + 1 - chunk.NextSequence) : null;
|
||
Interlocked.Exchange(ref _sampleLag, lag ?? -1);
|
||
|
||
var saturated = chunk.Observations.Count >= sampleCount;
|
||
if (saturated)
|
||
{
|
||
Interlocked.Increment(ref _saturatedChunks);
|
||
}
|
||
|
||
return saturated && lag is not 0;
|
||
}
|
||
|
||
/// <summary>One pump's adaptive-count state. Only that pump's task touches it.</summary>
|
||
private sealed class SampleCountState(int configured, long agentBufferSize)
|
||
{
|
||
/// <summary>The count the next stream opens with.</summary>
|
||
public int Count { get; set; } = configured;
|
||
|
||
/// <summary>The last count a stream actually delivered at.</summary>
|
||
public int LastDelivered { get; set; } = configured;
|
||
|
||
/// <summary>The most <see cref="Count"/> may be raised to; lowered to the last good count on a refusal.</summary>
|
||
public int Ceiling { get; set; } = SampleCountCeiling(configured, agentBufferSize);
|
||
|
||
/// <summary>Whether the at-the-ceiling Warning has been logged for this pump.</summary>
|
||
public bool CeilingWarned { get; set; }
|
||
}
|
||
|
||
/// <summary>
|
||
/// The count a behind stream should reopen with, or <see langword="null"/> when it is already at its ceiling
|
||
/// (logged once per pump).
|
||
/// </summary>
|
||
private int? NextSampleCount(SampleCountState state, MTConnectDriverOptions options, long cursor)
|
||
{
|
||
var sampleCount = state.Count;
|
||
var next = (int)Math.Min((long)sampleCount * 2, state.Ceiling);
|
||
if (next > sampleCount)
|
||
{
|
||
Interlocked.Increment(ref _sampleCountRaises);
|
||
_logger.LogInformation(
|
||
"MTConnect driver {DriverInstanceId} is falling behind {AgentUri}'s /sample stream (chunks full at count {Count}, lag {Lag} observation(s)); reopening at sequence {Cursor} with count {NextCount}.",
|
||
_driverInstanceId, AgentDisplay(options), sampleCount, DescribeLag(), cursor, next);
|
||
|
||
return next;
|
||
}
|
||
|
||
if (!state.CeilingWarned)
|
||
{
|
||
state.CeilingWarned = true;
|
||
_logger.LogWarning(
|
||
"MTConnect driver {DriverInstanceId} is still behind {AgentUri}'s /sample stream at its count ceiling {Count} (lag {Lag} observation(s)): the machine produces more than count / SampleIntervalMs observations a second, so values are backlog-stale and the stream will re-baseline (dropping intermediate transitions) whenever it falls out of the Agent's buffer. Lower SampleIntervalMs or raise SampleCount.",
|
||
_driverInstanceId, AgentDisplay(options), sampleCount, DescribeLag());
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
private string DescribeLag()
|
||
{
|
||
var lag = Interlocked.Read(ref _sampleLag);
|
||
return lag >= 0 ? lag.ToString(CultureInfo.InvariantCulture) : "unknown";
|
||
}
|
||
}
|