feat(mtconnect): /sample lag measurement + adaptive count (R06:T-5)
Subscription throughput was hard-capped at SampleCount / SampleIntervalMs with no lag detection: a machine producing more fell behind the ring buffer forever, re-baselining every buffer-length with every value backlog-stale. - The parser carries the header's optional lastSequence; each chunk measures lag = lastSequence + 1 - nextSequence and whether it was saturated (full count). - Three consecutive saturated chunks with a positive (or unknown) lag reopen the stream at the cursor reached with count doubled, up to 16x SampleCount (max 65536), through a new IMTConnectAgentClient.SampleAsync(from, count, ct) overload (default implementation ignores count). A raised stream that delivers nothing steps back to the last good count; at the ceiling one Warning per pump. - MTConnectDriver implements IDriverDiagnostics: sampleCount, sampleLag, sampleSaturatedChunks, sampleCountRaises on the /hosts bag. The cap is documented on SampleCount. Refs #743 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
|
||||
|
||||
/// <summary>
|
||||
/// Review 2026-10-21 R06:T-5 — <c>/sample</c> throughput is capped at
|
||||
/// <see cref="MTConnectDriverOptions.SampleCount"/> observations per chunk, one chunk per
|
||||
/// <see cref="MTConnectDriverOptions.SampleIntervalMs"/>. A machine producing more than that falls steadily behind
|
||||
/// the Agent's ring buffer: every buffer-length the driver re-baselines (losing every intermediate transition) and
|
||||
/// every delivered value is backlog-stale. This half measures the lag, surfaces it, and raises the per-stream
|
||||
/// <c>count</c> so the pump can catch up.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Measured per chunk.</b> <c>lag = lastSequence + 1 − nextSequence</c> — the observations the Agent holds
|
||||
/// that this stream has not been sent yet (unknown when the Agent omits <c>lastSequence</c>). A chunk is
|
||||
/// <b>saturated</b> when it carries the full <c>count</c>: the Agent had more and stopped. A stream is behind
|
||||
/// when <see cref="BehindChunksBeforeRaise"/> consecutive chunks are saturated with a positive (or unknown) lag.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Adapted per pump.</b> A behind stream is reopened at the cursor it reached with <c>count</c> doubled, up
|
||||
/// to <see cref="SampleCountCeiling"/>. A raised stream that then delivers nothing (an Agent that refuses the
|
||||
/// larger count) steps back to the last count that delivered and stops raising for that pump. At the ceiling,
|
||||
/// a stream still behind logs one Warning per pump. A new pump (a re-initialize, a resubscribe) starts again
|
||||
/// from the configured <see cref="MTConnectDriverOptions.SampleCount"/> — the count is an upper bound, so a
|
||||
/// raised one costs nothing while the Agent has little to send.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Surfaced</b> on the <see cref="IDriverDiagnostics"/> bag the <c>/hosts</c> page shows: the effective
|
||||
/// count, the last measured lag, saturated chunks and count raises.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed partial class MTConnectDriver : IDriverDiagnostics
|
||||
{
|
||||
/// <summary>Consecutive behind chunks before the count is raised — one saturated chunk is just a burst.</summary>
|
||||
internal const int BehindChunksBeforeRaise = 3;
|
||||
|
||||
/// <summary>The most the per-stream count is ever raised to, relative to the configured one.</summary>
|
||||
internal const int SampleCountMaxGrowth = 16;
|
||||
|
||||
/// <summary>Absolute ceiling on a raised count (a stock cppagent buffer holds 131072 observations).</summary>
|
||||
internal const int SampleCountAbsoluteCeiling = 65_536;
|
||||
|
||||
/// <summary>The count the current pump is streaming with. Diagnostics only.</summary>
|
||||
private int _effectiveSampleCount;
|
||||
|
||||
/// <summary>The last measured lag (observations), or -1 when unknown. Diagnostics only.</summary>
|
||||
private long _sampleLag = -1;
|
||||
|
||||
/// <summary>Chunks that carried the full count. Diagnostics only.</summary>
|
||||
private long _saturatedChunks;
|
||||
|
||||
/// <summary>Times a pump raised its count. Diagnostics only.</summary>
|
||||
private long _sampleCountRaises;
|
||||
|
||||
/// <summary>The ceiling a pump started with <paramref name="configured"/> may raise its count to.</summary>
|
||||
/// <param name="configured">The configured <see cref="MTConnectDriverOptions.SampleCount"/>.</param>
|
||||
/// <returns>The ceiling — never below <paramref name="configured"/>.</returns>
|
||||
internal static int SampleCountCeiling(int configured) =>
|
||||
Math.Max(configured, (int)Math.Min((long)configured * SampleCountMaxGrowth, SampleCountAbsoluteCeiling));
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyDictionary<string, string> GetDiagnostics()
|
||||
{
|
||||
var bag = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
var count = Volatile.Read(ref _effectiveSampleCount);
|
||||
if (count > 0)
|
||||
{
|
||||
bag["sampleCount"] = count.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
var lag = Interlocked.Read(ref _sampleLag);
|
||||
if (lag >= 0)
|
||||
{
|
||||
bag["sampleLag"] = lag.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
bag["sampleSaturatedChunks"] = Interlocked.Read(ref _saturatedChunks).ToString(CultureInfo.InvariantCulture);
|
||||
bag["sampleCountRaises"] = Interlocked.Read(ref _sampleCountRaises).ToString(CultureInfo.InvariantCulture);
|
||||
|
||||
return bag;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Measures one delivered chunk against the count its stream was opened with and records the result.
|
||||
/// </summary>
|
||||
/// <param name="chunk">The chunk just applied.</param>
|
||||
/// <param name="sampleCount">The count the stream was opened with.</param>
|
||||
/// <returns>Whether this chunk says the stream is behind the Agent.</returns>
|
||||
private bool MeasureSampleLag(MTConnectStreamsResult chunk, int sampleCount)
|
||||
{
|
||||
long? lag = chunk.LastSequence is { } last ? Math.Max(0, last + 1 - chunk.NextSequence) : null;
|
||||
Interlocked.Exchange(ref _sampleLag, lag ?? -1);
|
||||
|
||||
var saturated = chunk.Observations.Count >= sampleCount;
|
||||
if (saturated)
|
||||
{
|
||||
Interlocked.Increment(ref _saturatedChunks);
|
||||
}
|
||||
|
||||
return saturated && lag is not 0;
|
||||
}
|
||||
|
||||
/// <summary>One pump's adaptive-count state. Only that pump's task touches it.</summary>
|
||||
private sealed class SampleCountState(int configured)
|
||||
{
|
||||
/// <summary>The count the next stream opens with.</summary>
|
||||
public int Count { get; set; } = configured;
|
||||
|
||||
/// <summary>The last count a stream actually delivered at.</summary>
|
||||
public int LastDelivered { get; set; } = configured;
|
||||
|
||||
/// <summary>The most <see cref="Count"/> may be raised to; lowered to the last good count on a refusal.</summary>
|
||||
public int Ceiling { get; set; } = SampleCountCeiling(configured);
|
||||
|
||||
/// <summary>Whether the at-the-ceiling Warning has been logged for this pump.</summary>
|
||||
public bool CeilingWarned { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The count a behind stream should reopen with, or <see langword="null"/> when it is already at its ceiling
|
||||
/// (logged once per pump).
|
||||
/// </summary>
|
||||
private int? NextSampleCount(SampleCountState state, MTConnectDriverOptions options, long cursor)
|
||||
{
|
||||
var sampleCount = state.Count;
|
||||
var next = (int)Math.Min((long)sampleCount * 2, state.Ceiling);
|
||||
if (next > sampleCount)
|
||||
{
|
||||
Interlocked.Increment(ref _sampleCountRaises);
|
||||
_logger.LogInformation(
|
||||
"MTConnect driver {DriverInstanceId} is falling behind {AgentUri}'s /sample stream (chunks full at count {Count}, lag {Lag} observation(s)); reopening at sequence {Cursor} with count {NextCount}.",
|
||||
_driverInstanceId, options.AgentUri, sampleCount, DescribeLag(), cursor, next);
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
if (!state.CeilingWarned)
|
||||
{
|
||||
state.CeilingWarned = true;
|
||||
_logger.LogWarning(
|
||||
"MTConnect driver {DriverInstanceId} is still behind {AgentUri}'s /sample stream at its count ceiling {Count} (lag {Lag} observation(s)): the machine produces more than count / SampleIntervalMs observations a second, so values are backlog-stale and the stream will re-baseline (dropping intermediate transitions) whenever it falls out of the Agent's buffer. Lower SampleIntervalMs or raise SampleCount.",
|
||||
_driverInstanceId, options.AgentUri, sampleCount, DescribeLag());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private string DescribeLag()
|
||||
{
|
||||
var lag = Interlocked.Read(ref _sampleLag);
|
||||
return lag >= 0 ? lag.ToString(CultureInfo.InvariantCulture) : "unknown";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user