feat(drivers): per-driver poll-overrun meter on the shared OtOpcUa Meter; NoteOverrun + backoffCap doc notes
This commit is contained in:
Joseph Doherty
@@ -42,6 +42,19 @@ public static class OtOpcUaTelemetry
|
||||
Meter.CreateCounter<long>("otopcua.driver.lifecycle", unit: "{event}",
|
||||
description: "DriverInstanceActor lifecycle transitions (event=spawn|stop|fault).");
|
||||
|
||||
/// <summary>
|
||||
/// Poll cycles that ran longer than their subscription's publishing interval, as counted by
|
||||
/// <c>PollGroupEngine.OverrunCount</c>. <c>Core.Abstractions</c> deliberately carries no metrics
|
||||
/// dependency, so each poll driver wires this counter to the engine's <c>onOverrun</c> callback;
|
||||
/// this is the production consumer that surface was always missing. Tagged
|
||||
/// <c>driver.type</c> (a <c>DriverTypeNames</c> constant) + <c>driver.instance</c>.
|
||||
/// A non-zero and climbing rate means the device cannot sustain its configured poll rate — the
|
||||
/// loop is already running flat out.
|
||||
/// </summary>
|
||||
public static readonly Counter<long> PollOverruns =
|
||||
Meter.CreateCounter<long>("otopcua.driver.poll.overrun", unit: "{overrun}",
|
||||
description: "Poll cycles that ran longer than the poll interval (slow success and slow failure both count — see PollGroupEngine.NoteOverrun).");
|
||||
|
||||
// ---------------- VirtualTag / ScriptedAlarm engines ----------------
|
||||
|
||||
public static readonly Counter<long> VirtualTagEval =
|
||||
|
||||
@@ -44,10 +44,16 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
/// <c>consecutiveFailures > 0</c> with a <c>backoffCap</c> configured, the capped-exponential
|
||||
/// schedule wins, because a failing device must not be hammered at fixed rate.</para>
|
||||
///
|
||||
/// <para><b>No <c>Meter</c> lives here.</b> <c>Core.Abstractions</c> deliberately references
|
||||
/// nothing, so <see cref="OverrunCount"/> plus the <c>onOverrun</c> callback ARE the
|
||||
/// instrumentation surface — a driver that wants a metric wires its own meter to the callback.
|
||||
/// (Driver-level meters are a recorded follow-up, not part of this type.)</para>
|
||||
/// <para><b>No <c>Meter</c> lives here, and that is still deliberate.</b>
|
||||
/// <c>Core.Abstractions</c> references nothing, so <see cref="OverrunCount"/> plus the
|
||||
/// <c>onOverrun</c> callback ARE the instrumentation surface — a driver that wants a metric
|
||||
/// wires its own meter to the callback. The driver-level meter this note used to record as a
|
||||
/// follow-up is <b>delivered</b>: every one of the seven poll drivers now passes an
|
||||
/// <c>onOverrun</c> closure that bumps <c>OtOpcUaTelemetry.PollOverruns</c>
|
||||
/// (<c>otopcua.driver.poll.overrun</c>) on the shared OtOpcUa <c>Meter</c>, tagged with the
|
||||
/// driver's <c>DriverTypeNames</c> constant and its driver-instance id. The counter lives in
|
||||
/// <c>Commons.Observability</c>, on the consumer side of the reference graph — this type still
|
||||
/// owns no <c>Meter</c> and takes no metrics dependency.</para>
|
||||
///
|
||||
/// <para>Programmer errors and obviously-fatal exceptions (<see cref="OutOfMemoryException"/>,
|
||||
/// <see cref="ThreadAbortException"/>, <see cref="StackOverflowException"/>,
|
||||
@@ -85,7 +91,27 @@ public sealed class PollGroupEngine : IAsyncDisposable
|
||||
/// (05/STAB-8). When supplied, sustained reader failures stretch the retry cadence
|
||||
/// (interval, 2×, 4×, … saturating at this cap) instead of hammering a dead device every
|
||||
/// tick; a successful poll resets to the base interval. <c>null</c> (default) keeps the
|
||||
/// byte-identical fixed-interval cadence for consumers that have not opted in.</param>
|
||||
/// byte-identical fixed-interval cadence for consumers that have not opted in.
|
||||
/// <para><b>Units:</b> a <see cref="TimeSpan"/> ceiling on the post-poll <i>wait</i> — the
|
||||
/// longest gap the loop will ever leave between a failed poll and the next attempt. It is
|
||||
/// not a count, a multiplier, or a total-outage budget: a device that stays down waits
|
||||
/// <c>backoffCap</c> between every retry, forever.</para>
|
||||
/// <para><b>Interaction with <paramref name="minInterval"/> — it REPLACES the paced delay,
|
||||
/// it does not add to it.</b> The loop picks exactly one of two branches per cycle. With no
|
||||
/// failures outstanding (or no cap configured) the wait is the fixed-RATE remainder
|
||||
/// <c>max(interval − pollDuration, 0)</c>. With a cap configured AND
|
||||
/// <c>consecutiveFailures > 0</c>, the whole remainder computation is bypassed and the
|
||||
/// wait becomes <c>ConnectionBackoff.ComputeDelay(interval, failures, cap)</c> — the poll's
|
||||
/// own elapsed time is NOT subtracted, so the failure branch is a fixed DELAY (cycle =
|
||||
/// poll + backoff), deliberately unlike the fixed-rate success branch.</para>
|
||||
/// <para>The <c>interval</c> feeding that schedule is the per-subscription interval
|
||||
/// <i>already clamped</i> to <paramref name="minInterval"/> in <see cref="Subscribe"/>, so
|
||||
/// the first failure's wait equals the clamped interval (shift 0 → 1×) and the floor is
|
||||
/// honoured for free. One asymmetry worth knowing: <c>ComputeDelay</c> saturates at the cap
|
||||
/// unconditionally, so a <paramref name="backoffCap"/> set SMALLER than the clamped
|
||||
/// interval makes failures retry FASTER than the interval floor — the cap outranks
|
||||
/// <paramref name="minInterval"/> on the failure branch. Configure it above the floor
|
||||
/// (every driver here uses 30 s against a 100 ms floor).</para></param>
|
||||
/// <param name="onBatch">Optional per-cycle batch callback — invoked once per poll cycle that
|
||||
/// produced at least one change, carrying exactly that cycle's changed items in reader
|
||||
/// order, after all of the cycle's per-tag <paramref name="onChange"/> callbacks. The
|
||||
@@ -311,6 +337,16 @@ public sealed class PollGroupEngine : IAsyncDisposable
|
||||
/// Record a poll that took longer than its interval: bump <see cref="OverrunCount"/> and
|
||||
/// hand the measured duration to the optional <c>onOverrun</c> sink. Defensive against a
|
||||
/// sink that throws, for the same reason <see cref="ReportError"/> is.
|
||||
/// <para><b>Caveat — this CONFLATES a slow success with a slow failure.</b> It is called
|
||||
/// from the <c>finally</c> of both poll sites, so the only input is the measured elapsed
|
||||
/// time; a device that answers in 300 ms on a 200 ms interval and one that throws (or times
|
||||
/// out) after 300 ms both increment the counter and both raise <c>onOverrun</c> with the
|
||||
/// same duration. A consumer reading <c>otopcua.driver.poll.overrun</c> therefore learns
|
||||
/// "the loop cannot keep its cadence", NOT "the device is healthy but slow" — pair it with
|
||||
/// the driver's health surface (which <c>onError</c> already feeds) to tell the two apart.
|
||||
/// Disambiguating here would mean threading the cycle's outcome into this method and
|
||||
/// splitting the counter/callback by it; no consumer needs that yet, so it is deliberately
|
||||
/// not done.</para>
|
||||
/// </summary>
|
||||
private void NoteOverrun(TimeSpan interval, TimeSpan elapsed)
|
||||
{
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.AbCip.PlcFamilies;
|
||||
|
||||
@@ -161,7 +162,12 @@ public sealed class AbCipDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
onError: HandlePollError,
|
||||
backoffCap: PollBackoffCap,
|
||||
onBatch: (handle, items) =>
|
||||
OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)));
|
||||
OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)),
|
||||
// The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this
|
||||
// closure IS the production consumer of its overrun signal.
|
||||
onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1,
|
||||
new KeyValuePair<string, object?>("driver.type", DriverTypeNames.AbCip),
|
||||
new KeyValuePair<string, object?>("driver.instance", driverInstanceId)));
|
||||
_alarmProjection = new AbCipAlarmProjection(this, _options.AlarmPollInterval, _logger);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.PlcFamilies;
|
||||
|
||||
@@ -76,7 +77,12 @@ public sealed class AbLegacyDriver : IDriver, IReadable, IWritable, ITagDiscover
|
||||
onChange: (handle, tagRef, snapshot) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, tagRef, snapshot)),
|
||||
onError: HandlePollError,
|
||||
backoffCap: PollBackoffCap);
|
||||
backoffCap: PollBackoffCap,
|
||||
// The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this
|
||||
// closure IS the production consumer of its overrun signal.
|
||||
onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1,
|
||||
new KeyValuePair<string, object?>("driver.type", DriverTypeNames.AbLegacy),
|
||||
new KeyValuePair<string, object?>("driver.instance", driverInstanceId)));
|
||||
}
|
||||
|
||||
/// <summary>Upper bound on the poll-loop failure backoff — adopts the S7-proven 30 s cap fleet-wide (05/STAB-8).</summary>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS;
|
||||
@@ -121,7 +122,12 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
onError: HandlePollError,
|
||||
backoffCap: PollBackoffCap,
|
||||
onBatch: (handle, items) =>
|
||||
OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)));
|
||||
OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)),
|
||||
// The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this
|
||||
// closure IS the production consumer of its overrun signal.
|
||||
onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1,
|
||||
new KeyValuePair<string, object?>("driver.type", DriverTypeNames.FOCAS),
|
||||
new KeyValuePair<string, object?>("driver.instance", driverInstanceId)));
|
||||
}
|
||||
|
||||
/// <summary>True when the supplied DriverConfig JSON carries a real body. The bootstrapper always passes
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus;
|
||||
@@ -187,7 +188,12 @@ public sealed class ModbusDriver
|
||||
{
|
||||
if (_cycleBatchItems.TryRemove(handle, out var published) && published.Count > 0)
|
||||
OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, published));
|
||||
});
|
||||
},
|
||||
// The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this
|
||||
// closure IS the production consumer of its overrun signal.
|
||||
onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1,
|
||||
new KeyValuePair<string, object?>("driver.type", DriverTypeNames.Modbus),
|
||||
new KeyValuePair<string, object?>("driver.instance", driverInstanceId)));
|
||||
}
|
||||
|
||||
/// <summary>Upper bound on the poll-loop failure backoff — adopts the S7-proven 30 s cap fleet-wide (05/STAB-8).</summary>
|
||||
|
||||
@@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using S7.Net;
|
||||
using S7NetDataType = global::S7.Net.DataType;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.S7;
|
||||
@@ -69,7 +70,12 @@ public sealed class S7Driver
|
||||
minInterval: TimeSpan.FromMilliseconds(100),
|
||||
onError: HandlePollError,
|
||||
backoffCap: TimeSpan.FromSeconds(30),
|
||||
onBatch: (handle, items) => OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)));
|
||||
onBatch: (handle, items) => OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)),
|
||||
// The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this
|
||||
// closure IS the production consumer of its overrun signal.
|
||||
onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1,
|
||||
new KeyValuePair<string, object?>("driver.type", DriverTypeNames.S7),
|
||||
new KeyValuePair<string, object?>("driver.instance", driverInstanceId)));
|
||||
}
|
||||
// ---- ISubscribable + IHostConnectivityProbe state ----
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ using System.Data.Common;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Sql.Contracts;
|
||||
|
||||
@@ -228,7 +229,12 @@ public sealed class SqlDriver
|
||||
onChange: (handle, tagRef, snapshot) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, tagRef, snapshot)),
|
||||
onError: HandlePollError,
|
||||
backoffCap: PollBackoffCap);
|
||||
backoffCap: PollBackoffCap,
|
||||
// The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this
|
||||
// closure IS the production consumer of its overrun signal.
|
||||
onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1,
|
||||
new KeyValuePair<string, object?>("driver.type", DriverTypeNames.Sql),
|
||||
new KeyValuePair<string, object?>("driver.instance", driverInstanceId)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
|
||||
@@ -78,7 +79,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery
|
||||
onChange: (handle, tagRef, snapshot) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, tagRef, snapshot)),
|
||||
onError: HandlePollError,
|
||||
backoffCap: PollBackoffCap);
|
||||
backoffCap: PollBackoffCap,
|
||||
// The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this
|
||||
// closure IS the production consumer of its overrun signal.
|
||||
onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1,
|
||||
new KeyValuePair<string, object?>("driver.type", DriverTypeNames.TwinCAT),
|
||||
new KeyValuePair<string, object?>("driver.instance", driverInstanceId)));
|
||||
}
|
||||
|
||||
/// <summary>Upper bound on the poll-loop failure backoff — adopts the S7-proven 30 s cap fleet-wide (05/STAB-8).</summary>
|
||||
|
||||
@@ -670,6 +670,50 @@ public sealed class PollGroupEngineTests
|
||||
overruns.Count.ShouldBeLessThanOrEqualTo(1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The counter and the callback must be the SAME signal, not two approximations of it: every
|
||||
/// increment of <see cref="PollGroupEngine.OverrunCount"/> is matched by exactly one
|
||||
/// <c>onOverrun</c> invocation. This is what lets a driver's meter (the production consumer,
|
||||
/// <c>OtOpcUaTelemetry.PollOverruns</c>) be read as the counter itself — a callback that
|
||||
/// fired on only some increments, or twice on one, would silently mis-scale the metric.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The two existing pacing tests
|
||||
/// (<see cref="A_slow_reader_overrun_is_counted_and_the_next_poll_starts_promptly"/> /
|
||||
/// <see cref="A_fast_reader_never_overruns"/>) assert each side against its own loose
|
||||
/// threshold, so they would both still pass if the callback fired on, say, every other
|
||||
/// increment. This one compares the two directly, and does so only AFTER
|
||||
/// <c>UnsubscribeAsync</c> has drained the loop — while a poll is in flight the increment
|
||||
/// and the callback are two separate statements, so a live read could legitimately catch
|
||||
/// them one apart and the equality would be flaky rather than falsifiable.
|
||||
/// </remarks>
|
||||
[Fact]
|
||||
public async Task Every_overrun_increment_is_matched_by_exactly_one_onOverrun_callback()
|
||||
{
|
||||
async Task<IReadOnlyList<DataValueSnapshot>> Reader(IReadOnlyList<string> refs, CancellationToken ct)
|
||||
{
|
||||
await Task.Delay(120, ct);
|
||||
var now = DateTime.UtcNow;
|
||||
// A fresh value every poll, so the change path runs on every cycle too.
|
||||
return refs.Select(_ => new DataValueSnapshot(now.Ticks, 0u, now, now)).ToList();
|
||||
}
|
||||
|
||||
var overruns = new ConcurrentQueue<TimeSpan>();
|
||||
await using var engine = new PollGroupEngine(
|
||||
Reader,
|
||||
(_, _, _) => { },
|
||||
minInterval: TimeSpan.FromMilliseconds(20),
|
||||
onOverrun: e => overruns.Enqueue(e));
|
||||
|
||||
var handle = engine.Subscribe(["X"], TimeSpan.FromMilliseconds(40));
|
||||
await Task.Delay(1000, TestContext.Current.CancellationToken);
|
||||
await engine.UnsubscribeAsync(handle);
|
||||
|
||||
// Sanity: the scenario actually produced overruns, so the equality below is not vacuous.
|
||||
engine.OverrunCount.ShouldBeGreaterThanOrEqualTo(3);
|
||||
overruns.Count.ShouldBe((int)engine.OverrunCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The async teardown must be behaviourally identical to the sync one: the handle is known,
|
||||
/// the loop stops, and no further callback fires. Mirrors
|
||||
|
||||
Reference in New Issue
Block a user