feat(mtconnect): host-connectivity probe + instanceId rediscover (Task 13)

This commit is contained in:
Joseph Doherty
2026-07-24 16:56:31 -04:00
parent 13dd9fcd70
commit fa0e40796f
2 changed files with 1094 additions and 9 deletions
@@ -42,21 +42,24 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect;
/// <c>nextSequence</c>. /// <c>nextSequence</c>.
/// </para> /// </para>
/// <para> /// <para>
/// <b>Scope.</b> This type currently implements <see cref="IDriver"/>, /// <b>Scope.</b> This type implements <see cref="IDriver"/>, <see cref="IReadable"/>,
/// <see cref="IReadable"/>, <see cref="ISubscribable"/> and <see cref="ITagDiscovery"/>. /// <see cref="ISubscribable"/>, <see cref="ITagDiscovery"/>,
/// <c>IHostConnectivityProbe</c>/<c>IRediscoverable</c> (Task 13) are added on top of the /// <see cref="IHostConnectivityProbe"/> and <see cref="IRediscoverable"/> — the last two
/// state this class already caches — the probe model, the Agent <c>instanceId</c>, and the /// built on state this class already caches (the Agent <c>instanceId</c> and the probe
/// observation index. There is deliberately no <c>IWritable</c>: MTConnect is a read-only /// model). There is deliberately no <c>IWritable</c>: MTConnect is a read-only protocol,
/// protocol, which is also why every discovered leaf is /// which is also why every discovered leaf is
/// <see cref="SecurityClassification.ViewOnly"/>. /// <see cref="SecurityClassification.ViewOnly"/>.
/// </para> /// </para>
/// <para> /// <para>
/// <b>The read path never writes the shared observation index</b> — see /// <b>The read path never writes the shared observation index</b> — see
/// <see cref="ReadAsync"/>. That index has exactly one writer: the <c>/sample</c> pump /// <see cref="ReadAsync"/>. That index has exactly one writer: the <c>/sample</c> pump
/// (<see cref="RunSampleStreamAsync"/>). /// (<see cref="RunSampleStreamAsync"/>). The same single-writer discipline covers the two
/// Task-13 fields: the <b>connectivity probe loop</b> is the sole writer of
/// <see cref="HostState"/>, and it publishes <b>nothing</b> into
/// <see cref="AgentSession.ProbeModel"/> (see <see cref="RunProbeLoopAsync"/>).
/// </para> /// </para>
/// </remarks> /// </remarks>
public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDiscovery public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDiscovery, IHostConnectivityProbe, IRediscoverable
{ {
/// <summary> /// <summary>
/// Coarse per-entry byte weights behind <see cref="GetMemoryFootprint"/>. The contract asks /// Coarse per-entry byte weights behind <see cref="GetMemoryFootprint"/>. The contract asks
@@ -128,6 +131,13 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
private readonly ILogger<MTConnectDriver> _logger; private readonly ILogger<MTConnectDriver> _logger;
private readonly Func<MTConnectDriverOptions, IMTConnectAgentClient> _agentClientFactory; private readonly Func<MTConnectDriverOptions, IMTConnectAgentClient> _agentClientFactory;
/// <summary>
/// How the connectivity probe loop waits between ticks. Injected so the cadence is a seam
/// rather than a wall clock: the loop is a timer, and a suite that drove it by sleeping would
/// be flaky by construction. Defaults to <see cref="Task.Delay(TimeSpan, CancellationToken)"/>.
/// </summary>
private readonly Func<TimeSpan, CancellationToken, Task> _probeScheduler;
/// <summary> /// <summary>
/// Serializes the three lifecycle entry points against each other. Initialize / Reinitialize /// Serializes the three lifecycle entry points against each other. Initialize / Reinitialize
/// / Shutdown all swap the client and the served caches; overlapping them would let a /// / Shutdown all swap the client and the served caches; overlapping them would let a
@@ -183,6 +193,48 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
/// <summary>Live subscriptions by id. Guarded by <see cref="_subscriptionLock"/>.</summary> /// <summary>Live subscriptions by id. Guarded by <see cref="_subscriptionLock"/>.</summary>
private readonly Dictionary<long, SubscriptionState> _subscriptions = []; private readonly Dictionary<long, SubscriptionState> _subscriptions = [];
/// <summary>
/// Guards the two host-connectivity fields, which the probe loop writes and
/// <see cref="GetHostStatuses"/> reads from an arbitrary caller thread. Held for a field
/// read/write only — <b>never</b> across an await, and never while a status handler runs.
/// </summary>
private readonly Lock _probeLock = new();
/// <summary>
/// The Agent's last observed reachability. Written <b>only</b> by
/// <see cref="RunProbeLoopAsync"/>, so the state and the event stream can never disagree:
/// every value this field has ever held was announced by exactly one
/// <see cref="OnHostStatusChanged"/>. That is why a successful <see cref="InitializeAsync"/>
/// does not seed it <see cref="HostState.Running"/> — raising the event from there would run
/// caller code while the non-reentrant <see cref="_lifecycle"/> semaphore is held (the same
/// hazard that makes the pump, not <see cref="StartCoreAsync"/>, republish to subscribers),
/// and seeding it silently would leave a consumer that tracks the event stream disagreeing
/// with <see cref="GetHostStatuses"/>. <see cref="HostState.Unknown"/> until the first tick
/// is the honest answer. Guarded by <see cref="_probeLock"/>.
/// </summary>
private HostState _hostState = HostState.Unknown;
/// <summary>When <see cref="_hostState"/> last changed. Guarded by <see cref="_probeLock"/>.</summary>
private DateTime _hostStateChangedUtc = DateTime.UtcNow;
/// <summary>
/// Cancels the connectivity probe loop. Written only under <see cref="_lifecycle"/>, which
/// serializes every start and stop of the loop.
/// </summary>
private CancellationTokenSource? _probeCts;
/// <summary>The connectivity probe loop's task, or <c>null</c> when none runs. See <see cref="_probeCts"/>.</summary>
private Task? _probeTask;
/// <summary>
/// The Agent <c>instanceId</c> the last <see cref="OnRediscoveryNeeded"/> was raised for —
/// seeded at the commit point of <see cref="StartCoreAsync"/> with the id the priming
/// <c>/current</c> reported. This is what makes rediscovery fire once per <b>change</b>
/// rather than once per chunk: every chunk after a restart carries the new id, and a
/// re-baseline that fails leaves the pump still holding the old one.
/// </summary>
private long _announcedInstanceId;
private MTConnectDriverOptions _options; private MTConnectDriverOptions _options;
private MTConnectObservationIndex _index; private MTConnectObservationIndex _index;
private DriverHealth _health = new(DriverState.Unknown, null, null); private DriverHealth _health = new(DriverState.Unknown, null, null);
@@ -263,11 +315,17 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
/// <see cref="InitializeAsync"/>, never from this constructor. /// <see cref="InitializeAsync"/>, never from this constructor.
/// </param> /// </param>
/// <param name="logger">Optional logger; defaults to the null logger.</param> /// <param name="logger">Optional logger; defaults to the null logger.</param>
/// <param name="probeScheduler">
/// How the connectivity probe loop waits between ticks. Defaults to
/// <see cref="Task.Delay(TimeSpan, CancellationToken)"/>; tests inject a hand-driven ticker so
/// the probe's cadence is asserted deterministically instead of slept through.
/// </param>
public MTConnectDriver( public MTConnectDriver(
MTConnectDriverOptions options, MTConnectDriverOptions options,
string driverInstanceId, string driverInstanceId,
Func<MTConnectDriverOptions, IMTConnectAgentClient>? agentClientFactory = null, Func<MTConnectDriverOptions, IMTConnectAgentClient>? agentClientFactory = null,
ILogger<MTConnectDriver>? logger = null) ILogger<MTConnectDriver>? logger = null,
Func<TimeSpan, CancellationToken, Task>? probeScheduler = null)
{ {
ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(options);
ArgumentException.ThrowIfNullOrWhiteSpace(driverInstanceId); ArgumentException.ThrowIfNullOrWhiteSpace(driverInstanceId);
@@ -276,6 +334,7 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
_driverInstanceId = driverInstanceId; _driverInstanceId = driverInstanceId;
_logger = logger ?? NullLogger<MTConnectDriver>.Instance; _logger = logger ?? NullLogger<MTConnectDriver>.Instance;
_agentClientFactory = agentClientFactory ?? (o => new MTConnectAgentClient(o, logger)); _agentClientFactory = agentClientFactory ?? (o => new MTConnectAgentClient(o, logger));
_probeScheduler = probeScheduler ?? Task.Delay;
// Never null, so every consumer (Task 10's ReadAsync included) can ask it for a snapshot // Never null, so every consumer (Task 10's ReadAsync included) can ask it for a snapshot
// without a null check: an un-primed index answers BadWaitingForInitialData for an authored // without a null check: an un-primed index answers BadWaitingForInitialData for an authored
@@ -333,6 +392,14 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
} }
} }
/// <summary>
/// The connectivity probe loop's task while one is running, else <c>null</c>. Exposed to
/// tests as the deterministic teardown barrier — "the probe loop has stopped" is a task
/// completion, never a sleep — and as the proof that <c>Probe.Enabled = false</c> starts
/// nothing at all.
/// </summary>
internal Task? ProbeLoopTask => Volatile.Read(ref _probeTask);
/// <summary>The options currently in force — the constructor's, or the last document applied.</summary> /// <summary>The options currently in force — the constructor's, or the last document applied.</summary>
internal MTConnectDriverOptions EffectiveOptions => _options; internal MTConnectDriverOptions EffectiveOptions => _options;
@@ -362,6 +429,7 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
// enumerating a disposed client reads the disposal as a dropped connection and // enumerating a disposed client reads the disposal as a dropped connection and
// reconnects against an object that no longer exists. // reconnects against an object that no longer exists.
await StopSampleStreamAsync().ConfigureAwait(false); await StopSampleStreamAsync().ConfigureAwait(false);
await StopProbeLoopAsync().ConfigureAwait(false);
await TeardownCoreAsync().ConfigureAwait(false); await TeardownCoreAsync().ConfigureAwait(false);
await StartCoreAsync(options, existingClient: null, cancellationToken).ConfigureAwait(false); await StartCoreAsync(options, existingClient: null, cancellationToken).ConfigureAwait(false);
@@ -406,6 +474,7 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
var existing = Volatile.Read(ref _session)?.Client; var existing = Volatile.Read(ref _session)?.Client;
await StopSampleStreamAsync().ConfigureAwait(false); await StopSampleStreamAsync().ConfigureAwait(false);
await StopProbeLoopAsync().ConfigureAwait(false);
if (existing is null || RequiresClientRebuild(_options, incoming)) if (existing is null || RequiresClientRebuild(_options, incoming))
{ {
@@ -438,6 +507,7 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
var lastRead = ReadHealth().LastSuccessfulRead; var lastRead = ReadHealth().LastSuccessfulRead;
await StopSampleStreamAsync().ConfigureAwait(false); await StopSampleStreamAsync().ConfigureAwait(false);
await StopProbeLoopAsync().ConfigureAwait(false);
await TeardownCoreAsync().ConfigureAwait(false); await TeardownCoreAsync().ConfigureAwait(false);
WriteHealth(new DriverHealth(DriverState.Unknown, lastRead, null)); WriteHealth(new DriverHealth(DriverState.Unknown, lastRead, null));
@@ -969,6 +1039,351 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
private static bool IsCondition(string? category) => private static bool IsCondition(string? category) =>
category.AsSpan().Trim().Equals("CONDITION", StringComparison.OrdinalIgnoreCase); category.AsSpan().Trim().Equals("CONDITION", StringComparison.OrdinalIgnoreCase);
// ---- IHostConnectivityProbe: one host per Agent, on its own cheap tick ----
/// <inheritdoc/>
/// <remarks>
/// <b>Raised from the probe loop's task, never under a lock and never under
/// <see cref="_lifecycle"/>.</b> A handler is arbitrary caller code: one that blocks forever
/// blocks the loop with it (and therefore the teardown that awaits it), exactly as an
/// <see cref="OnDataChange"/> handler does to the pump. One that <i>throws</i> is absorbed.
/// </remarks>
public event EventHandler<HostStatusChangedEventArgs>? OnHostStatusChanged;
/// <summary>
/// The single host this driver instance talks to, named by its configured
/// <see cref="MTConnectDriverOptions.AgentUri"/>. Mirrors <c>ModbusDriver.HostName</c>'s
/// <c>host:port</c> convention: several MTConnect drivers in one server disambiguate on the
/// Admin dashboard by endpoint rather than by driver-instance id.
/// </summary>
public string HostName => _options.AgentUri;
/// <inheritdoc/>
/// <remarks>
/// One entry, always — an Agent is a single host, and the driver knows of it whether or not
/// it has been reached. Before the first probe tick (and for the whole life of an instance
/// with <c>Probe.Enabled = false</c>) that entry reads <see cref="HostState.Unknown"/>, which
/// is the truth: nothing has measured it. See <see cref="_hostState"/> for why a successful
/// initialize does not seed it <see cref="HostState.Running"/>.
/// </remarks>
public IReadOnlyList<HostConnectivityStatus> GetHostStatuses()
{
lock (_probeLock)
{
return [new HostConnectivityStatus(HostName, _hostState, _hostStateChangedUtc)];
}
}
/// <summary>
/// Starts the periodic connectivity probe, if the operator asked for one. <b>Caller must hold
/// <see cref="_lifecycle"/>.</b>
/// </summary>
/// <param name="client">The client this session owns; the loop dials that one, never a later one.</param>
/// <param name="options">The options this session started with.</param>
private void StartProbeLoopCore(IMTConnectAgentClient client, MTConnectDriverOptions options)
{
if (!options.Probe.Enabled)
{
return;
}
var cts = new CancellationTokenSource();
_probeCts = cts;
Volatile.Write(ref _probeTask, Task.Run(() => RunProbeLoopAsync(client, options, cts.Token), CancellationToken.None));
}
/// <summary>
/// The connectivity probe: wait, ask the Agent whether it is answering, publish the verdict,
/// repeat. Never throws — it is a detached background task, and an escaping exception would
/// silently end host reporting while <see cref="GetHostStatuses"/> kept serving the last
/// thing it saw.
/// </summary>
/// <remarks>
/// <para>
/// <b>It calls <see cref="IMTConnectAgentClient.ProbeAsync"/> directly and publishes
/// nothing.</b> Both halves of that are deliberate.
/// </para>
/// <para>
/// Routing it through <see cref="GetOrFetchProbeModelAsync"/> is the obvious shortcut and
/// it is <b>inert</b>: that accessor answers from the cache <see cref="InitializeAsync"/>
/// already warmed, so the loop would issue no request at all and report
/// <see cref="HostState.Running"/> forever no matter what the Agent was doing — a feature
/// that ticks, logs, and measures nothing.
/// </para>
/// <para>
/// Publishing the answer into <see cref="AgentSession.ProbeModel"/> is the opposite
/// mistake: it would put a second writer on a field the lifecycle owns under a
/// compare-and-swap, and would silently re-warm a cache that
/// <see cref="FlushOptionalCachesAsync"/> — or an Agent restart, see
/// <see cref="AnnounceAgentRestart"/> — deliberately dropped. The parsed model is
/// discarded; only the fact that a request completed is used.
/// </para>
/// <para>
/// <b>Why <c>/probe</c> rather than the <c>/sample</c> heartbeat.</b> The heartbeat is
/// free, but it only exists while something is subscribed — an instance with no
/// subscriptions would report <see cref="HostState.Unknown"/> indefinitely, and host
/// reachability would become a function of OPC UA client behaviour. A dedicated tick is
/// the one signal that is true whether or not anyone is watching. Its cost is a parse of
/// the device model per interval, which is why <see cref="MTConnectProbeOptions.Interval"/>
/// exists.
/// </para>
/// <para>
/// <b>It waits first, then probes.</b> <see cref="StartCoreAsync"/> has just completed a
/// successful <c>/probe</c> one moment earlier; a tick that fired immediately would spend
/// a second identical request to learn nothing, and would do it on every re-initialize.
/// </para>
/// <para>
/// <b>It never calls a lifecycle method</b> — see the <see cref="_lifecycle"/> remarks.
/// <see cref="StopProbeLoopAsync"/> awaits this task while holding that semaphore, so
/// reaching back into one would be a permanent hang with no exception and no log line.
/// </para>
/// </remarks>
private async Task RunProbeLoopAsync(
IMTConnectAgentClient client, MTConnectDriverOptions options, CancellationToken ct)
{
try
{
while (!ct.IsCancellationRequested)
{
try
{
await _probeScheduler(options.Probe.Interval, ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
if (ct.IsCancellationRequested)
{
return;
}
bool reachable;
try
{
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct);
deadline.CancelAfter(options.Probe.Timeout);
// The answer is deliberately discarded — see the remarks.
_ = await client.ProbeAsync(deadline.Token).ConfigureAwait(false);
reachable = true;
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return;
}
catch (Exception ex)
{
// Unreachable, unusable, disposed, or past its deadline — from the host's point
// of view these are one fact, and the data paths report the detail.
reachable = false;
_logger.LogDebug(
ex,
"MTConnect driver {DriverInstanceId} could not reach {AgentUri} on its connectivity probe.",
_driverInstanceId, options.AgentUri);
}
TransitionHostTo(reachable ? HostState.Running : HostState.Stopped);
}
}
catch (Exception ex)
{
_logger.LogError(
ex,
"MTConnect driver {DriverInstanceId} stopped its connectivity probe on an unexpected fault; host status for {AgentUri} will not be updated until the driver is re-initialized.",
_driverInstanceId, options.AgentUri);
}
}
/// <summary>
/// Publishes a host state and announces it — <b>only when it actually changed</b>. Raising on
/// every tick would emit one event per interval per Agent forever: Core would re-fan Bad
/// quality across the host's subtree each time, and an operator watching the feed could not
/// tell a new outage from an ongoing one.
/// </summary>
private void TransitionHostTo(HostState newState)
{
HostState old;
lock (_probeLock)
{
old = _hostState;
if (old == newState)
{
return;
}
_hostState = newState;
_hostStateChangedUtc = DateTime.UtcNow;
}
_logger.LogInformation(
"MTConnect driver {DriverInstanceId} host {AgentUri} went {Previous} -> {Current}.",
_driverInstanceId, HostName, old, newState);
// Outside the lock: a handler is caller code.
try
{
OnHostStatusChanged?.Invoke(this, new HostStatusChangedEventArgs(HostName, old, newState));
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"MTConnect driver {DriverInstanceId} caught a fault from a host-status subscriber for {AgentUri}; the probe continues.",
_driverInstanceId, HostName);
}
}
/// <summary>
/// Stops the connectivity probe loop and waits for it to be gone. Idempotent, and never
/// throws — it runs on teardown paths where a second exception would mask the first.
/// </summary>
/// <remarks>
/// <b>⚠️ This runs while <see cref="_lifecycle"/> is held</b> and it awaits the loop, on
/// exactly the same terms as <see cref="StopSampleStreamAsync"/>: safe only because the loop
/// never touches the lifecycle and every await it makes observes the token cancelled below.
/// The one thing outside that guarantee is an <see cref="OnHostStatusChanged"/> handler that
/// blocks forever — a consumer bug this driver cannot paper over.
/// </remarks>
private async Task StopProbeLoopAsync()
{
var cts = _probeCts;
var task = Volatile.Read(ref _probeTask);
_probeCts = null;
Volatile.Write(ref _probeTask, null);
if (cts is null)
{
return;
}
try
{
await cts.CancelAsync().ConfigureAwait(false);
}
catch (ObjectDisposedException)
{
// Raced another stop; the loop is already going away.
}
if (task is not null)
{
try
{
await task.ConfigureAwait(false);
}
catch (Exception ex)
{
// RunProbeLoopAsync is written not to throw, so this is belt-and-braces.
_logger.LogDebug(
ex,
"MTConnect driver {DriverInstanceId} swallowed a fault from its connectivity probe while stopping it.",
_driverInstanceId);
}
}
cts.Dispose();
}
// ---- IRediscoverable: the Agent instanceId watch ----
/// <inheritdoc/>
/// <remarks>
/// <para>
/// <b>This event is what makes <see cref="RediscoverPolicy"/> =
/// <see cref="DiscoveryRediscoverPolicy.Once"/> safe.</b> The runtime runs exactly one
/// discovery pass per connect, so an Agent that restarts mid-run and renumbers, adds, or
/// drops data items would otherwise leave a stale address space sitting behind a
/// perfectly Healthy driver.
/// </para>
/// <para>
/// <b>Raised from the <c>/sample</c> pump's task, outside every lock.</b> A handler that
/// throws is absorbed and logged — a consumer bug must not tear down the stream serving
/// every subscriber. A handler that <i>blocks</i> blocks the pump, and one that
/// synchronously awaits a lifecycle method on this driver deadlocks it: teardown waits
/// for the pump, and the pump is waiting inside the handler. Core's consumer schedules
/// the rediscovery rather than running it inline, which is the only safe shape.
/// </para>
/// </remarks>
public event EventHandler<RediscoveryEventArgs>? OnRediscoveryNeeded;
/// <summary>
/// Handles an Agent restart observed on the wire: drop the device model it invalidated, then
/// tell Core to re-discover — <b>once per distinct <c>instanceId</c></b>.
/// </summary>
/// <remarks>
/// <para>
/// <b>Dropping the cached <c>/probe</c> model is not housekeeping — it is what stops this
/// whole capability being inert.</b> Core's response to the event is to re-run
/// <see cref="DiscoverAsync"/>, which reads
/// <see cref="GetOrFetchProbeModelAsync"/>; with the cache still warm that call answers
/// from a model describing a process that no longer exists, and the "rediscovery" would
/// diff the stale tree against itself and change nothing. Dropped through the same
/// compare-and-swap <see cref="FlushOptionalCachesAsync"/> uses, so a lifecycle change
/// landing mid-drop wins rather than being undone.
/// </para>
/// <para>
/// <b>Once per change, not once per chunk.</b> Every chunk after a restart carries the new
/// id, and a re-baseline that fails leaves the pump still comparing against the old one —
/// so the guard is a latch on the id that was last announced, not on the pump's cursor
/// state. A second, different restart is a second announcement; an id the Agent reuses
/// after some other id is one too, because that is genuinely a new process.
/// </para>
/// </remarks>
/// <param name="instanceId">The <c>instanceId</c> the Agent's chunk reported.</param>
/// <param name="options">The options in force, for the log line's endpoint.</param>
private void AnnounceAgentRestart(long instanceId, MTConnectDriverOptions options)
{
if (Interlocked.Exchange(ref _announcedInstanceId, instanceId) == instanceId)
{
return;
}
InvalidateProbeModel();
_logger.LogInformation(
"MTConnect driver {DriverInstanceId} dropped its cached /probe device model for {AgentUri} and raised rediscovery: agent instanceId is now {InstanceId}.",
_driverInstanceId, options.AgentUri, instanceId);
try
{
OnRediscoveryNeeded?.Invoke(this, new RediscoveryEventArgs("MTConnect agent instanceId changed", null));
}
catch (Exception ex)
{
_logger.LogWarning(
ex,
"MTConnect driver {DriverInstanceId} caught a fault from a rediscovery subscriber; the /sample stream continues.",
_driverInstanceId);
}
}
/// <summary>
/// Drops the cached <c>/probe</c> device model so the next consumer re-fetches it. Same
/// compare-and-swap discipline as <see cref="FlushOptionalCachesAsync"/> — retried, because
/// unlike a flush this drop is a correctness requirement rather than a courtesy: losing the
/// race must not leave a model from a dead Agent installed.
/// </summary>
private void InvalidateProbeModel()
{
while (true)
{
var session = Volatile.Read(ref _session);
if (session?.ProbeModel is null)
{
return;
}
var dropped = session with { ProbeModel = null, ProbeModelDataItemCount = 0 };
if (ReferenceEquals(Interlocked.CompareExchange(ref _session, dropped, session), session))
{
return;
}
}
}
/// <summary> /// <summary>
/// Starts the shared <c>/sample</c> pump, if there is anything to pump and anything to pump /// Starts the shared <c>/sample</c> pump, if there is anything to pump and anything to pump
/// from. <b>Caller must hold <see cref="_subscriptionLock"/>.</b> /// from. <b>Caller must hold <see cref="_subscriptionLock"/>.</b>
@@ -1166,6 +1581,11 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
ObservationIndex.Clear(); ObservationIndex.Clear();
FanOut(SubscribedReferences(), ObservationIndex); FanOut(SubscribedReferences(), ObservationIndex);
// The driver's own state is consistent by this point, so it is safe to hand the
// restart to arbitrary caller code — which may call straight back into
// DiscoverAsync.
AnnounceAgentRestart(chunk.InstanceId, options);
return await RebaselineAsync(client, options, expected, instanceId, ct).ConfigureAwait(false); return await RebaselineAsync(client, options, expected, instanceId, ct).ConfigureAwait(false);
} }
@@ -1632,6 +2052,19 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
RequirePositive(options.SampleIntervalMs, nameof(MTConnectDriverOptions.SampleIntervalMs)); RequirePositive(options.SampleIntervalMs, nameof(MTConnectDriverOptions.SampleIntervalMs));
RequirePositive(options.SampleCount, nameof(MTConnectDriverOptions.SampleCount)); RequirePositive(options.SampleCount, nameof(MTConnectDriverOptions.SampleCount));
// Same 01/S-6 guard for Task 13's two knobs, gated on the feature that reads them. A 0
// interval turns the "periodic" probe into an unthrottled hot loop against the Agent, and
// a 0 (or negative) timeout cancels every probe before it can be answered — pinning the
// host to Stopped forever and reporting an outage that does not exist. Checked only when
// probing is ON because a disabled probe reads neither value, and faulting a deployment
// over a field the driver never touches would buy nothing; flipping Enabled back on is a
// config edit, and ReinitializeAsync runs this same validation.
if (options.Probe.Enabled)
{
RequirePositive(options.Probe.Interval, $"{nameof(MTConnectDriverOptions.Probe)}.{nameof(MTConnectProbeOptions.Interval)}");
RequirePositive(options.Probe.Timeout, $"{nameof(MTConnectDriverOptions.Probe)}.{nameof(MTConnectProbeOptions.Timeout)}");
}
var client = existingClient; var client = existingClient;
if (client is null) if (client is null)
{ {
@@ -1650,6 +2083,11 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
_options = options; _options = options;
built = null; built = null;
// The Agent this session talks to, as of now. A restart is measured against THIS, so a
// re-initialize against a different Agent process never re-announces the change the
// operator's own action caused.
Volatile.Write(ref _announcedInstanceId, current.InstanceId);
// One publish, so a concurrent lock-free reader sees the new client, the new probe // One publish, so a concurrent lock-free reader sees the new client, the new probe
// cache, and the pump's opening cursor together or none of them. // cache, and the pump's opening cursor together or none of them.
Volatile.Write( Volatile.Write(
@@ -1678,6 +2116,8 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
StartSampleStreamCore(republishOnStart: true); StartSampleStreamCore(republishOnStart: true);
} }
StartProbeLoopCore(client, options);
_logger.LogInformation( _logger.LogInformation(
"MTConnect driver {DriverInstanceId} initialized against {AgentUri}{DeviceScope}: {DeviceCount} device(s), {ObservationCount} primed observation(s), agent instanceId {InstanceId}.", "MTConnect driver {DriverInstanceId} initialized against {AgentUri}{DeviceScope}: {DeviceCount} device(s), {ObservationCount} primed observation(s), agent instanceId {InstanceId}.",
_driverInstanceId, _driverInstanceId,
@@ -2084,6 +2524,22 @@ public sealed class MTConnectDriver : IDriver, IReadable, ISubscribable, ITagDis
} }
} }
/// <summary>
/// The <see cref="TimeSpan"/> half of the 01/S-6 guard, for the connectivity probe's two
/// knobs. A non-positive interval is an unthrottled loop; a non-positive timeout cancels
/// every request before it can be answered.
/// </summary>
private static void RequirePositive(TimeSpan value, string optionName)
{
if (value <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(
nameof(MTConnectDriverOptions),
value,
$"{optionName} must be positive; a non-positive value would either spin the connectivity probe against the Agent or cancel every probe before it could be answered.");
}
}
/// <summary>Barrier-protected read of the cross-thread health snapshot.</summary> /// <summary>Barrier-protected read of the cross-thread health snapshot.</summary>
private DriverHealth ReadHealth() => Volatile.Read(ref _health); private DriverHealth ReadHealth() => Volatile.Read(ref _health);
@@ -0,0 +1,629 @@
using System.Collections.Concurrent;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests;
/// <summary>
/// Task 13 — <see cref="MTConnectDriver"/>'s two optional capabilities:
/// <see cref="IHostConnectivityProbe"/> (a periodic reachability tick that flips one host
/// Running ↔ Stopped) and <see cref="IRediscoverable"/> (the Agent <c>instanceId</c> watch that
/// is the ONLY thing making <see cref="DiscoveryRediscoverPolicy.Once"/> safe).
/// </summary>
/// <remarks>
/// <para>
/// <b>Nothing here waits on wall-clock time, and the probe is a timer.</b> The driver takes
/// its inter-tick delay through an injected scheduler seam, and this suite supplies
/// <see cref="ManualTicker"/>: the loop parks in the seam, the test releases exactly one
/// tick, and the ticker's next park is the proof that the probe which followed has already
/// completed. No test sleeps, no test polls a counter, and
/// <see cref="CannedAgentClient"/> stays timer-free.
/// </para>
/// <para>
/// <b>The load-bearing tests are the anti-inertness ones.</b> Both capabilities have a
/// plausible-looking implementation that compiles, ships, and does nothing:
/// a connectivity probe routed through the driver's cached <c>/probe</c> accessor never
/// touches the network once the cache is warm (so it reports Running forever, whatever the
/// Agent is doing), and a rediscovery signal raised without dropping that same cache makes
/// Core re-run discovery and receive the identical stale device tree. Both are pinned here
/// by asserting the Agent was actually called.
/// </para>
/// </remarks>
public sealed class MTConnectHostAndRediscoverTests
{
private const string AgentUri = "http://fixture-agent:5000";
/// <summary>The <c>instanceId</c> every canned fixture shares.</summary>
private const long FixtureInstanceId = 1655000000L;
/// <summary>A DIFFERENT <c>instanceId</c> — the Agent came back as a new process.</summary>
private const long RestartedInstanceId = 1655999999L;
/// <summary>A third <c>instanceId</c> — the Agent restarted twice.</summary>
private const long SecondRestartInstanceId = 1656111111L;
/// <summary>The cursor <c>Fixtures/current.xml</c> primes the pump with.</summary>
private const long PrimedNextSequence = 108L;
/// <summary>
/// Failure guard for the awaits a defect could turn into a permanent hang (a probe loop that
/// never parks, a teardown that deadlocks on the lifecycle semaphore). It exists to make a
/// hang red, not to assert a duration — no assertion is ever made about elapsed time.
/// </summary>
private static readonly TimeSpan Watchdog = TimeSpan.FromSeconds(10);
private static readonly DateTime ObservedAt = new(2026, 7, 24, 12, 30, 0, DateTimeKind.Utc);
/// <summary>The interval the probe tests author, asserted to reach the scheduler verbatim.</summary>
private static readonly TimeSpan AuthoredInterval = TimeSpan.FromSeconds(7);
private static CancellationToken Ct => TestContext.Current.CancellationToken;
// ---- fixtures ----
/// <summary>
/// Options with the background probe <b>disabled</b> — the default for the rediscovery half
/// of this suite, so no probe loop can add calls behind a <c>ProbeCallCount</c> assertion.
/// </summary>
private static MTConnectDriverOptions Opts(MTConnectProbeOptions? probe = null) =>
new()
{
AgentUri = AgentUri,
RequestTimeoutMs = 5000,
Probe = probe ?? new MTConnectProbeOptions { Enabled = false },
Tags =
[
new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64),
new MTConnectTagDefinition("dev1_execution", DriverDataType.String),
],
};
private static MTConnectProbeOptions Probing(bool enabled = true) =>
new() { Enabled = enabled, Interval = AuthoredInterval, Timeout = TimeSpan.FromSeconds(2) };
private static async Task<(MTConnectDriver Driver, CannedAgentClient Client)> InitializedDriverAsync(
CannedAgentClient? client = null,
MTConnectDriverOptions? options = null,
ManualTicker? ticker = null)
{
var agent = client ?? CannedAgentClient.FromFixtures();
var driver = new MTConnectDriver(
options ?? Opts(), "mt1", _ => agent, probeScheduler: ticker is null ? null : ticker.WaitAsync);
await driver.InitializeAsync("{}", Ct);
return (driver, agent);
}
/// <summary>
/// Brings a driver up with the probe loop wired to a manual ticker, and returns once the loop
/// is demonstrably parked in its first inter-tick wait (i.e. it has run zero probes).
/// </summary>
private static async Task<(MTConnectDriver Driver, CannedAgentClient Client, ManualTicker Ticker)>
ProbingDriverAsync(CannedAgentClient? client = null)
{
var ticker = new ManualTicker();
var (driver, agent) = await InitializedDriverAsync(client, Opts(Probing()), ticker);
await ticker.ParkedAsync().WaitAsync(Watchdog, Ct);
return (driver, agent, ticker);
}
/// <summary>
/// Releases exactly one probe tick and returns once the probe it triggered has completed —
/// proven by the loop parking again, not by a delay.
/// </summary>
private static async Task TickAsync(ManualTicker ticker)
{
ticker.Tick();
await ticker.ParkedAsync().WaitAsync(Watchdog, Ct);
}
private static ConcurrentQueue<RediscoveryEventArgs> RecordRediscovery(MTConnectDriver driver)
{
var seen = new ConcurrentQueue<RediscoveryEventArgs>();
((IRediscoverable)driver).OnRediscoveryNeeded += (_, e) => seen.Enqueue(e);
return seen;
}
private static ConcurrentQueue<HostStatusChangedEventArgs> RecordHostStatus(MTConnectDriver driver)
{
var seen = new ConcurrentQueue<HostStatusChangedEventArgs>();
((IHostConnectivityProbe)driver).OnHostStatusChanged += (_, e) => seen.Enqueue(e);
return seen;
}
private static MTConnectStreamsResult ChunkFrom(
long instanceId, long firstSequence, long nextSequence, params (string Id, string Value)[] observations) =>
new(
instanceId,
nextSequence,
firstSequence,
[.. observations.Select(o => new MTConnectObservation(o.Id, o.Value, ObservedAt))]);
/// <summary>A device model that is recognisably NOT the canned fixture's.</summary>
private static MTConnectProbeModel OtherModel() =>
new([
new MTConnectDevice(
"other-device",
"other-device",
[],
[new MTConnectDataItem("other_item", "other_item", "EVENT", "AVAILABILITY", null, null, null, null)]),
]);
// ---- IRediscoverable: the instanceId watch ----
/// <summary>
/// The plan's TDD case. <see cref="MTConnectDriver.RediscoverPolicy"/> is
/// <see cref="DiscoveryRediscoverPolicy.Once"/>, so the runtime runs exactly one discovery
/// pass per connect — an Agent that restarts and renumbers (or adds) data items would leave a
/// stale address space behind a Healthy driver. This event is the only thing that makes that
/// policy safe.
/// </summary>
[Fact]
public async Task InstanceId_change_raises_rediscovery()
{
var (driver, client) = await InitializedDriverAsync();
var seen = RecordRediscovery(driver);
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
client.CurrentAnswers.Enqueue(ChunkFrom(RestartedInstanceId, 40, 42, ("dev1_pos", "9.5")));
await client
.PumpAsync(ChunkFrom(RestartedInstanceId, 5000, 5005, ("dev1_pos", "5.5")))
.WaitAsync(Watchdog, Ct);
var raised = seen.ShouldHaveSingleItem();
raised.Reason.ShouldBe("MTConnect agent instanceId changed");
raised.ScopeHint.ShouldBeNull();
}
/// <summary>
/// Once per <b>change</b>, not once per chunk. The chunks that follow a restart all carry the
/// new <c>instanceId</c>; a driver that compared each chunk against the id it was
/// <i>initialized</i> with — or that simply re-raised on every chunk — would ask Core to
/// rebuild the whole address space on every heartbeat, forever.
/// </summary>
[Fact]
public async Task InstanceId_change_raises_rediscovery_once_per_change_not_once_per_chunk()
{
var (driver, client) = await InitializedDriverAsync();
var seen = RecordRediscovery(driver);
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
client.CurrentAnswers.Enqueue(ChunkFrom(RestartedInstanceId, 40, 42, ("dev1_pos", "9.5")));
await client
.PumpAsync(ChunkFrom(RestartedInstanceId, 5000, 5005, ("dev1_pos", "5.5")))
.WaitAsync(Watchdog, Ct);
seen.Count.ShouldBe(1);
// Two more chunks from the SAME restarted Agent, contiguous with the re-baseline.
await client.PumpAsync(ChunkFrom(RestartedInstanceId, 42, 45, ("dev1_pos", "10.5"))).WaitAsync(Watchdog, Ct);
await client.PumpAsync(ChunkFrom(RestartedInstanceId, 45, 48, ("dev1_pos", "11.5"))).WaitAsync(Watchdog, Ct);
seen.Count.ShouldBe(1);
// …but a SECOND restart is a second change, and must be announced.
client.CurrentAnswers.Enqueue(ChunkFrom(SecondRestartInstanceId, 10, 12, ("dev1_pos", "1.5")));
await client
.PumpAsync(ChunkFrom(SecondRestartInstanceId, 6000, 6005, ("dev1_pos", "2.5")))
.WaitAsync(Watchdog, Ct);
seen.Count.ShouldBe(2);
}
/// <summary>
/// An unchanged <c>instanceId</c> is the overwhelmingly common case — every ordinary chunk,
/// every heartbeat, and every ring-buffer re-baseline of a still-running Agent. None of them
/// may cost an address-space rebuild.
/// </summary>
[Fact]
public async Task Unchanged_instanceId_raises_no_rediscovery()
{
var (driver, client) = await InitializedDriverAsync();
var seen = RecordRediscovery(driver);
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
await client
.PumpAsync(ChunkFrom(FixtureInstanceId, PrimedNextSequence, 113, ("dev1_pos", "1.5")))
.WaitAsync(Watchdog, Ct);
// A heartbeat…
await client.PumpAsync(ChunkFrom(FixtureInstanceId, 1, 120)).WaitAsync(Watchdog, Ct);
// …and a ring-buffer gap, which re-baselines but is NOT a restart.
client.CurrentAnswers.Enqueue(ChunkFrom(FixtureInstanceId, 5000, 5005, ("dev1_pos", "2.5")));
await client
.PumpAsync(ChunkFrom(FixtureInstanceId, 5000, 5005, ("dev1_pos", "2.5")))
.WaitAsync(Watchdog, Ct);
seen.ShouldBeEmpty();
}
/// <summary>
/// <b>The anti-inertness pin for <see cref="IRediscoverable"/>.</b> A restart invalidates the
/// cached <c>/probe</c> device model — it describes a process that no longer exists. Raising
/// the event while keeping that cache would have Core dutifully re-run
/// <see cref="MTConnectDriver.DiscoverAsync"/> and receive the <i>identical stale tree</i>
/// from <c>GetOrFetchProbeModelAsync</c>'s warm cache: a feature that fires, logs, and
/// changes nothing.
/// </summary>
[Fact]
public async Task InstanceId_change_drops_the_cached_probe_model_so_rediscovery_sees_the_new_one()
{
var (driver, client) = await InitializedDriverAsync();
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
driver.CachedProbeModel.ShouldNotBeNull();
// The restarted Agent declares a different device model.
client.Probe = OtherModel();
client.CurrentAnswers.Enqueue(ChunkFrom(RestartedInstanceId, 40, 42, ("dev1_pos", "9.5")));
await client
.PumpAsync(ChunkFrom(RestartedInstanceId, 5000, 5005, ("dev1_pos", "5.5")))
.WaitAsync(Watchdog, Ct);
driver.CachedProbeModel.ShouldBeNull();
var probeCalls = client.ProbeCallCount;
var builder = new CapturingBuilder();
await driver.DiscoverAsync(builder, Ct);
// Discovery went back to the Agent, and streamed the NEW model rather than the old one.
client.ProbeCallCount.ShouldBe(probeCalls + 1);
builder.Variables.Select(v => v.Attr.FullName).ShouldBe(["other_item"]);
}
/// <summary>
/// A rediscovery handler is arbitrary caller code. One that throws is a bug in the consumer,
/// not a reason to stop streaming data to every subscriber on this driver.
/// </summary>
[Fact]
public async Task A_throwing_rediscovery_handler_does_not_kill_the_pump()
{
var (driver, client) = await InitializedDriverAsync();
var seen = new ConcurrentQueue<DataChangeEventArgs>();
driver.OnDataChange += (_, e) => seen.Enqueue(e);
((IRediscoverable)driver).OnRediscoveryNeeded += (_, _) =>
throw new InvalidOperationException("rediscovery subscriber blew up");
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
client.CurrentAnswers.Enqueue(ChunkFrom(RestartedInstanceId, 40, 42, ("dev1_pos", "9.5")));
await client
.PumpAsync(ChunkFrom(RestartedInstanceId, 5000, 5005, ("dev1_pos", "5.5")))
.WaitAsync(Watchdog, Ct);
// The pump survived the throw, re-baselined, and keeps delivering.
await client.PumpAsync(ChunkFrom(RestartedInstanceId, 42, 45, ("dev1_pos", "12.5"))).WaitAsync(Watchdog, Ct);
driver.SampleStreamTask!.IsCompleted.ShouldBeFalse();
seen.Where(e => e.FullReference == "dev1_pos").Last().Snapshot.Value.ShouldBe(12.5d);
}
// ---- IHostConnectivityProbe ----
/// <summary>
/// One host per Agent, named by the authored <c>AgentUri</c> — that is the identity the Admin
/// dashboard shows and the key Core scopes its Bad-quality fan-out by. Before the first tick
/// the honest answer is <see cref="HostState.Unknown"/>: the probe loop is the sole writer of
/// this field, and it has not run yet.
/// </summary>
[Fact]
public async Task Host_statuses_report_one_host_named_by_the_agent_uri()
{
var (driver, _, _) = await ProbingDriverAsync();
driver.HostName.ShouldBe(AgentUri);
var status = ((IHostConnectivityProbe)driver).GetHostStatuses().ShouldHaveSingleItem();
status.HostName.ShouldBe(AgentUri);
status.State.ShouldBe(HostState.Unknown);
}
/// <summary>
/// <b>The anti-inertness pin for <see cref="IHostConnectivityProbe"/>.</b> Every tick must
/// reach the Agent. A probe routed through the driver's cached-model accessor
/// (<c>GetOrFetchProbeModelAsync</c>) compiles, ships, and — with a cache warmed by
/// initialize — never issues a single request, reporting Running forever regardless of what
/// the Agent is doing. The call count is the only thing that can tell the two apart.
/// </summary>
[Fact]
public async Task Probe_loop_calls_the_agent_on_every_tick()
{
var (_, client, ticker) = await ProbingDriverAsync();
// Parked before the first tick: the loop waits, THEN probes, so initialize's own /probe is
// the only call so far.
client.ProbeCallCount.ShouldBe(1);
await TickAsync(ticker);
client.ProbeCallCount.ShouldBe(2);
await TickAsync(ticker);
client.ProbeCallCount.ShouldBe(3);
await TickAsync(ticker);
client.ProbeCallCount.ShouldBe(4);
// …and it waits the authored interval between them, rather than a hard-coded cadence.
ticker.LastInterval.ShouldBe(AuthoredInterval);
}
/// <summary>
/// Running ↔ Stopped, and the event fires on the <b>transition</b> only. A driver that raised
/// on every tick would emit one event per interval per Agent forever; Core would re-fan Bad
/// quality across the host's subtree each time, and an operator watching the feed could not
/// tell a new outage from an ongoing one.
/// </summary>
[Fact]
public async Task Probe_flips_the_host_on_transition_only()
{
var (driver, client, ticker) = await ProbingDriverAsync();
var seen = RecordHostStatus(driver);
client.ProbeFailure = new HttpRequestException("agent unreachable");
await TickAsync(ticker);
await TickAsync(ticker);
var down = seen.ShouldHaveSingleItem();
down.HostName.ShouldBe(AgentUri);
down.OldState.ShouldBe(HostState.Unknown);
down.NewState.ShouldBe(HostState.Stopped);
((IHostConnectivityProbe)driver).GetHostStatuses()[0].State.ShouldBe(HostState.Stopped);
client.ProbeFailure = null;
await TickAsync(ticker);
await TickAsync(ticker);
seen.Count.ShouldBe(2);
var up = seen.Last();
up.OldState.ShouldBe(HostState.Stopped);
up.NewState.ShouldBe(HostState.Running);
((IHostConnectivityProbe)driver).GetHostStatuses()[0].State.ShouldBe(HostState.Running);
}
/// <summary>
/// <b>The probe is a liveness check, never a cache writer.</b> Publishing its answer into
/// <c>AgentSession.ProbeModel</c> would put a second writer on the field the lifecycle owns
/// under a compare-and-swap, and would silently re-warm a cache that
/// <c>FlushOptionalCachesAsync</c> (or an Agent restart) deliberately dropped.
/// </summary>
[Fact]
public async Task Probe_ticks_never_publish_into_the_session_probe_cache()
{
var (driver, client, ticker) = await ProbingDriverAsync();
var original = driver.CachedProbeModel.ShouldNotBeNull();
// The Agent now answers /probe with a different model. Nothing the connectivity probe does
// may let that model reach the driver's cache.
client.Probe = OtherModel();
await TickAsync(ticker);
await TickAsync(ticker);
driver.CachedProbeModel.ShouldBeSameAs(original);
driver.GetMemoryFootprint().ShouldBeGreaterThan(0);
// …and a flushed cache stays flushed until a real consumer asks for it.
await driver.FlushOptionalCachesAsync(Ct);
await TickAsync(ticker);
driver.CachedProbeModel.ShouldBeNull();
}
/// <summary>
/// <b>Host connectivity is deliberately NOT an input to <c>DriverHealth</c>.</b> The two real
/// data paths (<c>/current</c> reads, the <c>/sample</c> pump) own health and will report the
/// same outage with the failure that actually mattered. The probe's deadline
/// (<c>Probe.Timeout</c>, 2 s) is far tighter than <c>RequestTimeoutMs</c> and its request is
/// the <i>largest</i> document the Agent serves, so letting it degrade the driver would flap
/// a perfectly working instance on a slow-but-alive Agent.
/// </summary>
[Fact]
public async Task An_unreachable_host_does_not_degrade_driver_health()
{
var (driver, client, ticker) = await ProbingDriverAsync();
client.ProbeFailure = new HttpRequestException("agent unreachable");
await TickAsync(ticker);
((IHostConnectivityProbe)driver).GetHostStatuses()[0].State.ShouldBe(HostState.Stopped);
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
// …and the converse: a reachable host does not clear a genuinely failing read path either.
client.ProbeFailure = null;
client.CurrentFailure = new HttpRequestException("connection refused");
await driver.ReadAsync(["dev1_pos"], Ct);
driver.GetHealth().State.ShouldBe(DriverState.Degraded);
await TickAsync(ticker);
((IHostConnectivityProbe)driver).GetHostStatuses()[0].State.ShouldBe(HostState.Running);
driver.GetHealth().State.ShouldBe(DriverState.Degraded);
}
/// <summary>
/// <c>Probe.Enabled = false</c> means no loop, no scheduler wait, and not one extra request.
/// An operator who turned probing off must not still be paying for it.
/// </summary>
[Fact]
public async Task Probe_disabled_starts_no_loop_and_makes_no_calls()
{
var ticker = new ManualTicker();
var (driver, client) = await InitializedDriverAsync(options: Opts(Probing(enabled: false)), ticker: ticker);
driver.ProbeLoopTask.ShouldBeNull();
ticker.Waits.ShouldBe(0);
client.ProbeCallCount.ShouldBe(1); // the initialize prime, and nothing else
// The capability is still implemented — it just has nothing to report.
((IHostConnectivityProbe)driver).GetHostStatuses()
.ShouldHaveSingleItem().State.ShouldBe(HostState.Unknown);
}
/// <summary>
/// Teardown stops the timer and waits for it. A loop left running would keep dialling an
/// Agent through a client its owner has already disposed — the "torn down but still
/// reporting" shape the rest of this driver's lifecycle is written to prevent.
/// </summary>
[Fact]
public async Task Shutdown_stops_the_probe_loop_and_makes_no_further_calls()
{
var (driver, client, ticker) = await ProbingDriverAsync();
await TickAsync(ticker);
var loop = driver.ProbeLoopTask.ShouldNotBeNull();
await driver.ShutdownAsync(Ct).WaitAsync(Watchdog, Ct);
loop.IsCompleted.ShouldBeTrue();
loop.IsFaulted.ShouldBeFalse();
driver.ProbeLoopTask.ShouldBeNull();
client.IsDisposed.ShouldBeTrue();
// The loop is provably gone (ShutdownAsync awaited it), so releasing another tick can reach
// nobody — no probe against a disposed client, and no further waits on the scheduler.
var calls = client.ProbeCallCount;
var waits = ticker.Waits;
ticker.Tick();
client.ProbeCallCount.ShouldBe(calls);
ticker.Waits.ShouldBe(waits);
}
/// <summary>
/// A re-initialize replaces the client, so it must replace the loop that dials it — the old
/// one holds a reference to a client the re-initialize may be about to dispose.
/// </summary>
[Fact]
public async Task Reinitialize_replaces_the_probe_loop()
{
var (driver, _, ticker) = await ProbingDriverAsync();
var first = driver.ProbeLoopTask.ShouldNotBeNull();
await driver.ReinitializeAsync("{}", Ct).WaitAsync(Watchdog, Ct);
first.IsCompleted.ShouldBeTrue();
var second = driver.ProbeLoopTask.ShouldNotBeNull();
second.ShouldNotBeSameAs(first);
// The replacement loop is live: it parks, ticks, and probes like the first.
await ticker.ParkedAsync().WaitAsync(Watchdog, Ct);
await TickAsync(ticker);
}
// ---- operator-authorable zeroes (arch-review 01/S-6) ----
/// <summary>
/// arch-review 01/S-6, applied to the two knobs Task 13 is the first consumer of. A
/// <c>0</c> interval turns the "periodic" probe into an unthrottled hot loop against the
/// Agent; a <c>0</c> timeout cancels every probe before it can be answered, pinning the host
/// to Stopped forever and reporting an outage that does not exist. Both are refused with the
/// same <c>RequirePositive</c> posture as the four timing knobs Task 9 guarded.
/// </summary>
[Theory]
[InlineData(0, 2000)]
[InlineData(-1, 2000)]
[InlineData(5000, 0)]
[InlineData(5000, -1)]
public async Task A_non_positive_probe_interval_or_timeout_is_refused(int intervalMs, int timeoutMs)
{
var options = Opts(new MTConnectProbeOptions
{
Enabled = true,
Interval = TimeSpan.FromMilliseconds(intervalMs),
Timeout = TimeSpan.FromMilliseconds(timeoutMs),
});
var driver = new MTConnectDriver(options, "mt1", _ => CannedAgentClient.FromFixtures());
await Should.ThrowAsync<ArgumentOutOfRangeException>(() => driver.InitializeAsync("{}", Ct));
driver.GetHealth().State.ShouldBe(DriverState.Faulted);
}
/// <summary>
/// …but only when probing is on. Faulting a deployment over a field the driver never reads
/// would be a new way for a deploy to fail with nothing gained; flipping
/// <c>Enabled</c> back on is a config edit that re-runs the same validation.
/// </summary>
[Fact]
public async Task A_non_positive_probe_interval_is_ignored_when_probing_is_disabled()
{
var options = Opts(new MTConnectProbeOptions
{
Enabled = false, Interval = TimeSpan.Zero, Timeout = TimeSpan.Zero,
});
var (driver, _) = await InitializedDriverAsync(options: options);
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
driver.ProbeLoopTask.ShouldBeNull();
}
/// <summary>
/// A hand-driven replacement for <see cref="Task.Delay(TimeSpan, CancellationToken)"/>: the
/// probe loop parks here, and a test releases exactly one tick at a time. The park signal is
/// what makes the suite deterministic — when the loop is parked again, the probe that
/// followed the previous release has demonstrably finished.
/// </summary>
/// <remarks>
/// <see cref="Tick"/> installs the <i>next</i> park signal <b>before</b> releasing the
/// current wait, so a loop that races ahead and parks again immediately cannot lose the
/// signal a test is about to await.
/// </remarks>
private sealed class ManualTicker
{
private readonly Lock _gate = new();
private TaskCompletionSource _parked = new(TaskCreationOptions.RunContinuationsAsynchronously);
private TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _waits;
/// <summary>How many times the loop has parked in this scheduler.</summary>
public int Waits => Volatile.Read(ref _waits);
/// <summary>The interval the loop most recently asked to wait for.</summary>
public TimeSpan LastInterval { get; private set; }
/// <summary>The scheduler seam handed to the driver.</summary>
public async Task WaitAsync(TimeSpan interval, CancellationToken ct)
{
TaskCompletionSource release;
lock (_gate)
{
LastInterval = interval;
Interlocked.Increment(ref _waits);
release = _release;
_parked.TrySetResult();
}
await release.Task.WaitAsync(ct).ConfigureAwait(false);
}
/// <summary>Completes once the loop is parked in the current wait.</summary>
public Task ParkedAsync()
{
lock (_gate)
{
return _parked.Task;
}
}
/// <summary>Releases the current wait so exactly one probe runs.</summary>
public void Tick()
{
lock (_gate)
{
var release = _release;
_release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
_parked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
release.TrySetResult();
}
}
}
}