Merge branch 'master' into worktree-agent-a720278a57ae41792

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 13:25:58 -05:00
69 changed files with 6709 additions and 915 deletions
@@ -5,11 +5,19 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
/// <summary>
/// Default <see cref="IGalaxyPlatformSource"/>: a platform-only <c>DiscoverHierarchy</c> sweep
/// over the gateway's <see cref="GalaxyRepositoryClient"/> — the request filters on
/// <c>template_chain_contains</c> (<c>$WinPlatform</c>, <c>$AppEngine</c>) and asks for no
/// attributes, so it is a few rows per galaxy rather than the full object + attribute dump the
/// address-space discovery pulls. Paged at the configured <c>Repository.DiscoverPageSize</c>.
/// over the gateway's <see cref="GalaxyRepositoryClient"/> — one sweep per platform template
/// (<c>$WinPlatform</c>, then <c>$AppEngine</c>), each filtered on that single
/// <c>template_chain_contains</c> token with no attributes, so it is a few rows per galaxy rather
/// than the full object + attribute dump the address-space discovery pulls. Paged at the
/// configured <c>Repository.DiscoverPageSize</c>.
/// </summary>
/// <remarks>
/// ⚠️ <b>One template per request (#864).</b> The gateway's shared
/// <c>GalaxyHierarchyProjector.MatchesFilters</c> ANDs the <c>template_chain_contains</c> entries —
/// an object must match <i>every</i> token — so a single request carrying both templates matches no
/// platform (a <c>$WinPlatform</c>'s chain has no <c>$AppEngine</c>, and vice versa) and the probe
/// watcher seeded 0 platforms in production. The sweeps are unioned and de-duplicated by tag name.
/// </remarks>
public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
{
/// <summary>Template-chain tokens that identify a platform / engine object.</summary>
@@ -32,15 +40,16 @@ public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
=> QueryAsync(_client.DiscoverHierarchyRawAsync, _pageSize, cancellationToken);
/// <summary>
/// The platform-only sweep behind a delegate so it is unit-testable without a channel. Every
/// request carries the template filter and <c>include_attributes = false</c>; the reply is
/// The platform-only sweeps behind a delegate so it is unit-testable without a channel. One
/// paged sweep per entry of <see cref="PlatformTemplates"/>; every request carries exactly that
/// one template token and <c>include_attributes = false</c>; the reply is
/// re-filtered on the returned template chain (a gateway that ignores the filter still
/// yields only platforms) and reduced to non-empty tag names.
/// </summary>
/// <param name="discover">The raw <c>DiscoverHierarchy</c> RPC.</param>
/// <param name="pageSize">Objects per page.</param>
/// <param name="cancellationToken">Cancellation token for the walk.</param>
/// <returns>Distinct platform tag names in gateway order.</returns>
/// <returns>Distinct platform tag names: <c>$WinPlatform</c> sweep first, then <c>$AppEngine</c>, each in gateway order.</returns>
internal static async Task<IReadOnlyList<string>> QueryAsync(
Func<DiscoverHierarchyRequest, CancellationToken, Task<DiscoverHierarchyReply>> discover,
int pageSize,
@@ -51,33 +60,37 @@ public sealed class GatewayGalaxyPlatformSource : IGalaxyPlatformSource
var names = new List<string>();
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var seenPageTokens = new HashSet<string>(StringComparer.Ordinal);
var pageToken = string.Empty;
do
foreach (var template in PlatformTemplates)
{
var request = new DiscoverHierarchyRequest
var seenPageTokens = new HashSet<string>(StringComparer.Ordinal);
var pageToken = string.Empty;
do
{
PageSize = effective,
PageToken = pageToken,
IncludeAttributes = false,
};
request.TemplateChainContains.Add(PlatformTemplates);
var request = new DiscoverHierarchyRequest
{
PageSize = effective,
PageToken = pageToken,
IncludeAttributes = false,
};
// Exactly ONE token per request: the gateway ANDs template filters (#864).
request.TemplateChainContains.Add(template);
var reply = await discover(request, cancellationToken).ConfigureAwait(false);
foreach (var obj in reply.Objects)
{
if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue;
if (seen.Add(obj.TagName)) names.Add(obj.TagName);
}
var reply = await discover(request, cancellationToken).ConfigureAwait(false);
foreach (var obj in reply.Objects)
{
if (string.IsNullOrEmpty(obj.TagName) || !IsPlatform(obj)) continue;
if (seen.Add(obj.TagName)) names.Add(obj.TagName);
}
pageToken = reply.NextPageToken ?? string.Empty;
if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
{
throw new InvalidOperationException(
$"Galaxy DiscoverHierarchy (platform query) returned a repeated page token '{pageToken}' — refusing to loop.");
pageToken = reply.NextPageToken ?? string.Empty;
if (!string.IsNullOrWhiteSpace(pageToken) && !seenPageTokens.Add(pageToken))
{
throw new InvalidOperationException(
$"Galaxy DiscoverHierarchy (platform query, {template}) returned a repeated page token '{pageToken}' — refusing to loop.");
}
}
while (!string.IsNullOrWhiteSpace(pageToken));
}
while (!string.IsNullOrWhiteSpace(pageToken));
return names;
}
@@ -551,6 +551,9 @@ public sealed class GalaxyDriver
// a cold gateway cache's first-load wait — longer than Galaxy's Tier-A Initialize budget. It
// runs in the background: InitializeAsync neither waits on it nor fails because of it.
await PrepareProbeSeedAsync(cancellationToken).ConfigureAwait(false);
// #865: the seed's probe advises start the event pump themselves (BuildProbeWatcher's
// pre-subscribe hook) — a driver with no authored tags never Reads/Subscribes, and the pump
// used to start only there, so every initial ScanState value went to no stream.
RequestProbeSeed("initialize");
_logger.LogInformation(
"GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}",
@@ -741,9 +744,12 @@ public sealed class GalaxyDriver
// The watcher publishes its probe handles into the driver's registry (reserved id) so the
// EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every
// probe event was dropped before it got anywhere near the watcher.
// #865: the watcher also starts the pump before every probe SubscribeBulk — the initial ScanState
// value only reaches a stream that is already attached (the gateway does not replay a fresh one).
private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) =>
new(subscriber, _hostStatuses, _subscriptions, _logger,
bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs);
bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs,
ensureEventStream: EnsureProbeEventStreamAsync);
/// <summary>
/// Build the production gw client + session + per-capability runtime components
@@ -1058,6 +1064,26 @@ public sealed class GalaxyDriver
// subscriptions to replay the dead pump used to stay in _eventPump, so every later
// Subscribe/Read reused it and was silently deaf while health stayed Healthy.
RestartEventPumpForReplay();
// #865: every re-advise below pushes its initial value once, to whatever stream is attached.
EventPump? replayPump;
lock (_pumpLock) replayPump = _eventPump;
if (replayPump is not null)
{
await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false);
// A replay whose fresh pump already died must FAIL, so the supervisor backs off and stays
// Degraded — succeeding would go Healthy with a dead stream, and the next advise's re-report
// would start a new cycle at attempt 1 (no backoff): a tight reconnect loop.
lock (_pumpLock) replayPump = _eventPump;
if (replayPump is { HasEnded: true })
{
// A teardown cancelling this replay can end the pump too: that is a clean cancellation.
cancellationToken.ThrowIfCancellationRequested();
// The synchronous case; a stream that ends a round trip later reports a transport failure
// while Replaying, which the supervisor itself turns into a failed attempt.
throw new InvalidOperationException(
"The Galaxy event stream ended during the reconnect replay — retrying after backoff.");
}
}
// Only subscriptions still holding the previous session's handles: one registered after the
// reopen is already live on the recreated session (re-subscribing it would double-advise).
@@ -1313,27 +1339,45 @@ public sealed class GalaxyDriver
{
old = _eventPump;
if (old is null) return; // pump never started — nothing to restart
// #865: a pump first started after the reopen's fence (no pump existed before the reconnect)
// already streams the recreated session, and advises registered under it may be waiting for
// their initial values on it — keep it, unless its loop has ended (a dead pump is always replaced,
// #859 P-1: a stream fault during Replaying is not re-reported by the supervisor).
if (old.Session == _subscriptions.CurrentSession && !old.HasEnded) return;
_eventPump = null;
// #865: until the replacement exists, AwaitCurrentEventStreamAsync must not start one of its
// own — two streams would race for the gateway's single-subscriber slot while the old one detaches.
_pumpRestarting = true;
}
// Detach + dispose the faulted pump outside the lock so a slow shutdown doesn't
// block a concurrent EnsureEventPumpStarted; the old loop already exited on fault.
old.OnDataChange -= OnPumpDataChange;
try { old.DisposeAsync().AsTask().GetAwaiter().GetResult(); }
catch (Exception ex)
try
{
_logger.LogWarning(ex,
"GalaxyDriver {InstanceId} faulted EventPump dispose failed during replay — continuing.",
_driverInstanceId);
}
// Detach + dispose the faulted pump outside the lock so a slow shutdown doesn't
// block a concurrent EnsureEventPumpStarted; the old loop already exited on fault.
old.OnDataChange -= OnPumpDataChange;
try { old.DisposeAsync().AsTask().GetAwaiter().GetResult(); }
catch (Exception ex)
{
_logger.LogWarning(ex,
"GalaxyDriver {InstanceId} faulted EventPump dispose failed during replay — continuing.",
_driverInstanceId);
}
// EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the
// recreate if the driver is being disposed or its runtime torn down — the teardown
// already released the pump and a fresh one here would leak past it.
if (_disposed || _runtimeStopping) return;
EnsureEventPumpStarted();
// EnsureEventPumpStarted creates + starts a fresh pump under the lock. Skip the
// recreate if the driver is being disposed or its runtime torn down — the teardown
// already released the pump and a fresh one here would leak past it.
if (_disposed || _runtimeStopping) return;
EnsureEventPumpStarted();
}
finally
{
lock (_pumpLock) _pumpRestarting = false;
}
}
/// <summary>True while <see cref="RestartEventPumpForReplay"/> is between dropping the old pump and starting its replacement.</summary>
private bool _pumpRestarting;
private void OnSupervisorStateChanged(object? sender, StateTransition transition)
{
// Reflect supervisor state in DriverHealth + transport forwarder.
@@ -1686,7 +1730,19 @@ public sealed class GalaxyDriver
var readSubscriber = _subscriber!;
// A live pump must exist (values arrive through it); the read listens on ReadDataChange, not
// on this instance, so it survives a reconnect's pump restart.
EnsureEventPumpStarted();
ThrowIfRuntimeStopping(); // the pump itself is started by AwaitCurrentEventStreamAsync below
// #865: a read waits ONLY for the initial value, which reaches only a current-session stream that is
// already requested. The internal deadline bounds this too; a deadline (not caller) cancel falls
// through so the read still answers per-ref, as it does for a deadline during the RPC.
int issueSession;
try
{
issueSession = await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!callerToken.IsCancellationRequested)
{
issueSession = _subscriptions.CurrentSession;
}
var subscriptionId = _subscriptions.NextSubscriptionId();
// Pre-allocate one TaskCompletionSource per full-reference so the OnDataChange
@@ -1723,7 +1779,7 @@ public sealed class GalaxyDriver
// window before the RPC and close it right after registration (the finally below), not after the
// value wait: a window held for the whole wait would park every unrelated unknown-handle event.
var earlyEventWindow = _subscriptions.BeginPendingSubscribe();
var session = _subscriptions.CurrentSession; // #861 — the session this SubscribeBulk is issued on
var session = issueSession; // #861/#865 — the session this SubscribeBulk is issued on (its stream verified)
try
{
// The RPC gets only the CALLER's token: the internal deadline bounds our wait, not the RPC, so a slow
@@ -1931,7 +1987,7 @@ public sealed class GalaxyDriver
// generation, so the Register below can tell its SubscribeBulk straddled a teardown.
var generation = Volatile.Read(ref _runtimeGeneration);
var subscriber = _subscriber;
var pump = EnsureEventPumpStarted();
ThrowIfRuntimeStopping(); // the pump is started per attempt by AwaitCurrentEventStreamAsync (#865)
var subscriptionId = _subscriptions.NextSubscriptionId();
if (fullReferences.Count == 0)
@@ -1964,6 +2020,10 @@ public sealed class GalaxyDriver
for (var attempt = 1; ; attempt++)
{
// #865: per attempt (a #861 retry is a new session): the advise's initial value reaches only a
// current-session stream that is already requested — the gateway does not replay a fresh stream.
var issueSession = await AwaitCurrentEventStreamAsync(cancellationToken).ConfigureAwait(false);
// #859: MXAccess's initial value for a new handle can reach the event pump before this
// reply is registered (the gateway gives no reply-before-event ordering, and the reply and
// the event ride separate RPCs). Open the early-event window BEFORE the RPC so such events
@@ -1972,7 +2032,7 @@ public sealed class GalaxyDriver
var earlyEventWindow = _subscriptions.BeginPendingSubscribe();
try
{
var session = _subscriptions.CurrentSession; // #861 — the session this SubscribeBulk is issued on
var session = issueSession; // #861/#865 — the session this SubscribeBulk is issued on (its stream verified)
var results = await subscriber
.SubscribeBulkAsync(dialledRefs, bufferedIntervalMs, cancellationToken)
.ConfigureAwait(false);
@@ -1981,7 +2041,6 @@ public sealed class GalaxyDriver
switch (TryRegisterForGeneration(subscriptionId, bindings, generation, session))
{
case RegisterOutcome.Registered:
_ = pump; // keep the pump alive for the subscription's lifetime
return new GalaxySubscriptionHandle(subscriptionId);
case RegisterOutcome.TornDown:
@@ -2080,11 +2139,25 @@ public sealed class GalaxyDriver
}
}
/// <summary>A Subscribe/Read is refused while a teardown is in progress (the pump would belong to a dying subscriber).</summary>
private void ThrowIfRuntimeStopping()
{
lock (_pumpLock)
{
if (_runtimeStopping)
{
throw new InvalidOperationException(
"GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync.");
}
}
}
/// <summary>
/// Lazily start the <see cref="EventPump"/> on the first subscribe. The pump is
/// shared across every subscription on this driver — fan-out happens through the
/// <see cref="SubscriptionRegistry"/> reverse map, not by spinning a pump per
/// subscription.
/// Lazily start the <see cref="EventPump"/> on the first advise. Since #865 the advise paths
/// (probe seed, Subscribe, Read) go through <see cref="AwaitCurrentEventStreamAsync"/>, and only the
/// replay replaces an existing pump. The pump is shared across every subscription on this driver —
/// fan-out happens through the <see cref="SubscriptionRegistry"/> reverse map, not by spinning a pump
/// per subscription.
/// </summary>
private EventPump EnsureEventPumpStarted()
{
@@ -2100,18 +2173,206 @@ public sealed class GalaxyDriver
"GalaxyDriver runtime is shutting down — no subscription can be served until the next InitializeAsync.");
}
if (_eventPump is not null) return _eventPump;
_eventPump = new EventPump(
// The fault callback carries the pump's identity, so a late fault from a pump that is no
// longer the live one (a stale pump of a reopened session) is not escalated (#865 review).
EventPump? created = null;
created = new EventPump(
_subscriber!, _subscriptions, _logger,
channelCapacity: _options.MxAccess.EventPumpChannelCapacity,
clientName: _options.MxAccess.ClientName,
onStreamFault: OnEventPumpStreamFault,
onStreamFault: cause => OnEventPumpStreamFault(created, cause),
onReplayGap: OnEventPumpReplayGap);
_eventPump = created;
_eventPump.OnDataChange += OnPumpDataChange;
_eventPump.Start();
return _eventPump;
}
}
/// <summary>
/// <see cref="EnsureEventPumpStarted"/> for infrastructure callers that must not fail because the
/// runtime is going away: null — and no pump — while a teardown is in progress or there is no subscriber.
/// </summary>
/// <returns>The running (or just started) pump; null when none may be started.</returns>
private EventPump? TryEnsureEventPumpStarted()
{
lock (_pumpLock)
{
if (_runtimeStopping || _disposed || _subscriber is null) return null;
try
{
return EnsureEventPumpStarted(); // re-enters _pumpLock (Lock is reentrant)
}
catch (InvalidOperationException)
{
// _runtimeStopping is written without _pumpLock; a teardown that flipped it between the
// check above and EnsureEventPumpStarted's own check is the same "no pump" answer.
return null;
}
}
}
/// <summary>
/// Upper bound on how long an advise waits for a current-session event stream (#865). Kept well under
/// Galaxy's Tier-A Read (2 s) / Subscribe (5 s) call budgets, so the "proceed anyway" fallback is
/// reachable before the resilience pipeline times the call out.
/// </summary>
internal TimeSpan StreamRequestWait { get; set; } = TimeSpan.FromSeconds(1);
/// <summary>
/// #865 — make sure an advise about to be issued has an event stream for its initial value, and return
/// the registry session to issue (and register) it under.
/// </summary>
/// <remarks>
/// <para>
/// A fresh StreamEvents (after_worker_sequence = 0) is never replayed, so an initial value pushed before
/// the stream attaches is lost — and a slow-changing attribute (ScanState, a setpoint) may not publish
/// again for a long time. So: start the pump if there is none, and wait until its stream call has started
/// (<see cref="EventPump.StreamRequested"/> — this narrows, it cannot close, the attach race: the gateway
/// offers no attach acknowledgement).
/// </para>
/// <para>
/// A pump bound to an OLDER session belongs to a gateway session a reconnect replaced (the reopen →
/// replay window): an advise now would go to the recreated session with no stream for it, and —
/// registered under the current session — the replay would not re-advise it. The pump is not replaced
/// here (only <see cref="ReplayAsync"/> restarts it, so two streams never race for the gateway's
/// single-subscriber slot); while the supervisor is actually reopening/replaying the advise WAITS for the
/// replay's pump. During a reconnect BACKOFF (TransportLost) no replay is imminent, so it does not wait.
/// The returned session is the one the pump was verified against: a reconnect after this check makes the
/// caller's registration fail as a session straddle (#861), whose retry comes back through here.
/// </para>
/// <para>
/// Bounded by <see cref="StreamRequestWait"/> per call (a #861 retry starts a new budget). On timeout the
/// caller proceeds with the current session — no worse than the pre-#865 behaviour, but note the cost:
/// such an advise registers as current, so the replay will not re-advise it, and its initial value is
/// lost until the attribute next changes.
/// </para>
/// <para>
/// A current-session pump whose loop has ENDED is not a stream (its <see cref="EventPump.StreamRequested"/>
/// completes on exit). If the supervisor already reports Healthy, its fault was swallowed while it was
/// recovering (it ignores faults outside Healthy) — the driver would be deaf while Healthy — so it is
/// reported again to start a new recovery cycle.
/// </para>
/// </remarks>
/// <param name="cancellationToken">Cancels the wait.</param>
/// <returns>The registry session to issue the advise under.</returns>
private async Task<int> AwaitCurrentEventStreamAsync(CancellationToken cancellationToken)
{
var elapsed = System.Diagnostics.Stopwatch.StartNew();
while (true)
{
var session = _subscriptions.CurrentSession;
EventPump? pump;
bool restarting;
lock (_pumpLock)
{
restarting = _pumpRestarting;
pump = restarting ? null : TryEnsureEventPumpStarted();
}
if (pump is null && !restarting) return session; // tearing down — the caller's own fences decide
var remaining = StreamRequestWait - elapsed.Elapsed;
if (pump is not null && pump.Session == session)
{
if (pump.HasEnded)
{
ReportEndedPump(pump);
return session;
}
if (remaining > TimeSpan.Zero)
{
try
{
await pump.StreamRequested.WaitAsync(remaining, cancellationToken).ConfigureAwait(false);
if (pump.HasEnded) ReportEndedPump(pump);
return session;
}
catch (TimeoutException)
{
// fall through to the log
}
}
else if (pump.StreamRequested.IsCompleted)
{
return session;
}
}
else if (remaining > TimeSpan.Zero && ReplayExpectedSoon())
{
// Reopen → replay window: the replay restarts the pump for the current session.
await Task.Delay(TimeSpan.FromMilliseconds(20), cancellationToken).ConfigureAwait(false);
continue;
}
LogNoCurrentStream();
return _subscriptions.CurrentSession;
}
}
/// <summary>
/// True when a stale pump will be replaced shortly: a reopen/replay attempt is running, or there is
/// no supervisor (the injected-seam path, where the replay is driven directly), or a restart is underway.
/// </summary>
private bool ReplayExpectedSoon()
{
if (Volatile.Read(ref _pumpRestarting)) return true;
var supervisor = _supervisor;
if (supervisor is null) return true;
// Not the state: between attempts (the backoff delay, up to MaxBackoff) it still reads
// Reopening/Replaying while no replay is coming any time soon.
return supervisor.AttemptInProgress;
}
/// <summary>A current-session pump's loop ended while the supervisor was not listening (see AwaitCurrentEventStreamAsync).</summary>
private void ReportEndedPump(EventPump pump)
{
var supervisor = _supervisor;
if (supervisor is null || _runtimeStopping || supervisor.CurrentState != ReconnectSupervisor.State.Healthy) return;
lock (_pumpLock)
{
if (!ReferenceEquals(_eventPump, pump)) return; // already replaced
}
_logger.LogWarning(
"GalaxyDriver {InstanceId} event stream has ended while the reconnect supervisor reports Healthy — re-reporting the transport failure.",
_driverInstanceId);
try { supervisor.ReportTransportFailure(new InvalidOperationException("The Galaxy event stream ended; no events can be received.")); }
catch (ObjectDisposedException) { /* driver disposing */ }
}
private long _lastNoStreamWarningTicks;
/// <summary>Warning at most once a minute while the supervisor is Healthy; Debug otherwise (an outage would flood).</summary>
private void LogNoCurrentStream()
{
var supervisor = _supervisor;
var now = Environment.TickCount64;
var last = Interlocked.Read(ref _lastNoStreamWarningTicks);
if ((supervisor is null || supervisor.CurrentState == ReconnectSupervisor.State.Healthy)
&& (last == 0 || now - last >= 60_000)
&& Interlocked.CompareExchange(ref _lastNoStreamWarningTicks, now, last) == last)
{
_logger.LogWarning(
"GalaxyDriver {InstanceId} had no started event stream for the current gateway session within {Wait} — advising anyway; initial values may be missed.",
_driverInstanceId, StreamRequestWait);
return;
}
_logger.LogDebug(
"GalaxyDriver {InstanceId} advising without a started current-session event stream (supervisor not Healthy or warning rate-limited).",
_driverInstanceId);
}
/// <summary>
/// #865 — the probe watcher's pre-subscribe hook (<see cref="AwaitCurrentEventStreamAsync"/>). A probe's
/// initial ScanState value is usually its only event, and a driver with no authored tags never
/// Reads/Subscribes, so without this its pump — and every host status — never existed.
/// </summary>
private Task<int> EnsureProbeEventStreamAsync(CancellationToken cancellationToken)
{
// A seed pass a teardown already cancelled must not start a pump after the teardown finished.
cancellationToken.ThrowIfCancellationRequested();
return AwaitCurrentEventStreamAsync(cancellationToken);
}
/// <summary>
/// Replay-gap callback for the <see cref="EventPump"/>. The gateway told us it evicted
/// events we can never receive, so every cached value may be stale by an unknown amount.
@@ -2195,7 +2456,12 @@ public sealed class GalaxyDriver
/// replay; without this hand-off a transient transport drop permanently kills
/// the event stream and <c>GetHealth()</c> keeps reporting Healthy.
/// </summary>
private void OnEventPumpStreamFault(Exception cause)
private int _escalatedStreamFaults;
/// <summary>Test seam (#865): stream faults that passed the superseded-pump filter (escalated toward the supervisor).</summary>
internal int EscalatedStreamFaultsForTests => Volatile.Read(ref _escalatedStreamFaults);
private void OnEventPumpStreamFault(EventPump? source, Exception cause)
{
if (_runtimeStopping)
{
@@ -2205,6 +2471,23 @@ public sealed class GalaxyDriver
return;
}
if (source is not null)
{
bool stale;
lock (_pumpLock) stale = !ReferenceEquals(_eventPump, source) || source.Session != _subscriptions.CurrentSession;
if (stale)
{
// A pump that is no longer the live one: the identity check does the work (the replay
// replaces/replaced it). The session part only matters between a reopen and its replay,
// where the supervisor ignores faults anyway — ⚠️ a future session bump while Healthy that
// keeps the same pump would make its real faults look stale here. Its fault says nothing
// about the current transport, and reported while Replaying it would fail a good replay.
_logger.LogDebug(cause, "GalaxyDriver {InstanceId} ignored a stream fault from a superseded event pump.", _driverInstanceId);
return;
}
}
Interlocked.Increment(ref _escalatedStreamFaults);
var supervisor = _supervisor;
if (supervisor is null)
{
@@ -44,6 +44,11 @@ public sealed class PerPlatformProbeWatcher : IDisposable
// driven directly (unit tests of the decoding table).
private readonly SubscriptionRegistry? _registry;
// #865: invoked before every probe SubscribeBulk so the driver's event stream is attached when the
// gateway pushes the initial ScanState value (a fresh StreamEvents does not replay earlier events,
// and ScanState rarely changes — a lost initial value leaves the host missing indefinitely).
private readonly Func<CancellationToken, Task<int>>? _ensureEventStream;
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
// Each handle carries the registry session that issued it (#859 I-A): item handles are per gateway
@@ -79,16 +84,24 @@ public sealed class PerPlatformProbeWatcher : IDisposable
/// <param name="registry">The driver's subscription registry; null to keep the handles private.</param>
/// <param name="logger">Optional logger for diagnostic messages.</param>
/// <param name="bufferedUpdateIntervalMs">Buffered update interval in milliseconds; must be >= 0.</param>
/// <param name="ensureEventStream">
/// Optional: awaited before every probe SubscribeBulk to make sure the event stream that carries
/// the initial ScanState value is running and requested for the current gateway session (the
/// driver starts its <see cref="EventPump"/> and waits for its StreamEvents call). Returns the
/// registry session the stream was verified against; the advise is issued and fenced under it.
/// </param>
internal PerPlatformProbeWatcher(
IGalaxySubscriber subscriber,
HostStatusAggregator aggregator,
SubscriptionRegistry? registry,
ILogger? logger = null,
int bufferedUpdateIntervalMs = 0)
int bufferedUpdateIntervalMs = 0,
Func<CancellationToken, Task<int>>? ensureEventStream = null)
{
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
_aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator));
_registry = registry;
_ensureEventStream = ensureEventStream;
_logger = logger ?? NullLogger.Instance;
if (bufferedUpdateIntervalMs < 0)
{
@@ -196,10 +209,18 @@ public sealed class PerPlatformProbeWatcher : IDisposable
{
var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
// #865: the stream must be live BEFORE the advise — the initial value is the probe's only
// event until the platform's scan state changes, and an unattached stream never sees it.
// The returned session is the one the stream was verified for: a reconnect after the check makes
// the session test below refuse the reply as a straddle, and the retry comes back through here.
int? streamSession = _ensureEventStream is null
? null
: await _ensureEventStream(cancellationToken).ConfigureAwait(false);
// #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until
// PublishBindingsLocked registers the handles below (window closes at method exit).
using var earlyEventWindow = _registry?.BeginPendingSubscribe();
var session = CurrentSession; // the session this SubscribeBulk is issued on
var session = streamSession ?? CurrentSession; // the session this SubscribeBulk is issued on
// PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
// when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
// changes are rare so a coarser interval is usually fine; deployments that need
@@ -148,6 +148,36 @@ internal sealed class EventPump : IAsyncDisposable
private readonly int _session;
/// <summary>
/// The registry session this pump was created for (#865). A pump of an older session was opened
/// against a gateway session a reconnect replaced; its stream will never carry the new session's events.
/// </summary>
internal int Session => _session;
/// <summary>True once the stream loop has exited (faulted, completed by the gateway, or disposed).</summary>
internal bool HasEnded => Volatile.Read(ref _ended);
// Set at the very start of the stream loop's finally — before StreamRequested is completed there — so a
// waiter released by a dying loop always sees it ended (the loop task itself completes only later).
private bool _ended;
private readonly TaskCompletionSource _streamRequested = new(TaskCreationOptions.RunContinuationsAsynchronously);
/// <summary>
/// Completes once the stream loop has STARTED its <c>StreamEvents</c> call (the first
/// <c>MoveNextAsync</c> runs the subscriber's iterator synchronously up to the gRPC call creation) —
/// or once the loop has ended without getting that far, so a waiter never hangs on a dead pump.
/// #865: the gateway does not replay a fresh stream, so a caller whose only expected event is the
/// initial value of an advise it is about to make waits on this first.
/// </summary>
/// <remarks>
/// ⚠️ This NARROWS the stream-attach race, it does not close it: the HTTP/2 request is sent
/// asynchronously and the gateway attaches the subscriber after auth, concurrently with any other
/// RPC. In practice the attach wins (an advise's initial value needs a worker IPC round trip plus an
/// MXAccess advise), but the gateway offers no attach acknowledgement to await.
/// </remarks>
public Task StreamRequested => _streamRequested.Task;
/// <summary>
/// Marker written into the channel to wake the dispatch loop after a registration promoted
/// parked early events (#859). Never dispatched itself; compared by reference.
@@ -179,10 +209,29 @@ internal sealed class EventPump : IAsyncDisposable
private async Task RunAsync(CancellationToken ct)
{
IAsyncEnumerator<MxEvent>? enumerator = null;
try
{
await foreach (var ev in _subscriber.StreamEventsAsync(ct).WithCancellation(ct).ConfigureAwait(false))
enumerator = _subscriber.StreamEventsAsync(ct).GetAsyncEnumerator(ct);
// Start the first MoveNext BEFORE awaiting it: that synchronous step issues the RPC, which is
// the moment StreamRequested reports (#865).
var next = enumerator.MoveNextAsync();
if (next.IsCompleted)
{
// Already decided (an item, an immediate end, or a fault): consume it now, so a stream that
// ended or faulted at once is seen as ended (finally) BEFORE StreamRequested releases anyone.
var hasFirst = await next.ConfigureAwait(false);
if (!hasFirst) Volatile.Write(ref _ended, true);
_streamRequested.TrySetResult();
next = new ValueTask<bool>(hasFirst);
}
else
{
_streamRequested.TrySetResult();
}
for (; await next.ConfigureAwait(false); next = enumerator.MoveNextAsync())
{
var ev = enumerator.Current;
if (ct.IsCancellationRequested) break;
EventsReceived.Add(1, _clientTag);
@@ -241,6 +290,21 @@ internal sealed class EventPump : IAsyncDisposable
}
finally
{
Volatile.Write(ref _ended, true);
// A loop that died before issuing the RPC must not strand a StreamRequested waiter.
_streamRequested.TrySetResult();
if (enumerator is not null)
{
try { await enumerator.DisposeAsync().ConfigureAwait(false); }
catch (Exception ex) when (ct.IsCancellationRequested || ex is OperationCanceledException)
{
// Disposing a cancelled stream may surface its cancellation — shutdown noise.
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Galaxy EventPump stream enumerator dispose failed.");
}
}
// Tell the dispatch loop the producer is done so it drains and exits.
_channel.Writer.TryComplete();
}
@@ -104,6 +104,18 @@ public sealed class ReconnectSupervisor : IDisposable
get { lock (_stateLock) return _state; }
}
private bool _attemptInProgress;
// Set (under _stateLock) by a ReportTransportFailure that arrives while Replaying; cleared on entering Replaying.
private bool _faultDuringReplay;
/// <summary>
/// True while a reopen or replay is actually running — false between attempts (the backoff delay,
/// during which the state still reads Reopening/Replaying) and while Healthy. #865: an advise waits
/// for the replay's pump only while one is imminent.
/// </summary>
public bool AttemptInProgress => Volatile.Read(ref _attemptInProgress);
/// <summary>True when CurrentState != Healthy. Drivers map this to DriverState.Degraded.</summary>
public bool IsDegraded
{
@@ -136,10 +148,16 @@ public sealed class ReconnectSupervisor : IDisposable
lock (_stateLock)
{
// Re-checked under the lock (StopAsync sets it under the same lock): a report that passed the
// check above while a stop ran must not start a loop nothing would ever cancel.
ObjectDisposedException.ThrowIf(_disposed, this);
_lastError = cause.Message;
if (_state != State.Healthy)
{
// Already recovering — nothing else to do.
// Already recovering — nothing else to do, except: a failure DURING the replay means the
// replay ran onto a transport that is already gone again (#865 — e.g. the restarted event
// stream refused or ended); the replay's success must not then be reported as Healthy.
if (_state == State.Replaying) _faultDuringReplay = true;
_logger.LogDebug("Transport failure reported during {State}: {Message}", _state, cause.Message);
return;
}
@@ -208,20 +226,32 @@ public sealed class ReconnectSupervisor : IDisposable
// === Reopening phase ===
lock (_stateLock) TransitionLocked(State.Reopening, _lastError);
// In progress from the reopen through the replay (no gap between the phases); cleared on any
// exit from the attempt — failure (then the backoff), cancellation or success.
Volatile.Write(ref _attemptInProgress, true);
try
{
await _reopen(ct).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
Volatile.Write(ref _attemptInProgress, false);
return;
}
catch (Exception ex)
{
Volatile.Write(ref _attemptInProgress, false);
_logger.LogWarning(ex, "Galaxy reopen failed (attempt {Attempt})", attempt);
lock (_stateLock) { _lastError = ex.Message; }
continue; // back to backoff + retry
}
// === Replaying phase ===
lock (_stateLock) TransitionLocked(State.Replaying, _lastError);
lock (_stateLock)
{
_faultDuringReplay = false;
TransitionLocked(State.Replaying, _lastError);
}
try
{
@@ -234,12 +264,32 @@ public sealed class ReconnectSupervisor : IDisposable
lock (_stateLock) { _lastError = ex.Message; }
continue; // back to backoff + retry
}
finally
{
Volatile.Write(ref _attemptInProgress, false);
}
// === Done ===
// === Done — unless the transport failed again while the replay ran ===
bool faultedDuringReplay;
string? replayFault;
lock (_stateLock)
{
_lastError = null;
TransitionLocked(State.Healthy, null);
faultedDuringReplay = _faultDuringReplay;
replayFault = _lastError;
if (!faultedDuringReplay)
{
_lastError = null;
TransitionLocked(State.Healthy, null);
}
}
if (faultedDuringReplay)
{
// Going Healthy here would hide a dead transport (the fault was swallowed above), and the
// next report would restart at attempt 1 with no backoff. Retry this cycle with backoff.
_logger.LogWarning(
"Galaxy replay completed but the transport failed again while it ran (attempt {Attempt}: {Fault}) — retrying",
attempt, replayFault);
continue;
}
_logger.LogInformation("Galaxy reconnect succeeded after {Attempt} attempt(s)", attempt);
return;
@@ -282,12 +332,14 @@ public sealed class ReconnectSupervisor : IDisposable
/// <returns>A task that completes once the recovery loop has exited.</returns>
public async Task StopAsync()
{
if (_disposed) return;
_disposed = true;
CancellationTokenSource? cts;
Task? loop;
lock (_stateLock) { cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null; }
lock (_stateLock)
{
if (_disposed) return;
_disposed = true;
cts = _loopCts; loop = _recoveryLoop; _loopCts = null; _recoveryLoop = null;
}
// The loop's own finally may have released the CTS already — a cancelled-then-disposed CTS
// throws on Cancel, and a disposed one is a no-op on Dispose.
@@ -314,6 +314,12 @@ internal sealed class SubscriptionRegistry
/// </summary>
internal Action? AfterMapWriteForTests { get; set; }
/// <summary>
/// Test seam (#865): invoked on <see cref="ResolveOrPark"/>'s lock-free fast path after the handle
/// lookup missed, before the open-window check.
/// </summary>
internal Action? AfterFastPathMissForTests { get; set; }
/// <summary>
/// Test seam (M-1): parks <paramref name="ev"/> unconditionally, producing the parked list for a
/// resolved handle that the atomic registration otherwise makes impossible.
@@ -496,7 +502,12 @@ internal sealed class SubscriptionRegistry
{
subscribers = ResolveSubscribers(ev.ItemHandle);
if (subscribers.Count > 0) return EarlyEventOutcome.Resolved;
if (Volatile.Read(ref _pendingSubscribes) == 0) return EarlyEventOutcome.Stale;
AfterFastPathMissForTests?.Invoke();
// #865 (found by its stress run): a miss is NOT decided here. The lookup above and an
// open-window check are not atomic — a subscribe can register the handle AND close its window
// in between, and declaring the event stale then lost its initial value for good (the first
// event of a bulk advise, while the dispatch thread was slow). Misses are rare; the locked path
// below re-does the lookup, the session check and the window check atomically.
}
lock (_parkGate)
@@ -256,14 +256,15 @@ internal sealed class HeidenhainSession : IAsyncDisposable
await PrepareTelemetryAsync(wire, ct).ConfigureAwait(false);
value = feature switch
{
"Execution" => Lsv2Data.U16(await _client.ReadRuntimeAsync(23, ct).ConfigureAwait(false)),
"ProgramState" => Lsv2Data.U16(await _client.ReadRuntimeAsync(26, ct).ConfigureAwait(false)),
// Each arm boxed explicitly so no numeric arm can widen another (lmxopcua#867).
"Execution" => (object)Lsv2Data.U16(await _client.ReadRuntimeAsync(23, ct).ConfigureAwait(false)),
"ProgramState" => (object)Lsv2Data.U16(await _client.ReadRuntimeAsync(26, ct).ConfigureAwait(false)),
"Program" => Lsv2Data.Program(await _client.ReadRuntimeAsync(24, ct).ConfigureAwait(false)),
"Overrides" => Lsv2Data.Overrides(await _client.ReadRuntimeAsync(25, ct).ConfigureAwait(false)),
"Axes" => Lsv2Data.Axes(await _client.ReadRuntimeAsync(22, ct).ConfigureAwait(false)),
"Tool" => Lsv2Data.Tool(await _client.ReadRuntimeAsync(51, ct).ConfigureAwait(false)),
HeidenhainCatalog.AlarmsFeature => await AlarmsAsync(ct).ConfigureAwait(false),
_ => await TimerAsync(feature, ct).ConfigureAwait(false),
_ => (object)await TimerAsync(feature, ct).ConfigureAwait(false),
};
}
@@ -444,24 +445,27 @@ internal sealed class HeidenhainSession : IAsyncDisposable
case double[] o when address.StartsWith("Overrides/", StringComparison.Ordinal):
return o[name switch { "Feed" => 0, "Spindle" => 1, _ => 2 }];
case Lsv2Axes a:
return name == "Names" ? a.Names : a.Positions;
return name == "Names" ? (object)a.Names : (object)a.Positions;
case Lsv2Tool t:
// lmxopcua#867: every arm is cast to object. Without it C# unifies long/int/double? to double?, so
// Tool/Number (Int64) and Tool/Index + Tool/AxisCode (Int32) were published as Double. A null
// geometry still boxes to null (BadNotSupported upstream).
return name switch
{
"Number" => t.Number,
"Index" => t.Index,
"AxisCode" => t.Axis,
"Length" => t.Length,
_ => t.Radius,
"Number" => (object)t.Number,
"Index" => (object)t.Index,
"AxisCode" => (object)t.Axis,
"Length" => (object?)t.Length,
_ => (object?)t.Radius,
};
case List<Lsv2Alarm> list:
return name switch
{
"Count" => list.Count,
"Messages" => list.Select(x => x.Message).ToArray(),
"Classes" => list.Select(x => x.ErrorClass).ToArray(),
"Groups" => list.Select(x => x.Group).ToArray(),
_ => list.Select(x => x.Number).ToArray(),
"Count" => (object)list.Count,
"Messages" => (object)list.Select(x => x.Message).ToArray(),
"Classes" => (object)list.Select(x => x.ErrorClass).ToArray(),
"Groups" => (object)list.Select(x => x.Group).ToArray(),
_ => (object)list.Select(x => x.Number).ToArray(),
};
case int i:
return i;
@@ -76,7 +76,10 @@ internal sealed class MTConnectObservationIndex
private const uint Good = 0x00000000u;
/// <summary>The Agent reported <c>UNAVAILABLE</c>, or supplied no text at all.</summary>
/// <summary>
/// The Agent reported <c>UNAVAILABLE</c>, or supplied no text at all for a non-string item
/// (an empty string-typed item is a Good <c>""</c> — lmxopcua#824).
/// </summary>
private const uint BadNoCommunication = 0x80310000u;
/// <summary>The tag is authored but the Agent has not yet reported it.</summary>
@@ -443,10 +446,8 @@ internal sealed class MTConnectObservationIndex
var serverTimestamp = Now;
var sourceTimestamp = DateTime.SpecifyKind(observation.TimestampUtc, DateTimeKind.Utc);
// No text at all is the degenerate spelling of "no value": MTConnect defines UNAVAILABLE as
// the only way to report absence, so an empty element is not an empty string value.
if (string.IsNullOrWhiteSpace(observation.Value) ||
string.Equals(observation.Value, UnavailableSentinel, StringComparison.Ordinal))
// UNAVAILABLE is MTConnect's one explicit "no value" sentinel — always BadNoCommunication.
if (string.Equals(observation.Value, UnavailableSentinel, StringComparison.Ordinal))
{
return new DataValueSnapshot(null, BadNoCommunication, sourceTimestamp, serverTimestamp);
}
@@ -456,7 +457,7 @@ internal sealed class MTConnectObservationIndex
// Publishing that as a Good string would be noise an operator would trust. Checked before
// the tag lookup because it is a fact about the wire shape, true whether or not the data
// item is authored — and after the sentinel, so an unavailable data set still reports the
// truthful no-comms.
// truthful no-comms, but BEFORE the empty-text test (#824).
if (observation.IsStructured)
{
return new DataValueSnapshot(null, BadNotSupported, sourceTimestamp, serverTimestamp);
@@ -464,6 +465,24 @@ internal sealed class MTConnectObservationIndex
_tags.TryGetValue(observation.DataItemId, out var tag);
// A genuinely EMPTY element (no text, or whitespace the parser trimmed away) is NOT the
// sentinel. Owner decision on lmxopcua#824 (Java parity): on a string-typed item it is a
// Good empty string — an Agent can legitimately report a blank Program/Message/Block.
// An unauthored item coerces as String (below), so it is served the same way.
// On any other type (numeric, Boolean, DateTime, any array) "" cannot be a value, so it
// stays Bad — and stays BadNoCommunication rather than BadTypeMismatch on purpose:
// the Agent supplied no value at all (the same meaning as UNAVAILABLE, just spelled
// degenerately), whereas BadTypeMismatch tells the operator the Agent sent text of the
// wrong type and points them at the tag's authored data type, which is not the fault.
// Checked AFTER the structured test above, so element-children with no text stay
// BadNotSupported rather than becoming a Good "".
if (string.IsNullOrWhiteSpace(observation.Value))
{
return tag is null or { IsArray: false, DriverDataType: DriverDataType.String or DriverDataType.Reference }
? new DataValueSnapshot(string.Empty, Good, sourceTimestamp, serverTimestamp)
: new DataValueSnapshot(null, BadNoCommunication, sourceTimestamp, serverTimestamp);
}
// TIME_SERIES vectors arrive as one whitespace-separated string. Checked AFTER the
// sentinel so an unavailable array tag still reports the truthful no-comms.
if (tag is { IsArray: true })
@@ -1,5 +1,4 @@
using System.Diagnostics;
using System.Net.Sockets;
using System.Text.Json;
using System.Text.Json.Serialization;
using TwinCAT.Ads;
@@ -8,59 +7,40 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT;
/// <summary>
/// Two-phase, degrade-guarded Test-Connect probe for the <see cref="TwinCATDriverOptions"/>-shaped
/// driver config. Phase 1: a bare TCP connect to the first device's AMS router host (first four
/// octets of the AMS Net ID) on the AMS port — fast rejection of unreachable targets. Phase 2:
/// a real ADS handshake — <c>AdsClient.Connect(netId, port)</c> + <c>ReadStateAsync</c> — to
/// confirm the endpoint speaks ADS and report the controller's run-state, not just that a TCP
/// socket opened.
/// Test-Connect probe for the <see cref="TwinCATDriverOptions"/>-shaped driver config. It takes
/// exactly the path the driver takes (<see cref="AdsTwinCATClient.ConnectAsync"/>):
/// <c>AdsClient.Connect(netId, port)</c> through the <b>local AMS router</b> (TwinCAT XAR, or
/// <c>Beckhoff.TwinCAT.Ads.TcpRouter</c> hosted in-process), followed by one read-only
/// <c>ReadStateAsync</c>, all inside the caller's time budget.
/// <para>
/// <b>Outcome classification</b> — three cases:
/// <list type="number">
/// <item>
/// <b>ADS connected + ReadState OK</b> → <c>Ok=true</c>, message <c>"ADS state: {AdsState}"</c>
/// (e.g. "Run" / "Config" / "Stop"), with latency.
/// </item>
/// <item>
/// <b>Route/auth rejection from a reachable router</b> — an <see cref="AdsErrorException"/>
/// (or a non-success <c>ReadStateAsync</c> result) whose <see cref="AdsErrorCode"/> means the
/// router answered but won't let us in (e.g. <see cref="AdsErrorCode.TargetPortNotFound"/>,
/// <see cref="AdsErrorCode.TargetMachineNotFound"/>, <see cref="AdsErrorCode.PortNotConnected"/>,
/// <see cref="AdsErrorCode.PortDisabled"/>) → <c>Ok=false</c>, message
/// <c>"Reachable at {host}:{port} but ADS handshake failed: {code} — check the target's ADS
/// route table authorizes this host"</c>. This is a TRUE red: the driver itself also needs
/// the route, so a green tick here would be a false positive.
/// </item>
/// <item>
/// <b>Handshake could not be ATTEMPTED on this host</b> — the managed AMS router cannot run
/// headless (Beckhoff's <c>AdsClient.Connect</c> throws a server exception
/// "Check for a running TwinCAT router instance!"), or a <see cref="PlatformNotSupportedException"/>
/// / <see cref="TypeInitializationException"/> / <see cref="DllNotFoundException"/> /
/// <see cref="NotSupportedException"/> surfaces, or <c>ReadStateAsync</c> reports a client-side
/// port-not-open status → <b>DEGRADE</b>: <c>Ok=true</c>, message
/// <c>"Reachable at {host}:{port} (ADS handshake unavailable on this host — TCP reachability
/// only)"</c>, with latency. The probe NEVER produces a result worse than the old TCP-only
/// probe.
/// </item>
/// <b>No TCP pre-check (#863).</b> The driver has no IP field: an AMS Net ID is a Beckhoff
/// identifier that the router maps to a host through its route table, and the AMS port
/// (801/851/…) is not a TCP port. The probe therefore never opens a socket of its own, and never
/// dials anything derived from the AMS Net ID. The former first-phase check dialled the first four
/// Net ID octets as an IPv4 address on the AMS port — a host unrelated to the target (often a
/// public internet address) on a port ADS does not use — so Test Connect timed out against every
/// real runtime.
/// </para>
/// <para>
/// <b>Outcome classification</b> — <c>Ok=true</c> only when the target answered:
/// <list type="bullet">
/// <item><b>ReadState OK</b> → <c>Ok=true</c>, <c>"ADS state: {AdsState}"</c>, with latency.</item>
/// <item><b>No local AMS router</b> — <c>AdsClient.Connect</c> cannot reach a router on this
/// host (Beckhoff's <c>AdsServerException</c> "Check for a running TwinCAT router
/// instance!"), the client port never opened, or the ADS client cannot load here →
/// <c>Ok=false</c>, "No local AMS router on this host …". The target was not contacted.</item>
/// <item><b>No route / target unreachable</b> — the router answered but has no route to the Net
/// ID, the target refused the route or port, or the router's own connection to the target
/// failed → <c>Ok=false</c>, "No route to … or target unreachable: {code} …".</item>
/// <item><b>Timeout</b> — a sync/device timeout code, or the caller's budget expired →
/// <c>Ok=false</c>, "Probe timed out after {n}s …".</item>
/// <item>Any other ADS error → <c>Ok=false</c>, "ADS handshake with … failed: {code}".</item>
/// </list>
/// </para>
/// </summary>
/// <remarks>
/// The line between case 2 (device-rejected → RED) and case 3 (can't-attempt → DEGRADE) is the
/// crux. Classification rests on the exception's identity: only an <see cref="AdsErrorException"/>
/// (or a result <see cref="AdsErrorCode"/>) carrying a route/target-port code is a RED — that is the
/// ADS router answering and refusing the route. Beckhoff's <c>TwinCAT.Ads.Server.AdsServerException</c>
/// ("running TwinCAT router instance!") derives from plain <see cref="Exception"/>, NOT from
/// <see cref="AdsErrorException"/>, so it is correctly classified as "can't attempt → DEGRADE".
/// When genuinely ambiguous, the probe DEGRADES (Ok=true, TCP-only note) rather than emit a false RED.
/// <para>
/// AMS Net ID format is six dot-separated octets (e.g. <c>192.168.1.10.1.1</c>); the first four
/// are typically the host IPv4 address by Beckhoff convention. The AMS router resolves the real IP
/// route server-side; the probe uses the first-four-octet heuristic for the TCP preflight target,
/// which is reliable for the overwhelming majority of production deployments.
/// </para>
/// The probe is read-only — <c>ReadStateAsync</c> never mutates PLC state — and always disposes the
/// <see cref="AdsClient"/>.
/// The probe is read-only — <c>ReadStateAsync</c> never mutates PLC state, and the probe sends no
/// AddRoute — and always disposes the <see cref="AdsClient"/>.
/// </remarks>
public sealed class TwinCATDriverProbe : IDriverProbe
{
@@ -71,22 +51,57 @@ public sealed class TwinCATDriverProbe : IDriverProbe
Converters = { new JsonStringEnumConverter() },
};
/// <summary>
/// AMS error codes that mean "the router answered but refused the route / target port" — a
/// genuine RED. The driver itself would also be denied, so a green tick would be a false
/// positive. Everything NOT in this set (client-side port/connection errors, sync timeouts,
/// router-not-initialised, etc.) is treated as "couldn't attempt the handshake" → DEGRADE.
/// </summary>
private static readonly HashSet<AdsErrorCode> _routeRejectCodes =
/// <summary>Codes meaning the router on this host is absent, inactive or never gave us a port.</summary>
private static readonly HashSet<AdsErrorCode> _noRouterCodes =
[
AdsErrorCode.ClientPortNotOpen,
AdsErrorCode.ClientNoAmsAddr,
AdsErrorCode.RouterNotInitialized,
AdsErrorCode.RouterNotActive,
];
/// <summary>
/// Codes meaning the router answered but cannot get us to the target: no route, the route or
/// port was refused, or the router's connection to the target host failed.
/// </summary>
private static readonly HashSet<AdsErrorCode> _noRouteCodes =
[
AdsErrorCode.TargetPortNotFound,
AdsErrorCode.TargetMachineNotFound,
AdsErrorCode.TargetPortNotFound,
AdsErrorCode.PortNotConnected,
AdsErrorCode.PortDisabled,
AdsErrorCode.AccessDenied,
AdsErrorCode.DeviceAccessDenied,
AdsErrorCode.HostUnreachable,
AdsErrorCode.WSA_NetworkUnreachable,
AdsErrorCode.WSA_HostDown,
AdsErrorCode.WSA_HostUnreachable,
AdsErrorCode.WSA_HostNotFound,
];
/// <summary>Codes meaning a route exists but the target did not answer in time.</summary>
private static readonly HashSet<AdsErrorCode> _timeoutCodes =
[
AdsErrorCode.ClientSyncTimeOut,
AdsErrorCode.SyncTimeOut,
AdsErrorCode.DeviceTimeOut,
AdsErrorCode.WSA_TimedOut,
AdsErrorCode.RouterFragmentTimeout,
];
/// <summary>
/// The ADS handshake: connect through the local AMS router and read the target's state.
/// Throws <see cref="AdsErrorException"/> for a non-success ReadState result. Swappable for tests.
/// </summary>
private readonly Func<TwinCATAmsAddress, TimeSpan, CancellationToken, Task<AdsState>> _readState;
/// <summary>Initializes the probe with the real ADS handshake through the local AMS router.</summary>
public TwinCATDriverProbe() : this(ReadStateViaLocalRouterAsync) { }
/// <summary>Test seam — substitutes the ADS handshake.</summary>
internal TwinCATDriverProbe(Func<TwinCATAmsAddress, TimeSpan, CancellationToken, Task<AdsState>> readState)
=> _readState = readState;
/// <inheritdoc />
public string DriverType => "TwinCAT";
@@ -98,110 +113,90 @@ public sealed class TwinCATDriverProbe : IDriverProbe
catch (Exception ex) { return new(false, $"Config JSON is invalid: {ex.Message}", null); }
if (opts is null) return new(false, "Config JSON deserialized to null.", null);
var (host, port, parsed) = ExtractTarget(opts);
if (parsed is null || string.IsNullOrWhiteSpace(host) || port <= 0)
return new(false, "Config has no host/port to probe.", null);
var target = TwinCATAmsAddress.TryParse(opts.Devices.FirstOrDefault()?.HostAddress);
if (target is null)
return new(false, "Config has no host/port to probe (expected a device hostAddress ads://{netId}:{port}).", null);
// Phase 1: bare TCP preflight — fast rejection for unreachable hosts. Messages here are
// UNCHANGED from the original TCP-only probe.
var label = $"{target.NetId}:{target.Port}";
var timedOut = $"Probe timed out after {timeout.TotalSeconds:F0}s waiting for ADS state from {label}";
var sw = Stopwatch.StartNew();
try
{
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
await socket.ConnectAsync(host, port, ct);
}
catch (SocketException ex)
{
return new(false, $"Connect failed: {ex.SocketErrorCode}", null);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
return new(false, $"Probe timed out after {timeout.TotalSeconds:F0}s.", null);
}
catch (Exception ex)
{
return new(false, ex.Message, null);
}
// Phase 2: real ADS handshake. Connect + ReadStateAsync. The crux is the three-way
// classification of how this can fail — see the class-doc and ClassifyHandshakeFailure.
var degradeNote = $"Reachable at {host}:{port} (ADS handshake unavailable on this host — TCP reachability only)";
try
{
using var client = new AdsClient();
// Bound the ADS round-trip by the caller's timeout (clamped >=1s, mirrors AdsTwinCATClient).
client.Timeout = (int)Math.Max(1_000, timeout.TotalMilliseconds);
// Connect can throw a server exception ("running TwinCAT router instance!") on a headless
// host with no AMS router — that is a can't-attempt DEGRADE, classified below.
client.Connect(AmsNetId.Parse(parsed.NetId), parsed.Port);
var state = await client.ReadStateAsync(ct).ConfigureAwait(false);
// Task.Run + WaitAsync keep the whole handshake — including the synchronous
// AdsClient.Connect — inside the caller's budget.
var state = await Task.Run(() => _readState(target, timeout, ct), ct)
.WaitAsync(timeout, ct).ConfigureAwait(false);
sw.Stop();
if (state.Succeeded)
return new(true, $"ADS state: {state.State.AdsState}", sw.Elapsed);
// Non-throwing failure carried in the result's error code.
return state.ErrorCode == AdsErrorCode.ClientPortNotOpen
? new(true, degradeNote, sw.Elapsed) // client never opened — DEGRADE
: ClassifyHandshakeFailure(state.ErrorCode, host, port, sw, degradeNote);
return new(true, $"ADS state: {state}", sw.Elapsed);
}
catch (TimeoutException)
{
return new(false, $"{timedOut} — the target did not answer.", null);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
// Caller timeout — keep the original timed-out message.
return new(false, $"Probe timed out after {timeout.TotalSeconds:F0}s.", null);
return new(false, $"{timedOut}.", null);
}
catch (AdsErrorException ex)
{
// The router answered with an ADS-level error. Route/auth rejection → RED; anything
// else (sync timeout, client port issues, …) → DEGRADE.
return ClassifyHandshakeFailure(ex.ErrorCode, host, port, sw, degradeNote);
return Classify(ex.ErrorCode, label, timedOut);
}
catch (Exception)
catch (Exception ex) when (IsNoRouter(ex))
{
// Everything else — TwinCAT.Ads.Server.AdsServerException ("running TwinCAT router
// instance!"), PlatformNotSupportedException, TypeInitializationException,
// DllNotFoundException, NotSupportedException, etc. — means the handshake could not be
// ATTEMPTED on this host. DEGRADE: never worse than the TCP-only probe.
sw.Stop();
return new(true, degradeNote, sw.Elapsed);
return new(false, NoRouterMessage(ex.Message), null);
}
catch (Exception ex)
{
return new(false, $"ADS handshake with {label} failed: {ex.Message}", null);
}
}
/// <summary>Default handshake — the same calls <see cref="AdsTwinCATClient.ConnectAsync"/> makes.</summary>
private static async Task<AdsState> ReadStateViaLocalRouterAsync(
TwinCATAmsAddress target, TimeSpan timeout, CancellationToken ct)
{
using var client = new AdsClient();
client.Timeout = (int)Math.Max(1_000, timeout.TotalMilliseconds);
client.Connect(AmsNetId.Parse(target.NetId), target.Port);
var state = await client.ReadStateAsync(ct).ConfigureAwait(false);
if (!state.Succeeded)
throw new AdsErrorException($"ReadState failed: {state.ErrorCode}", state.ErrorCode);
return state.State.AdsState;
}
/// <summary>Maps an ADS error code to one of the probe's failure classes.</summary>
internal static DriverProbeResult Classify(AdsErrorCode code, string label, string timedOut)
{
if (_noRouterCodes.Contains(code))
return new(false, NoRouterMessage(code.ToString()), null);
if (_noRouteCodes.Contains(code))
return new(false,
$"No route to {label} or target unreachable: {code} — check the local AMS router has a route to this AMS Net ID and the target's route table authorizes this host.",
null);
if (_timeoutCodes.Contains(code))
return new(false, $"{timedOut} ({code}) — the router has a route but the target did not answer.", null);
return new(false, $"ADS handshake with {label} failed: {code}", null);
}
private static string NoRouterMessage(string detail) =>
$"No local AMS router on this host — start TwinCAT XAR or host the Beckhoff TcpRouter; the target was not contacted ({detail}).";
/// <summary>
/// Classifies an ADS-level failure (from an <see cref="AdsErrorException"/> or a non-success
/// <c>ReadStateAsync</c> result). A route/target-port/access code means the router answered
/// but refused the route → RED. Any other code is treated as "couldn't attempt" → DEGRADE,
/// so the probe never under-reports a host with no usable ADS runtime.
/// True when the ADS client could not reach a router on this host, or cannot load here at all.
/// Beckhoff's <c>AdsServerException</c> / <c>AmsServerException</c> ("Check for a running TwinCAT
/// router instance!") derive from plain <see cref="Exception"/>, so they are matched by type name.
/// </summary>
private static DriverProbeResult ClassifyHandshakeFailure(
AdsErrorCode code, string host, int port, Stopwatch sw, string degradeNote)
private static bool IsNoRouter(Exception ex)
{
if (_routeRejectCodes.Contains(code))
return new(false,
$"Reachable at {host}:{port} but ADS handshake failed: {code} — check the target's ADS route table authorizes this host",
null);
sw.Stop();
return new(true, degradeNote, sw.Elapsed);
}
private static (string host, int port, TwinCATAmsAddress? parsed) ExtractTarget(TwinCATDriverOptions opts)
{
// Parse the first device's ads:// address. AMS Net ID is six-octet; by Beckhoff
// convention the first four octets are the host IPv4. Extract those as the TCP target.
var firstDevice = opts.Devices.FirstOrDefault();
if (firstDevice is null) return (string.Empty, 0, null);
var parsed = TwinCATAmsAddress.TryParse(firstDevice.HostAddress);
if (parsed is null) return (string.Empty, 0, null);
// NetId = "a.b.c.d.e.f" — take the first 4 octets as the host IP.
var parts = parsed.NetId.Split('.');
if (parts.Length < 4) return (string.Empty, 0, null);
var hostIp = string.Join('.', parts[0], parts[1], parts[2], parts[3]);
return (hostIp, parsed.Port, parsed);
for (var e = ex; e is not null; e = e.InnerException)
{
if (e is PlatformNotSupportedException or TypeInitializationException or DllNotFoundException)
return true;
var name = e.GetType().FullName;
if (name is "TwinCAT.Ads.Server.AdsServerException" or "TwinCAT.Ams.AmsServerException")
return true;
}
return false;
}
}