using System.Collections.Concurrent;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Health;
///
/// Subscribes the ScanState attribute of every $WinPlatform /
/// $AppEngine object the discoverer surfaced and translates ScanState
/// value-changes into per-host updates.
/// Ports the state machine in
/// Driver.Galaxy.Host/Backend/Stability/GalaxyRuntimeProbeManager.cs onto the
/// gateway subscription path.
///
///
/// Address grammar: each platform tag's probe address is
/// {platformTagName}.ScanState. The watcher subscribes that address through
/// ; the EventPump (PR 4.4) routes inbound
/// OnDataChange events back via . State decoding:
///
/// - Quality < 192 (Good) → .
/// - Value 1, true, or "Running" → .
/// - Value 0, false, or "Stopped" → .
/// - Anything else with Good quality → .
///
/// is idempotent — call it after every
/// Discover / Rediscover. Newly-added platforms are subscribed; removed ones are
/// unsubscribed and dropped from the aggregator.
///
public sealed class PerPlatformProbeWatcher : IDisposable
{
public const string ProbeSuffix = ".ScanState";
private readonly IGalaxySubscriber _subscriber;
private readonly HostStatusAggregator _aggregator;
private readonly ILogger _logger;
private readonly int _bufferedUpdateIntervalMs;
// The driver's fan-out registry. The EventPump resolves every inbound event through it, so
// the probe handles must be registered there (under SubscriptionRegistry.ProbeSubscriptionId)
// or every ScanState event is dropped as "stale after unsubscribe". Null when the watcher is
// driven directly (unit tests of the decoding table).
private readonly SubscriptionRegistry? _registry;
// #865: invoked before every probe SubscribeBulk so the driver's event stream is attached when the
// gateway pushes the initial ScanState value (a fresh StreamEvents does not replay earlier events,
// and ScanState rarely changes — a lost initial value leaves the host missing indefinitely).
private readonly Func? _ensureEventStream;
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
// Each handle carries the registry session that issued it (#859 I-A): item handles are per gateway
// session and the next session reuses the numbers, so an old-session handle must never be published,
// unsubscribed or purged against the current session.
private readonly ConcurrentDictionary _itemHandlesByPlatform =
new(StringComparer.OrdinalIgnoreCase);
private readonly Lock _syncLock = new();
// Written under _syncLock by StopAsync; read under it wherever a write-back must not outlive a stop.
private volatile bool _disposed;
/// Initializes a new instance of the PerPlatformProbeWatcher class.
/// The Galaxy subscriber for managing probe subscriptions.
/// The host status aggregator for tracking platform connectivity.
/// Optional logger for diagnostic messages.
/// Buffered update interval in milliseconds; must be >= 0.
public PerPlatformProbeWatcher(
IGalaxySubscriber subscriber,
HostStatusAggregator aggregator,
ILogger? logger = null,
int bufferedUpdateIntervalMs = 0)
: this(subscriber, aggregator, registry: null, logger, bufferedUpdateIntervalMs)
{
}
///
/// Driver-side ctor: the watcher publishes its probe handles into
/// under so the
/// fans ScanState events out to .
///
/// The Galaxy subscriber for managing probe subscriptions.
/// The host status aggregator for tracking platform connectivity.
/// The driver's subscription registry; null to keep the handles private.
/// Optional logger for diagnostic messages.
/// Buffered update interval in milliseconds; must be >= 0.
///
/// Optional: awaited before every probe SubscribeBulk to make sure the event stream that carries
/// the initial ScanState value is running and requested (the driver starts its
/// and waits for its StreamEvents call to be issued).
///
internal PerPlatformProbeWatcher(
IGalaxySubscriber subscriber,
HostStatusAggregator aggregator,
SubscriptionRegistry? registry,
ILogger? logger = null,
int bufferedUpdateIntervalMs = 0,
Func? ensureEventStream = null)
{
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
_aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator));
_registry = registry;
_ensureEventStream = ensureEventStream;
_logger = logger ?? NullLogger.Instance;
if (bufferedUpdateIntervalMs < 0)
{
throw new ArgumentOutOfRangeException(nameof(bufferedUpdateIntervalMs),
"bufferedUpdateIntervalMs must be >= 0; 0 means use the gw's default cadence.");
}
_bufferedUpdateIntervalMs = bufferedUpdateIntervalMs;
}
/// Current platform → probe-handle bindings, as the registry sees them. Test seam.
internal IReadOnlyList CurrentBindings => BuildBindings();
private IReadOnlyList BuildBindings() =>
[.. _itemHandlesByPlatform.Select(kvp => new TagBinding(kvp.Key + ProbeSuffix, kvp.Value.ItemHandle))];
/// The bindings to publish for : another session's handles bind as 0 (no events).
private IReadOnlyList BuildBindingsFor(int session) =>
[.. _itemHandlesByPlatform.Select(kvp => new TagBinding(
kvp.Key + ProbeSuffix, kvp.Value.Session == session ? kvp.Value.ItemHandle : 0))];
private int CurrentSession => _registry?.CurrentSession ?? 0;
///
/// Mirror the tracked handle set into the driver registry (no-op without one, or once stopped —
/// a stopped watcher must never write its entry back into a registry the teardown cleared).
///
private void PublishBindings()
{
lock (_syncLock) PublishBindingsLocked(CurrentSession);
}
/// False when the registry's session is no longer (nothing published).
private bool PublishBindingsLocked(int session)
{
if (_registry is null || _disposed) return true;
if (_itemHandlesByPlatform.IsEmpty)
{
_registry.Remove(SubscriptionRegistry.ProbeSubscriptionId);
return true;
}
// Fenced (#859 I-A): only handles issued by `session` are bound, and only while it is current.
return _registry.TryUpsert(SubscriptionRegistry.ProbeSubscriptionId, BuildBindingsFor(session), session);
}
///
/// Re-issue SubscribeBulk for every tracked platform and replace the handles — the
/// reconnect replay path, where the gateway session was recreated and every pre-reconnect
/// handle is dead. Platforms the gateway now rejects keep a 0 handle (no events expected).
///
/// A cancellation token to cancel the operation.
/// The number of probe references re-subscribed.
public async Task ResubscribeAsync(CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(_disposed, this);
var platforms = _itemHandlesByPlatform.Keys.ToArray();
if (platforms.Length == 0) return 0;
await SubscribeAndRecordAsync(platforms, cancellationToken).ConfigureAwait(false);
return platforms.Length;
}
///
/// The reconnect replay: re-issue SubscribeBulk only for platforms whose handle predates the
/// registry's current session, then republish. A platform a seed already advised on the new
/// session is left alone (re-subscribing it would double-advise and leak a handle), and every
/// reconnect calls this regardless of the shared registry entry's session — that entry is
/// re-stamped by any probe publish, so it cannot say which platforms are stale (#859).
///
/// A cancellation token to cancel the operation.
/// The number of probe references re-subscribed.
public async Task ResubscribeStaleAsync(CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(_disposed, this);
string[] stale;
lock (_syncLock)
{
var session = CurrentSession;
stale = [.. _itemHandlesByPlatform.Where(kvp => kvp.Value.Session != session).Select(kvp => kvp.Key)];
}
if (stale.Length == 0) return 0;
await SubscribeAndRecordAsync(stale, cancellationToken).ConfigureAwait(false);
return stale.Length;
}
private async Task SubscribeAndRecordAsync(IReadOnlyList platforms, CancellationToken cancellationToken)
{
// #861 / #859 I-A: a reply that straddled a gateway reconnect carries dead handles; re-issue.
for (var attempt = 1; ; attempt++)
{
if (await TrySubscribeAndRecordAsync(platforms, cancellationToken).ConfigureAwait(false)) return;
if (attempt >= MaxSessionAttempts)
{
_logger.LogWarning(
"PerPlatformProbeWatcher probe subscribe straddled {Attempts} gateway reconnects — giving up until the next replay/seed", attempt);
return;
}
}
}
private const int MaxSessionAttempts = 3;
/// False when the reply straddled a session change (nothing recorded; retry).
private async Task TrySubscribeAndRecordAsync(IReadOnlyList platforms, CancellationToken cancellationToken)
{
var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
// #865: the stream must be live BEFORE the advise — the initial value is the probe's only
// event until the platform's scan state changes, and an unattached stream never sees it.
if (_ensureEventStream is not null) await _ensureEventStream(cancellationToken).ConfigureAwait(false);
// #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until
// PublishBindingsLocked registers the handles below (window closes at method exit).
using var earlyEventWindow = _registry?.BeginPendingSubscribe();
var session = CurrentSession; // the session this SubscribeBulk is issued on
// PR 6.3 — use the configured bufferedUpdateIntervalMs (defaults to 0 = gw cadence
// when the driver hasn't overridden MxAccess.PublishingIntervalMs). Probe ScanState
// changes are rare so a coarser interval is usually fine; deployments that need
// tighter health visibility can dial it down through GalaxyDriverOptions.
var results = await _subscriber.SubscribeBulkAsync(
probeAddresses, _bufferedUpdateIntervalMs, cancellationToken).ConfigureAwait(false);
var acquired = new (string Platform, int ItemHandle, string? Error)[platforms.Count];
for (var i = 0; i < platforms.Count; i++)
{
var match = results.FirstOrDefault(r => string.Equals(
r.TagAddress, probeAddresses[i], StringComparison.OrdinalIgnoreCase));
acquired[i] = (platforms[i], match is { WasSuccessful: true } ? match.ItemHandle : 0,
match?.ErrorMessage ?? "");
}
// Re-check the stop AFTER the await, atomically with the record + publish: a StopAsync that
// ran while SubscribeBulk was in flight already cleared the map and the registry entry, so
// recording now would resurrect the probe entry into the next runtime's registry.
bool stopped;
lock (_syncLock)
{
stopped = _disposed;
if (!stopped)
{
if (_registry is not null && _registry.CurrentSession != session)
{
// Straddled a reconnect: these handles died with the old session and their numbers may
// be reused — neither record, publish nor unsubscribe them.
return false;
}
foreach (var (platform, itemHandle, _) in acquired)
_itemHandlesByPlatform[platform] = new ProbeHandle(itemHandle, session);
if (!PublishBindingsLocked(session)) return false;
}
}
if (stopped)
{
var orphans = acquired.Where(a => a.ItemHandle > 0).Select(a => a.ItemHandle).ToArray();
if (orphans.Length > 0)
{
try { await _subscriber.UnsubscribeBulkAsync(orphans, CancellationToken.None).ConfigureAwait(false); }
catch (Exception ex)
{
_logger.LogWarning(ex,
"PerPlatformProbeWatcher could not release {Count} probe handle(s) acquired after stop", orphans.Length);
}
}
return true;
}
foreach (var (platform, itemHandle, error) in acquired)
{
if (itemHandle <= 0)
{
_logger.LogWarning("PerPlatformProbeWatcher subscribe failed for {Platform}: {Error}", platform, error);
}
}
return true;
}
///
/// Snapshot of platform tag names currently watched. Read under the sync lock so a platform
/// is never visible here before has published its probe
/// binding into the driver registry (the record and the publish are one locked step): a
/// caller that sees the platform and then raises a ScanState event must find it routable,
/// not dropped by the pump as "stale after unsubscribe" (#856).
///
public IReadOnlyCollection WatchedPlatforms
{
get { lock (_syncLock) return [.. _itemHandlesByPlatform.Keys]; }
}
///
/// Reconcile the watched platform set against .
/// Subscribes new entries, unsubscribes dropped ones. Calling with the same set is
/// a no-op.
///
/// The platform tag names to synchronize.
/// A cancellation token to cancel the operation.
/// A task that represents the asynchronous operation.
public async Task SyncPlatformsAsync(
IEnumerable platformTagNames, CancellationToken cancellationToken)
{
ObjectDisposedException.ThrowIf(_disposed, this);
ArgumentNullException.ThrowIfNull(platformTagNames);
var desired = new HashSet(platformTagNames, StringComparer.OrdinalIgnoreCase);
// Compute deltas under the lock so concurrent SyncPlatformsAsync calls don't
// race on the membership view.
List toAdd;
List<(string Platform, int ItemHandle)> toRemove;
int removeSession;
bool droppedWithoutRelease;
lock (_syncLock)
{
toAdd = [.. desired.Where(p => !_itemHandlesByPlatform.ContainsKey(p))];
// Captured before the RPC (#859 I-B); only handles issued by the CURRENT session are released —
// an older session's numbers may now belong to unrelated live items (#859 I-A).
removeSession = CurrentSession;
toRemove = [.. _itemHandlesByPlatform
.Where(kvp => !desired.Contains(kvp.Key)
&& kvp.Value.ItemHandle > 0 && kvp.Value.Session == removeSession)
.Select(kvp => (kvp.Key, kvp.Value.ItemHandle))];
var deadRemovals = _itemHandlesByPlatform
.Where(kvp => !desired.Contains(kvp.Key) && !(kvp.Value.ItemHandle > 0 && kvp.Value.Session == removeSession))
.Select(kvp => kvp.Key).ToArray();
// Dropped platforms with no releasable handle (rejected, or an older session's dead number):
// forget them without any unsubscribe.
foreach (var platform in deadRemovals)
{
_itemHandlesByPlatform.TryRemove(platform, out _);
_aggregator.Remove(platform);
}
droppedWithoutRelease = deadRemovals.Length > 0;
// Drop removed entries from the membership map up-front so a concurrent
// OnProbeValueChanged for them is silently ignored. The unsubscribe RPC
// runs outside the lock.
foreach (var (platform, _) in toRemove)
{
_itemHandlesByPlatform.TryRemove(platform, out _);
_aggregator.Remove(platform);
}
}
if (droppedWithoutRelease && toRemove.Count == 0) PublishBindings(); // keep the fan-out map in step
if (toRemove.Count > 0)
{
// Drop the dead handles from the fan-out map before the (slow, fallible) unsubscribe.
PublishBindings();
try
{
await _subscriber.UnsubscribeBulkAsync(
[.. toRemove.Select(t => t.ItemHandle)], cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"PerPlatformProbeWatcher unsubscribe failed for {Count} probe(s); aggregator entries already cleared.",
toRemove.Count);
}
finally
{
// #859 I-3 interim: no trailing event for a removed probe handle may be promoted onto
// a reused handle number.
_registry?.PurgeParked([.. toRemove.Select(t => t.ItemHandle)], removeSession);
}
}
if (toAdd.Count == 0) return;
await SubscribeAndRecordAsync(toAdd, cancellationToken).ConfigureAwait(false);
}
///
/// Route an OnDataChange for a probe address into the aggregator. The EventPump
/// (PR 4.4) calls this; tests can drive it directly to exercise the state machine
/// without spinning a real gw. Foreign references (anything not ending in
/// , or a probe for a platform we're not tracking) are
/// silently dropped.
///
/// The full reference path of the probe attribute.
/// The probe value to decode.
/// The quality byte for the value.
public void OnProbeValueChanged(string fullReference, object? value, byte qualityByte)
{
if (_disposed) return;
ArgumentNullException.ThrowIfNull(fullReference);
if (!fullReference.EndsWith(ProbeSuffix, StringComparison.OrdinalIgnoreCase)) return;
var platform = fullReference[..^ProbeSuffix.Length];
if (!_itemHandlesByPlatform.ContainsKey(platform)) return;
var state = DecodeState(value, qualityByte);
_aggregator.Update(new HostConnectivityStatus(platform, state, DateTime.UtcNow));
}
///
/// Decode a ScanState value + raw quality byte to a .
/// Public for tests that want to pin the decoding table.
///
/// The probe value to decode.
/// The quality byte for the value.
/// The decoded host state.
public static HostState DecodeState(object? value, byte qualityByte)
{
if (qualityByte < 192) return HostState.Unknown;
return value switch
{
bool b => b ? HostState.Running : HostState.Stopped,
int i => i == 1 ? HostState.Running : i == 0 ? HostState.Stopped : HostState.Faulted,
short s => s == 1 ? HostState.Running : s == 0 ? HostState.Stopped : HostState.Faulted,
long l => l == 1 ? HostState.Running : l == 0 ? HostState.Stopped : HostState.Faulted,
string str when string.Equals(str, "Running", StringComparison.OrdinalIgnoreCase) => HostState.Running,
string str when string.Equals(str, "Stopped", StringComparison.OrdinalIgnoreCase) => HostState.Stopped,
_ => HostState.Faulted,
};
}
///
/// Stops the watcher and unsubscribes every tracked platform's probe handle on the gateway
/// (best-effort — a transport error is logged and swallowed). The async counterpart of
/// ; the driver's ShutdownAsync uses it so the unsubscribe
/// round-trip is awaited rather than blocked on. Idempotent.
///
/// Bounds the unsubscribe round-trip.
/// A task that completes once the handles have been released (or the attempt gave up).
public async Task StopAsync(CancellationToken cancellationToken)
{
int[] liveHandles;
lock (_syncLock)
{
if (_disposed) return;
_disposed = true;
var session = CurrentSession; // never release an older session's (reusable) numbers
liveHandles = [.. _itemHandlesByPlatform.Values
.Where(h => h.ItemHandle > 0 && h.Session == session).Select(h => h.ItemHandle)];
_itemHandlesByPlatform.Clear();
_registry?.Remove(SubscriptionRegistry.ProbeSubscriptionId);
}
if (liveHandles.Length > 0)
{
try { await _subscriber.UnsubscribeBulkAsync(liveHandles, cancellationToken).ConfigureAwait(false); }
catch (Exception ex) { _logger.LogWarning(ex, "PerPlatformProbeWatcher stop unsubscribe failed"); }
}
}
/// Disposes the probe watcher and unsubscribes all tracked platforms (blocking on the round-trip).
public void Dispose() => StopAsync(CancellationToken.None).GetAwaiter().GetResult();
/// A probe item handle and the registry session that issued it.
/// The gateway item handle (0 = rejected).
/// The registry session the handle belongs to.
private readonly record struct ProbeHandle(int ItemHandle, int Session);
}