5a7a08deca
DriverInstanceActor.DisconnectObserved had no production sender. The only paths into Reconnecting were a failed *initial* connect and an operator's Force Reconnect, so a driver that connected and then lost its device stayed Connected indefinitely: publishing Bad for every tag, never re-initialising, and NOT recovering when the device returned. PublishHealthSnapshot read GetHealth() for reporting only and never acted on a driver reporting itself unhealthy. Both comms-loss quality paths -- #477's native-condition annotation and #481's scripted-alarm input fan-out -- hang off ConnectivityChanged, so both were correctly wired to a signal that never fired. The Connected health tick now evaluates a watchdog and self-sends DisconnectObserved when the driver reports a non-Healthy state AND its LastSuccessfulRead has been frozen for DefaultCommsLossTimeout (90s). The conjunction is the design, and the issue's own proposal -- "non-Healthy for N consecutive polls" -- is wrong. Degraded means *connected but some data is bad*; several drivers write health per tag inside the read loop (ModbusDriver.ReadAsync), so one unreadable address latches Degraded forever against a perfectly reachable device, and a state-only watchdog would reconnect that driver every 90s indefinitely -- worse than the bug. LastSuccessfulRead is the honest evidence: preserved across Degraded/Faulted writes, advanced by any success. The state check remains the second conjunct so an idle Healthy driver with nothing subscribed is not declared dead for not reading. Verified across all 12 drivers that a sustained failure does reach a non-Healthy state -- nine poll drivers via Degraded, MQTT via live connection state, Galaxy via its reconnect supervisor. CalculationDriver reports Healthy while running and so can never trip, which is correct: it has no transport. Also: the Reconnecting retry now calls ShutdownAsync before InitializeAsync. ModbusDriver (_transport) and MqttDriver (_connection) overwrite an IAsyncDisposable field inside InitializeAsync without disposing the previous one, so re-initialising without a teardown leaks a socket per attempt -- latent while only a manual Force Reconnect reached this path, but the watchdog makes it automatic and unbounded (~360 leaks for an hour-long outage at the 10s retry). Tests: 8 new in DriverInstanceActorCommsLossWatchdogTests, each absence assertion carrying a positive control (a proven count of processed health-poll ticks). Falsified five ways -- dropping either conjunct reds exactly the test written for it, and removing the teardown, the null baseline, or the watchdog call each reds its own guard. Live gate PASSED on docker-dev (Modbus + otopcua-pymodbus-standard), run twice: - fixture stopped -> watchdog fires at ~108s, "declaring comms lost" - fan-out observed at Debug: "pushed BadNoCommunication for 2 mux ref(s)" (#481 Layer 4) and the #477 native-condition annotation on a driver owning conditions - fixture restarted -> reconnected in ~8s with NO node restart; both the raw (ns=2) and UNS (ns=3) NodeIds read Good with fresh timestamps Claude-Session: https://claude.ai/code/session_015p7wGqy3YpZNCpDzTpGMKo
191 lines
9.7 KiB
C#
191 lines
9.7 KiB
C#
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Shared <see cref="IDriver"/> stub harness used by <c>DriverInstanceActorTests</c> and
|
|
/// <c>DriverInstanceActorWriteAndSubscribeTests</c>. Promoted from the superset copy in
|
|
/// <c>DriverInstanceActorTests</c> so both suites compile against a single definition.
|
|
/// </summary>
|
|
internal class StubDriver : IDriver
|
|
{
|
|
/// <summary>Gets or sets a value indicating whether initialization should throw.</summary>
|
|
public bool InitializeShouldThrow { get; set; }
|
|
/// <summary>Gets the number of times initialization was called.</summary>
|
|
public int InitializeCount;
|
|
/// <summary>Gets the number of times reinitialization was called.</summary>
|
|
public int ReinitializeCount;
|
|
|
|
private readonly object _initConfigsLock = new();
|
|
/// <summary>Every config string passed to <see cref="InitializeAsync"/>, in call order.</summary>
|
|
public List<string> InitConfigs { get; } = new();
|
|
/// <summary>Optional per-config init behaviour. When set, it fully owns the init outcome for that
|
|
/// config (await/throw); <see cref="InitializeShouldThrow"/> is ignored. Null ⇒ legacy behaviour.</summary>
|
|
public Func<string, Task>? InitBehavior { get; set; }
|
|
|
|
/// <summary>Gets the driver instance ID.</summary>
|
|
public string DriverInstanceId => "stub-driver-1";
|
|
/// <summary>Gets the driver type.</summary>
|
|
public string DriverType => "Stub";
|
|
|
|
/// <summary>Initializes the driver with the specified configuration JSON.</summary>
|
|
/// <param name="driverConfigJson">The driver configuration JSON.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public virtual async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref InitializeCount);
|
|
lock (_initConfigsLock) InitConfigs.Add(driverConfigJson);
|
|
if (InitBehavior is not null) { await InitBehavior(driverConfigJson); return; }
|
|
if (InitializeShouldThrow) throw new InvalidOperationException("stub-init-fail");
|
|
}
|
|
|
|
/// <summary>Reinitializes the driver with the specified configuration JSON.</summary>
|
|
/// <param name="driverConfigJson">The driver configuration JSON.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref ReinitializeCount);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Number of <see cref="ShutdownAsync"/> calls — a child actor stopping raises it, so a test
|
|
/// can observe a driver child being torn down (e.g. by the #516 config-change respawn).</summary>
|
|
public int ShutdownCount;
|
|
|
|
/// <summary>Shuts down the driver.</summary>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public virtual Task ShutdownAsync(CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref ShutdownCount);
|
|
return Task.CompletedTask;
|
|
}
|
|
/// <summary>Gets the health status of the driver.</summary>
|
|
/// <remarks>
|
|
/// Virtual so the #527 comms-loss watchdog suite can drive health directly. Note the
|
|
/// <see cref="DateTime.UtcNow"/>: this stub is permanently Healthy with an always-fresh
|
|
/// <c>LastSuccessfulRead</c>, so the watchdog can never fire for it — which is why every existing
|
|
/// test built on <see cref="StubDriver"/> is unaffected by the watchdog, and equally why a watchdog
|
|
/// test built on it would be vacuous.
|
|
/// </remarks>
|
|
public virtual DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A <see cref="StubDriver"/> that also implements <see cref="IWritable"/>, recording every
|
|
/// write call and returning a configurable status code.
|
|
/// </summary>
|
|
internal sealed class WritableStubDriver : StubDriver, IWritable
|
|
{
|
|
/// <summary>Gets or sets the next status code to return from write operations.</summary>
|
|
public uint NextStatusCode { get; set; } = 0u;
|
|
/// <summary>Gets the list of write requests received.</summary>
|
|
public List<WriteRequest> Writes { get; } = new();
|
|
|
|
/// <summary>Writes the specified requests.</summary>
|
|
/// <param name="writes">The write requests.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task<IReadOnlyList<WriteResult>> WriteAsync(
|
|
IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
|
|
{
|
|
Writes.AddRange(writes);
|
|
IReadOnlyList<WriteResult> results = writes.Select(_ => new WriteResult(NextStatusCode)).ToList();
|
|
return Task.FromResult(results);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// A <see cref="StubDriver"/> that also implements <see cref="ISubscribable"/>, firing
|
|
/// <see cref="OnDataChange"/> on demand and counting subscribe calls.
|
|
/// </summary>
|
|
internal sealed class SubscribableStubDriver : StubDriver, ISubscribable
|
|
{
|
|
/// <summary>Occurs when data changes.</summary>
|
|
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
|
|
|
private readonly StubHandle _handle = new();
|
|
|
|
/// <summary>Gets the number of subscribers to OnDataChange.</summary>
|
|
public int OnDataChangeSubscriberCount => OnDataChange?.GetInvocationList().Length ?? 0;
|
|
|
|
/// <summary>Number of times <see cref="SubscribeAsync"/> was called (re-subscribe asserts).</summary>
|
|
public int SubscribeCount;
|
|
/// <summary>The reference set passed to the most recent <see cref="SubscribeAsync"/> call.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedRefs;
|
|
|
|
/// <summary>Number of times <see cref="UnsubscribeAsync"/> was called — the observable for a live
|
|
/// subscription being torn down (an EMPTY desired set drops the handle rather than re-subscribing).</summary>
|
|
public int UnsubscribeCount;
|
|
|
|
/// <summary>When true, <see cref="UnsubscribeAsync"/> genuinely yields (<c>await Task.Yield()</c>)
|
|
/// before completing, so a <c>ConfigureAwait(false)</c> continuation in the actor resumes off the
|
|
/// Akka ActorContext on a thread-pool thread — reproducing the no-ActorContext race that a
|
|
/// synchronously-completed stub task hides (the continuation otherwise runs inline).</summary>
|
|
public bool UnsubscribeYields { get; set; }
|
|
|
|
/// <summary>
|
|
/// How many of the first <see cref="SubscribeAsync"/> calls throw before one succeeds. Default 0
|
|
/// (always succeed) leaves existing behaviour untouched.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// This is how a test reaches the state the #488 reconcile exists for: the actor is Connected and
|
|
/// holds a desired ref set, but the subscribe that should have established the handle failed, so
|
|
/// <c>_subscriptionHandle</c> is null and nothing else will ever re-assert it — there is no
|
|
/// connect transition coming, and no deploy.
|
|
/// </remarks>
|
|
public int SubscribeFailuresBeforeSuccess { get; set; }
|
|
|
|
/// <summary>Subscribes to the specified full references.</summary>
|
|
/// <param name="fullReferences">The full references to subscribe to.</param>
|
|
/// <param name="publishingInterval">The publishing interval.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task<ISubscriptionHandle> SubscribeAsync(
|
|
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken)
|
|
{
|
|
var attempt = Interlocked.Increment(ref SubscribeCount);
|
|
LastSubscribedRefs = fullReferences;
|
|
if (attempt <= SubscribeFailuresBeforeSuccess)
|
|
{
|
|
throw new InvalidOperationException($"stub subscribe failure #{attempt}");
|
|
}
|
|
|
|
return Task.FromResult<ISubscriptionHandle>(_handle);
|
|
}
|
|
|
|
/// <summary>Unsubscribes from the specified subscription handle.</summary>
|
|
/// <param name="handle">The subscription handle.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public async Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref UnsubscribeCount);
|
|
if (UnsubscribeYields)
|
|
{
|
|
// Complete the awaited task from a fresh background thread that has NO Akka actor
|
|
// cell on it, so the caller's `ConfigureAwait(false)` continuation resumes on a
|
|
// clean thread-pool thread where InternalCurrentActorCellKeeper.Current is null —
|
|
// a deterministic repro of the real async-backend no-ActorContext race.
|
|
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
_ = Task.Run(() => tcs.SetResult());
|
|
await tcs.Task.ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>Fires a data change event with the specified parameters.</summary>
|
|
/// <param name="fullRef">The full reference of the data that changed.</param>
|
|
/// <param name="value">The new value.</param>
|
|
/// <param name="statusCode">The OPC UA status code.</param>
|
|
public void FireDataChange(string fullRef, object? value, uint statusCode)
|
|
{
|
|
var snapshot = new DataValueSnapshot(value, statusCode, DateTime.UtcNow, DateTime.UtcNow);
|
|
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, snapshot));
|
|
}
|
|
}
|
|
|
|
/// <summary>Minimal <see cref="ISubscriptionHandle"/> for use by <see cref="SubscribableStubDriver"/>.</summary>
|
|
internal sealed class StubHandle : ISubscriptionHandle
|
|
{
|
|
/// <summary>Gets the diagnostic ID of the subscription.</summary>
|
|
public string DiagnosticId => "stub-sub";
|
|
}
|