5067509dd4
A data re-subscribe (every deploy's bulk SetDesiredSubscriptions push to a kept driver) dropped the prior data subscription through the full DetachSubscription, which also unhooked OnAlarmEvent and forgot the alarm handle. On any deploy that did not respawn the driver, the alarm subscription kept logging "established" while every event reached nobody until a reconnect or respawn. The data path now uses DetachDataSubscription; only a lost connection and PostStop detach both. The #488 health-poll reconcile only looked at data, so an alarm subscribe that failed on (re)connect gated the feed for the life of the connection. ReconcileAlarmSubscription now re-attaches a missing handler and re-subscribes a stale/missing alarm handle every tick. OpcUaClient removes a half-built alarm subscription from its session when SubscribeAlarmsAsync throws, so the new retries cannot accumulate. Tests: actor-level data-resubscribe + reconcile-retry tests and an end-to-end NativeAlarmFeedSurvivesDeployTests through DriverHostActor; all three fail against the unfixed actor. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
409 lines
20 KiB
C#
409 lines
20 KiB
C#
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Shared <see cref="IDriver"/> stub harness used by <c>DriverInstanceActorTests</c> and
|
|
/// <c>DriverInstanceActorWriteAndSubscribeTests</c>. Promoted from the superset copy in
|
|
/// <c>DriverInstanceActorTests</c> so both suites compile against a single definition.
|
|
/// </summary>
|
|
internal class StubDriver : IDriver
|
|
{
|
|
/// <summary>Gets or sets a value indicating whether initialization should throw.</summary>
|
|
public bool InitializeShouldThrow { get; set; }
|
|
/// <summary>Gets the number of times initialization was called.</summary>
|
|
public int InitializeCount;
|
|
/// <summary>Gets the number of times reinitialization was called.</summary>
|
|
public int ReinitializeCount;
|
|
|
|
private readonly object _initConfigsLock = new();
|
|
/// <summary>Every config string passed to <see cref="InitializeAsync"/>, in call order.</summary>
|
|
public List<string> InitConfigs { get; } = new();
|
|
/// <summary>Optional per-config init behaviour. When set, it fully owns the init outcome for that
|
|
/// config (await/throw); <see cref="InitializeShouldThrow"/> is ignored. Null ⇒ legacy behaviour.</summary>
|
|
public Func<string, Task>? InitBehavior { get; set; }
|
|
|
|
/// <summary>Gets the driver instance ID.</summary>
|
|
public string DriverInstanceId => "stub-driver-1";
|
|
/// <summary>Gets the driver type.</summary>
|
|
public string DriverType => "Stub";
|
|
|
|
/// <summary>Initializes the driver with the specified configuration JSON.</summary>
|
|
/// <param name="driverConfigJson">The driver configuration JSON.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public virtual async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref InitializeCount);
|
|
lock (_initConfigsLock) InitConfigs.Add(driverConfigJson);
|
|
if (InitBehavior is not null) { await InitBehavior(driverConfigJson); return; }
|
|
if (InitializeShouldThrow) throw new InvalidOperationException("stub-init-fail");
|
|
}
|
|
|
|
/// <summary>Reinitializes the driver with the specified configuration JSON.</summary>
|
|
/// <param name="driverConfigJson">The driver configuration JSON.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref ReinitializeCount);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Number of <see cref="ShutdownAsync"/> calls — a child actor stopping raises it, so a test
|
|
/// can observe a driver child being torn down (e.g. by the #516 config-change respawn).</summary>
|
|
public int ShutdownCount;
|
|
|
|
/// <summary>Shuts down the driver.</summary>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public virtual Task ShutdownAsync(CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref ShutdownCount);
|
|
return Task.CompletedTask;
|
|
}
|
|
/// <summary>Gets the health status of the driver.</summary>
|
|
/// <remarks>
|
|
/// Virtual so the #527 comms-loss watchdog suite can drive health directly. Note the
|
|
/// <see cref="DateTime.UtcNow"/>: this stub is permanently Healthy with an always-fresh
|
|
/// <c>LastSuccessfulRead</c>, so the watchdog can never fire for it — which is why every existing
|
|
/// test built on <see cref="StubDriver"/> is unaffected by the watchdog, and equally why a watchdog
|
|
/// test built on it would be vacuous.
|
|
/// </remarks>
|
|
public virtual DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A <see cref="StubDriver"/> that also implements <see cref="IWritable"/>, recording every
|
|
/// write call and returning a configurable status code.
|
|
/// </summary>
|
|
internal sealed class WritableStubDriver : StubDriver, IWritable
|
|
{
|
|
/// <summary>Gets or sets the next status code to return from write operations.</summary>
|
|
public uint NextStatusCode { get; set; } = 0u;
|
|
/// <summary>Gets the list of write requests received.</summary>
|
|
public List<WriteRequest> Writes { get; } = new();
|
|
|
|
/// <summary>Writes the specified requests.</summary>
|
|
/// <param name="writes">The write requests.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task<IReadOnlyList<WriteResult>> WriteAsync(
|
|
IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
|
|
{
|
|
Writes.AddRange(writes);
|
|
IReadOnlyList<WriteResult> results = writes.Select(_ => new WriteResult(NextStatusCode)).ToList();
|
|
return Task.FromResult(results);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// A <see cref="StubDriver"/> that also implements <see cref="ISubscribable"/>, firing
|
|
/// <see cref="OnDataChange"/> on demand and counting subscribe calls.
|
|
/// </summary>
|
|
internal sealed class SubscribableStubDriver : StubDriver, ISubscribable
|
|
{
|
|
/// <summary>Occurs when data changes.</summary>
|
|
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
|
|
|
private readonly StubHandle _handle = new();
|
|
|
|
/// <summary>Gets the number of subscribers to OnDataChange.</summary>
|
|
public int OnDataChangeSubscriberCount => OnDataChange?.GetInvocationList().Length ?? 0;
|
|
|
|
/// <summary>Number of times <see cref="SubscribeAsync"/> was called (re-subscribe asserts).</summary>
|
|
public int SubscribeCount;
|
|
/// <summary>The reference set passed to the most recent <see cref="SubscribeAsync"/> call.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedRefs;
|
|
|
|
/// <summary>Number of times <see cref="UnsubscribeAsync"/> was called — the observable for a live
|
|
/// subscription being torn down (an EMPTY desired set drops the handle rather than re-subscribing).</summary>
|
|
public int UnsubscribeCount;
|
|
|
|
/// <summary>When true, <see cref="UnsubscribeAsync"/> genuinely yields (<c>await Task.Yield()</c>)
|
|
/// before completing, so a <c>ConfigureAwait(false)</c> continuation in the actor resumes off the
|
|
/// Akka ActorContext on a thread-pool thread — reproducing the no-ActorContext race that a
|
|
/// synchronously-completed stub task hides (the continuation otherwise runs inline).</summary>
|
|
public bool UnsubscribeYields { get; set; }
|
|
|
|
/// <summary>
|
|
/// How many of the first <see cref="SubscribeAsync"/> calls throw before one succeeds. Default 0
|
|
/// (always succeed) leaves existing behaviour untouched.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// This is how a test reaches the state the #488 reconcile exists for: the actor is Connected and
|
|
/// holds a desired ref set, but the subscribe that should have established the handle failed, so
|
|
/// <c>_subscriptionHandle</c> is null and nothing else will ever re-assert it — there is no
|
|
/// connect transition coming, and no deploy.
|
|
/// </remarks>
|
|
public int SubscribeFailuresBeforeSuccess { get; set; }
|
|
|
|
/// <summary>Subscribes to the specified full references.</summary>
|
|
/// <param name="fullReferences">The full references to subscribe to.</param>
|
|
/// <param name="publishingInterval">The publishing interval.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task<ISubscriptionHandle> SubscribeAsync(
|
|
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken)
|
|
{
|
|
var attempt = Interlocked.Increment(ref SubscribeCount);
|
|
LastSubscribedRefs = fullReferences;
|
|
if (attempt <= SubscribeFailuresBeforeSuccess)
|
|
{
|
|
throw new InvalidOperationException($"stub subscribe failure #{attempt}");
|
|
}
|
|
|
|
return Task.FromResult<ISubscriptionHandle>(_handle);
|
|
}
|
|
|
|
/// <summary>Unsubscribes from the specified subscription handle.</summary>
|
|
/// <param name="handle">The subscription handle.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public async Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref UnsubscribeCount);
|
|
if (UnsubscribeYields)
|
|
{
|
|
// Complete the awaited task from a fresh background thread that has NO Akka actor
|
|
// cell on it, so the caller's `ConfigureAwait(false)` continuation resumes on a
|
|
// clean thread-pool thread where InternalCurrentActorCellKeeper.Current is null —
|
|
// a deterministic repro of the real async-backend no-ActorContext race.
|
|
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
_ = Task.Run(() => tcs.SetResult());
|
|
await tcs.Task.ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>Fires a data change event with the specified parameters.</summary>
|
|
/// <param name="fullRef">The full reference of the data that changed.</param>
|
|
/// <param name="value">The new value.</param>
|
|
/// <param name="statusCode">The OPC UA status code.</param>
|
|
public void FireDataChange(string fullRef, object? value, uint statusCode)
|
|
{
|
|
var snapshot = new DataValueSnapshot(value, statusCode, DateTime.UtcNow, DateTime.UtcNow);
|
|
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, snapshot));
|
|
}
|
|
}
|
|
|
|
/// <summary>Minimal <see cref="ISubscriptionHandle"/> for use by <see cref="SubscribableStubDriver"/>.</summary>
|
|
internal sealed class StubHandle : ISubscriptionHandle
|
|
{
|
|
/// <summary>Gets the diagnostic ID of the subscription.</summary>
|
|
public string DiagnosticId => "stub-sub";
|
|
}
|
|
|
|
/// <summary>
|
|
/// A per-instance recording driver for the restart / re-subscribe suite (03/S16, U6): unlike
|
|
/// <see cref="SubscribableStubDriver"/> it carries the <b>spec's</b> <see cref="DriverInstanceId"/>
|
|
/// (<see cref="StubDriver"/> hard-codes a constant on a non-virtual member), and it counts and records the
|
|
/// alarm subscription alongside the value one — the two travel in a single
|
|
/// <c>SetDesiredSubscriptions</c> but through two different child code paths, so both need observing.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// <b>The falsification bar these counters exist for:</b> a restart test must assert a FUNCTION the new
|
|
/// incarnation performs — it subscribed, it re-initialised — never that the incarnation exists.
|
|
/// Existence is what
|
|
/// <c>DriverHostActorReconcileTests.RestartDriver_respawns_the_child_without_an_actor_name_collision</c>
|
|
/// asserts, and that test passes against the very defect 03/S16 names.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b><see cref="FireMalformedDataChange"/> is the supervision crash seam</b> — see
|
|
/// <c>DriverHostActorRestartTests</c>'s file remarks for why the obvious alternatives do not work.
|
|
/// </para>
|
|
/// </remarks>
|
|
internal sealed class RecordingSubscribableDriver : IDriver, ISubscribable, IAlarmSource
|
|
{
|
|
private readonly Lock _gate = new();
|
|
private readonly StubHandle _handle = new();
|
|
private readonly RecordingAlarmHandle _alarmHandle = new();
|
|
private int _initializeCount;
|
|
private int _shutdownCount;
|
|
private int _subscribeCount;
|
|
private int _subscribeAlarmsCount;
|
|
private IReadOnlyList<string>? _lastSubscribedRefs;
|
|
private IReadOnlyList<string>? _lastSubscribedAlarmRefs;
|
|
|
|
/// <summary>Initializes a new recording driver stamped with the spec's identity.</summary>
|
|
/// <param name="driverInstanceId">The driver instance id from the spec that created it.</param>
|
|
/// <param name="driverType">The driver type from the spec that created it.</param>
|
|
public RecordingSubscribableDriver(string driverInstanceId, string driverType)
|
|
{
|
|
DriverInstanceId = driverInstanceId;
|
|
DriverType = driverType;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public string DriverInstanceId { get; }
|
|
|
|
/// <inheritdoc />
|
|
public string DriverType { get; }
|
|
|
|
/// <summary>Number of <see cref="InitializeAsync"/> calls. A supervision restart re-uses the SAME driver
|
|
/// object (the Props closure captured it), so this going 1 to 2 is how the reprovision is observed.</summary>
|
|
public int InitializeCount => Volatile.Read(ref _initializeCount);
|
|
|
|
/// <summary>Number of <see cref="ShutdownAsync"/> calls — a restart's <c>PostStop</c> raises it.</summary>
|
|
public int ShutdownCount => Volatile.Read(ref _shutdownCount);
|
|
|
|
/// <summary>Number of <see cref="SubscribeAsync"/> calls.</summary>
|
|
public int SubscribeCount => Volatile.Read(ref _subscribeCount);
|
|
|
|
/// <summary>Number of <see cref="SubscribeAlarmsAsync"/> calls.</summary>
|
|
public int SubscribeAlarmsCount => Volatile.Read(ref _subscribeAlarmsCount);
|
|
|
|
/// <summary>The refs passed to the most recent <see cref="SubscribeAsync"/>, or null if never called.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedRefs
|
|
{
|
|
get { lock (_gate) return _lastSubscribedRefs; }
|
|
}
|
|
|
|
/// <summary>The refs passed to the most recent <see cref="SubscribeAlarmsAsync"/>, or null if never called.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedAlarmRefs
|
|
{
|
|
get { lock (_gate) return _lastSubscribedAlarmRefs; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// When set, <see cref="InitializeAsync"/> awaits it instead of completing inline — so the actor
|
|
/// stays in <c>Connecting</c> for as long as the test wants. Needed because several defects only
|
|
/// bite a child that has NOT yet connected (<c>InitializeRequested</c> is unhandled in
|
|
/// <c>Connected</c>, so a test built on an already-connected child passes vacuously).
|
|
/// </summary>
|
|
public TaskCompletionSource? InitializeGate { get; set; }
|
|
|
|
/// <inheritdoc />
|
|
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _initializeCount);
|
|
if (InitializeGate is { } gate) await gate.Task.ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public Task ShutdownAsync(CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _shutdownCount);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
|
|
|
/// <inheritdoc />
|
|
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
|
|
|
/// <inheritdoc />
|
|
public Task<ISubscriptionHandle> SubscribeAsync(
|
|
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _subscribeCount);
|
|
lock (_gate) _lastSubscribedRefs = fullReferences.ToArray();
|
|
return Task.FromResult<ISubscriptionHandle>(_handle);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public event EventHandler<AlarmEventArgs>? OnAlarmEvent;
|
|
|
|
/// <inheritdoc />
|
|
public Task<IAlarmSubscriptionHandle> SubscribeAlarmsAsync(
|
|
IReadOnlyList<string> sourceNodeIds, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _subscribeAlarmsCount);
|
|
lock (_gate) _lastSubscribedAlarmRefs = sourceNodeIds.ToArray();
|
|
return Task.FromResult<IAlarmSubscriptionHandle>(_alarmHandle);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public Task AcknowledgeAsync(
|
|
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <summary>Fires a well-formed data change (the ordinary production path).</summary>
|
|
/// <param name="fullRef">The reference whose value changed.</param>
|
|
/// <param name="value">The new value.</param>
|
|
public void FireDataChange(string fullRef, object? value) =>
|
|
OnDataChange?.Invoke(this, new DataChangeEventArgs(
|
|
_handle, fullRef, new DataValueSnapshot(value, 0u, DateTime.UtcNow, DateTime.UtcNow)));
|
|
|
|
/// <summary>
|
|
/// Raises <see cref="OnDataChange"/> with a NULL <c>Snapshot</c> — the supervision crash seam.
|
|
/// <c>DataChangeEventArgs</c> is a positional record with no null validation, and the actor's
|
|
/// <c>OnDataChangeForward</c> dereferences <c>msg.Snapshot</c> unguarded ON THE ACTOR THREAD, so this
|
|
/// produces a genuine unhandled exception inside the child, which Akka's default decider turns into a
|
|
/// Restart. The fault itself is arbitrary; what is under test is the RECOVERY.
|
|
/// </summary>
|
|
/// <param name="fullRef">The reference to name in the malformed event.</param>
|
|
public void FireMalformedDataChange(string fullRef) =>
|
|
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, null!));
|
|
|
|
/// <summary>Raises <see cref="OnAlarmEvent"/>; present so the event is not merely written-to-never-read.</summary>
|
|
/// <param name="conditionId">The condition id to raise.</param>
|
|
/// <param name="kind">The transition kind to report.</param>
|
|
public void FireAlarm(string conditionId, AlarmTransitionKind kind = AlarmTransitionKind.Unspecified) =>
|
|
OnAlarmEvent?.Invoke(this, new AlarmEventArgs(
|
|
_alarmHandle, conditionId, conditionId, "AlarmCondition", "test", AlarmSeverity.High, DateTime.UtcNow,
|
|
Kind: kind));
|
|
|
|
/// <summary>Live subscribers on <see cref="OnAlarmEvent"/> — an alarm subscription with no handler
|
|
/// attached delivers nothing (#662), so this is the other half of "the feed is up".</summary>
|
|
public int AlarmSubscriberCount => OnAlarmEvent?.GetInvocationList().Length ?? 0;
|
|
|
|
private sealed class RecordingAlarmHandle : IAlarmSubscriptionHandle
|
|
{
|
|
/// <inheritdoc />
|
|
public string DiagnosticId => "recording-alarm-sub";
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// An <see cref="IDriverFactory"/> that hands out <see cref="RecordingSubscribableDriver"/>s and KEEPS
|
|
/// every one it created, in creation order. The list — not a single "the driver" reference — is the point:
|
|
/// an operator Restart tears the child down and spawns a new one around a NEW driver object, so the
|
|
/// assertion subject is <c>Created[1]</c>, while a SUPERVISION restart re-uses <c>Created[0]</c> (the Props
|
|
/// closure captured it) and part of the assertion is that <c>Created.Count</c> stayed 1.
|
|
/// </summary>
|
|
internal sealed class RecordingSubscribableDriverFactory : IDriverFactory
|
|
{
|
|
private readonly string _supportedType;
|
|
private readonly Lock _gate = new();
|
|
private readonly List<RecordingSubscribableDriver> _created = new();
|
|
|
|
private readonly bool _gateInitialize;
|
|
|
|
/// <summary>Initializes a factory that materialises exactly one driver type.</summary>
|
|
/// <param name="supportedType">The driver type this factory answers for.</param>
|
|
/// <param name="gateInitialize">When true, every created driver gets an un-completed
|
|
/// <see cref="RecordingSubscribableDriver.InitializeGate"/>, so its child actor stays in
|
|
/// <c>Connecting</c> until the test completes it.</param>
|
|
public RecordingSubscribableDriverFactory(string supportedType, bool gateInitialize = false)
|
|
{
|
|
_supportedType = supportedType;
|
|
_gateInitialize = gateInitialize;
|
|
}
|
|
|
|
/// <summary>Every driver this factory created, in creation order.</summary>
|
|
public IReadOnlyList<RecordingSubscribableDriver> Created
|
|
{
|
|
get { lock (_gate) return _created.ToArray(); }
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
|
|
{
|
|
if (!string.Equals(driverType, _supportedType, StringComparison.Ordinal)) return null;
|
|
var driver = new RecordingSubscribableDriver(driverInstanceId, driverType);
|
|
if (_gateInitialize)
|
|
driver.InitializeGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
lock (_gate) _created.Add(driver);
|
|
return driver;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public IReadOnlyCollection<string> SupportedTypes => new[] { _supportedType };
|
|
}
|