Joseph Doherty
2b7100cf19
A ReprovisionRequested in flight from an old incarnation could reach the NEW child after HandleRestartDriver swapped it in. In Connecting that child DOES handle InitializeRequested, so it started a second concurrent InitializeAsync, racing the #527 dispose-before-overwrite. Seen RED: InitializeCount == 2 on a child held in Connecting by a gated init (an already-connected child swallows InitializeRequested and passes vacuously).
403 lines
20 KiB
C#
403 lines
20 KiB
C#
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Shared <see cref="IDriver"/> stub harness used by <c>DriverInstanceActorTests</c> and
|
|
/// <c>DriverInstanceActorWriteAndSubscribeTests</c>. Promoted from the superset copy in
|
|
/// <c>DriverInstanceActorTests</c> so both suites compile against a single definition.
|
|
/// </summary>
|
|
internal class StubDriver : IDriver
|
|
{
|
|
/// <summary>Gets or sets a value indicating whether initialization should throw.</summary>
|
|
public bool InitializeShouldThrow { get; set; }
|
|
/// <summary>Gets the number of times initialization was called.</summary>
|
|
public int InitializeCount;
|
|
/// <summary>Gets the number of times reinitialization was called.</summary>
|
|
public int ReinitializeCount;
|
|
|
|
private readonly object _initConfigsLock = new();
|
|
/// <summary>Every config string passed to <see cref="InitializeAsync"/>, in call order.</summary>
|
|
public List<string> InitConfigs { get; } = new();
|
|
/// <summary>Optional per-config init behaviour. When set, it fully owns the init outcome for that
|
|
/// config (await/throw); <see cref="InitializeShouldThrow"/> is ignored. Null ⇒ legacy behaviour.</summary>
|
|
public Func<string, Task>? InitBehavior { get; set; }
|
|
|
|
/// <summary>Gets the driver instance ID.</summary>
|
|
public string DriverInstanceId => "stub-driver-1";
|
|
/// <summary>Gets the driver type.</summary>
|
|
public string DriverType => "Stub";
|
|
|
|
/// <summary>Initializes the driver with the specified configuration JSON.</summary>
|
|
/// <param name="driverConfigJson">The driver configuration JSON.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public virtual async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref InitializeCount);
|
|
lock (_initConfigsLock) InitConfigs.Add(driverConfigJson);
|
|
if (InitBehavior is not null) { await InitBehavior(driverConfigJson); return; }
|
|
if (InitializeShouldThrow) throw new InvalidOperationException("stub-init-fail");
|
|
}
|
|
|
|
/// <summary>Reinitializes the driver with the specified configuration JSON.</summary>
|
|
/// <param name="driverConfigJson">The driver configuration JSON.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref ReinitializeCount);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>Number of <see cref="ShutdownAsync"/> calls — a child actor stopping raises it, so a test
|
|
/// can observe a driver child being torn down (e.g. by the #516 config-change respawn).</summary>
|
|
public int ShutdownCount;
|
|
|
|
/// <summary>Shuts down the driver.</summary>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public virtual Task ShutdownAsync(CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref ShutdownCount);
|
|
return Task.CompletedTask;
|
|
}
|
|
/// <summary>Gets the health status of the driver.</summary>
|
|
/// <remarks>
|
|
/// Virtual so the #527 comms-loss watchdog suite can drive health directly. Note the
|
|
/// <see cref="DateTime.UtcNow"/>: this stub is permanently Healthy with an always-fresh
|
|
/// <c>LastSuccessfulRead</c>, so the watchdog can never fire for it — which is why every existing
|
|
/// test built on <see cref="StubDriver"/> is unaffected by the watchdog, and equally why a watchdog
|
|
/// test built on it would be vacuous.
|
|
/// </remarks>
|
|
public virtual DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A <see cref="StubDriver"/> that also implements <see cref="IWritable"/>, recording every
|
|
/// write call and returning a configurable status code.
|
|
/// </summary>
|
|
internal sealed class WritableStubDriver : StubDriver, IWritable
|
|
{
|
|
/// <summary>Gets or sets the next status code to return from write operations.</summary>
|
|
public uint NextStatusCode { get; set; } = 0u;
|
|
/// <summary>Gets the list of write requests received.</summary>
|
|
public List<WriteRequest> Writes { get; } = new();
|
|
|
|
/// <summary>Writes the specified requests.</summary>
|
|
/// <param name="writes">The write requests.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task<IReadOnlyList<WriteResult>> WriteAsync(
|
|
IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
|
|
{
|
|
Writes.AddRange(writes);
|
|
IReadOnlyList<WriteResult> results = writes.Select(_ => new WriteResult(NextStatusCode)).ToList();
|
|
return Task.FromResult(results);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// A <see cref="StubDriver"/> that also implements <see cref="ISubscribable"/>, firing
|
|
/// <see cref="OnDataChange"/> on demand and counting subscribe calls.
|
|
/// </summary>
|
|
internal sealed class SubscribableStubDriver : StubDriver, ISubscribable
|
|
{
|
|
/// <summary>Occurs when data changes.</summary>
|
|
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
|
|
|
private readonly StubHandle _handle = new();
|
|
|
|
/// <summary>Gets the number of subscribers to OnDataChange.</summary>
|
|
public int OnDataChangeSubscriberCount => OnDataChange?.GetInvocationList().Length ?? 0;
|
|
|
|
/// <summary>Number of times <see cref="SubscribeAsync"/> was called (re-subscribe asserts).</summary>
|
|
public int SubscribeCount;
|
|
/// <summary>The reference set passed to the most recent <see cref="SubscribeAsync"/> call.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedRefs;
|
|
|
|
/// <summary>Number of times <see cref="UnsubscribeAsync"/> was called — the observable for a live
|
|
/// subscription being torn down (an EMPTY desired set drops the handle rather than re-subscribing).</summary>
|
|
public int UnsubscribeCount;
|
|
|
|
/// <summary>When true, <see cref="UnsubscribeAsync"/> genuinely yields (<c>await Task.Yield()</c>)
|
|
/// before completing, so a <c>ConfigureAwait(false)</c> continuation in the actor resumes off the
|
|
/// Akka ActorContext on a thread-pool thread — reproducing the no-ActorContext race that a
|
|
/// synchronously-completed stub task hides (the continuation otherwise runs inline).</summary>
|
|
public bool UnsubscribeYields { get; set; }
|
|
|
|
/// <summary>
|
|
/// How many of the first <see cref="SubscribeAsync"/> calls throw before one succeeds. Default 0
|
|
/// (always succeed) leaves existing behaviour untouched.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// This is how a test reaches the state the #488 reconcile exists for: the actor is Connected and
|
|
/// holds a desired ref set, but the subscribe that should have established the handle failed, so
|
|
/// <c>_subscriptionHandle</c> is null and nothing else will ever re-assert it — there is no
|
|
/// connect transition coming, and no deploy.
|
|
/// </remarks>
|
|
public int SubscribeFailuresBeforeSuccess { get; set; }
|
|
|
|
/// <summary>Subscribes to the specified full references.</summary>
|
|
/// <param name="fullReferences">The full references to subscribe to.</param>
|
|
/// <param name="publishingInterval">The publishing interval.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public Task<ISubscriptionHandle> SubscribeAsync(
|
|
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken)
|
|
{
|
|
var attempt = Interlocked.Increment(ref SubscribeCount);
|
|
LastSubscribedRefs = fullReferences;
|
|
if (attempt <= SubscribeFailuresBeforeSuccess)
|
|
{
|
|
throw new InvalidOperationException($"stub subscribe failure #{attempt}");
|
|
}
|
|
|
|
return Task.FromResult<ISubscriptionHandle>(_handle);
|
|
}
|
|
|
|
/// <summary>Unsubscribes from the specified subscription handle.</summary>
|
|
/// <param name="handle">The subscription handle.</param>
|
|
/// <param name="cancellationToken">Cancellation token for the operation.</param>
|
|
public async Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref UnsubscribeCount);
|
|
if (UnsubscribeYields)
|
|
{
|
|
// Complete the awaited task from a fresh background thread that has NO Akka actor
|
|
// cell on it, so the caller's `ConfigureAwait(false)` continuation resumes on a
|
|
// clean thread-pool thread where InternalCurrentActorCellKeeper.Current is null —
|
|
// a deterministic repro of the real async-backend no-ActorContext race.
|
|
var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
_ = Task.Run(() => tcs.SetResult());
|
|
await tcs.Task.ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <summary>Fires a data change event with the specified parameters.</summary>
|
|
/// <param name="fullRef">The full reference of the data that changed.</param>
|
|
/// <param name="value">The new value.</param>
|
|
/// <param name="statusCode">The OPC UA status code.</param>
|
|
public void FireDataChange(string fullRef, object? value, uint statusCode)
|
|
{
|
|
var snapshot = new DataValueSnapshot(value, statusCode, DateTime.UtcNow, DateTime.UtcNow);
|
|
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, snapshot));
|
|
}
|
|
}
|
|
|
|
/// <summary>Minimal <see cref="ISubscriptionHandle"/> for use by <see cref="SubscribableStubDriver"/>.</summary>
|
|
internal sealed class StubHandle : ISubscriptionHandle
|
|
{
|
|
/// <summary>Gets the diagnostic ID of the subscription.</summary>
|
|
public string DiagnosticId => "stub-sub";
|
|
}
|
|
|
|
/// <summary>
|
|
/// A per-instance recording driver for the restart / re-subscribe suite (03/S16, U6): unlike
|
|
/// <see cref="SubscribableStubDriver"/> it carries the <b>spec's</b> <see cref="DriverInstanceId"/>
|
|
/// (<see cref="StubDriver"/> hard-codes a constant on a non-virtual member), and it counts and records the
|
|
/// alarm subscription alongside the value one — the two travel in a single
|
|
/// <c>SetDesiredSubscriptions</c> but through two different child code paths, so both need observing.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// <b>The falsification bar these counters exist for:</b> a restart test must assert a FUNCTION the new
|
|
/// incarnation performs — it subscribed, it re-initialised — never that the incarnation exists.
|
|
/// Existence is what
|
|
/// <c>DriverHostActorReconcileTests.RestartDriver_respawns_the_child_without_an_actor_name_collision</c>
|
|
/// asserts, and that test passes against the very defect 03/S16 names.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b><see cref="FireMalformedDataChange"/> is the supervision crash seam</b> — see
|
|
/// <c>DriverHostActorRestartTests</c>'s file remarks for why the obvious alternatives do not work.
|
|
/// </para>
|
|
/// </remarks>
|
|
internal sealed class RecordingSubscribableDriver : IDriver, ISubscribable, IAlarmSource
|
|
{
|
|
private readonly Lock _gate = new();
|
|
private readonly StubHandle _handle = new();
|
|
private readonly RecordingAlarmHandle _alarmHandle = new();
|
|
private int _initializeCount;
|
|
private int _shutdownCount;
|
|
private int _subscribeCount;
|
|
private int _subscribeAlarmsCount;
|
|
private IReadOnlyList<string>? _lastSubscribedRefs;
|
|
private IReadOnlyList<string>? _lastSubscribedAlarmRefs;
|
|
|
|
/// <summary>Initializes a new recording driver stamped with the spec's identity.</summary>
|
|
/// <param name="driverInstanceId">The driver instance id from the spec that created it.</param>
|
|
/// <param name="driverType">The driver type from the spec that created it.</param>
|
|
public RecordingSubscribableDriver(string driverInstanceId, string driverType)
|
|
{
|
|
DriverInstanceId = driverInstanceId;
|
|
DriverType = driverType;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public string DriverInstanceId { get; }
|
|
|
|
/// <inheritdoc />
|
|
public string DriverType { get; }
|
|
|
|
/// <summary>Number of <see cref="InitializeAsync"/> calls. A supervision restart re-uses the SAME driver
|
|
/// object (the Props closure captured it), so this going 1 to 2 is how the reprovision is observed.</summary>
|
|
public int InitializeCount => Volatile.Read(ref _initializeCount);
|
|
|
|
/// <summary>Number of <see cref="ShutdownAsync"/> calls — a restart's <c>PostStop</c> raises it.</summary>
|
|
public int ShutdownCount => Volatile.Read(ref _shutdownCount);
|
|
|
|
/// <summary>Number of <see cref="SubscribeAsync"/> calls.</summary>
|
|
public int SubscribeCount => Volatile.Read(ref _subscribeCount);
|
|
|
|
/// <summary>Number of <see cref="SubscribeAlarmsAsync"/> calls.</summary>
|
|
public int SubscribeAlarmsCount => Volatile.Read(ref _subscribeAlarmsCount);
|
|
|
|
/// <summary>The refs passed to the most recent <see cref="SubscribeAsync"/>, or null if never called.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedRefs
|
|
{
|
|
get { lock (_gate) return _lastSubscribedRefs; }
|
|
}
|
|
|
|
/// <summary>The refs passed to the most recent <see cref="SubscribeAlarmsAsync"/>, or null if never called.</summary>
|
|
public IReadOnlyList<string>? LastSubscribedAlarmRefs
|
|
{
|
|
get { lock (_gate) return _lastSubscribedAlarmRefs; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// When set, <see cref="InitializeAsync"/> awaits it instead of completing inline — so the actor
|
|
/// stays in <c>Connecting</c> for as long as the test wants. Needed because several defects only
|
|
/// bite a child that has NOT yet connected (<c>InitializeRequested</c> is unhandled in
|
|
/// <c>Connected</c>, so a test built on an already-connected child passes vacuously).
|
|
/// </summary>
|
|
public TaskCompletionSource? InitializeGate { get; set; }
|
|
|
|
/// <inheritdoc />
|
|
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _initializeCount);
|
|
if (InitializeGate is { } gate) await gate.Task.ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public Task ShutdownAsync(CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _shutdownCount);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
|
|
|
/// <inheritdoc />
|
|
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
|
|
|
/// <inheritdoc />
|
|
public Task<ISubscriptionHandle> SubscribeAsync(
|
|
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _subscribeCount);
|
|
lock (_gate) _lastSubscribedRefs = fullReferences.ToArray();
|
|
return Task.FromResult<ISubscriptionHandle>(_handle);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public event EventHandler<AlarmEventArgs>? OnAlarmEvent;
|
|
|
|
/// <inheritdoc />
|
|
public Task<IAlarmSubscriptionHandle> SubscribeAlarmsAsync(
|
|
IReadOnlyList<string> sourceNodeIds, CancellationToken cancellationToken)
|
|
{
|
|
Interlocked.Increment(ref _subscribeAlarmsCount);
|
|
lock (_gate) _lastSubscribedAlarmRefs = sourceNodeIds.ToArray();
|
|
return Task.FromResult<IAlarmSubscriptionHandle>(_alarmHandle);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <inheritdoc />
|
|
public Task AcknowledgeAsync(
|
|
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
/// <summary>Fires a well-formed data change (the ordinary production path).</summary>
|
|
/// <param name="fullRef">The reference whose value changed.</param>
|
|
/// <param name="value">The new value.</param>
|
|
public void FireDataChange(string fullRef, object? value) =>
|
|
OnDataChange?.Invoke(this, new DataChangeEventArgs(
|
|
_handle, fullRef, new DataValueSnapshot(value, 0u, DateTime.UtcNow, DateTime.UtcNow)));
|
|
|
|
/// <summary>
|
|
/// Raises <see cref="OnDataChange"/> with a NULL <c>Snapshot</c> — the supervision crash seam.
|
|
/// <c>DataChangeEventArgs</c> is a positional record with no null validation, and the actor's
|
|
/// <c>OnDataChangeForward</c> dereferences <c>msg.Snapshot</c> unguarded ON THE ACTOR THREAD, so this
|
|
/// produces a genuine unhandled exception inside the child, which Akka's default decider turns into a
|
|
/// Restart. The fault itself is arbitrary; what is under test is the RECOVERY.
|
|
/// </summary>
|
|
/// <param name="fullRef">The reference to name in the malformed event.</param>
|
|
public void FireMalformedDataChange(string fullRef) =>
|
|
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, null!));
|
|
|
|
/// <summary>Raises <see cref="OnAlarmEvent"/>; present so the event is not merely written-to-never-read.</summary>
|
|
/// <param name="conditionId">The condition id to raise.</param>
|
|
public void FireAlarm(string conditionId) =>
|
|
OnAlarmEvent?.Invoke(this, new AlarmEventArgs(
|
|
_alarmHandle, conditionId, conditionId, "AlarmCondition", "test", AlarmSeverity.High, DateTime.UtcNow));
|
|
|
|
private sealed class RecordingAlarmHandle : IAlarmSubscriptionHandle
|
|
{
|
|
/// <inheritdoc />
|
|
public string DiagnosticId => "recording-alarm-sub";
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// An <see cref="IDriverFactory"/> that hands out <see cref="RecordingSubscribableDriver"/>s and KEEPS
|
|
/// every one it created, in creation order. The list — not a single "the driver" reference — is the point:
|
|
/// an operator Restart tears the child down and spawns a new one around a NEW driver object, so the
|
|
/// assertion subject is <c>Created[1]</c>, while a SUPERVISION restart re-uses <c>Created[0]</c> (the Props
|
|
/// closure captured it) and part of the assertion is that <c>Created.Count</c> stayed 1.
|
|
/// </summary>
|
|
internal sealed class RecordingSubscribableDriverFactory : IDriverFactory
|
|
{
|
|
private readonly string _supportedType;
|
|
private readonly Lock _gate = new();
|
|
private readonly List<RecordingSubscribableDriver> _created = new();
|
|
|
|
private readonly bool _gateInitialize;
|
|
|
|
/// <summary>Initializes a factory that materialises exactly one driver type.</summary>
|
|
/// <param name="supportedType">The driver type this factory answers for.</param>
|
|
/// <param name="gateInitialize">When true, every created driver gets an un-completed
|
|
/// <see cref="RecordingSubscribableDriver.InitializeGate"/>, so its child actor stays in
|
|
/// <c>Connecting</c> until the test completes it.</param>
|
|
public RecordingSubscribableDriverFactory(string supportedType, bool gateInitialize = false)
|
|
{
|
|
_supportedType = supportedType;
|
|
_gateInitialize = gateInitialize;
|
|
}
|
|
|
|
/// <summary>Every driver this factory created, in creation order.</summary>
|
|
public IReadOnlyList<RecordingSubscribableDriver> Created
|
|
{
|
|
get { lock (_gate) return _created.ToArray(); }
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
|
|
{
|
|
if (!string.Equals(driverType, _supportedType, StringComparison.Ordinal)) return null;
|
|
var driver = new RecordingSubscribableDriver(driverInstanceId, driverType);
|
|
if (_gateInitialize)
|
|
driver.InitializeGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
lock (_gate) _created.Add(driver);
|
|
return driver;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public IReadOnlyCollection<string> SupportedTypes => new[] { _supportedType };
|
|
}
|