Merge branch 'master' into remed/lane-E
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,153 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Telemetry;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 U2: <c>TelemetryDial:CallTimeoutSeconds</c> was bound, defaulted and documented, and consumed
|
||||
/// nowhere. It now bounds a dial's <b>establishment</b>: a dial that has delivered no event by the deadline is
|
||||
/// cut and re-dialled with a failure streak. A live node answers a fresh subscribe at once (the node-local hub
|
||||
/// replays its health/resilience snapshots), so the deadline bites only a dial that never connects — a
|
||||
/// black-holed host, where the underlying socket connect has no timeout of its own. An established stream
|
||||
/// is deliberately NOT bounded: HTTP/2 keepalive pings own its liveness, and the wire has no heartbeat, so an
|
||||
/// idle timeout would churn every quiet node.
|
||||
/// </summary>
|
||||
public sealed class TelemetryDialDeadlineTests : TestKit
|
||||
{
|
||||
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(8);
|
||||
|
||||
[Fact]
|
||||
public void A_dial_that_delivers_no_event_by_the_deadline_is_cancelled_and_redialled()
|
||||
{
|
||||
var dial = new FakeDialLoop();
|
||||
Spawn(dial, callTimeoutSeconds: 1);
|
||||
|
||||
var first = dial.WaitForCount(1, Timeout).Single();
|
||||
first.Ct.IsCancellationRequested.ShouldBeFalse("the deadline has not elapsed yet");
|
||||
|
||||
// The stalled dial is cut at the deadline...
|
||||
AwaitCondition(() => first.Ct.IsCancellationRequested, Timeout);
|
||||
// ...and the supervisor re-dials (a stalled dial is a transport failure, never a silent end).
|
||||
var invocations = dial.WaitForCount(2, Timeout);
|
||||
invocations[1].CorrelationId.ShouldBe("central-node-1-2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_dial_that_delivers_an_event_before_the_deadline_is_left_open()
|
||||
{
|
||||
var dial = new FakeDialLoop { DeliverOnOpen = true };
|
||||
var sinks = new Sinks();
|
||||
Spawn(dial, callTimeoutSeconds: 1, sinks);
|
||||
|
||||
var first = dial.WaitForCount(1, Timeout).Single();
|
||||
AwaitCondition(() => sinks.HealthStore.Upserts.Count == 1, Timeout);
|
||||
|
||||
// Positive control: well past the deadline, the established stream is untouched.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2.5));
|
||||
first.Ct.IsCancellationRequested.ShouldBeFalse("an established stream must never be cut by the dial deadline");
|
||||
dial.Invocations.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
private IActorRef Spawn(FakeDialLoop dial, int callTimeoutSeconds, Sinks? sinks = null)
|
||||
{
|
||||
sinks ??= new Sinks();
|
||||
var options = new TelemetryDialOptions { ContactRefreshSeconds = 3600, CallTimeoutSeconds = callTimeoutSeconds };
|
||||
return Sys.ActorOf(TelemetryDialSupervisor.Props(
|
||||
() => Task.FromResult<IReadOnlyList<TelemetryDialTarget>>(
|
||||
new[] { new TelemetryDialTarget("node-1", "http://h1:5300") }),
|
||||
dial.Loop,
|
||||
sinks.AlarmBroadcaster,
|
||||
sinks.ScriptBroadcaster,
|
||||
sinks.HealthStore,
|
||||
sinks.ResilienceStore,
|
||||
options));
|
||||
}
|
||||
|
||||
private static DriverHealthChanged Health(string id) =>
|
||||
new("C1", id, "Healthy", DateTime.UtcNow, null, 0, DateTime.UtcNow);
|
||||
|
||||
private sealed class DialInvocation
|
||||
{
|
||||
private readonly TaskCompletionSource _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public required string CorrelationId { get; init; }
|
||||
public required CancellationToken Ct { get; init; }
|
||||
public Task Completion => _completion.Task;
|
||||
public void Complete() => _completion.TrySetResult();
|
||||
}
|
||||
|
||||
/// <summary>Records each invocation; optionally delivers one health event on open; ends only when cancelled.</summary>
|
||||
private sealed class FakeDialLoop
|
||||
{
|
||||
private readonly ConcurrentQueue<DialInvocation> _invocations = new();
|
||||
|
||||
public bool DeliverOnOpen { get; init; }
|
||||
|
||||
public IReadOnlyList<DialInvocation> Invocations => _invocations.ToArray();
|
||||
|
||||
public TelemetryDialLoop Loop => async (_, correlationId, onMapped, _, ct) =>
|
||||
{
|
||||
var invocation = new DialInvocation { CorrelationId = correlationId, Ct = ct };
|
||||
_invocations.Enqueue(invocation);
|
||||
if (DeliverOnOpen) onMapped(Health("drv-1"));
|
||||
await using var reg = ct.Register(() => invocation.Complete());
|
||||
await invocation.Completion;
|
||||
};
|
||||
|
||||
public IReadOnlyList<DialInvocation> WaitForCount(int count, TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var snapshot = _invocations.ToArray();
|
||||
if (snapshot.Length >= count) return snapshot;
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
|
||||
throw new Xunit.Sdk.XunitException(
|
||||
$"Expected at least {count} dial invocations within {timeout}, saw {_invocations.Count}.");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Sinks
|
||||
{
|
||||
public NullBroadcaster<AlarmTransitionEvent> AlarmBroadcaster { get; } = new();
|
||||
public NullBroadcaster<ScriptLogEntry> ScriptBroadcaster { get; } = new();
|
||||
public RecordingHealthStore HealthStore { get; } = new();
|
||||
public NullResilienceStore ResilienceStore { get; } = new();
|
||||
}
|
||||
|
||||
private sealed class NullBroadcaster<T> : IInProcessBroadcaster<T>
|
||||
{
|
||||
public event Action<T>? Received;
|
||||
public event Action<bool>? ConnectionStateChanged;
|
||||
public bool IsConnected { get; private set; }
|
||||
public void Publish(T item) => Received?.Invoke(item);
|
||||
public void SetConnected(bool connected) { IsConnected = connected; ConnectionStateChanged?.Invoke(connected); }
|
||||
}
|
||||
|
||||
private sealed class RecordingHealthStore : IDriverStatusSnapshotStore
|
||||
{
|
||||
public ConcurrentQueue<DriverHealthChanged> Upserts { get; } = new();
|
||||
public event Action<DriverHealthChanged>? SnapshotChanged;
|
||||
public void Upsert(DriverHealthChanged snapshot) { Upserts.Enqueue(snapshot); SnapshotChanged?.Invoke(snapshot); }
|
||||
public bool TryGet(string driverInstanceId, out DriverHealthChanged snapshot) { snapshot = null!; return false; }
|
||||
public IReadOnlyCollection<DriverHealthChanged> GetAll() => Upserts.ToArray();
|
||||
}
|
||||
|
||||
private sealed class NullResilienceStore : IDriverResilienceStatusStore
|
||||
{
|
||||
public event Action<DriverResilienceStatusChanged>? SnapshotChanged;
|
||||
public void Upsert(DriverResilienceStatusChanged snapshot) => SnapshotChanged?.Invoke(snapshot);
|
||||
public IReadOnlyList<DriverResilienceStatusChanged> GetForInstance(string driverInstanceId) => Array.Empty<DriverResilienceStatusChanged>();
|
||||
public IReadOnlyCollection<DriverResilienceStatusChanged> GetAll() => Array.Empty<DriverResilienceStatusChanged>();
|
||||
}
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// R08:1.1 (#736): the server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>) is advertised as
|
||||
/// <c>HistoryServerCapabilities/MaxReturnDataValues</c> (i=11273) through <see cref="HistoryPaging.AdvertiseMaxReturnDataValues"/>,
|
||||
/// read back the way a client would — through the SDK master node manager — over a real server.
|
||||
/// </summary>
|
||||
public sealed class HistoryServerCapabilitiesAdvertisementTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
|
||||
{
|
||||
private async Task<uint> ReadAdvertisedAsync()
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.Read, identity: null);
|
||||
var nodesToRead = new ReadValueIdCollection
|
||||
{
|
||||
new ReadValueId { NodeId = VariableIds.HistoryServerCapabilities_MaxReturnDataValues, AttributeId = Attributes.Value },
|
||||
};
|
||||
var (values, _) = await fx.Nm.Server.NodeManager.ReadAsync(context, 0, TimestampsToReturn.Neither, nodesToRead, TestContext.Current.CancellationToken);
|
||||
values.Count.ShouldBe(1);
|
||||
StatusCode.IsGood(values[0].StatusCode).ShouldBeTrue(values[0].StatusCode.ToString());
|
||||
return (uint)values[0].Value;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_cap_is_advertised_as_MaxReturnDataValues_and_a_re_sync_overwrites_it()
|
||||
{
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 10_000).ShouldBeTrue();
|
||||
(await ReadAdvertisedAsync()).ShouldBe(10_000u);
|
||||
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 2_500).ShouldBeTrue();
|
||||
(await ReadAdvertisedAsync()).ShouldBe(2_500u);
|
||||
|
||||
// Unbounded (the opt-out) advertises Part 11's "no limit".
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 0).ShouldBeTrue();
|
||||
(await ReadAdvertisedAsync()).ShouldBe(0u);
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, -5).ShouldBeTrue();
|
||||
(await ReadAdvertisedAsync()).ShouldBe(0u);
|
||||
}
|
||||
finally
|
||||
{
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 0);
|
||||
fx.Gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_server_is_a_quiet_false()
|
||||
{
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(null, 10_000).ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
+436
@@ -0,0 +1,436 @@
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// R08:1.1 (#736): the paged-vs-unpaged fuzz for the server raw cap. Every paging change in this repo has had boundary
|
||||
/// bugs that the hand-written cases missed (silent truncation, an endless cursor, an empty page between every value), so
|
||||
/// the cap is proved the only way that has worked: random series (bursts, gaps, sub-millisecond stamps, tie clusters),
|
||||
/// random windows (empty, on a tie, one step past a tie, between samples, forward / reverse / end-only, with and without
|
||||
/// bounds) and random caps — <c>NumValuesPerNode</c> 0 / 1 / 2 / small / huge, the server cap 3 … 1000 or off, the
|
||||
/// tie-cluster budget 3 / 8 / 65536 — paged through a REAL SDK node manager against a wire-faithful fake of each backend
|
||||
/// (SqlServer: tick precision, native reverse, <c>TOP</c>; QuestDB: µs truncation of bounds AND stamps, native reverse,
|
||||
/// <c>LIMIT</c>; Gateway: forward only, <c>[start, end)</c> after its own boundary-row filtering, walked back by the
|
||||
/// server) that enforces the backend half of the cap exactly as the real sources do (a read asking for 0 or more than
|
||||
/// the cap reads cap + 1 rows and is refused when the window holds more).
|
||||
/// <para>
|
||||
/// The oracle is the same node manager's one-shot answer with the cap OFF (<c>NumValuesPerNode = 0</c>, no limit),
|
||||
/// which for a plain read is also checked against the series itself. The concatenation of the pages must equal it
|
||||
/// exactly; the read must terminate within the number of values plus two pages; no page may exceed the cap; the fake
|
||||
/// must never see a request for 0 or more than the cap (the "bounded by the same cap" invariant); and a
|
||||
/// <c>BadTooManyOperations</c> is accepted only when a tie cluster (or, on the Gateway shape, one millisecond) is
|
||||
/// larger than the budget that the cap and <c>MaxTieClusterOverfetch</c> leave — after an exact prefix. The run is
|
||||
/// non-vacuous by construction: it asserts that empty windows, mid-tie windows, reverse reads, <c>NumValuesPerNode</c>
|
||||
/// of 1 and 2, the smallest server cap (3) and every shape occurred, and that most iterations were exact.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Iterations: <c>OTOPCUA_PAGING_FUZZ_ITERATIONS</c> (default 10 000, the lane gate). Seeds are stable, so a failure
|
||||
/// names a reproducible seed.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class NodeManagerHistoryReadRawCapFuzzTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>The backend the fake is shaped like.</summary>
|
||||
public enum Shape { SqlServer, QuestDb, Gateway }
|
||||
|
||||
private enum Direction { Forward, Reverse, EndOnly }
|
||||
|
||||
private static int Iterations =>
|
||||
int.TryParse(Environment.GetEnvironmentVariable("OTOPCUA_PAGING_FUZZ_ITERATIONS"), out var n) && n > 0 ? n : 10_000;
|
||||
|
||||
[Fact]
|
||||
public async Task Paged_equals_unpaged_on_random_series_windows_and_caps_and_always_terminates()
|
||||
{
|
||||
var iterations = Iterations;
|
||||
var counts = new Dictionary<string, int>();
|
||||
void Count(string key) => counts[key] = counts.GetValueOrDefault(key) + 1;
|
||||
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
for (var seed = 0; seed < iterations; seed++)
|
||||
{
|
||||
var rng = new Random(seed);
|
||||
var shape = (Shape)(seed % 3);
|
||||
var resolution = shape == Shape.QuestDb ? 10L : 1L;
|
||||
var series = RandomSeries(rng, resolution);
|
||||
var cap = rng.Next(10) switch { 0 => 0, 1 or 2 => 3, 3 => 4, 4 => 5, 5 => 7, 6 => 10, 7 => 16, 8 => 33, _ => 1000 };
|
||||
var n = rng.Next(9) switch { 0 or 1 => 0u, 2 or 3 => 1u, 4 or 5 => 2u, 6 => (uint)rng.Next(3, 20), 7 => (uint)rng.Next(20, 200), _ => 100_000u };
|
||||
var tieBudget = rng.Next(4) switch { 0 => 3, 1 => 8, _ => 65536 };
|
||||
var direction = rng.Next(5) switch { 0 => Direction.Reverse, 1 => Direction.EndOnly, _ => Direction.Forward };
|
||||
if (direction == Direction.EndOnly && n == 0) direction = Direction.Forward; // Part 11: one bound only with a cap
|
||||
var bounds = direction != Direction.EndOnly && rng.Next(4) == 0;
|
||||
|
||||
var a = PickBoundary(rng, series, resolution);
|
||||
var b = PickBoundary(rng, series, resolution);
|
||||
var (lo, hi) = a <= b ? (a, b) : (b, a);
|
||||
var (start, end) = direction switch
|
||||
{
|
||||
Direction.Forward => (lo, hi),
|
||||
Direction.Reverse => (hi, lo),
|
||||
_ => (DateTime.MinValue, hi),
|
||||
};
|
||||
var tag = $"seed {seed} {shape} cap {cap} N {n} tie {tieBudget} {direction} bounds {bounds} [{start:O} .. {end:O}]";
|
||||
|
||||
// The walk-back (Gateway shape only: reverse / end-only reads, and the bound lookups of any bounded read) refuses
|
||||
// a millisecond holding more samples than its budget — computed up front, over the whole series, because the
|
||||
// walk and the lookups read around the window.
|
||||
var walks = shape == Shape.Gateway && (direction != Direction.Forward || bounds);
|
||||
var msMax = walks ? MillisecondMax(series) : 0;
|
||||
|
||||
// The oracle: the one-shot answer with the cap off (and, for a plain read, the series itself).
|
||||
var truthSource = new CappedFake(series, shape, cap: 0);
|
||||
fx.Nm.HistoricalSourceForTest(truthSource, tieBudget);
|
||||
// An end-only read is paged backward from EndTime (the node manager flips it to a reverse read down to the
|
||||
// beginning of time); its oracle is the explicit reverse read down to the Unix epoch — the walk-back's floor,
|
||||
// and Part 11 (the SDK checks it) needs both bounds set when NumValuesPerNode = 0.
|
||||
var (truth, truthError, truthCp) = direction == Direction.EndOnly
|
||||
? Read(hi, DateTime.UnixEpoch, 0, bounds: false, cp: null)
|
||||
: Read(start, end, 0, bounds, cp: null);
|
||||
if (truthError.StatusCode.Code == StatusCodes.BadTooManyOperations)
|
||||
{
|
||||
// Only the walk-back can refuse an uncapped one-shot read, and only past its budget (max(1, M) with no cap).
|
||||
(walks && msMax > Math.Max(1, tieBudget)).ShouldBeTrue($"the oracle read was refused: {tag} (ms max {msMax})");
|
||||
Count("oracle refused");
|
||||
continue;
|
||||
}
|
||||
truthError.StatusCode.Code.ShouldBe(StatusCodes.Good, $"oracle: {tag}");
|
||||
truthCp.ShouldBeNull($"oracle (no cap, N = 0) is one page: {tag}");
|
||||
if (!bounds)
|
||||
Render(truth).ShouldBe(Render(Oracle(series, start, end, direction, resolution)), $"oracle vs series: {tag}");
|
||||
truthSource.BadRequests.ShouldBe(0, $"a reverse window reached a forward-only backend on the oracle read: {tag}");
|
||||
|
||||
var windowTies = LargestTieCluster(truth);
|
||||
if (truth.Count == 0) Count("empty");
|
||||
if (windowTies >= 2) Count("ties in window");
|
||||
if (EndsOnTie(series, start, end, direction, resolution)) Count("window ends on a tie");
|
||||
if (direction == Direction.Reverse) Count("reverse");
|
||||
if (direction == Direction.EndOnly) Count("end-only");
|
||||
if (n == 1) Count("N=1");
|
||||
if (n == 2) Count("N=2");
|
||||
if (n == 0) Count("N=0");
|
||||
if (cap == 3) Count("cap=3");
|
||||
if (bounds) Count("bounds");
|
||||
Count(shape.ToString());
|
||||
|
||||
// A boundary tie cluster as large as the cap leaves a resumed page no room: the server steps past it with one
|
||||
// empty page carrying a continuation point (the documented #694 ceiling behaviour) — the one empty page allowed.
|
||||
var oversizedClusters = cap > 0
|
||||
? truth.Where(v => !(v.Value is null && v.StatusCode.Code == StatusCodes.BadBoundNotFound))
|
||||
.GroupBy(v => v.SourceTimestamp.Ticks).Count(g => g.Count() >= Math.Max(2, cap - 1))
|
||||
: 0;
|
||||
|
||||
// The read under test.
|
||||
var source = new CappedFake(series, shape, cap);
|
||||
fx.Nm.HistoricalSourceForTest(source, tieBudget);
|
||||
var pages = new List<IReadOnlyList<DataValue>>();
|
||||
var collected = new List<DataValue>();
|
||||
byte[]? cp = null;
|
||||
StatusCode status;
|
||||
do
|
||||
{
|
||||
var (values, error, next) = Read(start, end, n, bounds, cp);
|
||||
status = error.StatusCode;
|
||||
if (!StatusCode.IsGood(status)) break;
|
||||
pages.Add(values);
|
||||
collected.AddRange(values);
|
||||
cp = next;
|
||||
pages.Count.ShouldBeLessThanOrEqualTo(truth.Count + 2 + oversizedClusters, $"paging must terminate: {tag} pages {Sizes(pages)}");
|
||||
}
|
||||
while (cp is not null);
|
||||
|
||||
source.OverCapRequests.ShouldBe(0, $"the server asked the backend for 0 or more than the cap: {tag}");
|
||||
source.BadRequests.ShouldBe(0, $"a reverse window reached a forward-only backend: {tag}");
|
||||
|
||||
if (status.Code == StatusCodes.BadTooManyOperations)
|
||||
{
|
||||
// Legitimate only past the budget the cap leaves: a tie cluster larger than min(M, cap − 1), or on the
|
||||
// Gateway shape a millisecond holding more than min(M, cap − 2) samples (the walk-back's budget).
|
||||
var overfetchBound = cap > 0 ? Math.Min(Math.Max(1, tieBudget), Math.Max(1, cap - 1)) : Math.Max(1, tieBudget);
|
||||
var walkBudget = cap > 0 ? Math.Max(1, Math.Min(tieBudget, cap - 2)) : Math.Max(1, tieBudget);
|
||||
var legit = windowTies > overfetchBound || (walks && msMax > walkBudget);
|
||||
legit.ShouldBeTrue($"refused with no tie cluster over {overfetchBound} (max {windowTies}; ms max {msMax} vs {walkBudget}): {tag}");
|
||||
Render(collected).ShouldBe(Render(truth.Take(collected.Count)), $"the prefix before the refusal: {tag}");
|
||||
Count("refused");
|
||||
continue;
|
||||
}
|
||||
|
||||
status.Code.ShouldBe(StatusCodes.Good, $"{tag}");
|
||||
Render(collected).ShouldBe(Render(truth), $"paged ≠ unpaged: {tag} pages {Sizes(pages)}");
|
||||
if (cap > 0)
|
||||
{
|
||||
// A bounded read whose data fits one page carries its start bound, the data and its end bound: the node
|
||||
// manager fits the end bound against the client's N but not against the server cap (follow-up F2), so that
|
||||
// one page may hold cap + 1 values; every other page is at most the cap.
|
||||
var bothBoundsOnOnePage = bounds && pages.Count == 1;
|
||||
pages.ShouldAllBe(p => p.Count <= cap + (bothBoundsOnOnePage ? 1 : 0), $"a page over the cap: {tag} pages {Sizes(pages)}");
|
||||
if (bothBoundsOnOnePage && pages[0].Count == cap + 1) Count("bounded single page at cap+1 (F2)");
|
||||
}
|
||||
if (n > 0) pages.ShouldAllBe(p => p.Count <= n, $"a page over N: {tag} pages {Sizes(pages)}");
|
||||
for (var i = 0; i < pages.Count - 1; i++)
|
||||
{
|
||||
if (pages[i].Count > 0) continue;
|
||||
// Allowed only as the step past a boundary tie cluster the cap cannot page within.
|
||||
var previousLast = i > 0 && pages[i - 1].Count > 0 ? pages[i - 1][^1].SourceTimestamp.Ticks : (long?)null;
|
||||
var clusterAtBoundary = previousLast is { } t ? truth.Count(v => v.SourceTimestamp.Ticks == t) : 0;
|
||||
(cap > 0 && clusterAtBoundary >= Math.Max(2, cap - 1)).ShouldBeTrue(
|
||||
$"an empty page with a continuation point (page {i}, boundary cluster {clusterAtBoundary}): {tag} pages {Sizes(pages)}");
|
||||
Count("empty step over an oversized tie cluster");
|
||||
}
|
||||
if (pages.Count > 1) Count("paged");
|
||||
if (pages.Count > 1 && n == 0) Count("paged at N=0");
|
||||
// A cost guard against a read explosion, not a budget. A native shape reads once per page plus the bound
|
||||
// lookups. The Gateway shape's walk-back has its own cost profile — ~30 reads a page on a dense tag
|
||||
// (#691/#693), far more on a sparse series under a tiny tie budget (seed 320: 82 reads with NO cap, seed 6428:
|
||||
// 120+) — so it is judged RELATIVE to the same request with the cap off (the pre-#736 behaviour), per page:
|
||||
// the cap must not make the walk explode compared to before. The baseline asks for the page size the cap
|
||||
// imposes (N = cap − 1): an uncapped N = 0 reverse read used to be ONE int.MaxValue forward read — the very
|
||||
// memory vector the cap removes — so "what N = 0 cost before" is not the comparison.
|
||||
var allowedReads = 8 * pages.Count + 8;
|
||||
if (shape == Shape.Gateway)
|
||||
{
|
||||
var baseline = new CappedFake(series, shape, cap: 0);
|
||||
fx.Nm.HistoricalSourceForTest(baseline, tieBudget);
|
||||
var baseN = cap > 0 && (n == 0 || n > (uint)cap) ? (uint)(cap - 1) : n;
|
||||
var basePages = 0;
|
||||
byte[]? baseCp = null;
|
||||
do
|
||||
{
|
||||
var (_, baseError, next) = Read(start, end, baseN, bounds, baseCp);
|
||||
if (!StatusCode.IsGood(baseError.StatusCode)) break;
|
||||
basePages++;
|
||||
baseCp = next;
|
||||
}
|
||||
while (baseCp is not null && basePages <= truth.Count + 2 + oversizedClusters);
|
||||
allowedReads = 2 * baseline.Reads * Math.Max(1, pages.Count) / Math.Max(1, basePages) + 32 * pages.Count + 16;
|
||||
}
|
||||
source.Reads.ShouldBeLessThanOrEqualTo(allowedReads,
|
||||
$"backend reads exploded: {source.Reads} for {pages.Count} pages (allowed {allowedReads}): {tag}");
|
||||
Count("exact");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
fx.Nm.HistoricalSourceForTest(NullHistorianDataSource.Instance, 65536);
|
||||
fx.Gate.Release();
|
||||
}
|
||||
|
||||
// Non-vacuous: every category the brief names occurred, and most runs were exact.
|
||||
var summary = string.Join(", ", counts.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key}={kv.Value}"));
|
||||
foreach (var required in new[] { "empty", "ties in window", "window ends on a tie", "reverse", "end-only", "N=1", "N=2", "N=0", "cap=3", "bounds", "paged", "paged at N=0", "SqlServer", "QuestDb", "Gateway" })
|
||||
counts.GetValueOrDefault(required).ShouldBeGreaterThan(iterations / 200, $"{required} under-represented: {summary}");
|
||||
counts.GetValueOrDefault("exact").ShouldBeGreaterThan(iterations * 6 / 10, summary);
|
||||
TestContext.Current.TestOutputHelper?.WriteLine($"raw-cap fuzz: {iterations} iterations — {summary}");
|
||||
}
|
||||
|
||||
// --- generators -----------------------------------------------------------------------------
|
||||
|
||||
private static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue;
|
||||
|
||||
private static DateTime Trunc(DateTime t, long resolution) => new(t.Ticks - t.Ticks % resolution, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>Random ascending series: bursts, gaps, sub-millisecond stamps and tie clusters; stamps stored at the resolution.</summary>
|
||||
private static List<DataValueSnapshot> RandomSeries(Random rng, long resolution)
|
||||
{
|
||||
var list = new List<DataValueSnapshot>();
|
||||
var t = T0.Ticks - rng.Next(0, 3) * TimeSpan.TicksPerHour;
|
||||
var count = rng.Next(1, 120);
|
||||
var v = 0;
|
||||
while (list.Count < count)
|
||||
{
|
||||
t += rng.Next(6) switch
|
||||
{
|
||||
0 => rng.Next(1, 10), // sub-millisecond (sub-µs on QuestDB: collapses onto a tie)
|
||||
1 => rng.Next(1, 5) * TimeSpan.TicksPerMillisecond, // dense
|
||||
2 => rng.Next(1, 3) * TimeSpan.TicksPerSecond,
|
||||
3 => rng.Next(1, 30) * TimeSpan.TicksPerMinute, // a gap
|
||||
_ => rng.Next(1, 1000) * TimeSpan.TicksPerMillisecond,
|
||||
};
|
||||
var ties = rng.Next(7) == 0 ? rng.Next(2, 9) : 1;
|
||||
for (var k = 0; k < ties; k++)
|
||||
{
|
||||
var ts = Trunc(new DateTime(t, DateTimeKind.Utc), resolution);
|
||||
var status = rng.Next(12) == 0 ? StatusCodes.Bad : StatusCodes.Good;
|
||||
list.Add(new DataValueSnapshot(status == StatusCodes.Bad ? null : (double)v, status, ts, ts));
|
||||
v++;
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>A window bound: before / after the data, exactly on a sample (a tie, maybe), one step past one, a tick before one, or between two.</summary>
|
||||
private static DateTime PickBoundary(Random rng, List<DataValueSnapshot> series, long resolution)
|
||||
{
|
||||
var first = Ts(series[0]);
|
||||
var last = Ts(series[^1]);
|
||||
var i = rng.Next(series.Count);
|
||||
var t = Ts(series[i]);
|
||||
switch (rng.Next(9))
|
||||
{
|
||||
case 0: return first.AddSeconds(-rng.Next(1, 100));
|
||||
case 1: return last.AddSeconds(rng.Next(1, 100));
|
||||
case 2:
|
||||
case 3: return t;
|
||||
case 4: return t.AddTicks(resolution);
|
||||
case 5: return t.AddTicks(-1); // sub-resolution: a microsecond backend truncates it back ONTO the sample
|
||||
case 6: return t.AddMilliseconds(1);
|
||||
default:
|
||||
var next = i + 1 < series.Count ? Ts(series[i + 1]) : t.AddSeconds(10);
|
||||
return new DateTime(t.Ticks + (long)((next.Ticks - t.Ticks) * rng.NextDouble()), DateTimeKind.Utc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The series' own answer for a plain (no bounds) read: forward <c>[start, end)</c>, point, reverse / end-only <c>(end, start]</c> newest first.</summary>
|
||||
private static List<DataValue> Oracle(List<DataValueSnapshot> series, DateTime start, DateTime end, Direction direction, long resolution)
|
||||
{
|
||||
var s = Trunc(start, resolution);
|
||||
var e = Trunc(end, resolution);
|
||||
IEnumerable<DataValueSnapshot> window = direction switch
|
||||
{
|
||||
// start == end is Part 11's point read whichever way it was generated: the node manager reads it forward.
|
||||
Direction.Forward or Direction.Reverse when s == e => series.Where(x => Ts(x) == s),
|
||||
Direction.Forward => series.Where(x => Ts(x) >= s && Ts(x) < e),
|
||||
Direction.Reverse => series.Where(x => Ts(x) > e && Ts(x) <= s).Reverse(),
|
||||
_ => series.Where(x => Ts(x) <= e).Reverse(),
|
||||
};
|
||||
return window.Select(x => new DataValue(new Variant(x.Value), new StatusCode(x.StatusCode), Ts(x), Ts(x))).ToList();
|
||||
}
|
||||
|
||||
/// <summary>The largest tie cluster among the returned stored values (a bound placeholder is not a stored value).</summary>
|
||||
private static int LargestTieCluster(IReadOnlyList<DataValue> truth) =>
|
||||
truth.Where(v => !(v.Value is null && v.StatusCode.Code == StatusCodes.BadBoundNotFound))
|
||||
.GroupBy(v => v.SourceTimestamp.Ticks).Select(g => g.Count()).DefaultIfEmpty(0).Max();
|
||||
|
||||
/// <summary>The most samples within one millisecond anywhere in the series (the walk-back's refusal rule).</summary>
|
||||
private static int MillisecondMax(List<DataValueSnapshot> series)
|
||||
{
|
||||
var msMax = 0;
|
||||
for (int i = 0, j = 0; i < series.Count; i++)
|
||||
{
|
||||
while (Ts(series[i]).Ticks - Ts(series[j]).Ticks >= TimeSpan.TicksPerMillisecond) j++;
|
||||
msMax = Math.Max(msMax, i - j + 1);
|
||||
}
|
||||
|
||||
return msMax;
|
||||
}
|
||||
|
||||
private static bool EndsOnTie(List<DataValueSnapshot> series, DateTime start, DateTime end, Direction direction, long resolution)
|
||||
{
|
||||
var edge = Trunc(direction == Direction.Forward ? end : direction == Direction.Reverse ? start : end, resolution);
|
||||
return series.Count(x => Ts(x) == edge) >= 2;
|
||||
}
|
||||
|
||||
// --- helpers ----------------------------------------------------------------------------------
|
||||
|
||||
private static string Sizes(IEnumerable<IReadOnlyList<DataValue>> pages) => "[" + string.Join(",", pages.Select(p => p.Count)) + "]";
|
||||
|
||||
private static string Render(IEnumerable<DataValue> values) =>
|
||||
string.Join(" | ", values.Select(v => $"{v.SourceTimestamp.Ticks}={v.Value ?? "null"}/{v.StatusCode.Code:X8}"));
|
||||
|
||||
private (IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp) Read(DateTime start, DateTime end, uint max, bool bounds, byte[]? cp)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
StartTime = start, EndTime = end, NumValuesPerNode = max, ReturnBounds = bounds, IsReadModified = false,
|
||||
};
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = fx.NodeId, ContinuationPoint = cp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
fx.Nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
|
||||
|
||||
var values = new List<DataValue>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
values.AddRange(((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues);
|
||||
return (values, errors[0], results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A series-backed historian shaped like one backend, enforcing the backend half of the server raw cap exactly as
|
||||
/// the real data sources do: it reports the cap as <see cref="MaxRawValuesPerRead"/>; a read asking for 0 or more than
|
||||
/// the cap reads cap + 1 rows and throws <see cref="HistoryReadLimitExceededException"/> when the window holds more
|
||||
/// (never a truncated answer). It also records every such request (<see cref="OverCapRequests"/>) — the server must
|
||||
/// never issue one — and every reverse window sent to the forward-only shape (<see cref="BadRequests"/>).
|
||||
/// </summary>
|
||||
private sealed class CappedFake(IReadOnlyList<DataValueSnapshot> series, Shape shape, int cap) : IHistorianDataSource
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
public int Reads { get; private set; }
|
||||
public int OverCapRequests { get; private set; }
|
||||
public int BadRequests { get; private set; }
|
||||
|
||||
public bool SupportsReverseRaw => shape != Shape.Gateway;
|
||||
|
||||
public long TimestampResolutionTicks => shape == Shape.QuestDb ? 10 : 1;
|
||||
|
||||
public int MaxRawValuesPerRead => cap;
|
||||
|
||||
private DateTime Trunc(DateTime t) => NodeManagerHistoryReadRawCapFuzzTests.Trunc(t, TimestampResolutionTicks);
|
||||
|
||||
public Task<HistorianRead> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
|
||||
{
|
||||
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
|
||||
lock (_gate)
|
||||
{
|
||||
Reads++;
|
||||
if (probe) OverCapRequests++;
|
||||
if (shape == Shape.Gateway && startUtc > endUtc) BadRequests++;
|
||||
}
|
||||
|
||||
if (shape == Shape.Gateway && startUtc > endUtc)
|
||||
throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)");
|
||||
|
||||
startUtc = Trunc(startUtc);
|
||||
endUtc = Trunc(endUtc);
|
||||
List<DataValueSnapshot> window;
|
||||
if (startUtc > endUtc) window = series.Where(s => Ts(s) > endUtc && Ts(s) <= startUtc).Reverse().ToList();
|
||||
else if (startUtc == endUtc) window = series.Where(s => Ts(s) == startUtc).ToList();
|
||||
else window = series.Where(s => Ts(s) >= startUtc && Ts(s) < endUtc).ToList();
|
||||
|
||||
var top = probe ? cap + 1 : maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var page = window.Take(top).ToList();
|
||||
if (probe && page.Count > cap)
|
||||
throw new HistoryReadLimitExceededException(cap, $"more than MaxRawValuesPerRead ({cap}) samples");
|
||||
return Task.FromResult(new HistorianRead(page, null));
|
||||
}
|
||||
|
||||
public Task<HistorianRead> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistorianRead> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
|
||||
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test-only wiring shared by the paging fuzz: swap the source and the tie budget, with a fresh continuation store.</summary>
|
||||
internal static class NodeManagerFuzzExtensions
|
||||
{
|
||||
public static void HistoricalSourceForTest(this OtOpcUaNodeManager nm, IHistorianDataSource source, int tieBudget)
|
||||
{
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
nm.MaxTieClusterOverfetch = tieBudget;
|
||||
nm.HistorianDataSource = source;
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,15 @@ public sealed class RawPageLimitTests
|
||||
[InlineData(5000u, 983, 984, 0u, 0u, true)] // the ties alone fill a read: never a 0 (uncapped) request
|
||||
[InlineData(5000u, 5000, 984, 0u, 0u, true)] // never an underflow
|
||||
[InlineData(5000u, 0, 1, 0u, 0u, true)]
|
||||
// R08:1.1 (#736): the server raw cap (default 10 000) is the limit every backend reports — an uncapped read pages at it.
|
||||
[InlineData(0u, 0, 10_000, 9_999u, 10_000u, false)] // NumValuesPerNode = 0: a first page of cap − 1 + the lookahead row
|
||||
[InlineData(0u, 1, 10_000, 9_998u, 10_000u, false)] // a resumed page re-reads its one boundary tie
|
||||
[InlineData(100_000u, 0, 10_000, 9_999u, 10_000u, false)] // a client cap above the server cap is bounded by it
|
||||
[InlineData(10_000u, 0, 10_000, 9_999u, 10_000u, false)] // exactly the cap: still cap − 1 (the lookahead row needs room)
|
||||
[InlineData(500u, 0, 10_000, 500u, 501u, false)] // under the cap: unchanged
|
||||
[InlineData(0u, 0, 3, 2u, 3u, false)] // the smallest usable cap (MinMaxRawValuesPerRead = 3): a first page of two
|
||||
[InlineData(0u, 1, 3, 1u, 3u, false)] // … and a resumed page still carries one value beside its boundary tie
|
||||
[InlineData(0u, 1, 2, 0u, 0u, true)] // at 2 a resumed page has NO room — every value would cost an empty page (why 3 is the floor)
|
||||
public void Plan_fits_page_ties_and_lookahead_into_the_limit(uint want, int k, int limit, uint page, uint request, bool noRoom)
|
||||
{
|
||||
var plan = RawPageLimit.Plan(want, k, limit);
|
||||
|
||||
+6
-6
@@ -64,7 +64,7 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
{
|
||||
published.Enqueue(e);
|
||||
var quality = e.Snapshot.StatusCode == 0u ? OpcUaStatus.Good : OpcUaStatus.Bad;
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
|
||||
mux.Tell(ValuePublishes.Single(
|
||||
"calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
};
|
||||
|
||||
@@ -75,7 +75,7 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
// compute dedups) until the computed value has flowed all the way through.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
|
||||
mux.Tell(ValuePublishes.Single(
|
||||
"src-drv", "src/a", 21.0, OpcUaStatus.Good, Ts));
|
||||
|
||||
// Leg 1 — dep flow: src/a=21 → A computes 42 and publishes on calc/engine/out.
|
||||
@@ -118,14 +118,14 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
{
|
||||
published.Enqueue(e);
|
||||
var quality = e.Snapshot.StatusCode == 0u ? OpcUaStatus.Good : OpcUaStatus.Bad;
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
mux.Tell(ValuePublishes.Single("calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
};
|
||||
var adapter = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(driver, mux), "dep-adapter2");
|
||||
|
||||
// Leg 1 — before redeploy: src/a=2 → keep=20, remove=7.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/a", 2.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/a", 2.0, OpcUaStatus.Good, Ts));
|
||||
published.ShouldContain(e => e.FullReference == "calc/keep" && Equals(e.Snapshot.Value, 20.0));
|
||||
published.ShouldContain(e => e.FullReference == "calc/remove" && Equals(e.Snapshot.Value, 7.0));
|
||||
}, PresenceBudget, TimeSpan.FromMilliseconds(100));
|
||||
@@ -150,8 +150,8 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
// new result; the REMOVED tag never publishes a value derived from the post-redeploy input.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/b", 50.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/a", 3.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/b", 50.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/a", 3.0, OpcUaStatus.Good, Ts));
|
||||
|
||||
// (a) new dep B flows through the RE-REGISTERED interest → new tag computes: 50 + 100 = 150.
|
||||
published.ShouldContain(e => e.FullReference == "calc/add" && Equals(e.Snapshot.Value, 150.0));
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 U1: <c>GetDiagnostics</c> answered a placeholder — <c>Guid.Empty</c>, "Spawned", zero devices and
|
||||
/// "now" for every child, in every behaviour — so the snapshot looked live and said nothing. It now carries
|
||||
/// each child's row id, instance id, last-published health state, device counts and last change time.
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly NodeId TestNode = NodeId.Parse("node-1:4053");
|
||||
private const string DriverId = "DI-1";
|
||||
|
||||
[Fact]
|
||||
public async Task A_connected_child_reports_its_row_id_name_health_state_and_device_count()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var rowId = Guid.NewGuid();
|
||||
var before = DateTime.UtcNow.AddSeconds(-1);
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus");
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevA, new[] { (DriverId, "Line1-Modbus", "dev1", "speed", "Float", false) },
|
||||
Array.Empty<(string, string, string, string, string?)>(),
|
||||
enabled: true, driverRowIds: new Dictionary<string, Guid> { [DriverId] = rowId });
|
||||
var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
|
||||
|
||||
var snapshot = await AwaitDiagnosticsAsync(host, s => s.Drivers.Any(d => d.State == "Healthy"));
|
||||
|
||||
snapshot.NodeId.ShouldBe(TestNode);
|
||||
snapshot.CurrentRevision.ShouldBe(RevA);
|
||||
snapshot.RunningFromCache.ShouldBeFalse();
|
||||
var driver = snapshot.Drivers.ShouldHaveSingleItem();
|
||||
driver.DriverInstanceId.ShouldBe(rowId, "the real row id, not Guid.Empty");
|
||||
driver.Name.ShouldBe(DriverId, "the string instance id every other surface keys on");
|
||||
driver.State.ShouldBe("Healthy");
|
||||
driver.ConnectedDevices.ShouldBe(1);
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
driver.LastChangeUtc.ShouldBeGreaterThan(before);
|
||||
driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stubbed_child_reports_Stubbed_with_no_devices()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
// The factory answers for a type the artifact does not use, so the host stubs the child.
|
||||
var factory = new RecordingSubscribableDriverFactory("NotModbus");
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevA, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) },
|
||||
Array.Empty<(string, string, string, string, string?)>(), enabled: true);
|
||||
var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
|
||||
|
||||
var snapshot = await AwaitDiagnosticsAsync(host, s => s.Drivers.Count == 1);
|
||||
|
||||
var driver = snapshot.Drivers.Single();
|
||||
driver.State.ShouldBe("Stubbed");
|
||||
driver.ConnectedDevices.ShouldBe(0);
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
}
|
||||
|
||||
private async Task<NodeDiagnosticsSnapshot> AwaitDiagnosticsAsync(IActorRef host, Func<NodeDiagnosticsSnapshot, bool> ready)
|
||||
{
|
||||
NodeDiagnosticsSnapshot? snapshot = null;
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
snapshot = await host.Ask<NodeDiagnosticsSnapshot>(new GetDiagnostics(CorrelationId.NewId()), PresenceBudget);
|
||||
ready(snapshot).ShouldBeTrue();
|
||||
}, PresenceBudget);
|
||||
return snapshot!;
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L2: <c>DriverHostActor._lastReprovisionUtc</c> was never pruned — a driver removed by a deploy
|
||||
/// after a supervision crash kept its stamp, so the map grew with every id ever reprovisioned-then-removed,
|
||||
/// and (the observable half) a driver removed and later re-added inherited the OLD stamp: its first crash
|
||||
/// after the re-add was deferred by the backoff as if it were crash-looping, although the new incarnation had
|
||||
/// never crashed. <c>StopChild</c> now drops the stamp.
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorReprovisionStampTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
||||
private static readonly RevisionHash RevC = RevisionHash.Parse(new string('c', 64));
|
||||
private static readonly NodeId TestNode = NodeId.Parse("node-1:4053");
|
||||
private const string DriverId = "DI-1";
|
||||
private const string SpeedPath = "Plant/Modbus/dev1/speed";
|
||||
|
||||
[Fact]
|
||||
public void A_driver_removed_and_re_added_does_not_inherit_the_old_incarnations_reprovision_stamp()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
// Far longer than the presence budget below: an inherited stamp would defer the re-added driver's
|
||||
// reprovision past it, which is the failure this test reports.
|
||||
var backoff = TimeSpan.FromSeconds(60);
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus");
|
||||
var withDriver = new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) };
|
||||
var none = Array.Empty<(string, string, string, string, string?)>();
|
||||
|
||||
var revA = V3ArtifactSeeder.SeedV3Deployment(db, RevA, withDriver, none, enabled: true);
|
||||
var (host, _, _, coordinator) = V3ArtifactSeeder.SpawnHostAndApplyWithCoordinator(
|
||||
this, db, TestNode, revA, RevA, factory, reprovisionBackoff: backoff);
|
||||
AwaitAssert(() => factory.Created.Count.ShouldBe(1), PresenceBudget);
|
||||
AwaitAssert(() => factory.Created[0].SubscribeCount.ShouldBe(1), PresenceBudget);
|
||||
|
||||
// Crash 1 on the first incarnation: reprovisioned immediately, which stamps the id.
|
||||
factory.Created[0].FireMalformedDataChange(SpeedPath);
|
||||
AwaitAssert(() => factory.Created[0].InitializeCount.ShouldBe(2), PresenceBudget);
|
||||
|
||||
// Deploy B removes the driver (StopChild), deploy C re-adds it: a brand-new driver object.
|
||||
var revB = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevB, Array.Empty<(string, string, string, string, string, bool)>(), none, enabled: true);
|
||||
V3ArtifactSeeder.Dispatch(host, coordinator, revB, RevB);
|
||||
var revC = V3ArtifactSeeder.SeedV3Deployment(db, RevC, withDriver, none, enabled: true);
|
||||
V3ArtifactSeeder.Dispatch(host, coordinator, revC, RevC);
|
||||
AwaitAssert(() => factory.Created.Count.ShouldBe(2), PresenceBudget);
|
||||
var reAdded = factory.Created[1];
|
||||
AwaitAssert(() => reAdded.SubscribeCount.ShouldBe(1), PresenceBudget);
|
||||
|
||||
// Its FIRST crash must reprovision immediately — it has no history of its own.
|
||||
reAdded.FireMalformedDataChange(SpeedPath);
|
||||
AwaitAssert(
|
||||
() => reAdded.InitializeCount.ShouldBe(
|
||||
2, "the re-added driver's first reprovision was deferred by the REMOVED incarnation's stamp"),
|
||||
duration: TimeSpan.FromSeconds(5));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Event;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 H5: the <c>Stale</c> behaviour registered no receive for a driver child's value/alarm bursts (nor
|
||||
/// for a late <c>ArtifactForApply</c> / <c>ApplyWedged</c> pipe-back), so any that landed there hit
|
||||
/// <c>Unhandled</c> — dead-letter noise and a dropped message with no log naming why. They are now explicit
|
||||
/// drops. The two private pipe-back records cannot be constructed from a test; the two public child
|
||||
/// messages are pinned here, and the private pair by the same registration pattern (compile-time).
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorStaleDropTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("stale-drop-test");
|
||||
|
||||
[Fact]
|
||||
public void Child_value_and_alarm_bursts_landing_in_Stale_are_dropped_not_unhandled()
|
||||
{
|
||||
var unhandled = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(unhandled.Ref, typeof(UnhandledMessage));
|
||||
|
||||
// ThrowingDbFactory ⇒ Bootstrap's catch ⇒ Become(Stale).
|
||||
var host = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, CreateTestProbe().Ref,
|
||||
localRoles: new HashSet<string> { "driver" }));
|
||||
// Barrier: Stale answers RouteNodeWrite synchronously, so a reply proves the host is in Stale.
|
||||
var asker = CreateTestProbe();
|
||||
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget).Reason!.ShouldContain("stale");
|
||||
|
||||
// Positive control: an unknown message IS reported as unhandled by this host.
|
||||
host.Tell("not a message the host knows");
|
||||
unhandled.ExpectMsg<UnhandledMessage>(m => ReferenceEquals(m.Recipient, host) || m.Recipient.Equals(host), PresenceBudget);
|
||||
|
||||
host.Tell(new DriverInstanceActor.AttributeValueBatchPublished(
|
||||
"DI-1", new[] { new DriverInstanceActor.AttributeValueItem("Plant/x/y", 1.0, OpcUaStatus.Good, DateTime.UtcNow) }));
|
||||
host.Tell(new DriverInstanceActor.AttributeAlarmPublished(
|
||||
"DI-1", new AlarmEventArgs(new Handle(), "Plant/x/alarm", "Plant/x/alarm", "AlarmCondition", "test",
|
||||
AlarmSeverity.High, DateTime.UtcNow)));
|
||||
// Barrier again, then the absence window.
|
||||
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget);
|
||||
|
||||
unhandled.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
private sealed class Handle : IAlarmSubscriptionHandle
|
||||
{
|
||||
public string DiagnosticId => "stale-test";
|
||||
}
|
||||
|
||||
private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext() => throw new InvalidOperationException("db unreachable");
|
||||
}
|
||||
}
|
||||
+288
@@ -0,0 +1,288 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 P2: <c>WriteAttribute</c> was a <c>ReceiveAsync</c>, which suspends the mailbox for the whole
|
||||
/// device round-trip (up to the 5 s bound). Writes therefore serialised — the SECOND of two operator writes to
|
||||
/// a ~3 s device missed the host's 8 s Ask and was reverted although the device took it — and every data
|
||||
/// change queued behind the write. Writes now run off the mailbox under a per-driver in-flight cap; the
|
||||
/// subscribe/unsubscribe pair that must serialise against the subscription handle stays <c>ReceiveAsync</c>.
|
||||
/// </summary>
|
||||
public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly TimeSpan WriteLatency = TimeSpan.FromMilliseconds(1200);
|
||||
|
||||
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(
|
||||
IDriver driver, int? maxWritesInFlight = null, TimeSpan? queuedWriteBudget = null, TimeSpan? writeTimeout = null)
|
||||
{
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
|
||||
driver, commsLossTimeout: TimeSpan.Zero, maxWritesInFlight: maxWritesInFlight,
|
||||
queuedWriteBudget: queuedWriteBudget, writeTimeout: writeTimeout));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => m.Connected, PresenceBudget);
|
||||
return (actor, parent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Two_concurrent_slow_writes_both_complete_within_one_device_round_trip()
|
||||
{
|
||||
var driver = new SlowWritableDriver { Latency = WriteLatency };
|
||||
var (actor, _) = Connect(driver);
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var askBudget = TimeSpan.FromSeconds(8); // the host's Ask
|
||||
var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-1", 1), askBudget);
|
||||
var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-2", 2), askBudget);
|
||||
var results = await Task.WhenAll(first, second);
|
||||
sw.Stop();
|
||||
|
||||
results.ShouldAllBe(r => r.Success);
|
||||
driver.Writes.Select(w => w.FullReference).ShouldBe(new[] { "tag-1", "tag-2" }, ignoreOrder: true);
|
||||
// Both were inside the driver at once — that is the proof; the wall clock is only a sanity ceiling
|
||||
// (serialised writes would take two latencies, and the host's Ask is 8 s).
|
||||
driver.MaxConcurrentWrites.ShouldBe(2, "the second write waited for the first — writes are still serialised on the mailbox");
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(8));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Data_changes_are_delivered_while_a_write_is_pending()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, parent) = Connect(driver);
|
||||
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
|
||||
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
|
||||
|
||||
var pending = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 7), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
|
||||
driver.FireDataChange("tag-a", 3.5);
|
||||
var published = parent.ExpectMsg<DriverInstanceActor.AttributeValueBatchPublished>(PresenceBudget);
|
||||
published.Items.Single().Value.ShouldBe(3.5);
|
||||
pending.IsCompleted.ShouldBeFalse("the write is still held — the data change did not wait behind it");
|
||||
|
||||
// Positive control: releasing the device completes the write with its real outcome.
|
||||
driver.ReleaseWrites();
|
||||
(await pending).Success.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Writes_beyond_the_in_flight_cap_queue_and_drain_in_order()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, _) = Connect(driver, maxWritesInFlight: 2);
|
||||
|
||||
var asks = Enumerable.Range(0, 4)
|
||||
.Select(i => actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute($"tag-{i}", i), PresenceBudget))
|
||||
.ToArray();
|
||||
|
||||
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
|
||||
// Settle: a third write entering the driver would show up here.
|
||||
await Task.Delay(300);
|
||||
driver.WritesEntered.ShouldBe(2, "the cap let a third write into the driver");
|
||||
|
||||
driver.ReleaseWrites();
|
||||
var results = await Task.WhenAll(asks);
|
||||
results.ShouldAllBe(r => r.Success);
|
||||
// Entry order: the first two slots went to the first two writes, the queued pair entered only after a
|
||||
// slot freed. Within each pair the order is unspecified — distinct tags run concurrently on the pool;
|
||||
// only SAME-tag order is guaranteed (pinned by the same-tag test below).
|
||||
var entered = driver.EnteredRefs;
|
||||
entered.Count.ShouldBe(4);
|
||||
entered.Take(2).ShouldBe(new[] { "tag-0", "tag-1" }, ignoreOrder: true);
|
||||
entered.Skip(2).ShouldBe(new[] { "tag-2", "tag-3" }, ignoreOrder: true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix round 1: writes to the SAME tag must keep their order — two concurrent writes v1, v2 could land as
|
||||
/// v1. A same-tag write waits behind its tag while other tags still run concurrently.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Writes_to_the_same_tag_are_serialised_in_order_while_other_tags_run_concurrently()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, _) = Connect(driver, maxWritesInFlight: 4);
|
||||
|
||||
var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
|
||||
// Positive control: a DIFFERENT tag is not held behind tag-a.
|
||||
var other = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-b", 9), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
|
||||
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-b" });
|
||||
|
||||
await Task.Delay(300);
|
||||
driver.WritesEntered.ShouldBe(2, "the second tag-a write entered the driver while the first was still inside it");
|
||||
second.IsCompleted.ShouldBeFalse();
|
||||
|
||||
driver.ReleaseWrites();
|
||||
(await Task.WhenAll(first, second, other)).ShouldAllBe(r => r.Success);
|
||||
driver.Writes.Where(w => w.FullReference == "tag-a").Select(w => w.Value).ShouldBe(new object[] { 1, 2 });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix round 1: a write that sat in the queue longer than the host's Ask budget has already been reported
|
||||
/// as a timeout (and reverted) upstream — writing it late would push a stale value to the device.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_queued_write_older_than_the_budget_is_failed_BadTimeout_instead_of_written_late()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, _) = Connect(driver, maxWritesInFlight: 1, queuedWriteBudget: TimeSpan.FromMilliseconds(300));
|
||||
|
||||
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
var stale = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
|
||||
|
||||
await Task.Delay(600);
|
||||
driver.ReleaseWrites();
|
||||
|
||||
(await inFlight).Success.ShouldBeTrue();
|
||||
var staleResult = await stale;
|
||||
staleResult.Success.ShouldBeFalse();
|
||||
staleResult.Reason!.ShouldContain("800A0000", customMessage: "BadTimeout");
|
||||
driver.EnteredRefs.ShouldBe(new[] { "tag-0" }, "the stale write must never reach the device");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix round 2: a token-ignoring hung write held its tag forever, so every later write to that tag piled up
|
||||
/// in the queue. The write is bounded pessimistically; on overrun it answers BadTimeout and releases its
|
||||
/// slot and tag, and a later write to the same tag reaches the device.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_hung_token_ignoring_write_is_abandoned_and_releases_its_tag_for_a_later_write()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true, IgnoreCancellation = true };
|
||||
var (actor, _) = Connect(driver, maxWritesInFlight: 1, writeTimeout: TimeSpan.FromMilliseconds(300));
|
||||
|
||||
var hung = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
|
||||
var hungResult = await hung;
|
||||
hungResult.Success.ShouldBeFalse();
|
||||
hungResult.Reason!.ShouldContain("800A0000", customMessage: "BadTimeout");
|
||||
driver.MaxConcurrentWrites.ShouldBe(1);
|
||||
|
||||
// The tag (and the single slot) are free again: a later tag-a write reaches the device even though the
|
||||
// abandoned one is still inside it.
|
||||
var later = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
|
||||
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-a" });
|
||||
driver.MaxConcurrentWrites.ShouldBe(2, "positive control: the abandoned write really is still wedged inside the driver");
|
||||
|
||||
driver.ReleaseWrites();
|
||||
(await later).Success.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Queued_writes_fail_fast_when_the_driver_leaves_Connected()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, parent) = Connect(driver, maxWritesInFlight: 1);
|
||||
|
||||
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
var queued = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.ForceReconnect());
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => !m.Connected, PresenceBudget);
|
||||
|
||||
var queuedResult = await queued;
|
||||
queuedResult.Success.ShouldBeFalse();
|
||||
queuedResult.Reason.ShouldBe("driver not connected");
|
||||
driver.WritesEntered.ShouldBe(1, "a queued write must never reach a driver that is reconnecting");
|
||||
|
||||
// The in-flight one still answers with the device's real outcome once it is released.
|
||||
driver.ReleaseWrites();
|
||||
(await inFlight).Success.ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>A writable + subscribable driver whose writes either take <see cref="Latency"/> or hold until released.</summary>
|
||||
private sealed class SlowWritableDriver : IDriver, IWritable, ISubscribable
|
||||
{
|
||||
private readonly StubHandle _handle = new();
|
||||
private readonly ConcurrentQueue<WriteRequest> _writes = new();
|
||||
private readonly ConcurrentQueue<string> _enteredRefs = new();
|
||||
private TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private int _entered;
|
||||
private int _inside;
|
||||
private int _maxInside;
|
||||
|
||||
public TimeSpan Latency { get; init; }
|
||||
public bool HoldWrites { get; init; }
|
||||
/// <summary>When true a held write ignores its cancellation token — a wedged driver.</summary>
|
||||
public bool IgnoreCancellation { get; init; }
|
||||
|
||||
public IReadOnlyList<WriteRequest> Writes => _writes.ToArray();
|
||||
public IReadOnlyList<string> EnteredRefs => _enteredRefs.ToArray();
|
||||
public int WritesEntered => Volatile.Read(ref _entered);
|
||||
public int MaxConcurrentWrites => Volatile.Read(ref _maxInside);
|
||||
|
||||
public string DriverInstanceId => "slow-writer-1";
|
||||
public string DriverType => "SlowWriter";
|
||||
|
||||
public void ReleaseWrites() => _release.TrySetResult();
|
||||
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
||||
|
||||
public async Task<IReadOnlyList<WriteResult>> WriteAsync(IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _entered);
|
||||
foreach (var w in writes) _enteredRefs.Enqueue(w.FullReference);
|
||||
var inside = Interlocked.Increment(ref _inside);
|
||||
int seen;
|
||||
while (inside > (seen = Volatile.Read(ref _maxInside)))
|
||||
Interlocked.CompareExchange(ref _maxInside, inside, seen);
|
||||
try
|
||||
{
|
||||
if (HoldWrites && IgnoreCancellation) await _release.Task;
|
||||
else if (HoldWrites) await _release.Task.WaitAsync(cancellationToken);
|
||||
else if (Latency > TimeSpan.Zero) await Task.Delay(Latency, cancellationToken);
|
||||
foreach (var w in writes) _writes.Enqueue(w);
|
||||
return writes.Select(_ => new WriteResult(OpcUaStatus.Good)).ToList();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Decrement(ref _inside);
|
||||
}
|
||||
}
|
||||
|
||||
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
||||
|
||||
public Task<ISubscriptionHandle> SubscribeAsync(
|
||||
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<ISubscriptionHandle>(_handle);
|
||||
|
||||
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public void FireDataChange(string fullRef, object? value) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(
|
||||
_handle, fullRef, new DataValueSnapshot(value, OpcUaStatus.Good, DateTime.UtcNow, DateTime.UtcNow)));
|
||||
}
|
||||
}
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L1 / P4: a driver child that stops for good must <b>dispose</b> its <see cref="IDriver"/>
|
||||
/// (the runtime only ever called <c>ShutdownAsync</c>, so a Calculation driver's compiled-script ALCs and an
|
||||
/// OpcUaClient driver's LoggerFactory + gate + CTS leaked on every respawn), its <c>ShutdownAsync</c> must be
|
||||
/// bounded (it used to block a dispatcher thread for as long as a dead socket took to answer), and a
|
||||
/// <b>supervision restart</b> must NOT dispose — the Props closure re-uses the same driver object for the
|
||||
/// restarted incarnation, which then has to serve through it.
|
||||
/// </summary>
|
||||
public sealed class DriverInstanceActorDisposeTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Stopping_the_child_shuts_the_driver_down_and_then_disposes_it_exactly_once()
|
||||
{
|
||||
var driver = new DisposableRecordingDriver();
|
||||
var parent = CreateTestProbe();
|
||||
parent.IgnoreMessages(m => m is DriverInstanceActor.ConnectivityChanged);
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
AwaitCondition(() => driver.InitializeCount == 1, PresenceBudget);
|
||||
|
||||
Watch(actor);
|
||||
Sys.Stop(actor);
|
||||
ExpectTerminated(actor, PresenceBudget);
|
||||
|
||||
driver.ShutdownCount.ShouldBe(1);
|
||||
driver.DisposeCount.ShouldBe(1, "a child stopped for good must dispose the driver it wrapped");
|
||||
driver.Trace.ToArray().ShouldBe(new[] { "init", "shutdown", "dispose" }, "shutdown must land before dispose");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_supervision_restart_keeps_the_driver_undisposed_and_the_restarted_child_still_serves()
|
||||
{
|
||||
var driver = new DisposableRecordingDriver();
|
||||
var parent = CreateTestProbe();
|
||||
// The probe parent's own strategy STOPS a failing child; the production host (no strategy anywhere under
|
||||
// src/Server) gets Akka's default decider, which RESTARTS it — the route under test.
|
||||
var actor = parent.ChildActorOf(
|
||||
DriverInstanceActor.Props(driver), new OneForOneStrategy(_ => Directive.Restart));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
// Barrier: Subscribe is only handled once Connected (it is dropped in Connecting).
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => m.Connected, PresenceBudget);
|
||||
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
|
||||
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
|
||||
|
||||
// The crash seam (see DriverHostActorRestartTests): a null snapshot NREs on the actor thread, and the
|
||||
// probe parent's default decider restarts the child in place around the SAME driver object.
|
||||
driver.FireMalformedDataChange("tag-a");
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ReprovisionRequested, PresenceBudget);
|
||||
|
||||
driver.ShutdownCount.ShouldBe(1, "PreRestart runs PostStop, which shuts the driver down");
|
||||
driver.DisposeCount.ShouldBe(0, "a restart re-uses the driver object — disposing it would kill the new incarnation");
|
||||
|
||||
// Positive control: the restarted incarnation serves through the undisposed driver. The host would
|
||||
// re-provision it exactly like this (InitializeRequested + the desired set).
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
|
||||
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
|
||||
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
|
||||
driver.FireDataChange("tag-a", 42.0);
|
||||
var published = (DriverInstanceActor.AttributeValueBatchPublished)parent.FishForMessage(
|
||||
m => m is DriverInstanceActor.AttributeValueBatchPublished, PresenceBudget);
|
||||
published.Items.Single().FullReference.ShouldBe("tag-a");
|
||||
published.Items.Single().Value.ShouldBe(42.0);
|
||||
|
||||
// ...and the eventual real stop still disposes it — once.
|
||||
Watch(actor);
|
||||
Sys.Stop(actor);
|
||||
ExpectTerminated(actor, PresenceBudget);
|
||||
driver.DisposeCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_hung_ShutdownAsync_is_abandoned_at_the_bound_and_the_driver_is_still_disposed()
|
||||
{
|
||||
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
// Ignores its token on purpose: a wedged socket/COM proxy does not honour cancellation either.
|
||||
var driver = new DisposableRecordingDriver { ShutdownBehavior = _ => never.Task };
|
||||
var bound = TimeSpan.FromMilliseconds(300);
|
||||
var parent = CreateTestProbe();
|
||||
parent.IgnoreMessages(m => m is DriverInstanceActor.ConnectivityChanged);
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver, shutdownTimeout: bound));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
AwaitCondition(() => driver.InitializeCount == 1, PresenceBudget);
|
||||
|
||||
Watch(actor);
|
||||
var sw = Stopwatch.StartNew();
|
||||
Sys.Stop(actor);
|
||||
ExpectTerminated(actor, PresenceBudget);
|
||||
sw.Stop();
|
||||
|
||||
driver.ShutdownCount.ShouldBe(1, "the shutdown was attempted (positive control for the bound)");
|
||||
never.Task.IsCompleted.ShouldBeFalse("the shutdown is still hung — the stop did not wait for it");
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the stop must not be pinned by the hung shutdown");
|
||||
driver.DisposeCount.ShouldBe(1, "a shutdown that timed out must not skip the dispose");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A subscribable driver that records its lifecycle and is <see cref="IAsyncDisposable"/>. Deliberately
|
||||
/// NOT also <see cref="IDisposable"/>: the actor prefers the async dispose when both exist, and a double
|
||||
/// implementation would hide a double-dispose.
|
||||
/// </summary>
|
||||
private sealed class DisposableRecordingDriver : IDriver, ISubscribable, IAsyncDisposable
|
||||
{
|
||||
private readonly StubHandle _handle = new();
|
||||
private int _initializeCount;
|
||||
private int _shutdownCount;
|
||||
private int _disposeCount;
|
||||
|
||||
public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }
|
||||
|
||||
public List<string> Trace { get; } = new();
|
||||
|
||||
public int InitializeCount => Volatile.Read(ref _initializeCount);
|
||||
public int ShutdownCount => Volatile.Read(ref _shutdownCount);
|
||||
public int DisposeCount => Volatile.Read(ref _disposeCount);
|
||||
|
||||
public string DriverInstanceId => "disposable-1";
|
||||
public string DriverType => "Disposable";
|
||||
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _initializeCount);
|
||||
lock (Trace) Trace.Add("init");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _shutdownCount);
|
||||
lock (Trace) Trace.Add("shutdown");
|
||||
return ShutdownBehavior is { } behavior ? behavior(cancellationToken) : Task.CompletedTask;
|
||||
}
|
||||
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Interlocked.Increment(ref _disposeCount);
|
||||
lock (Trace) Trace.Add("dispose");
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
||||
|
||||
public Task<ISubscriptionHandle> SubscribeAsync(
|
||||
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<ISubscriptionHandle>(_handle);
|
||||
|
||||
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public void FireDataChange(string fullRef, object? value) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(
|
||||
_handle, fullRef, new DataValueSnapshot(value, OpcUaStatus.Good, DateTime.UtcNow, DateTime.UtcNow)));
|
||||
|
||||
/// <summary>The supervision crash seam — see <see cref="RecordingSubscribableDriver.FireMalformedDataChange"/>.</summary>
|
||||
public void FireMalformedDataChange(string fullRef) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, null!));
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 H2: the <c>Reconnecting</c> retry timer fired every <c>reconnectInterval</c> whether or not the
|
||||
/// previous attempt was still running — <c>_initGeneration</c> only discarded the older attempt's RESULT. With an
|
||||
/// Initialize bound longer than the cadence (Tier C's 15 s vs. the 10 s timer, or the test harness's unbounded
|
||||
/// invoker) attempt N+1's <c>ShutdownAsync</c> ran while attempt N's <c>InitializeAsync</c> was still inside the
|
||||
/// driver: the dispose-before-overwrite race the drivers' rule file forbids, once per overlap, for as long as
|
||||
/// the device stayed down. The retry is now single-flight.
|
||||
/// </summary>
|
||||
public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Shutdown_and_Initialize_never_overlap_while_the_init_outlasts_the_retry_cadence()
|
||||
{
|
||||
// A 400 ms failing init against a 50 ms retry cadence: without the gate the timer starts a new
|
||||
// Shutdown+Initialize pair every 50 ms on top of the one still running (eight-deep overlap).
|
||||
var driver = new InterleavingDriver(initDuration: TimeSpan.FromMilliseconds(400), failUntilAttempt: 4);
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
|
||||
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero));
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
|
||||
// Positive control: the gate skips ticks, it does not stop retrying — the 4th attempt succeeds and the
|
||||
// actor connects.
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
|
||||
|
||||
driver.Attempts.ShouldBeGreaterThanOrEqualTo(4);
|
||||
driver.Overlaps.ShouldBe(0,
|
||||
$"ShutdownAsync entered while an InitializeAsync was still running. Interleaving: {driver.Interleaving}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_pre_reconnect_teardown_that_hangs_is_abandoned_at_the_bound_and_the_init_still_runs()
|
||||
{
|
||||
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 2)
|
||||
{
|
||||
// Only the SECOND shutdown (the reconnect teardown) hangs; the first attempt has none.
|
||||
ShutdownBehavior = _ => never.Task,
|
||||
};
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
|
||||
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
|
||||
shutdownTimeout: TimeSpan.FromMilliseconds(200)));
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
|
||||
never.Task.IsCompleted.ShouldBeFalse("the teardown is still hung — the attempt did not wait for it");
|
||||
driver.Attempts.ShouldBe(2);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix round 1: the single-flight gate only cleared on the attempt's RESULT, and the invoker's timeout is
|
||||
/// token-only — a driver whose InitializeAsync ignores its token wedged the reconnect loop forever. The
|
||||
/// attempt is now bounded pessimistically (<c>initializeTimeout</c>); an overrun is abandoned, reported
|
||||
/// as a failure, and the next tick retries.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_token_ignoring_never_completing_init_is_abandoned_and_a_later_attempt_connects()
|
||||
{
|
||||
var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 1)
|
||||
{
|
||||
// Attempt 1 hangs forever and ignores cancellation; attempt 2 completes.
|
||||
HangOnAttempt = 1,
|
||||
};
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
|
||||
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
|
||||
initializeTimeout: TimeSpan.FromMilliseconds(300)));
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
|
||||
driver.Attempts.ShouldBe(2);
|
||||
driver.HungAttemptStillRunning.ShouldBeTrue("positive control: the first attempt really was wedged");
|
||||
}
|
||||
|
||||
/// <summary>Records every Shutdown/Initialize entry and exit so an overlap is a fact, not an inference.</summary>
|
||||
private sealed class InterleavingDriver : IDriver
|
||||
{
|
||||
private readonly TimeSpan _initDuration;
|
||||
private readonly int _failUntilAttempt;
|
||||
private readonly Lock _gate = new();
|
||||
private readonly List<string> _events = new();
|
||||
private int _attempts;
|
||||
private int _initsInProgress;
|
||||
private int _overlaps;
|
||||
|
||||
public InterleavingDriver(TimeSpan initDuration, int failUntilAttempt)
|
||||
{
|
||||
_initDuration = initDuration;
|
||||
_failUntilAttempt = failUntilAttempt;
|
||||
}
|
||||
|
||||
public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }
|
||||
|
||||
/// <summary>The 1-based attempt that never completes and ignores its token (0 = none).</summary>
|
||||
public int HangOnAttempt { get; init; }
|
||||
|
||||
private readonly TaskCompletionSource _never = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public bool HungAttemptStillRunning => !_never.Task.IsCompleted && Volatile.Read(ref _initsInProgress) > 0;
|
||||
|
||||
public int Attempts => Volatile.Read(ref _attempts);
|
||||
public int Overlaps => Volatile.Read(ref _overlaps);
|
||||
public string Interleaving { get { lock (_gate) return string.Join(" ", _events); } }
|
||||
|
||||
public string DriverInstanceId => "interleaving-1";
|
||||
public string DriverType => "Interleaving";
|
||||
|
||||
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
var attempt = Interlocked.Increment(ref _attempts);
|
||||
Interlocked.Increment(ref _initsInProgress);
|
||||
Record($"init{attempt}>");
|
||||
try
|
||||
{
|
||||
if (attempt == HangOnAttempt) await _never.Task;
|
||||
if (_initDuration > TimeSpan.Zero) await Task.Delay(_initDuration, CancellationToken.None);
|
||||
if (attempt < _failUntilAttempt) throw new InvalidOperationException($"init attempt {attempt} refused");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Record($"<init{attempt}");
|
||||
Interlocked.Decrement(ref _initsInProgress);
|
||||
}
|
||||
}
|
||||
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (Volatile.Read(ref _initsInProgress) > 0) Interlocked.Increment(ref _overlaps);
|
||||
Record("shutdown>");
|
||||
try
|
||||
{
|
||||
if (ShutdownBehavior is { } behavior) await behavior(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Record("<shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
||||
|
||||
private void Record(string e) { lock (_gate) _events.Add(e); }
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,8 @@ public static class V3ArtifactSeeder
|
||||
/// is parsed out of <c>TagConfig.alarm</c> by <c>TagConfigIntent</c>), which the alarm-ref half of the
|
||||
/// restart suite needs — an alarm-bearing raw tag is excluded from the VALUE subscription set and carried
|
||||
/// in the alarm one instead, so the two travel through different child code paths.</param>
|
||||
/// <param name="driverRowIds">Optional <c>DriverInstanceRowId</c> (the DB row Guid) per driver id; absent
|
||||
/// ids get <see cref="Guid.Empty"/>, which is what an artifact without the property parses to.</param>
|
||||
/// <param name="taglessDrivers">Extra <c>DriverInstance</c> ids carrying NO devices and NO tags. The
|
||||
/// only way to express a driver that is a running child yet absent from both the value- and alarm-ref
|
||||
/// maps — the <c>_children.Keys</c> arm of <c>ApplyDesiredSubscriptions</c>'s union, which must still
|
||||
@@ -58,14 +60,16 @@ public static class V3ArtifactSeeder
|
||||
(string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs,
|
||||
bool enabled = false,
|
||||
IReadOnlyDictionary<string, string>? tagConfigByTag = null,
|
||||
string[]? taglessDrivers = null)
|
||||
string[]? taglessDrivers = null,
|
||||
IReadOnlyDictionary<string, Guid>? driverRowIds = null)
|
||||
{
|
||||
Guid RowId(string id) => driverRowIds is not null && driverRowIds.TryGetValue(id, out var g) ? g : Guid.Empty;
|
||||
var drivers = rawTags
|
||||
.Select(t => (t.Driver, t.DriverName))
|
||||
.Distinct()
|
||||
.Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })
|
||||
.Select(d => new { DriverInstanceRowId = RowId(d.Driver), DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })
|
||||
.Concat((taglessDrivers ?? Array.Empty<string>())
|
||||
.Select(id => new { DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }))
|
||||
.Select(id => new { DriverInstanceRowId = RowId(id), DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }))
|
||||
.ToArray();
|
||||
var devices = rawTags
|
||||
.Select(t => (t.Driver, t.Device))
|
||||
@@ -201,7 +205,37 @@ public static class V3ArtifactSeeder
|
||||
return (actor, mux, coordinator);
|
||||
}
|
||||
|
||||
private static void Dispatch(IActorRef actor, TestProbe coordinator, DeploymentId deploymentId, RevisionHash rev)
|
||||
/// <summary>The same arrangement as <see cref="SpawnHostAndApply"/>, also returning the coordinator probe so a
|
||||
/// test can <see cref="Dispatch"/> FURTHER deployments to the same host (remove a driver, re-add it).</summary>
|
||||
/// <param name="kit">The calling TestKit, supplying the ActorSystem + probes.</param>
|
||||
/// <param name="db">The factory whose deployment was seeded by <see cref="SeedV3Deployment"/>.</param>
|
||||
/// <param name="localNode">The host's local node id.</param>
|
||||
/// <param name="deploymentId">The deployment to dispatch.</param>
|
||||
/// <param name="rev">The revision hash the deployment was sealed under.</param>
|
||||
/// <param name="driverFactory">Optional driver factory forwarded to <c>DriverHostActor.Props</c>.</param>
|
||||
/// <param name="reprovisionBackoff">Optional per-driver supervision-reprovision rate limit.</param>
|
||||
/// <returns>The host actor plus the publish, mux and coordinator probes wired into it.</returns>
|
||||
public static (IActorRef Actor, TestProbe Publish, TestProbe Mux, TestProbe Coordinator) SpawnHostAndApplyWithCoordinator(
|
||||
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
|
||||
NodeId localNode, DeploymentId deploymentId, RevisionHash rev,
|
||||
IDriverFactory? driverFactory = null,
|
||||
TimeSpan? reprovisionBackoff = null)
|
||||
{
|
||||
var publish = kit.CreateCleanOutcomePublishProbe();
|
||||
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory, reprovisionBackoff);
|
||||
|
||||
Dispatch(actor, coordinator, deploymentId, rev);
|
||||
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(RuntimeActorTestBase.PresenceBudget);
|
||||
|
||||
return (actor, publish, mux, coordinator);
|
||||
}
|
||||
|
||||
/// <summary>Dispatches <paramref name="deploymentId"/> to <paramref name="actor"/> and waits for its Applied ack.</summary>
|
||||
/// <param name="actor">The host.</param>
|
||||
/// <param name="coordinator">The coordinator probe the host acks to.</param>
|
||||
/// <param name="deploymentId">The deployment to dispatch.</param>
|
||||
/// <param name="rev">Its revision hash.</param>
|
||||
public static void Dispatch(IActorRef actor, TestProbe coordinator, DeploymentId deploymentId, RevisionHash rev)
|
||||
{
|
||||
actor.Tell(new DispatchDeployment(deploymentId, rev, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(RuntimeActorTestBase.PresenceBudget)
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Builds the ONE message shape a driver child sends for its values — a
|
||||
/// <see cref="DriverInstanceActor.AttributeValueBatchPublished"/> — for tests that push a single value into the
|
||||
/// dependency mux by hand. The per-value <see cref="DriverInstanceActor.AttributeValuePublished"/> has no
|
||||
/// production sender (02/U4); tests should not keep one alive either.
|
||||
/// </summary>
|
||||
public static class ValuePublishes
|
||||
{
|
||||
/// <summary>A one-item batch for <paramref name="fullReference"/>.</summary>
|
||||
/// <param name="driverInstanceId">The publishing driver's id.</param>
|
||||
/// <param name="fullReference">The value's reference (RawPath).</param>
|
||||
/// <param name="value">The value.</param>
|
||||
/// <param name="statusCode">The full OPC UA status code.</param>
|
||||
/// <param name="timestampUtc">The source timestamp.</param>
|
||||
/// <returns>The batch message.</returns>
|
||||
public static DriverInstanceActor.AttributeValueBatchPublished Single(
|
||||
string driverInstanceId, string fullReference, object? value, uint statusCode, DateTime timestampUtc) =>
|
||||
new(driverInstanceId, new[] { new DriverInstanceActor.AttributeValueItem(fullReference, value, statusCode, timestampUtc) });
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 P7: <see cref="DbHealthProbeActor"/> ran its SELECT synchronously on the actor thread, so a
|
||||
/// hanging SQL connect/command held the dispatcher thread for the implicit 15 s / 30 s SqlClient timeouts and
|
||||
/// queued every later tick (and every <c>GetStatus</c> Ask) behind it. The probe now runs off the mailbox
|
||||
/// and pipes its outcome back, so the cached answer stays available while a probe hangs.
|
||||
/// </summary>
|
||||
public sealed class DbHealthProbeOffThreadTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public async Task GetStatus_answers_from_cache_while_a_probe_is_hung()
|
||||
{
|
||||
var factory = new HangingDbFactory();
|
||||
try
|
||||
{
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(factory));
|
||||
AwaitCondition(() => factory.Entered > 0, PresenceBudget);
|
||||
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(3));
|
||||
|
||||
status.Reachable.ShouldBeFalse();
|
||||
status.LastError.ShouldBe("not probed yet");
|
||||
factory.Entered.ShouldBe(1, "a second tick must not stack a second probe behind a hung one");
|
||||
}
|
||||
finally
|
||||
{
|
||||
factory.Release();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_probe_that_completes_updates_the_cached_status()
|
||||
{
|
||||
// Positive control for the off-thread path: the piped-back outcome lands in the cache.
|
||||
var db = NewInMemoryDbFactory();
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(db));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(3));
|
||||
status.Reachable.ShouldBeTrue();
|
||||
}, PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>Blocks every <c>CreateDbContext</c> until released — a connect that never answers.</summary>
|
||||
private sealed class HangingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
private readonly ManualResetEventSlim _release = new(false);
|
||||
private int _entered;
|
||||
|
||||
public int Entered => Volatile.Read(ref _entered);
|
||||
|
||||
public void Release() => _release.Set();
|
||||
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
{
|
||||
Interlocked.Increment(ref _entered);
|
||||
_release.Wait();
|
||||
throw new InvalidOperationException("released without a database");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,18 +15,22 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
|
||||
|
||||
public sealed class HealthProbeActorTests : RuntimeActorTestBase
|
||||
{
|
||||
/// <summary>Verifies that the DB health probe actor returns reachable status against an in-memory database.</summary>
|
||||
/// <summary>Verifies that the DB health probe actor returns reachable status against an in-memory database.
|
||||
/// The probe runs off the mailbox (02/P7), so the first Ask may still see "not probed yet" — poll.</summary>
|
||||
[Fact]
|
||||
public async Task DbHealthProbeActor_returns_reachable_against_in_memory_db()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(db));
|
||||
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, PresenceBudget);
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, PresenceBudget);
|
||||
|
||||
status.Reachable.ShouldBeTrue();
|
||||
status.LastError.ShouldBeNull();
|
||||
status.Reachable.ShouldBeTrue();
|
||||
status.LastError.ShouldBeNull();
|
||||
}, PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the peer OPC UA probe actor reports Ok true against a live listener.</summary>
|
||||
|
||||
@@ -168,4 +168,68 @@ public sealed class ServerHistorianOptionsTests
|
||||
}.Validate();
|
||||
Assert.Empty(w);
|
||||
}
|
||||
|
||||
// ── MaxRawValuesPerRead (R08:1.1, #736) ─────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void MaxRawValuesPerRead_defaults_to_ten_thousand_and_binds()
|
||||
{
|
||||
var o = new ServerHistorianOptions();
|
||||
Assert.Equal(10_000, o.MaxRawValuesPerRead);
|
||||
Assert.Equal(10_000, ServerHistorianOptions.DefaultMaxRawValuesPerRead);
|
||||
Assert.Equal(10_000, o.EffectiveMaxRawValuesPerRead);
|
||||
|
||||
var bound = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?> { ["ServerHistorian:MaxRawValuesPerRead"] = "2500" })
|
||||
.Build()
|
||||
.GetSection(ServerHistorianOptions.SectionName)
|
||||
.Get<ServerHistorianOptions>()!;
|
||||
Assert.Equal(2500, bound.MaxRawValuesPerRead);
|
||||
Assert.Equal(2500, bound.EffectiveMaxRawValuesPerRead);
|
||||
Assert.Empty(new ServerHistorianOptions
|
||||
{
|
||||
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = 2500,
|
||||
}.Validate());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(-5)]
|
||||
public void Nonpositive_MaxRawValuesPerRead_is_unbounded_and_warns_when_enabled(int value)
|
||||
{
|
||||
var o = new ServerHistorianOptions
|
||||
{
|
||||
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = value,
|
||||
};
|
||||
Assert.Equal(0, o.EffectiveMaxRawValuesPerRead);
|
||||
var w = o.Validate();
|
||||
var m = Assert.Single(w);
|
||||
Assert.Contains("MaxRawValuesPerRead", m);
|
||||
Assert.Contains("unbounded", m);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1)]
|
||||
[InlineData(2)]
|
||||
public void A_MaxRawValuesPerRead_below_three_is_raised_to_three_and_warns_when_enabled(int value)
|
||||
{
|
||||
// A resumed page's read must carry a value, the boundary tie it re-reads and the lookahead row that says "more
|
||||
// follows": at 2 it has no room and every value costs an empty page with a continuation point (the #692 defect).
|
||||
var o = new ServerHistorianOptions
|
||||
{
|
||||
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = value,
|
||||
};
|
||||
Assert.Equal(3, o.EffectiveMaxRawValuesPerRead);
|
||||
Assert.Equal(ServerHistorianOptions.MinMaxRawValuesPerRead, o.EffectiveMaxRawValuesPerRead);
|
||||
var m = Assert.Single(o.Validate());
|
||||
Assert.Contains("MaxRawValuesPerRead", m);
|
||||
Assert.Equal(3, new ServerHistorianOptions { MaxRawValuesPerRead = 3 }.EffectiveMaxRawValuesPerRead);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Disabled_with_an_odd_MaxRawValuesPerRead_is_silent()
|
||||
{
|
||||
Assert.Empty(new ServerHistorianOptions { Enabled = false, MaxRawValuesPerRead = 0 }.Validate());
|
||||
Assert.Empty(new ServerHistorianOptions { Enabled = false, MaxRawValuesPerRead = 1 }.Validate());
|
||||
}
|
||||
}
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L4: <see cref="TelemetryLocalHub.Evict"/>'s tombstone set only ever shrank on a re-spawn of the
|
||||
/// SAME id, so a driver removed for good kept its id forever. The tombstone is load-bearing (#609 — it holds
|
||||
/// back the removed child's late writes), so it cannot simply be cleared; it is now timestamped and swept
|
||||
/// once older than <see cref="TelemetryLocalHub.EvictionTombstoneTtl"/>, which outlives any in-flight
|
||||
/// capability call by a wide margin.
|
||||
/// </summary>
|
||||
public sealed class TelemetryLocalHubTombstoneSweepTests
|
||||
{
|
||||
private static DriverHealthChanged Health(string instanceId) =>
|
||||
new("cluster-a", instanceId, "Healthy", null, null, 0, DateTime.UtcNow);
|
||||
|
||||
private static List<TelemetryItem> Drain(ITelemetrySubscription sub)
|
||||
{
|
||||
var items = new List<TelemetryItem>();
|
||||
while (sub.Reader.TryRead(out var item)) items.Add(item);
|
||||
return items;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_tombstone_holds_late_writes_until_the_ttl_then_is_swept()
|
||||
{
|
||||
var clock = new ManualTime { Now = new DateTimeOffset(2026, 10, 22, 12, 0, 0, TimeSpan.Zero) };
|
||||
var hub = new TelemetryLocalHub(clock);
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
hub.Evict("drv-a");
|
||||
|
||||
// Inside the TTL: the late write is dropped (the #609 contract) — and another eviction's sweep does
|
||||
// not lift a tombstone that is still young.
|
||||
clock.Now += TelemetryLocalHub.EvictionTombstoneTtl / 2;
|
||||
hub.Evict("drv-b");
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using (var sub = hub.Subscribe(16))
|
||||
Drain(sub).ShouldBeEmpty("a late write inside the TTL must still be held back");
|
||||
|
||||
// Past the TTL: the next eviction sweeps drv-a's tombstone and its writes land again.
|
||||
clock.Now += TelemetryLocalHub.EvictionTombstoneTtl;
|
||||
hub.Evict("drv-c");
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using (var sub = hub.Subscribe(16))
|
||||
Drain(sub).OfType<TelemetryItem.Health>().Single().E.DriverInstanceId.ShouldBe("drv-a");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Readmit_lifts_a_tombstone_before_its_ttl()
|
||||
{
|
||||
var clock = new ManualTime { Now = DateTimeOffset.UtcNow };
|
||||
var hub = new TelemetryLocalHub(clock);
|
||||
hub.Evict("drv-a");
|
||||
hub.Readmit("drv-a");
|
||||
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using var sub = hub.Subscribe(16);
|
||||
Drain(sub).Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
private sealed class ManualTime : TimeProvider
|
||||
{
|
||||
public DateTimeOffset Now { get; set; }
|
||||
public override DateTimeOffset GetUtcNow() => Now;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user