using System.Diagnostics;
using Microsoft.Extensions.Logging.Abstractions;
using Opc.Ua;
using Opc.Ua.Server;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
///
/// Arch-review 03/S3 — the node-manager's HistoryRead arms must serve a multi-node batch with BOUNDED
/// per-node parallelism, a per-request deadline, and a process-wide concurrent-batch limiter, rather
/// than block-bridging the gateway per node sequentially on an SDK request thread. Boots the same real
/// harness as and drives the
/// public HistoryRead with a delaying fake that records the
/// maximum observed concurrent reads (deterministic — no raw-timing dependence for the primary
/// assertions).
///
public sealed class NodeManagerHistoryReadConcurrencyTests : IDisposable
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private readonly string _pkiRoot = Path.Combine(
Path.GetTempPath(),
$"otopcua-historyread-conc-{Guid.NewGuid():N}");
///
/// RED repro (03/S3): a single Raw HistoryRead naming 8 historized nodes against a 150 ms-per-read
/// backend must serve at least two nodes concurrently. On the current sequential foreach the
/// max observed concurrency is pinned at 1, so this fails.
///
[Fact]
public async Task Raw_batch_is_served_with_bounded_parallelism()
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
var fake = new ConcurrencyTrackingHistorianDataSource(TimeSpan.FromMilliseconds(150));
nm.HistorianDataSource = fake;
var nodeIds = MaterializeHistorizedNodes(nm, count: 8);
var details = new ReadRawModifiedDetails
{
StartTime = DateTime.UtcNow.AddHours(-1),
EndTime = DateTime.UtcNow,
NumValuesPerNode = 10,
IsReadModified = false,
};
var sw = Stopwatch.StartNew();
await Task.Run(() => InvokeHistoryRead(server, nm, details, nodeIds), Ct)
.WaitAsync(TimeSpan.FromSeconds(15), Ct);
sw.Stop();
// Primary (deterministic): the batch was served with >1 node in flight at once.
fake.MaxObservedConcurrency.ShouldBeGreaterThanOrEqualTo(2);
await host.DisposeAsync();
}
///
/// Parallelism is bounded ABOVE by HistoryReadBatchParallelism: with the property set to 3 and
/// 12 nodes in flight, no more than 3 reads are ever concurrent (03/S3).
///
[Fact]
public async Task Parallelism_is_bounded_above()
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
nm.HistoryReadBatchParallelism = 3;
var fake = new ConcurrencyTrackingHistorianDataSource(TimeSpan.FromMilliseconds(100));
nm.HistorianDataSource = fake;
var nodeIds = MaterializeHistorizedNodes(nm, count: 12);
var details = new ReadRawModifiedDetails
{
StartTime = DateTime.UtcNow.AddHours(-1),
EndTime = DateTime.UtcNow,
NumValuesPerNode = 10,
IsReadModified = false,
};
await Task.Run(() => InvokeHistoryRead(server, nm, details, nodeIds), Ct)
.WaitAsync(TimeSpan.FromSeconds(15), Ct);
fake.MaxObservedConcurrency.ShouldBeGreaterThanOrEqualTo(2);
fake.MaxObservedConcurrency.ShouldBeLessThanOrEqualTo(3);
await host.DisposeAsync();
}
///
/// One node's backend throw does not poison a parallel batch: that node surfaces
/// BadHistoryOperationUnsupported while the other seven return Good (per-node isolation holds
/// under parallelism — 03/S3).
///
[Fact]
public async Task One_node_failure_does_not_poison_a_parallel_batch()
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
var src = DateTime.UtcNow.AddSeconds(-5);
var srv = DateTime.UtcNow;
// Node index 3's tagname throws; all others return a single good sample.
var fake = new PerTagnameFake(tagname => tagname == "WW.Tag3"
? throw new InvalidOperationException("backend boom for Tag3")
: new HistorianRead(new[] { new DataValueSnapshot(1.0f, StatusCodes.Good, src, srv) }, null));
nm.HistorianDataSource = fake;
var nodeIds = MaterializeHistorizedNodes(nm, count: 8);
var details = new ReadRawModifiedDetails
{
StartTime = DateTime.UtcNow.AddHours(-1),
EndTime = DateTime.UtcNow,
NumValuesPerNode = 10,
IsReadModified = false,
};
var (results, errors) = await Task.Run(() => InvokeHistoryRead(server, nm, details, nodeIds), Ct)
.WaitAsync(TimeSpan.FromSeconds(15), Ct);
for (var i = 0; i < 8; i++)
{
if (i == 3)
{
StatusCode.IsBad(errors[i].StatusCode).ShouldBeTrue();
errors[i].StatusCode.Code.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
}
else
{
errors[i].StatusCode.Code.ShouldBe(StatusCodes.Good);
results[i].StatusCode.Code.ShouldBe(StatusCodes.Good);
}
}
await host.DisposeAsync();
}
///
/// A per-request deadline bounds a hung backend: reads that only complete on token cancellation
/// surface BadTimeout for every node and the call returns within the deadline (plus slack) —
/// it does not hang an SDK request thread indefinitely (03/S3).
///
[Fact]
public async Task Deadline_bounds_a_hung_backend()
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
nm.HistoryReadDeadline = TimeSpan.FromMilliseconds(500);
nm.HistorianDataSource = new HangingHistorianDataSource();
var nodeIds = MaterializeHistorizedNodes(nm, count: 4);
var details = new ReadRawModifiedDetails
{
StartTime = DateTime.UtcNow.AddHours(-1),
EndTime = DateTime.UtcNow,
NumValuesPerNode = 10,
IsReadModified = false,
};
var (_, errors) = await Task.Run(() => InvokeHistoryRead(server, nm, details, nodeIds), Ct)
.WaitAsync(TimeSpan.FromSeconds(10), Ct);
for (var i = 0; i < 4; i++)
errors[i].StatusCode.Code.ShouldBe(StatusCodes.BadTimeout);
await host.DisposeAsync();
}
///
/// A saturated process-wide batch limiter degrades a second concurrent HistoryRead to
/// BadTooManyOperations (rather than exhausting the SDK request-thread pool); once the parked
/// batch completes the slot frees and a later read succeeds (03/S3).
///
[Fact]
public async Task Saturated_limiter_rejects_with_BadTooManyOperations()
{
var (host, server) = await BootAsync();
var nm = server.NodeManager!;
nm.MaxConcurrentHistoryReadBatches = 1;
nm.HistoryReadDeadline = TimeSpan.FromMilliseconds(200);
var fake = new GatedRawFake();
nm.HistorianDataSource = fake;
var ids = MaterializeHistorizedNodes(nm, count: 2);
var details = new ReadRawModifiedDetails
{
StartTime = DateTime.UtcNow.AddHours(-1),
EndTime = DateTime.UtcNow,
NumValuesPerNode = 10,
IsReadModified = false,
};
// First batch parks in the gated fake, holding the single limiter permit.
var first = Task.Run(() => InvokeHistoryRead(server, nm, details, ids[0]), Ct);
var spin = Stopwatch.StartNew();
while (fake.Entered == 0 && spin.Elapsed < TimeSpan.FromSeconds(5))
await Task.Delay(10, Ct);
fake.Entered.ShouldBe(1);
// Second concurrent batch: the limiter is saturated → all handles BadTooManyOperations.
var (_, errors2) = await Task.Run(() => InvokeHistoryRead(server, nm, details, ids[1]), Ct)
.WaitAsync(TimeSpan.FromSeconds(5), Ct);
errors2[0].StatusCode.Code.ShouldBe(StatusCodes.BadTooManyOperations);
fake.Entered.ShouldBe(1, "the rejected batch must never reach the backend");
// Release the parked batch; the slot frees and a later read succeeds.
fake.Release();
await first.WaitAsync(TimeSpan.FromSeconds(10), Ct);
var (_, errors3) = await Task.Run(() => InvokeHistoryRead(server, nm, details, ids[1]), Ct)
.WaitAsync(TimeSpan.FromSeconds(10), Ct);
errors3[0].StatusCode.Code.ShouldBe(StatusCodes.Good);
await host.DisposeAsync();
}
/// Materialises historized Float variables and returns their NodeIds.
private static NodeId[] MaterializeHistorizedNodes(OtOpcUaNodeManager nm, int count)
{
var ids = new NodeId[count];
for (var i = 0; i < count; i++)
{
var key = $"eq/tag{i}";
nm.EnsureVariable(key, parentFolderNodeId: null, displayName: $"Tag{i}", dataType: "Float",
writable: false, historianTagname: $"WW.Tag{i}");
ids[i] = nm.TryGetVariable(key)!.NodeId;
}
return ids;
}
/// Issues one HistoryRead batch for all supplied node ids in order.
private static (IList Results, IList Errors) InvokeHistoryRead(
OtOpcUaSdkServer server, OtOpcUaNodeManager nm, HistoryReadDetails details, params NodeId[] nodeIds)
{
var context = new OperationContext(
new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
var nodesToRead = nodeIds.Select(id => new HistoryReadValueId { NodeId = id }).ToList();
var results = Enumerable.Repeat(null!, nodeIds.Length).ToList();
var errors = Enumerable.Repeat(null!, nodeIds.Length).ToList();
nm.HistoryRead(
context,
details,
TimestampsToReturn.Both,
releaseContinuationPoints: false,
nodesToRead,
results,
errors);
return (results, errors);
}
///
/// A delaying fake historian source that tracks the maximum number of reads in flight at once
/// (via interlocked live-counter) so a test can prove per-node parallelism deterministically.
///
private sealed class ConcurrencyTrackingHistorianDataSource(TimeSpan delay) : IHistorianDataSource
{
private int _current;
private int _max;
/// The highest number of concurrently in-flight reads observed.
public int MaxObservedConcurrency => Volatile.Read(ref _max);
private async Task TrackAsync(CancellationToken ct)
{
var now = Interlocked.Increment(ref _current);
UpdateMax(now);
try
{
await Task.Delay(delay, ct).ConfigureAwait(false);
}
finally
{
Interlocked.Decrement(ref _current);
}
return new HistorianRead(
new[] { new DataValueSnapshot(1.0f, StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow) }, null);
}
private void UpdateMax(int observed)
{
int cur;
while ((cur = Volatile.Read(ref _max)) < observed &&
Interlocked.CompareExchange(ref _max, observed, cur) != cur)
{
}
}
public Task ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
CancellationToken cancellationToken) => TrackAsync(cancellationToken);
public Task ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
HistoryAggregateType aggregate, CancellationToken cancellationToken) => TrackAsync(cancellationToken);
public Task ReadAtTimeAsync(
string fullReference, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) =>
TrackAsync(cancellationToken);
public Task ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
CancellationToken cancellationToken) =>
Task.FromResult(new HistoricalEventsResult(Array.Empty(), null));
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose()
{
}
}
/// A fake whose reads complete ONLY on token cancellation — used to prove the per-request
/// deadline surfaces BadTimeout rather than hanging.
private sealed class HangingHistorianDataSource : IHistorianDataSource
{
private static async Task HangAsync(CancellationToken ct)
{
await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
return new HistorianRead(Array.Empty(), null);
}
public Task ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
CancellationToken cancellationToken) => HangAsync(cancellationToken);
public Task ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
HistoryAggregateType aggregate, CancellationToken cancellationToken) => HangAsync(cancellationToken);
public Task ReadAtTimeAsync(
string fullReference, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) =>
HangAsync(cancellationToken);
public async Task ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
CancellationToken cancellationToken)
{
await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false);
return new HistoricalEventsResult(Array.Empty(), null);
}
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose() { }
}
/// A fake whose raw read parks (awaits a ) until the test
/// releases it — used to hold the process-wide batch limiter permit while a second batch is issued.
private sealed class GatedRawFake : IHistorianDataSource
{
private readonly TaskCompletionSource _gate = new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _entered;
/// Number of raw reads that reached the backend body.
public int Entered => Volatile.Read(ref _entered);
/// Releases the parked read so its awaiting batch can complete.
public void Release() => _gate.TrySetResult();
public async Task ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
CancellationToken cancellationToken)
{
Interlocked.Increment(ref _entered);
// Deliberately ignore the per-request deadline token: the first batch must keep HOLDING the
// single limiter permit until the test releases it, so the second batch deterministically hits
// a saturated limiter (rather than the first batch's own deadline freeing the slot early).
await _gate.Task.ConfigureAwait(false);
return new HistorianRead(
new[] { new DataValueSnapshot(1.0f, StatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow) }, null);
}
public Task ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
NullHistorianDataSource.Instance.ReadProcessedAsync(fullReference, startUtc, endUtc, interval, aggregate, cancellationToken);
public Task ReadAtTimeAsync(
string fullReference, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) =>
NullHistorianDataSource.Instance.ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken);
public Task ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
CancellationToken cancellationToken) =>
NullHistorianDataSource.Instance.ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose() { }
}
/// A per-tagname fake that delegates each raw read to a caller-supplied func (which may throw
/// to simulate a per-node backend failure). Processed/AtTime/Events delegate to the null source.
private sealed class PerTagnameFake(Func rawHandler) : IHistorianDataSource
{
public Task ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
CancellationToken cancellationToken) => Task.FromResult(rawHandler(fullReference));
public Task ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
NullHistorianDataSource.Instance.ReadProcessedAsync(fullReference, startUtc, endUtc, interval, aggregate, cancellationToken);
public Task ReadAtTimeAsync(
string fullReference, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) =>
NullHistorianDataSource.Instance.ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken);
public Task ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
CancellationToken cancellationToken) =>
NullHistorianDataSource.Instance.ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose() { }
}
private async Task<(OpcUaApplicationHost Host, OtOpcUaSdkServer Server)> BootAsync()
{
var host = new OpcUaApplicationHost(
new OpcUaApplicationHostOptions
{
ApplicationName = "OtOpcUa.HistoryReadConcTest",
ApplicationUri = $"urn:OtOpcUa.HistoryReadConcTest:{Guid.NewGuid():N}",
OpcUaPort = AllocateFreePort(),
PublicHostname = "localhost",
PkiStoreRoot = _pkiRoot,
},
NullLogger.Instance);
var server = new OtOpcUaSdkServer();
await host.StartAsync(server, Ct);
return (host, server);
}
private static int AllocateFreePort()
{
using var listener = new System.Net.Sockets.TcpListener(System.Net.IPAddress.Loopback, 0);
listener.Start();
var port = ((System.Net.IPEndPoint)listener.LocalEndpoint).Port;
listener.Stop();
return port;
}
/// Cleans up the PKI root directory.
public void Dispose()
{
if (Directory.Exists(_pkiRoot))
{
try { Directory.Delete(_pkiRoot, recursive: true); }
catch { /* best-effort cleanup */ }
}
}
}