Merge branch 'master' into remed/lane-E

This commit is contained in:
Joseph Doherty
2026-10-22 10:48:41 -04:00
51 changed files with 3658 additions and 128 deletions
+32 -1
View File
@@ -40,6 +40,17 @@
<PackageVersion Include="Microsoft.AspNetCore.DataProtection.EntityFrameworkCore" Version="10.0.7" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.0" />
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="10.0.0" />
<!--
Surgical transitive pin #4. GHSA-334f-vvp4-7g6p (CVE-2026-70159, NU1903 high) flags
Microsoft.AspNetCore.SignalR.Common 10.0.0..10.0.12, which SignalR.Client 10.0.0 pulls in via
SignalR.Client.Core. 10.0.13 is the first fixed version, pinned by direct PackageReference in
AdminUI (the one project that references SignalR.Client). Bumping SignalR.Client itself to
10.0.13 was tried and rejected: SignalR.Client.Core 10.0.13 floors Microsoft.Extensions.Logging /
DependencyInjection at 10.0.13, which the AdminUI shared framework absorbs but non-Web consumers
downstream (OpcUaServer.IntegrationTests, which references Logging.Abstractions 10.0.7 directly)
do not, producing an NU1605 downgrade.
-->
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Common" Version="10.0.13" />
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Core" Version="1.2.0" />
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="10.0.7" />
<!--
@@ -149,8 +160,28 @@
Microsoft.Extensions.* and, through the EFCore adapter, Microsoft.EntityFrameworkCore at
10.0.10, forcing a family-wide servicing bump (NU1605 downgrade cascade) that deserves its own
reviewed change.
2026-10-22: raised 10.0.10 -> 10.0.13. Three more NU1903 advisories (GHSA-7hqp-7c5m-4vq6,
GHSA-c399-9853-6jmj, GHSA-r955-8q5j-3h83; vulnerable <= 10.0.12) landed against 10.0.10, again
red only on a fresh restore (~67 projects). 10.0.13 is the first fixed version. Same pin site,
same reasoning against the DataProtection parent bump.
-->
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.10" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.13" />
<!--
Surgical transitive pin #3. GHSA-846f-f764-76mx (NU1903 high) + GHSA-2fmv-rv2c-59x7 (NU1902
moderate) flag System.Security.Cryptography.Pkcs <= 10.0.12 and 9.0.0..9.0.20. Two chains bring
it in, both entering at Core.Configuration:
- System.Security.Cryptography.Xml 10.0.x -> Pkcs 10.0.x (same version as Xml), and
- Microsoft.EntityFrameworkCore.SqlServer 10.0.7 -> Microsoft.Data.SqlClient 6.1.1 -> Pkcs 9.0.4.
In Web-SDK consumers (AdminUI, ControlPlane, Host, Security, ...) the ASP.NET Core shared
framework prunes the Xml chain, so the SqlClient 9.0.4 floor is what resolves there (~9
projects); everywhere else the Xml chain's 10.0.10 won (~67 projects). Raising the Xml pin alone
only fixes the second group, so Pkcs is pinned explicitly at 10.0.13 (first fixed 10.x) by a
direct PackageReference in Core.Configuration. Its net10.0 dependency group is empty, so the pin
floors nothing else. Bumping Microsoft.Data.SqlClient / EFCore.SqlServer instead would be the
family-wide servicing bump the Xml note above declines.
-->
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="10.0.13" />
<PackageVersion Include="System.CommandLine" Version="2.0.5" />
<PackageVersion Include="System.Data.SqlClient" Version="4.9.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.11.0" />
+56
View File
@@ -0,0 +1,56 @@
# Baseline 2026-10-22 @ ad2b5c40 — Release, filter Category!=E2E&Category!=LiveIntegration
# Result: all assemblies passed (0 failed). Run concurrently with 6 Wave-1 worktree builds.
Passed! - Failed: 0, Passed: 126, Skipped: 0, Total: 126, Duration: 251 ms - ZB.MOM.WW.OtOpcUa.Client.UI.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 202, Skipped: 0, Total: 202, Duration: 1 s - ZB.MOM.WW.OtOpcUa.Client.Shared.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 16, Skipped: 0, Total: 16, Duration: 91 ms - ZB.MOM.WW.OtOpcUa.Core.VirtualTags.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 138, Skipped: 0, Total: 138, Duration: 2 s - ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 51, Skipped: 0, Total: 51, Duration: 5 s - ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 359, Skipped: 0, Total: 359, Duration: 7 s - ZB.MOM.WW.OtOpcUa.Commons.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 95, Skipped: 0, Total: 95, Duration: 8 s - ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 455, Skipped: 0, Total: 455, Duration: 6 s - ZB.MOM.WW.OtOpcUa.Driver.Ab.Transport.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 437, Skipped: 0, Total: 437, Duration: 8 s - ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 359, Skipped: 1, Total: 360, Duration: 7 s - ZB.MOM.WW.OtOpcUa.Core.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 21, Skipped: 0, Total: 21, Duration: 196 ms - ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browser.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 66, Skipped: 0, Total: 66, Duration: 4 s - ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 1, Skipped: 217, Total: 218, Duration: 249 ms[xUnit.net 00:00:10.55] ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.IntegrationTests.AbLegacyIndirectAddressingTests.Slc500_indirect_word_read_and_write_resolve_through_the_index [SKIP]
Passed! - Failed: 0, Passed: 353, Skipped: 8, Total: 361, Duration: 780 ms - ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 752, Skipped: 0, Total: 752, Duration: 9 s - ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 194, Skipped: 0, Total: 194, Duration: 11 s - ZB.MOM.WW.OtOpcUa.Configuration.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 139, Skipped: 0, Total: 139, Duration: 12 s - ZB.MOM.WW.OtOpcUa.Client.CLI.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 597, Skipped: 0, Total: 597, Duration: 9 s - ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 81, Skipped: 0, Total: 81, Duration: 346 ms - ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 14, Skipped: 0, Total: 14, Duration: 4 s - ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests.dll (net10.0)
Passed! - Failed: 0, Passed: 269, Skipped: 0, Total: 269, Duration: 667 ms - ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 16, Skipped: 10, Total: 26, Duration: 10 s - ZB.MOM.WW.OtOpcUa.Driver.AbCip.IntegrationTests.dll (net10.0)
Passed! - Failed: 0, Passed: 511, Skipped: 0, Total: 511, Duration: 5 s - ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 1, Skipped: 39, Total: 40, Duration: 96 ms - ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests.dll (net10.0)
Passed! - Failed: 0, Passed: 161, Skipped: 0, Total: 161, Duration: 193 ms - ZB.MOM.WW.OtOpcUa.Driver.Modbus.Addressing.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 42, Skipped: 0, Total: 42, Duration: 4 s - ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 10, Skipped: 0, Total: 10, Duration: 1 s - ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Browser.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 465, Skipped: 0, Total: 465, Duration: 3 s - ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 16, Skipped: 0, Total: 16, Duration: 5 s - ZB.MOM.WW.OtOpcUa.Driver.MTConnect.IntegrationTests.dll (net10.0)
Passed! - Failed: 0, Passed: 254, Skipped: 0, Total: 254, Duration: 3 s - ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 47, Skipped: 0, Total: 47, Duration: 149 ms - ZB.MOM.WW.OtOpcUa.Driver.AbCip.Cli.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 48, Skipped: 0, Total: 48, Duration: 106 ms - ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Cli.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 84, Skipped: 0, Total: 84, Duration: 3 s - ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 363, Skipped: 0, Total: 363, Duration: 1 s - ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 49, Skipped: 0, Total: 49, Duration: 449 ms - ZB.MOM.WW.OtOpcUa.Driver.Cli.Common.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 52, Skipped: 0, Total: 52, Duration: 394 ms - ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 563, Skipped: 0, Total: 563, Duration: 6 s - ZB.MOM.WW.OtOpcUa.Driver.S7.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 72, Skipped: 0, Total: 72, Duration: 245 ms - ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 59, Skipped: 0, Total: 59, Duration: 88 ms - ZB.MOM.WW.OtOpcUa.Driver.S7.Cli.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 70, Skipped: 0, Total: 70, Duration: 263 ms - ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 635, Skipped: 0, Total: 635, Duration: 21 s - ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 89, Skipped: 0, Total: 89, Duration: 6 s - ZB.MOM.WW.OtOpcUa.Security.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 420, Skipped: 0, Total: 420, Duration: 20 s - ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 22, Skipped: 0, Total: 22, Duration: 7 s - ZB.MOM.WW.OtOpcUa.Analyzers.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 250, Skipped: 0, Total: 250, Duration: 47 s - ZB.MOM.WW.OtOpcUa.Cluster.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 199, Skipped: 0, Total: 199, Duration: 16 s - ZB.MOM.WW.OtOpcUa.Host.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 1674, Skipped: 0, Total: 1674, Duration: 25 s - ZB.MOM.WW.OtOpcUa.AdminUI.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 825, Skipped: 0, Total: 825, Duration: 1 m 20 s - ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 453, Skipped: 0, Total: 453, Duration: 1 m 30 s - ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 837, Skipped: 3, Total: 840, Duration: 1 m 31 s - ZB.MOM.WW.OtOpcUa.Runtime.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 278, Skipped: 0, Total: 278, Duration: 1 m 37 s - ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 267, Skipped: 14, Total: 281, Duration: 3 m 39 s - ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.dll (net10.0)
Passed! - Failed: 0, Passed: 992, Skipped: 1, Total: 993, Duration: 4 m 17 s - ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests.dll (net10.0)
Passed! - Failed: 0, Passed: 124, Skipped: 0, Total: 124, Duration: 4 m 28 s - ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.dll (net10.0)
@@ -72,7 +72,13 @@ public sealed class TelemetryDialOptions
/// <summary>How often central refreshes its set of dialable node contacts, in seconds.</summary>
public int ContactRefreshSeconds { get; set; } = 60;
/// <summary>Per-call deadline for a gRPC telemetry dial, in seconds.</summary>
/// <summary>
/// Deadline, in seconds, for a gRPC telemetry dial to deliver its first event. A live node answers a
/// fresh subscribe at once (it replays its health/resilience snapshots), so a dial still silent at the
/// deadline is one that never connected — it is cancelled and re-dialled. Bounds establishment only;
/// an established stream is kept alive by HTTP/2 keepalive pings, not by this value. Values below 1
/// are treated as 1.
/// </summary>
public int CallTimeoutSeconds { get; set; } = 30;
}
@@ -188,6 +188,13 @@ public sealed class DiscoveryDriverBrowser : IUniversalDriverBrowser
}
}
/// <summary>
/// Final teardown of a throwaway capture/probe driver: a bounded <see cref="IDriver.ShutdownAsync"/>
/// followed by a bounded dispose. The dispose is NOT optional and runs even after a failed or timed-out
/// shutdown: a Galaxy driver's <c>ShutdownAsync</c> releases none of its gRPC runtime (channels, MX
/// session, stream loops) — only <c>DisposeAsync</c> does — so a browse that stopped at shutdown leaked a
/// full runtime per <c>/raw</c> browse and per <see cref="CanBrowse"/> probe (02/L1, 06/G-1).
/// </summary>
private async Task BoundedShutdownAsync(IDriver driver, string driverType)
{
try
@@ -199,7 +206,25 @@ public sealed class DiscoveryDriverBrowser : IUniversalDriverBrowser
catch (Exception ex)
{
_logger.LogWarning(ex,
"Browse-capture shutdown for {DriverType} failed/timed out; instance abandoned", driverType);
"Browse-capture shutdown for {DriverType} failed/timed out; disposing the instance anyway", driverType);
}
try
{
switch (driver)
{
case IAsyncDisposable asyncDisposable:
await asyncDisposable.DisposeAsync().AsTask().WaitAsync(_shutdownTimeout).ConfigureAwait(false);
break;
case IDisposable disposable:
disposable.Dispose();
break;
}
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Browse-capture dispose for {DriverType} failed/timed out; instance abandoned", driverType);
}
}
@@ -28,6 +28,10 @@
Directory.Packages.props. This is the one project where the DataProtection chain
enters the repo; every other consumer reaches it through a ProjectReference here. -->
<PackageReference Include="System.Security.Cryptography.Xml"/>
<!-- Surgical transitive pin — see the System.Security.Cryptography.Pkcs block in
Directory.Packages.props. Microsoft.Data.SqlClient (via EFCore.SqlServer) enters here
too and pulls a vulnerable Pkcs 9.0.4 wherever the shared framework prunes the Xml chain. -->
<PackageReference Include="System.Security.Cryptography.Pkcs"/>
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions"/>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions"/>
<PackageReference Include="Polly.Core"/>
@@ -67,6 +67,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
/// <summary>The tie-cluster limit the crowding re-reads honour — exposed so the factory wiring is testable.</summary>
internal int MaxPreWindowRows => _maxPreWindowRows;
/// <summary>
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one raw read may
/// return; 0 = none. See <see cref="MaxRawValuesPerRead"/> and <see cref="ReadRawWindowAsync"/>.
/// </summary>
private readonly int _maxRawValuesPerRead;
/// <summary>A strict gateway's row ceiling, learned from its first refusal.</summary>
private readonly GatewayRowCeiling _ceiling;
private readonly GatewayTagPresenceCache _tags;
@@ -96,11 +102,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
/// The most rows before a raw window a crowding re-read makes room for (<c>ServerHistorian:MaxTieClusterOverfetch</c>,
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
/// </param>
/// <param name="maxRawValuesPerRead">
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/> (bounded further by a strict
/// gateway's learned ceiling), so the server pages at it. A read asking for more (or for everything) asks the gateway
/// for cap + 1 rows and is refused when the window holds more, never truncated. &lt;= 0 = unbounded (the pre-#736
/// behaviour: an uncapped read is sent as <see cref="int.MaxValue"/>).
/// </param>
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0)
{
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
_atTimeFromRaw = atTimeFromRaw;
ArgumentNullException.ThrowIfNull(client);
ArgumentNullException.ThrowIfNull(logger);
@@ -203,6 +217,13 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
/// a cancelled gateway stream evicts its pooled historian session). <c>ReturnBounds</c> is not plumbed to the
/// data source on any backend, so bounding values are never synthesised here.
/// </para>
/// <para>
/// <b>The server raw cap</b> (R08:1.1, #736). With a cap, a read asking for everything (<c>0</c>) or for more than
/// the cap runs as a capped read of cap + 1 — one more than it may return — so an over-cap window is DETECTED and
/// refused (<see cref="HistoryReadLimitExceededException"/>) rather than answered truncated (the server's paging
/// reads a short answer as the last page) and never sent as <see cref="int.MaxValue"/>. The server itself never asks
/// for more than the cap (it pages at <see cref="MaxRawValuesPerRead"/>); this bounds a caller that bypasses it.
/// </para>
/// </remarks>
private async Task<List<HistorianSample>> ReadRawWindowAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
@@ -210,30 +231,39 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
{
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
if (cap == 0)
var serverCap = _maxRawValuesPerRead;
// Under the server raw cap, a read asking for everything or for more than the cap runs as a capped read of
// cap + 1 (budgetCap); `cap` stays what the client asked for, and a refusal names the cap that applied (limit).
var probe = serverCap > 0 && (cap == 0 || cap > serverCap);
var budgetCap = probe ? (int)Math.Min(int.MaxValue, (long)serverCap + 1) : cap;
var limit = probe ? serverCap : cap;
var advice = probe
? $"set NumValuesPerNode at or below ServerHistorian:MaxRawValuesPerRead ({serverCap}) to page"
: cap == 0 ? "set NumValuesPerNode to page" : "lower NumValuesPerNode";
if (budgetCap == 0)
{
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
if (all.Result.Complete) return all.Result.Samples;
// Only reachable on a strict gateway: the ceiling re-read delivered the whole ceiling without reaching
// the window end — the window may hold more, and "all values" is never answered from a truncated read.
throw new HistoryReadLimitExceededException(cap,
throw new HistoryReadLimitExceededException(limit,
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
"or narrow the window.")
{ PreWindowRows = Crowding(all.Result.PreWindowRows) };
}
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
var asked = (int)Math.Min(int.MaxValue, (long)budgetCap + RawAggregateReader.PreWindowAllowance);
var pass = await PassAsync(asked).ConfigureAwait(false);
if (pass.Result.Complete) return pass.Result.Samples;
if (pass.Result.Complete) return Bounded(pass.Result.Samples);
// A strict gateway's ceiling already bounded that read: a larger budget would only be refused again. The
// refusal carries the pre-window rows it saw: the server re-reads the page that many values smaller.
if (pass.Budget < asked)
throw new HistoryReadLimitExceededException(cap,
throw new HistoryReadLimitExceededException(limit,
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
"NumValuesPerNode or narrow the window.")
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); {advice}, " +
"or narrow the window.")
{ PreWindowRows = Crowding(pass.Result.PreWindowRows) };
// The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
@@ -245,25 +275,25 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows);
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
{
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
var budget = (int)Math.Min(int.MaxValue, (long)budgetCap + room + RawAggregateReader.PreWindowAllowance);
// doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
if (_ceiling.Learned is var known and > 0 && budget > known)
{
var available = known - cap - RawAggregateReader.PreWindowAllowance;
var available = known - budgetCap - RawAggregateReader.PreWindowAllowance;
if (available <= 0) break;
room = Math.Min(room, available);
budget = cap + room + RawAggregateReader.PreWindowAllowance;
budget = budgetCap + room + RawAggregateReader.PreWindowAllowance;
}
if (budget <= pass.Budget) break; // no more room, never retry the same full read
pass = await PassAsync(budget).ConfigureAwait(false);
if (pass.Result.Complete) return pass.Result.Samples;
if (pass.Result.Complete) return Bounded(pass.Result.Samples);
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room));
}
throw new HistoryReadLimitExceededException(cap,
throw new HistoryReadLimitExceededException(limit,
$"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
{ PreWindowRows = preWindow };
@@ -271,10 +301,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause.
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
// The cap + 1 probe came back with one sample too many: the window holds more than the server raw cap.
List<HistorianSample> Bounded(List<HistorianSample> samples)
{
if (probe && samples.Count > serverCap)
throw new HistoryReadLimitExceededException(serverCap,
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({serverCap}) " +
"samples; set NumValuesPerNode at or below it to page, or narrow the window.");
return samples;
}
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names.
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
_ceiling.ReadAsync(budget,
b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken),
b => ReadRawPassAsync(fullReference, startUtc, endUtc, budgetCap, b, cancellationToken),
(r, _) => !r.Complete);
}
@@ -531,13 +571,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
public bool SupportsReverseRaw => false;
/// <inheritdoc />
// 0 until a strict gateway (AllowStreamingBeyondCeiling=false) has refused a budget and
// named its ceiling (GatewayRowCeiling); then that ceiling less the
// RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row — so a page of this
// size asks for exactly the ceiling. Back to 0 when a re-probe finds the gateway streaming again.
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
: 0;
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none), bounded further by a strict gateway's
// (AllowStreamingBeyondCeiling=false) learned ceiling once a refusal has named it (GatewayRowCeiling) — that
// ceiling less the RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row, so a page of
// this size asks for exactly the ceiling. The learned part is forgotten when a re-probe finds the gateway streaming
// again; the cap stays. 0 only with no cap and no learned ceiling.
public int MaxRawValuesPerRead
{
get
{
var learned = _ceiling.Learned is var c and > 0 ? Math.Max(1, c - RawAggregateReader.PreWindowAllowance) : 0;
var cap = _maxRawValuesPerRead;
return cap > 0 && learned > 0 ? Math.Min(cap, learned) : cap > 0 ? cap : learned;
}
}
/// <inheritdoc />
// false: the gateway's ReadEvents streams ascending only. The server never sends it a
@@ -44,7 +44,9 @@ public static class GatewayHistorian
// AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch).
atTimeFromRaw: options.GatewayAtTimeFromRaw,
// A crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
maxPreWindowRows: options.MaxTieClusterOverfetch);
maxPreWindowRows: options.MaxTieClusterOverfetch,
// The server raw cap (R08:1.1): the page limit the server reads at, and the bound on every raw read.
maxRawValuesPerRead: options.EffectiveMaxRawValuesPerRead);
}
/// <summary>
@@ -66,6 +66,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
private readonly TimeProvider _time;
private readonly HistorianHealthCounters _health = new();
private readonly int _maxTimeWeightedRawRows;
private readonly int _maxRawValuesPerRead;
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
/// <param name="conn">Connection helper.</param>
@@ -74,11 +75,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
/// <param name="log">Optional logger.</param>
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (&lt;= 0 = unbounded).</param>
/// <param name="maxRawValuesPerRead">
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/>, so the server pages at it.
/// A read asking for more (or for everything) is refused when the window holds more, never truncated.
/// &lt;= 0 = unbounded (the pre-#736 behaviour).
/// </param>
internal QuestDbHistorianDataSource(
QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null,
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows,
int maxRawValuesPerRead = 0)
{
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
@@ -93,12 +102,21 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
// (int.MaxValue); larger values clamp. At most top samples, ContinuationPoint always null: the node
// manager pages by re-reading from the last returned timestamp inclusive, which the (ts, tag) dedup
// key makes exact.
// The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads
// cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused
// (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging
// reads a short answer as the last page, so a silent LIMIT would lose data. The server itself never asks for more
// than the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it.
/// <inheritdoc />
public async Task<HistoryReadResult> ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fullReference);
var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
var cap = _maxRawValuesPerRead;
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
var top = probe
? (int)Math.Min(int.MaxValue, (long)cap + 1)
: maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
try
{
var samples = new List<DataValueSnapshot>();
@@ -129,9 +147,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
}
}
if (probe && samples.Count > cap)
throw new HistoryReadLimitExceededException(cap,
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " +
"set NumValuesPerNode at or below it to page, or narrow the window.");
RecordSuccess();
return new HistoryReadResult(samples, ContinuationPoint: null);
}
catch (HistoryReadLimitExceededException)
{
// A client-request limit (the server raw cap), not a backend fault: not counted as a failure.
throw;
}
catch (Exception ex)
{
RecordFailure(ex);
@@ -360,9 +388,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
/// <inheritdoc />
public bool SupportsReverseRaw => true;
// 0: a capped raw read is one bounded statement - no backend row ceiling.
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a
// client's NumValuesPerNode = 0 is served as pages of at most this many values. QuestDB itself has no row ceiling -
// a capped raw read is one bounded statement.
/// <inheritdoc />
public int MaxRawValuesPerRead => 0;
public int MaxRawValuesPerRead => _maxRawValuesPerRead;
// Reverse events reads are served natively (ORDER BY ... DESC).
/// <inheritdoc />
@@ -35,14 +35,17 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
/// <summary>Creates the provider over the <c>ServerHistorian:QuestDb</c> options (validated elsewhere).</summary>
/// <param name="options">The <c>ServerHistorian:QuestDb</c> options.</param>
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (&lt;= 0 = unbounded).</param>
/// <param name="maxRawValuesPerRead">The shared <c>ServerHistorian:MaxRawValuesPerRead</c> raw cap (&lt;= 0 = unbounded; R08:1.1).</param>
public QuestDbHistorianProvider(QuestDbHistorianOptions options,
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
_maxRawValuesPerRead = maxRawValuesPerRead;
}
private readonly int _maxTimeWeightedRawRows;
private readonly int _maxRawValuesPerRead;
/// <inheritdoc />
public string Name => "QuestDb";
@@ -75,7 +78,8 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
{
var s = Get(sp);
return new QuestDbHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows,
maxRawValuesPerRead: _maxRawValuesPerRead);
}
/// <inheritdoc />
@@ -61,6 +61,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
private readonly HistorianHealthCounters _health = new();
private readonly int _maxTimeWeightedRawRows;
private readonly int _maxRawValuesPerRead;
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
/// <param name="conn">Connection helper.</param>
@@ -69,11 +70,19 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
/// <param name="log">Optional logger.</param>
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (&lt;= 0 = unbounded).</param>
/// <param name="maxRawValuesPerRead">
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/>, so the server pages at it.
/// A read asking for more (or for everything) is refused when the window holds more, never truncated.
/// &lt;= 0 = unbounded (the pre-#736 behaviour).
/// </param>
internal SqlServerHistorianDataSource(
SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap, SqlServerTagCatalog catalog, ILogger? log = null,
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows,
int maxRawValuesPerRead = 0)
{
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
@@ -90,13 +99,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
// <c>0</c> is the node manager's "unbounded" sentinel; values above <see cref="int.MaxValue"/> clamp.
// The node manager pages by re-reading from the last returned timestamp inclusive and trimming it; the
// <c>(TagId, SourceTimeUtc)</c> primary key rules out tie clusters, so that is exact.
// The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads
// cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused
// (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging
// reads a short answer as the last page, so a silent TOP would lose data. The server itself never asks for more than
// the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it.
// </remarks>
/// <inheritdoc />
public async Task<HistoryReadResult> ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(fullReference);
var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
var cap = _maxRawValuesPerRead;
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
var top = probe
? (int)Math.Min(int.MaxValue, (long)cap + 1)
: maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
try
{
var tagId = await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
@@ -126,9 +144,19 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
}
}
if (probe && samples.Count > cap)
throw new HistoryReadLimitExceededException(cap,
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " +
"set NumValuesPerNode at or below it to page, or narrow the window.");
RecordSuccess();
return new HistoryReadResult(samples, ContinuationPoint: null);
}
catch (HistoryReadLimitExceededException)
{
// A client-request limit (the server raw cap), not a backend fault: not counted as a failure.
throw;
}
catch (Exception ex)
{
RecordFailure(ex);
@@ -319,9 +347,13 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
/// <inheritdoc />
public bool SupportsReverseRaw => true;
// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling.</remarks>
// <remarks>
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a
// client's NumValuesPerNode = 0 is served as pages of at most this many values. SQL Server itself has no row
// ceiling — a capped raw read is one bounded statement.
// </remarks>
/// <inheritdoc />
public int MaxRawValuesPerRead => 0;
public int MaxRawValuesPerRead => _maxRawValuesPerRead;
// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>).</remarks>
/// <inheritdoc />
@@ -31,14 +31,17 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
/// <summary>Creates the provider over the <c>ServerHistorian:SqlServer</c> options (validated elsewhere).</summary>
/// <param name="options">The <c>ServerHistorian:SqlServer</c> options.</param>
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (&lt;= 0 = unbounded).</param>
/// <param name="maxRawValuesPerRead">The shared <c>ServerHistorian:MaxRawValuesPerRead</c> raw cap (&lt;= 0 = unbounded; R08:1.1).</param>
public SqlServerHistorianProvider(SqlServerHistorianOptions options,
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
_maxRawValuesPerRead = maxRawValuesPerRead;
}
private readonly int _maxTimeWeightedRawRows;
private readonly int _maxRawValuesPerRead;
/// <inheritdoc />
public string Name => "SqlServer";
@@ -69,7 +72,8 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
{
var s = Get(sp);
return new SqlServerHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows,
maxRawValuesPerRead: _maxRawValuesPerRead);
}
/// <inheritdoc />
@@ -26,6 +26,10 @@
provider's DbProviderFactory (NpgsqlFactory / OracleClientFactory); both packages are fully
managed ADO.NET drivers, so adding them costs no native client install anywhere. -->
<PackageReference Include="Microsoft.Data.SqlClient" />
<!-- Surgical transitive pin — see the System.Security.Cryptography.Pkcs block in
Directory.Packages.props. SqlClient 6.1.1 pulls a vulnerable Pkcs 9.0.4; this is the
second place the SqlClient chain enters the repo (Core.Configuration is the first). -->
<PackageReference Include="System.Security.Cryptography.Pkcs" />
<PackageReference Include="Npgsql" />
<PackageReference Include="Oracle.ManagedDataAccess.Core" />
</ItemGroup>
@@ -1,4 +1,6 @@
using Akka.Actor;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
@@ -13,7 +15,9 @@ namespace ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
/// Sends <see cref="GetDiagnostics"/>; expects a <see cref="NodeDiagnosticsSnapshot"/> reply.
///
/// On timeout or any other failure (peer down, GetDiagnostics handler missing) returns an
/// empty snapshot so the UI degrades to "no data" instead of throwing.
/// empty snapshot so the UI degrades to "no data" instead of throwing. The failure is logged at
/// Warning, since the snapshot shape cannot carry it: an empty snapshot reads the same as a node with
/// no drivers. Consumed by the <c>/fleet/diagnostics</c> page.
/// </summary>
public sealed class FleetDiagnosticsClient : IFleetDiagnosticsClient
{
@@ -21,29 +25,38 @@ public sealed class FleetDiagnosticsClient : IFleetDiagnosticsClient
private readonly ActorSystem _system;
private readonly string _systemName;
private readonly ILogger<FleetDiagnosticsClient> _log;
/// <summary>Initializes a new FleetDiagnosticsClient with the given actor system and cluster options.</summary>
/// <param name="system">The Akka actor system.</param>
/// <param name="options">Cluster configuration options.</param>
public FleetDiagnosticsClient(ActorSystem system, IOptions<AkkaClusterOptions> options)
/// <param name="log">Logger for the degraded-to-empty path; defaults to a no-op logger.</param>
public FleetDiagnosticsClient(
ActorSystem system, IOptions<AkkaClusterOptions> options, ILogger<FleetDiagnosticsClient>? log = null)
{
_system = system;
_systemName = options.Value.SystemName;
_log = log ?? NullLogger<FleetDiagnosticsClient>.Instance;
}
/// <inheritdoc />
public async Task<NodeDiagnosticsSnapshot> GetDiagnosticsAsync(NodeId nodeId, CancellationToken ct)
{
var selection = _system.ActorSelection($"akka.tcp://{_systemName}@{nodeId.Value}/user/driver-host");
try
{
// Inside the try: a malformed node value makes ActorSelection throw, and that must degrade to
// "unavailable" like a dead node does, not surface through the page's error boundary.
var selection = _system.ActorSelection($"akka.tcp://{_systemName}@{nodeId.Value}/user/driver-host");
using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
linked.CancelAfter(AskTimeout);
return await selection.Ask<NodeDiagnosticsSnapshot>(
new GetDiagnostics(CorrelationId.NewId()), AskTimeout, linked.Token);
}
catch (Exception)
catch (Exception ex) when (!ct.IsCancellationRequested)
{
_log.LogWarning(ex,
"Fleet diagnostics for node {NodeId} unavailable (no answer within {Timeout}s); returning an empty snapshot",
nodeId.Value, AskTimeout.TotalSeconds);
return new NodeDiagnosticsSnapshot(
nodeId,
CurrentRevision: null,
@@ -0,0 +1,238 @@
@page "/fleet/diagnostics"
@* Per-node driver diagnostics, fetched on demand from the selected driver node's DriverHostActor over
Akka (IFleetDiagnosticsClient → GetDiagnostics → NodeDiagnosticsSnapshot). The node answers from
its live child set + each child's last-published health; the "running from cache" flag is the one
fact /hosts cannot show (a node booted from its artifact cache looks healthy but is frozen).
Reach it as /fleet/diagnostics?node=<host:port> (the NodeId shown on /fleet and /hosts), or pick
a driver-role member from the selector. *@
@attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)]
@rendermode RenderMode.InteractiveServer
@using ZB.MOM.WW.OtOpcUa.Cluster
@using ZB.MOM.WW.OtOpcUa.Commons.Interfaces
@using ZB.MOM.WW.OtOpcUa.Commons.Types
@inject IClusterRoleInfo Cluster
@inject IFleetDiagnosticsClient Diagnostics
@inject NavigationManager Nav
@implements IDisposable
<PageErrorBoundary>
<div class="d-flex justify-content-between align-items-center mb-3">
<h4 class="mb-0">Node diagnostics</h4>
<a class="btn btn-sm btn-outline-secondary" href="/fleet">Back to fleet status</a>
</div>
<div class="d-flex align-items-center mb-3 gap-2 flex-wrap">
<label class="small text-muted mb-0" for="diag-node">Node</label>
<select id="diag-node" class="form-select form-select-sm" style="width:auto"
value="@_selectedNode" @onchange="OnNodeSelected">
<option value="">— select a driver node —</option>
@foreach (var member in _members)
{
<option value="@member">@member</option>
}
@if (!string.IsNullOrEmpty(_selectedNode) && !_members.Contains(_selectedNode))
{
<option value="@_selectedNode">@_selectedNode (not an Up member)</option>
}
</select>
<input type="text" class="form-control form-control-sm" style="width:220px"
placeholder="or type host:port…" @bind="_typedNode" @bind:event="oninput"
@onkeydown="OnTypedKeyDown" />
<button class="btn btn-sm btn-outline-primary" @onclick="UseTypedNodeAsync"
disabled="@(string.IsNullOrWhiteSpace(_typedNode))">Go</button>
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync"
disabled="@(_loading || string.IsNullOrEmpty(_selectedNode))">
@if (_loading) { <span class="spinner-border spinner-border-sm me-1" /> }
Refresh
</button>
<span class="text-muted small">
Last fetched: @(_fetchedUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
</span>
</div>
@if (string.IsNullOrEmpty(_selectedNode))
{
<section class="panel notice rise" style="animation-delay:.02s">
Pick a driver-role node to fetch its driver diagnostics. The node answers from its own
DriverHostActor, so this works for any node reachable over Akka remoting from this admin node.
</section>
}
else if (_snapshot is null)
{
<p>@(_loading ? "Fetching…" : "No snapshot yet.")</p>
}
else
{
<section class="agg-grid rise" style="animation-delay:.02s">
<div class="agg-card">
<div class="agg-label">Node</div>
<div class="agg-value mono">@_snapshot.NodeId.Value</div>
</div>
<div class="agg-card">
<div class="agg-label">Revision</div>
<div class="agg-value mono" title="@(_snapshot.CurrentRevision?.Value ?? "")">
@(_snapshot.CurrentRevision is { } rev ? DisplayText.Abbreviate(rev.Value, 12) : "—")
</div>
</div>
<div class="agg-card @(_snapshot.RunningFromCache ? "caution" : "")">
<div class="agg-label">Config source</div>
<div class="agg-value">@(_snapshot.RunningFromCache ? "cache (frozen)" : "live")</div>
</div>
<div class="agg-card">
<div class="agg-label">Drivers</div>
<div class="agg-value numeric">@_snapshot.Drivers.Count</div>
</div>
<div class="agg-card alert">
<div class="agg-label">Faulted devices</div>
<div class="agg-value numeric">@_snapshot.Drivers.Sum(d => d.FaultedDevices)</div>
</div>
</section>
@if (_snapshot.RunningFromCache)
{
<section class="panel notice rise" style="animation-delay:.05s">
This node booted its configuration from its local artifact cache because central was unreachable.
It serves its last-known-good address space, but no deployment can reach it until central is back.
</section>
}
<section class="panel rise" style="animation-delay:.08s">
<div class="panel-head">
Drivers
<span class="text-muted small ms-2">as of @_snapshot.AsOfUtc.ToString("u")</span>
</div>
@if (_snapshot.Drivers.Count == 0)
{
<div class="p-3 text-muted">
Empty snapshot: the node has no driver children — or it did not answer within the client's
3 s budget (an unreachable node degrades to an empty snapshot; see the AdminUI log for the
FleetDiagnosticsClient warning).
</div>
}
else
{
<div class="table-wrap">
<table class="data-table">
<thead>
<tr>
<th>Driver instance</th>
<th>Row id</th>
<th>State</th>
<th class="numeric">Connected devices</th>
<th class="numeric">Faulted devices</th>
<th title="When the driver child last published a changed health snapshot (its spawn time until the first)">Last health publish</th>
</tr>
</thead>
<tbody>
@foreach (var d in _snapshot.Drivers.OrderBy(d => d.Name, StringComparer.OrdinalIgnoreCase))
{
<tr>
<td>@d.Name</td>
<td><span class="mono" title="@d.DriverInstanceId">@(d.DriverInstanceId == Guid.Empty ? "—" : DisplayText.Abbreviate(d.DriverInstanceId.ToString(), 13))</span></td>
<td><span class="chip @StateChipClass(d.State)">@d.State</span></td>
<td class="numeric">@d.ConnectedDevices</td>
<td class="numeric">@d.FaultedDevices</td>
<td>@d.LastChangeUtc.ToString("u")</td>
</tr>
}
</tbody>
</table>
</div>
}
</section>
}
</PageErrorBoundary>
@code {
[SupplyParameterFromQuery(Name = "node")]
public string? Node { get; set; }
private IReadOnlyList<string> _members = Array.Empty<string>();
private string _selectedNode = "";
private string _typedNode = "";
private NodeDiagnosticsSnapshot? _snapshot;
private bool _loading;
private DateTime? _fetchedUtc;
// Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
private readonly CancellationTokenSource _cts = new();
private volatile bool _disposed;
protected override async Task OnParametersSetAsync()
{
_members = Cluster.MembersWithRole(RoleParser.Driver)
.Select(n => n.Value)
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
.ToList();
var requested = Node?.Trim() ?? "";
if (!string.Equals(requested, _selectedNode, StringComparison.Ordinal))
{
_selectedNode = requested;
_snapshot = null;
if (!string.IsNullOrEmpty(_selectedNode)) await LoadAsync();
}
}
private Task OnNodeSelected(ChangeEventArgs e)
{
var value = e.Value?.ToString() ?? "";
Navigate(value);
return Task.CompletedTask;
}
private Task UseTypedNodeAsync()
{
Navigate(_typedNode.Trim());
return Task.CompletedTask;
}
private Task OnTypedKeyDown(KeyboardEventArgs e)
{
if (e.Key == "Enter") Navigate(_typedNode.Trim());
return Task.CompletedTask;
}
// The node id lives in the URL so a diagnostics view is linkable from /fleet and survives a reload.
private void Navigate(string node) =>
Nav.NavigateTo(string.IsNullOrEmpty(node)
? "/fleet/diagnostics"
: "/fleet/diagnostics?node=" + Uri.EscapeDataString(node));
private async Task RefreshAsync() => await LoadAsync();
private async Task LoadAsync()
{
if (_loading || string.IsNullOrEmpty(_selectedNode)) return;
_loading = true;
try
{
// The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
_snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
_fetchedUtc = DateTime.UtcNow;
}
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
{
// Circuit gone mid-fetch — a clean exit, not a fault.
}
finally
{
_loading = false;
if (!_disposed) StateHasChanged();
}
}
private static string StateChipClass(string state) => state switch
{
"Healthy" => "chip-ok",
"Degraded" or "Connecting" or "Reconnecting" => "chip-caution",
"Faulted" => "chip-alert",
_ => "chip-idle",
};
public void Dispose()
{
_disposed = true;
_cts.Cancel();
_cts.Dispose();
}
}
@@ -129,6 +129,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
Receive<TelemetryReceived>(HandleTelemetryReceived);
Receive<StreamStopped>(HandleStreamStopped);
Receive<Reconnect>(HandleReconnect);
Receive<DialDeadline>(HandleDialDeadline);
// A faulted node-source task pipes here. Log and wait for the next periodic refresh — a stale
// node set is far better than crashing the supervisor and dropping every live dialer.
@@ -251,6 +252,9 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
return;
}
// The stream is over; a still-armed establishment deadline for it must not fire into the reconnect.
Timers.Cancel(DeadlineTimerKey(msg.NodeId));
if (dialer.Connected)
{
dialer.Connected = false;
@@ -294,6 +298,34 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
}
}
/// <summary>
/// <c>TelemetryDial:CallTimeoutSeconds</c>: a dial that has delivered no event by the deadline is cut and
/// treated as a transport failure (reconnect with a failure streak). <c>Connected</c> flips on the first
/// routed event, on this thread, so a dial that connected in time is left alone — the deadline bounds
/// ESTABLISHMENT only. A live node answers a fresh subscribe at once (the node-local hub replays its
/// health/resilience snapshots), so what this catches is a dial that never connects: a black-holed host,
/// where the socket connect itself has no timeout and the stream would otherwise hang forever without a
/// reconnect. An established stream is never bounded here — the wire has no heartbeat, keepalive pings own
/// its liveness, and an idle cut would churn every quiet node.
/// </summary>
private void HandleDialDeadline(DialDeadline msg)
{
if (!_dialers.TryGetValue(msg.NodeId, out var dialer) || msg.Generation != dialer.Generation || dialer.Connected)
{
return;
}
_log.Warning(
"Telemetry dial to node {NodeId} ({Endpoint}) delivered nothing within {Deadline}s; cancelling it and re-dialing",
msg.NodeId, dialer.Target.Endpoint, DialDeadlineFor().TotalSeconds);
dialer.Cancel();
HandleStreamStopped(new StreamStopped(
msg.NodeId, msg.Generation,
new TimeoutException($"no telemetry event within {DialDeadlineFor().TotalSeconds:F0}s of dialing")));
}
private TimeSpan DialDeadlineFor() => TimeSpan.FromSeconds(Math.Max(1, _options.CallTimeoutSeconds));
private void StartDialer(string nodeId)
{
var dialer = _dialers[nodeId];
@@ -306,6 +338,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
var cts = new CancellationTokenSource();
dialer.Cts = cts;
dialer.Connected = false;
// One deadline per generation; a superseded tick is dropped by the generation guard in the handler.
Timers.StartSingleTimer(DeadlineTimerKey(nodeId), new DialDeadline(nodeId, generation), DialDeadlineFor());
var self = Self;
var target = dialer.Target;
@@ -357,6 +391,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
{
dialer.Cancel();
Timers.Cancel(ReconnectTimerKey(nodeId));
Timers.Cancel(DeadlineTimerKey(nodeId));
RecomputePill();
}
}
@@ -433,6 +468,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
private static string ReconnectTimerKey(string nodeId) => $"telemetry-reconnect-{nodeId}";
private static string DeadlineTimerKey(string nodeId) => $"telemetry-dial-deadline-{nodeId}";
/// <summary>Per-node dialer bookkeeping. Mutated only on the actor thread.</summary>
private sealed class NodeDialer(TelemetryDialTarget target)
{
@@ -496,4 +533,10 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
/// <summary>Backoff timer tick: start a fresh dialer for the node.</summary>
/// <param name="NodeId">The node id to reconnect.</param>
public sealed record Reconnect(string NodeId);
/// <summary>Dial-establishment deadline tick (<c>TelemetryDial:CallTimeoutSeconds</c>); see
/// <see cref="HandleDialDeadline"/>.</summary>
/// <param name="NodeId">The node whose dial is being checked.</param>
/// <param name="Generation">The dialer generation the deadline was armed for.</param>
public sealed record DialDeadline(string NodeId, int Generation);
}
@@ -9,6 +9,12 @@
<ItemGroup>
<FrameworkReference Include="Microsoft.AspNetCore.App"/>
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client"/>
<!-- Surgical transitive pin — see the Microsoft.AspNetCore.SignalR.Common block in
Directory.Packages.props (GHSA-334f-vvp4-7g6p on SignalR.Common 10.0.0). Inside AdminUI
the shared framework already supplies it (hence NU1510), but the PackageReference flows
to the non-Web consumers (Host.Tests, AdminUI.Tests, OpcUaServer.IntegrationTests) where
SignalR.Client 10.0.0 would otherwise resolve the vulnerable 10.0.0. -->
<PackageReference Include="Microsoft.AspNetCore.SignalR.Common" NoWarn="NU1510"/>
<PackageReference Include="ZB.MOM.WW.Theme"/>
<PackageReference Include="ZB.MOM.WW.Secrets.Ui"/>
</ItemGroup>
@@ -45,8 +45,8 @@ public static class HistorianBackendRegistry
new Dictionary<string, Func<ServerHistorianOptions, IConfiguration, IHistorianBackendProvider>>(StringComparer.OrdinalIgnoreCase)
{
[ServerHistorianOptions.GatewayBackendName] = (o, _) => new GatewayHistorianProvider(o),
[SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows),
[QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows),
[SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows, o.EffectiveMaxRawValuesPerRead),
[QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows, o.EffectiveMaxRawValuesPerRead),
};
/// <summary>The registered backend names, for validation messages.</summary>
@@ -1,3 +1,5 @@
using Opc.Ua;
using Opc.Ua.Server;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
@@ -110,24 +112,62 @@ internal enum HistoryCursorKind
/// <c>NumValuesPerNode</c> samples with a null continuation point — so paging is synthesised here,
/// time-based:
/// <list type="bullet">
/// <item>A page that returns EXACTLY the requested cap (<c>NumValuesPerNode &gt; 0</c>) MAY have
/// more behind it ⇒ emit a continuation point.</item>
/// <item>A short page (fewer than the cap) is the last page ⇒ no continuation point.</item>
/// <item><c>NumValuesPerNode == 0</c> means "all values, no limit" (OPC UA Part 11) ⇒ never page;
/// return everything in one shot.</item>
/// <item>A page whose read returned the lookahead row past it MAY have more behind it ⇒ emit a continuation
/// point (<see cref="IsFullPage"/> judges a read against what was asked for).</item>
/// <item>A short read (no lookahead row) is the last page ⇒ no continuation point.</item>
/// <item><c>NumValuesPerNode == 0</c> means "all values" (OPC UA Part 11) — but not "in one shot": the page is
/// sized to the backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> by <see cref="RawPageLimit"/>
/// (the server raw cap <c>ServerHistorian:MaxRawValuesPerRead</c>, default 10 000, and/or a strict gateway's
/// learned ceiling; review R08:1.1, #736), and a full page carries a continuation point like any other. Only
/// with no limit at all (<c>0</c>, the opt-out) is everything returned in one shot.</item>
/// </list>
/// All methods are static + pure so they unit-test without a server, a session, or the SDK.
/// All paging methods are static + pure so they unit-test without a server, a session, or the SDK;
/// <see cref="AdvertiseMaxReturnDataValues"/> is the one SDK-facing member.
/// </summary>
internal static class HistoryPaging
{
/// <summary>
/// Decide whether a just-returned page is "full" and therefore MAY be followed by more data —
/// the signal to emit a continuation point. A page is full when the client asked for a finite
/// cap (<paramref name="numValuesPerNode"/> &gt; 0) and the backend returned exactly that many
/// samples. A short page (or an unlimited <c>0</c> request) is terminal.
/// Advertise the server raw cap as <c>HistoryServerCapabilities/MaxReturnDataValues</c> (i=11273; Part 11 §5.4.2:
/// "the maximum number of values that can be returned by the Server for each Node", 0 = no limit). The cap is
/// the most values one HistoryRead-Raw page returns (<c>ServerHistorian:MaxRawValuesPerRead</c>), so a client can
/// size its <c>NumValuesPerNode</c> and expect a continuation point past it. Idempotent; a re-sync with a
/// different cap overwrites. Never throws: a missing diagnostics manager or node, or any SDK failure writing the
/// property, leaves the SDK default (0) and returns <c>false</c> for the caller to log — HistoryRead itself still
/// enforces the cap.
/// </summary>
/// <param name="returnedCount">The number of samples the backend returned for this page.</param>
/// <param name="numValuesPerNode">The client's per-page cap; <c>0</c> means "no limit".</param>
/// <param name="server">The running server.</param>
/// <param name="maxRawValuesPerRead">The cap (&lt;= 0 advertises 0 = no limit).</param>
/// <returns><c>true</c> when the property was written.</returns>
public static bool AdvertiseMaxReturnDataValues(IServerInternal? server, int maxRawValuesPerRead)
{
if (server?.DiagnosticsNodeManager is not DiagnosticsNodeManager diagnostics) return false;
try
{
lock (diagnostics.Lock)
{
var capabilities = diagnostics.GetDefaultHistoryCapabilities();
if (capabilities?.MaxReturnDataValues is not { } property) return false;
property.Value = (uint)Math.Max(0, maxRawValuesPerRead);
property.ClearChangeMasks(diagnostics.SystemContext, false);
return true;
}
}
catch (Exception)
{
return false; // advisory only: the caller logs; the cap is enforced by HistoryRead regardless
}
}
/// <summary>
/// Decide whether a just-returned read is "full" and therefore MAY be followed by more data —
/// the signal to emit a continuation point. A read is full when it was issued with a finite request
/// cap (<paramref name="numValuesPerNode"/> &gt; 0 — the <c>RawPagePlan.RequestCap</c>: page + boundary ties +
/// lookahead, which under a backend limit or the server raw cap is always finite, uncapped client reads included)
/// and the backend returned at least that many samples. A short read is terminal; so is a read issued with a
/// request cap of <c>0</c>, which only happens with no limit at all (one shot of the whole window).
/// </summary>
/// <param name="returnedCount">The number of samples the backend returned for this read.</param>
/// <param name="numValuesPerNode">The request cap the read was issued with; <c>0</c> means "no limit".</param>
/// <returns><c>true</c> when a continuation point should be emitted; otherwise <c>false</c>.</returns>
public static bool IsFullPage(int returnedCount, uint numValuesPerNode) =>
numValuesPerNode > 0 && returnedCount >= numValuesPerNode;
@@ -11,17 +11,22 @@ internal readonly record struct RawPagePlan(int Limit, uint PageCap, uint Reques
/// <summary>
/// sizes a HistoryRead-Raw page to what the backend can return in one read
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — a strict HistorianGateway's learned row ceiling, less
/// its boundary-row allowance). With a limit the read — page P + boundary ties k + the lookahead row — is at most L,
/// so P = min(want, L − k − 1) (all values: P = L − k − 1); a full page carries a continuation point like any other, as
/// Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Previously a cap above a strict gateway's ceiling
/// (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>.
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — the server raw cap
/// <c>ServerHistorian:MaxRawValuesPerRead</c>, default 10 000, which every backend reports (review R08:1.1, #736),
/// and/or a strict HistorianGateway's learned row ceiling less its boundary-row allowance, whichever is lower). With
/// a limit the read — page P + boundary ties k + the lookahead row — is at most L, so P = min(want, L − k − 1) (all
/// values: P = L − k − 1, i.e. a first page of 9 999 at the default cap); a full page carries a continuation point like
/// any other, as Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Previously a cap above a strict
/// gateway's ceiling (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>, and before
/// #736 an uncapped read on a backend without a ceiling read the whole window in one shot.
/// </summary>
/// <remarks>
/// <para>
/// <b>Never a silent truncation.</b> The backend still answers a read in full or refuses it; only the server's page is
/// smaller, and a page is judged full against what was asked for. A read the backend cannot serve at all (a tie
/// cluster larger than one read) stays <c>BadTooManyOperations</c>.
/// cluster larger than one read) stays <c>BadTooManyOperations</c>. The backends enforce the same cap themselves
/// (a read asking for more than L, or for everything, is refused when the window holds more than L rows), so this
/// planner is what keeps the server's own reads inside it: with L &gt; 0 no read it plans ever asks for 0 or more than L.
/// </para>
/// <para>
/// <b>One re-read, planned again.</b> A refused read (<see cref="HistoryReadLimitExceededException"/>) is re-planned and
@@ -0,0 +1,73 @@
using System.Collections.Concurrent;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
/// <summary>The last health snapshot a driver child published, as kept by <see cref="ChildHealthLedger"/>.</summary>
/// <param name="State">The driver's reported state.</param>
/// <param name="LastSuccessfulReadUtc">When the driver last read successfully, if ever.</param>
/// <param name="LastError">The driver's last error, if any.</param>
/// <param name="HostStatuses">Per-host connectivity for a multi-host driver, or null when the driver has none.</param>
/// <param name="PublishedUtc">When the snapshot was published — the child's dedup means this is the last CHANGE.</param>
public sealed record ChildHealthRecord(
DriverState State,
DateTime? LastSuccessfulReadUtc,
string? LastError,
IReadOnlyList<HostConnectivityStatus>? HostStatuses,
DateTime PublishedUtc);
/// <summary>
/// An <see cref="IDriverHealthPublisher"/> decorator that remembers the last snapshot per driver instance
/// and forwards every publish to the real publisher. <see cref="DriverHostActor"/> wraps the publisher it
/// hands its children in one of these so <c>GetDiagnostics</c> can answer from the children's own health
/// (state, host counts, last change) instead of a placeholder — the children publish on their own actor
/// threads, hence the concurrent map.
/// </summary>
/// <remarks>
/// Deliberately NOT a message to the host: the children already publish through this seam on every
/// change, so a parallel <c>Tell</c> would double every health transition's traffic for one consumer.
/// Entries are removed by the host when it stops a child; a late publish from a stopping child may re-add
/// one, which the host filters against its live child set when it answers.
/// </remarks>
public sealed class ChildHealthLedger : IDriverHealthPublisher
{
private readonly IDriverHealthPublisher _inner;
private readonly ConcurrentDictionary<string, ChildHealthRecord> _last = new(StringComparer.Ordinal);
/// <summary>Wraps <paramref name="inner"/>.</summary>
/// <param name="inner">The publisher every snapshot is forwarded to.</param>
public ChildHealthLedger(IDriverHealthPublisher inner)
{
ArgumentNullException.ThrowIfNull(inner);
_inner = inner;
}
/// <inheritdoc />
public void Publish(
string clusterId,
string driverInstanceId,
DriverHealth health,
int errorCount5Min,
DateTime? rediscoveryNeededUtc = null,
string? rediscoveryReason = null,
IReadOnlyList<HostConnectivityStatus>? hostStatuses = null,
SkippedTagReport? skippedTags = null,
IReadOnlyDictionary<string, string>? diagnostics = null)
{
_last[driverInstanceId] = new ChildHealthRecord(
health.State, health.LastSuccessfulRead, health.LastError, hostStatuses, DateTime.UtcNow);
_inner.Publish(
clusterId, driverInstanceId, health, errorCount5Min, rediscoveryNeededUtc, rediscoveryReason,
hostStatuses, skippedTags, diagnostics);
}
/// <summary>The last snapshot published for <paramref name="driverInstanceId"/>, or null if none yet.</summary>
/// <param name="driverInstanceId">The driver instance.</param>
/// <returns>The record, or null.</returns>
public ChildHealthRecord? TryGet(string driverInstanceId) =>
_last.TryGetValue(driverInstanceId, out var record) ? record : null;
/// <summary>Forgets <paramref name="driverInstanceId"/>'s snapshot (the child was stopped).</summary>
/// <param name="driverInstanceId">The driver instance.</param>
public void Forget(string driverInstanceId) => _last.TryRemove(driverInstanceId, out _);
}
@@ -195,7 +195,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
private readonly IReadOnlySet<string> _localRoles;
private readonly IActorRef? _dependencyMux;
private readonly IActorRef? _opcUaPublishActor;
private readonly IDriverHealthPublisher _healthPublisher;
/// <summary>The publisher handed to every child — a <see cref="ChildHealthLedger"/> over the configured one, so
/// <see cref="HandleGetDiagnostics"/> answers from the children's own last-published health.</summary>
private readonly ChildHealthLedger _healthPublisher;
private readonly IVirtualTagEvaluator _virtualTagEvaluator;
private readonly IHistoryWriter _historyWriter;
private readonly IActorRef? _virtualTagHostOverride;
@@ -594,6 +596,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
private sealed record ChildEntry(IActorRef Actor, DriverInstanceSpec Spec, bool Stubbed)
{
/// <summary>When this child was spawned — the diagnostics' last-change time until it publishes health.</summary>
public DateTime SpawnedUtc { get; } = DateTime.UtcNow;
// Convenience accessors for sites that don't need the full spec.
/// <summary>Gets the driver type name from the child's spec.</summary>
public string DriverType => Spec.DriverType;
@@ -886,7 +891,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
_localRoles = localRoles ?? new HashSet<string>(StringComparer.Ordinal);
_dependencyMux = dependencyMux;
_opcUaPublishActor = opcUaPublishActor;
_healthPublisher = healthPublisher ?? NullDriverHealthPublisher.Instance;
_healthPublisher = new ChildHealthLedger(healthPublisher ?? NullDriverHealthPublisher.Instance);
_virtualTagEvaluator = virtualTagEvaluator ?? NullVirtualTagEvaluator.Instance;
_historyWriter = historyWriter ?? NullHistoryWriter.Instance;
_virtualTagHostOverride = virtualTagHostOverride;
@@ -2489,6 +2494,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// A child connectivity transition while the host is Stale has no live address space to annotate — drop
// it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
Receive<DriverInstanceActor.ConnectivityChanged>(_ => { });
// Likewise a child's value / alarm burst: Stale is entered from Bootstrap's catch before any child is
// spawned, so none should arrive — but the orphan-replay BeginDirectApply in that same try block has
// already started a pipe-back before the catch can run, so "cannot happen" rests on ordering. Explicit
// drops, so a burst that does land is a logged drop rather than an Unhandled dead-letter.
Receive<DriverInstanceActor.AttributeValueBatchPublished>(m =>
_log.Debug("DriverHost {Node}: dropping a {Count}-item value burst from {Id} while Stale (no address space)",
_localNode, m.Items.Count, m.DriverInstanceId));
Receive<DriverInstanceActor.AttributeAlarmPublished>(m =>
_log.Debug("DriverHost {Node}: dropping a native alarm transition ({Cond}) from {Id} while Stale",
_localNode, m.Args.ConditionId, m.DriverInstanceId));
// A late artifact pipe-back from the apply that was in flight when the host fell Stale: that apply is
// lost (the coordinator's deadline and the boot-time orphan replay recover it). Warning, not Debug —
// an apply outcome is being discarded.
Receive<ArtifactForApply>(m =>
_log.Warning("DriverHost {Node}: dropping the artifact for {Id} (rev {Rev}) that arrived while Stale; the apply is abandoned",
_localNode, m.DeploymentId, m.Revision));
Receive<ApplyWedged>(m =>
_log.Debug("DriverHost {Node}: late apply-wedge tick for {Id} while Stale; dropping", _localNode, m.DeploymentId));
// Likewise the 03/S15 recovery audit: a timer armed before the host went Stale has nothing to report
// on (no latch was set while Stale, and the rebuild re-establishes every node's status anyway).
Receive<CommsRecoveryAudit>(_ => { });
@@ -2496,16 +2519,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Timers.StartPeriodicTimer("retry-db", RetryConfigDbConnection.Instance, ReconnectInterval);
}
/// <summary>
/// Answers <see cref="GetDiagnostics"/> from the live child set and each child's last-published health
/// (<see cref="ChildHealthLedger"/>). Used to be a placeholder — <c>Guid.Empty</c>, "Spawned", zero
/// devices, "now" — that looked live in every behaviour and was read by nobody (02/U1); the AdminUI
/// fleet-diagnostics page is its consumer.
/// </summary>
private void HandleGetDiagnostics(GetDiagnostics msg)
{
var drivers = _children
.Select(kv => new DriverInstanceDiagnostics(
DriverInstanceId: Guid.Empty,
Name: kv.Key,
State: kv.Value.Stubbed ? "Stubbed" : "Spawned",
ConnectedDevices: 0,
FaultedDevices: 0,
LastChangeUtc: DateTime.UtcNow))
.Select(kv => ToDiagnostics(kv.Key, kv.Value, _healthPublisher.TryGet(kv.Key)))
.ToArray();
var snapshot = new NodeDiagnosticsSnapshot(
NodeId: _localNode,
@@ -2516,6 +2539,39 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
Sender.Tell(snapshot);
}
/// <summary>
/// One child's diagnostics row. <c>Name</c> carries the string driver-instance id — the identity every
/// other surface (/hosts, telemetry, the artifact) keys on — and <c>DriverInstanceId</c> the DB row Guid;
/// the wire record has no slot for the authored display name. Device counts come from the driver's host
/// statuses when it is an <c>IHostConnectivityProbe</c> (Running = connected; Stopped/Faulted = faulted;
/// Unknown = neither); a single-host driver counts as one device, connected while it reports Healthy or
/// Degraded and faulted while it reports Faulted. A child that has not published yet reports
/// "Connecting" (or "Stubbed") with its spawn time as the last change.
/// </summary>
private static DriverInstanceDiagnostics ToDiagnostics(string driverInstanceId, ChildEntry entry, ChildHealthRecord? health)
{
var name = driverInstanceId;
if (entry.Stubbed)
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Stubbed", 0, 0, entry.SpawnedUtc);
if (health is null)
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Connecting", 0, 0, entry.SpawnedUtc);
int connected, faulted;
if (health.HostStatuses is { Count: > 0 } hosts)
{
connected = hosts.Count(h => h.State == HostState.Running);
faulted = hosts.Count(h => h.State is HostState.Stopped or HostState.Faulted);
}
else
{
connected = health.State is DriverState.Healthy or DriverState.Degraded ? 1 : 0;
faulted = health.State == DriverState.Faulted ? 1 : 0;
}
return new DriverInstanceDiagnostics(
entry.Spec.DriverInstanceRowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc);
}
private void HandleDispatchFromSteady(DispatchDeployment msg)
{
// The short-circuit claims a CLEAN Applied, so it is only honest while the current revision's
@@ -3771,6 +3827,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
// id lifts it (the tracker via _invokerFactory.Create, the hub via Readmit).
_invokerFactory.Release(driverInstanceId);
_telemetryHub?.Evict(driverInstanceId);
// The reprovision rate-limit stamp belongs to the incarnation history that just ended. Kept, it grew the
// map by one entry per id ever reprovisioned-then-removed AND made a later re-add of the same id inherit a
// backoff it never earned (its first crash was deferred as a "crash loop").
_lastReprovisionUtc.Remove(driverInstanceId);
Timers.Cancel(ReprovisionTimerKey(driverInstanceId));
_healthPublisher.Forget(driverInstanceId);
_log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
}
@@ -3821,6 +3883,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
StopDependencyAdapter(msg.DriverInstanceId);
Context.Stop(entry.Actor);
_children.Remove(msg.DriverInstanceId);
// The old incarnation's health must not read as the new one's until it publishes its own.
_healthPublisher.Forget(msg.DriverInstanceId);
// Respawn from the same spec the last reconcile used — preserves RowId, Name, ClusterId.
SpawnChild(entry.Spec);
@@ -1,3 +1,4 @@
using System.Diagnostics;
using Akka.Actor;
using Akka.Event;
using ZB.MOM.WW.OtOpcUa.Cluster;
@@ -184,6 +185,52 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// </remarks>
public static readonly TimeSpan DefaultCommsLossTimeout = TimeSpan.FromSeconds(90);
/// <summary>
/// Bound on the driver's <see cref="IDriver.ShutdownAsync"/> (and its dispose) when the child stops or
/// restarts, and on the teardown that precedes every reconnect attempt. <c>PostStop</c> runs on a
/// dispatcher thread and blocks on it, so a driver whose shutdown waits on a dead socket / COM proxy used
/// to pin that thread for as long as the wait took — a few such stops starved the default dispatcher and
/// wedged <c>CoordinatedShutdown</c> and the host's own restart cascade. A shutdown that overruns is
/// abandoned (logged), never awaited further.
/// </summary>
public static readonly TimeSpan DefaultShutdownTimeout = TimeSpan.FromSeconds(5);
/// <summary>
/// Default cap on the writes this actor runs against its driver at once. Writes run OFF the mailbox
/// (a <c>ReceiveAsync</c> used to suspend it for the whole device round-trip, so writes serialised and
/// every data change queued behind them — the second of two operator writes to a slow device missed the
/// host's 8 s Ask and was reverted although the device took it). The cap keeps a burst from flooding a
/// device with concurrent commands; writes beyond it queue in order and start as slots free.
/// </summary>
public const int DefaultMaxWritesInFlight = 4;
/// <summary>
/// How long a queued write may wait for a slot before it is failed <c>BadTimeout</c> instead of being
/// written. The host's write Ask is 8 s: a write older than that has already been reported as a timeout
/// (and its node reverted) upstream, so writing it late would push a stale value to the device.
/// </summary>
public static readonly TimeSpan DefaultQueuedWriteBudget = TimeSpan.FromSeconds(8);
/// <summary>
/// Pessimistic bound on one <see cref="IDriver.InitializeAsync"/> attempt. The invoker's tier timeout is
/// token-only (optimistic), so a driver whose initialise ignores its token would otherwise hold the
/// single-flight gate forever and wedge the reconnect loop. 2× the longest tier (Tier C: 15 s).
/// </summary>
public static readonly TimeSpan DefaultInitializeTimeout = TimeSpan.FromSeconds(30);
/// <summary>
/// Pessimistic bound on one write round-trip, on top of the 5 s token-based write cap. A driver whose
/// <see cref="IWritable.WriteAsync"/> ignores its token would otherwise hold its slot AND its tag forever,
/// piling every later write to that tag up in the queue. 2× the write cap.
/// </summary>
public static readonly TimeSpan DefaultWriteTimeout = TimeSpan.FromSeconds(10);
/// <summary>Self-sent by a write task once it has replied, releasing its slot and its tag.</summary>
/// <param name="TagId">The tag the write held.</param>
/// <param name="Incarnation">The actor incarnation that launched it — a late completion from a previous
/// incarnation (after a supervision restart) must not free the new incarnation's slot or tag.</param>
private sealed record WriteCompleted(string TagId, int Incarnation);
private readonly IDriver _driver;
/// <summary>Phase 6.1 resilience seam: every guarded driver-capability call this actor makes is
@@ -204,6 +251,47 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// <summary>Comms-loss watchdog threshold; <see cref="TimeSpan.Zero"/> or negative disables it.</summary>
private readonly TimeSpan _commsLossTimeout;
/// <summary>Bound on the driver shutdown / dispose; see <see cref="DefaultShutdownTimeout"/>.</summary>
private readonly TimeSpan _shutdownTimeout;
/// <summary>Cap on concurrent writes; see <see cref="DefaultMaxWritesInFlight"/>.</summary>
private readonly int _maxWritesInFlight;
/// <summary>Writes currently inside the driver. Actor-thread only (incremented on launch, decremented on
/// the self-sent <see cref="WriteCompleted"/>).</summary>
private int _writesInFlight;
/// <summary>Writes waiting for a slot (or for their tag), in arrival order, stamped with arrival time.
/// Failed fast on every exit from <c>Connected</c>, so a queued write never reaches a driver that is
/// reconnecting; failed <c>BadTimeout</c> at dequeue once older than <see cref="_queuedWriteBudget"/>.</summary>
private readonly LinkedList<(WriteAttribute Write, IActorRef ReplyTo, DateTime EnqueuedUtc)> _pendingWrites = new();
/// <summary>Tags with a write currently inside the driver. A second write to the same tag waits in the
/// queue behind it — two concurrent writes v1, v2 to one tag could otherwise land as v1.</summary>
private readonly HashSet<string> _tagsInFlight = new(StringComparer.Ordinal);
/// <summary>See <see cref="DefaultQueuedWriteBudget"/>.</summary>
private readonly TimeSpan _queuedWriteBudget;
/// <summary>See <see cref="DefaultInitializeTimeout"/>.</summary>
private readonly TimeSpan _initializeTimeout;
/// <summary>See <see cref="DefaultWriteTimeout"/>.</summary>
private readonly TimeSpan _writeTimeout;
/// <summary>This actor object's identity for its own off-mailbox completions (drawn from the process-wide
/// generation source, so it is unique across supervision restarts of the same child).</summary>
private readonly int _incarnation;
/// <summary>
/// Set by <see cref="PreRestart"/> so the <see cref="PostStop"/> it triggers skips the dispose. A
/// supervision restart recreates the actor object around the SAME <see cref="IDriver"/> (the Props closure
/// captured it) and the host then re-initialises that driver — disposing it here would hand the new
/// incarnation a dead object. Only a real stop (deploy removal, operator Restart, respawn on config change,
/// host shutdown) disposes.
/// </summary>
private bool _restarting;
/// <summary>When the actor last entered <c>Connected</c>. The comms-loss watchdog's baseline for a
/// driver that has never reported a successful read, so "connected but never read anything" is
/// measurable rather than indefinitely undefined.</summary>
@@ -218,6 +306,26 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// on the actor thread, so no lock is needed.</summary>
private int _initGeneration;
/// <summary>Process-wide source of attempt generations. A supervision restart recreates the actor object
/// (fresh fields) while a prior incarnation's attempt may still be running on the same driver; a per-actor
/// counter restarting at 0 would let that late result collide with the new incarnation's first attempt.</summary>
private static int s_initGenerationSource;
/// <summary>
/// True while an <see cref="InitializeAsync"/> attempt is running (set when it is launched, cleared by its
/// piped <see cref="InitializeSucceeded"/>/<see cref="InitializeFailed"/>). The <c>Reconnecting</c> retry
/// timer fires on its cadence regardless of whether the previous attempt finished, and the generation
/// token only discards an older attempt's RESULT — so with an Initialize bound longer than the cadence
/// (Tier C: 15 s vs. 10 s) attempt N+1's teardown ran while attempt N was still inside the driver: the
/// dispose-before-overwrite race the drivers' rule file forbids, once per overlap. A tick that lands while
/// an attempt is in flight is skipped; the next tick after the result retries. Actor-thread only.
/// </summary>
private bool _initInFlight;
/// <summary>Cancellation for the in-flight attempt, so <see cref="PostStop"/> can cancel it before the
/// shutdown rather than let it race the teardown on a restart.</summary>
private CancellationTokenSource? _initCts;
/// <summary>
/// Timestamps of recent Faulted-state transitions; used to compute the 5-minute error count.
/// No lock needed — every read/write site runs inside an Akka message handler, which is
@@ -307,6 +415,17 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// <param name="commsLossTimeout">Optional comms-loss watchdog threshold; defaults to
/// <see cref="DefaultCommsLossTimeout"/>. <see cref="TimeSpan.Zero"/> disables the watchdog — used by
/// tests that need a Connected driver to stay Connected while reporting non-Healthy.</param>
/// <param name="shutdownTimeout">Optional bound on the driver's shutdown / dispose on stop and on the
/// pre-reconnect teardown; defaults to <see cref="DefaultShutdownTimeout"/>. Exists so a test can prove a
/// hung shutdown is abandoned without waiting 5 s.</param>
/// <param name="maxWritesInFlight">Optional cap on concurrent writes against the driver; defaults to
/// <see cref="DefaultMaxWritesInFlight"/>. Writes beyond it queue in arrival order.</param>
/// <param name="queuedWriteBudget">Optional bound on how long a queued write may wait before it is failed
/// <c>BadTimeout</c>; defaults to <see cref="DefaultQueuedWriteBudget"/>.</param>
/// <param name="initializeTimeout">Optional pessimistic bound on one initialise attempt; defaults to
/// <see cref="DefaultInitializeTimeout"/>.</param>
/// <param name="writeTimeout">Optional pessimistic bound on one write round-trip; defaults to
/// <see cref="DefaultWriteTimeout"/>.</param>
/// <returns>A <see cref="Akka.Actor.Props"/> instance configured to create the wrapped <see cref="DriverInstanceActor"/>.</returns>
public static Props Props(
IDriver driver,
@@ -316,7 +435,12 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
string? clusterId = null,
IDriverCapabilityInvoker? invoker = null,
TimeSpan? healthPollInterval = null,
TimeSpan? commsLossTimeout = null) =>
TimeSpan? commsLossTimeout = null,
TimeSpan? shutdownTimeout = null,
int? maxWritesInFlight = null,
TimeSpan? queuedWriteBudget = null,
TimeSpan? initializeTimeout = null,
TimeSpan? writeTimeout = null) =>
Akka.Actor.Props.Create(() => new DriverInstanceActor(
driver,
reconnectInterval ?? DefaultReconnectInterval,
@@ -325,7 +449,12 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
clusterId ?? string.Empty,
invoker,
healthPollInterval,
commsLossTimeout));
commsLossTimeout,
shutdownTimeout,
maxWritesInFlight,
queuedWriteBudget,
initializeTimeout,
writeTimeout));
/// <summary>
/// Returns true when the driver should boot in DEV-STUB mode based on host platform and
@@ -362,6 +491,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
/// subscription reconcile; defaults to <see cref="HealthPollInterval"/> when null.</param>
/// <param name="commsLossTimeout">Comms-loss watchdog threshold; defaults to
/// <see cref="DefaultCommsLossTimeout"/> when null, and is disabled when zero or negative.</param>
/// <param name="shutdownTimeout">Bound on the driver's shutdown / dispose; defaults to
/// <see cref="DefaultShutdownTimeout"/> when null.</param>
/// <param name="maxWritesInFlight">Cap on concurrent writes; defaults to
/// <see cref="DefaultMaxWritesInFlight"/> when null or non-positive.</param>
/// <param name="queuedWriteBudget">Bound on a queued write's wait; defaults to
/// <see cref="DefaultQueuedWriteBudget"/> when null or non-positive.</param>
/// <param name="initializeTimeout">Pessimistic bound on one initialise attempt; defaults to
/// <see cref="DefaultInitializeTimeout"/> when null or non-positive.</param>
/// <param name="writeTimeout">Pessimistic bound on one write round-trip; defaults to
/// <see cref="DefaultWriteTimeout"/> when null or non-positive.</param>
public DriverInstanceActor(
IDriver driver,
TimeSpan reconnectInterval,
@@ -370,8 +509,18 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
string? clusterId = null,
IDriverCapabilityInvoker? invoker = null,
TimeSpan? healthPollInterval = null,
TimeSpan? commsLossTimeout = null)
TimeSpan? commsLossTimeout = null,
TimeSpan? shutdownTimeout = null,
int? maxWritesInFlight = null,
TimeSpan? queuedWriteBudget = null,
TimeSpan? initializeTimeout = null,
TimeSpan? writeTimeout = null)
{
_writeTimeout = writeTimeout is { } wt && wt > TimeSpan.Zero ? wt : DefaultWriteTimeout;
_incarnation = Interlocked.Increment(ref s_initGenerationSource);
_maxWritesInFlight = maxWritesInFlight is > 0 ? maxWritesInFlight.Value : DefaultMaxWritesInFlight;
_queuedWriteBudget = queuedWriteBudget is { } qb && qb > TimeSpan.Zero ? qb : DefaultQueuedWriteBudget;
_initializeTimeout = initializeTimeout is { } it && it > TimeSpan.Zero ? it : DefaultInitializeTimeout;
_driver = driver;
_invoker = invoker ?? NullDriverCapabilityInvoker.Instance;
_driverInstanceId = driver.DriverInstanceId;
@@ -380,6 +529,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
_reconnectInterval = reconnectInterval;
_healthPollInterval = healthPollInterval ?? HealthPollInterval;
_commsLossTimeout = commsLossTimeout ?? DefaultCommsLossTimeout;
_shutdownTimeout = shutdownTimeout is { } bound && bound > TimeSpan.Zero ? bound : DefaultShutdownTimeout;
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
new KeyValuePair<string, object?>("event", startStubbed ? "spawn_stub" : "spawn"),
new KeyValuePair<string, object?>("driver_type", driver.DriverType));
@@ -421,6 +571,18 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Context.Parent.Tell(new ReprovisionRequested(_driverInstanceId, reason.GetType().Name));
}
/// <summary>
/// Flags the restart before the base implementation runs <see cref="PostStop"/>, so that PostStop
/// shuts the driver down (the host re-initialises it on <see cref="ReprovisionRequested"/>) but does
/// NOT dispose it — see <see cref="_restarting"/>.
/// </summary>
/// <inheritdoc />
protected override void PreRestart(Exception reason, object message)
{
_restarting = true;
base.PreRestart(reason, message);
}
private void Stubbed()
{
// Stubbed drivers accept the standard message contracts but return deterministic
@@ -448,6 +610,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
// "write timeout". Synchronous Receive: Sender.Tell on the actor thread is safe.
Receive<WriteAttribute>(_ =>
Sender.Tell(new WriteAttributeResult(false, "driver not connected")));
// A write launched while Connected can finish after the transition out of it; its slot still frees.
Receive<WriteCompleted>(OnWriteCompleted);
// An ack arriving while still connecting can't reach the upstream alarm system; drop it (the ack is
// fire-and-forget — no reply to surface — and the OPC UA condition state already committed locally).
Receive<RouteAlarmAck>(_ =>
@@ -455,6 +619,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Receive<InitializeSucceeded>(msg =>
{
if (msg.Generation != _initGeneration) return;
CompleteInitAttempt();
_log.Info("DriverInstance {Id}: connected", _driverInstanceId);
Become(Connected);
PublishHealthSnapshot();
@@ -465,6 +630,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Receive<InitializeFailed>(msg =>
{
if (msg.Generation != _initGeneration) return;
CompleteInitAttempt();
_log.Warning("DriverInstance {Id}: initialize failed: {Reason}", _driverInstanceId, msg.Reason);
RecordFault();
Become(Reconnecting);
@@ -514,10 +680,14 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
{
_log.Info("DriverInstance {Id}: ForceReconnect requested by admin; re-entering Reconnecting", _driverInstanceId);
DetachSubscription();
FailQueuedWrites("driver not connected");
Become(Reconnecting);
PublishHealthSnapshot();
});
ReceiveAsync<WriteAttribute>(HandleWriteAsync);
// Synchronous Receive: the device call runs off the mailbox (see StartOrQueueWrite), so data changes
// and further writes are never head-of-line blocked behind a slow device.
Receive<WriteAttribute>(msg => StartOrQueueWrite(msg, Sender));
Receive<WriteCompleted>(OnWriteCompleted);
ReceiveAsync<RouteAlarmAck>(HandleAcknowledgeAsync);
ReceiveAsync<Subscribe>(HandleSubscribeAsync);
ReceiveAsync<Unsubscribe>(_ => UnsubscribeAsync());
@@ -745,6 +915,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
// timeout on an inbound write to a transiently-down driver). Synchronous Receive.
Receive<WriteAttribute>(_ =>
Sender.Tell(new WriteAttributeResult(false, "driver not connected")));
Receive<WriteCompleted>(OnWriteCompleted);
// An ack arriving while reconnecting can't reach the upstream alarm system; drop it (fire-and-forget,
// no reply — the OPC UA condition state already committed locally on the Part 9 ack).
Receive<RouteAlarmAck>(_ =>
@@ -752,6 +923,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Receive<InitializeSucceeded>(msg =>
{
if (msg.Generation != _initGeneration) return;
CompleteInitAttempt();
Timers.Cancel("retry-connect");
_log.Info("DriverInstance {Id}: reconnected", _driverInstanceId);
Become(Connected);
@@ -760,9 +932,13 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
AttachAlarmSource();
SubscribeDesiredAlarms();
});
// A failure here is a no-op regardless of generation — the retry timer keeps trying the
// current config; only a (generation-matched) InitializeSucceeded transitions state.
Receive<InitializeFailed>(_ => { /* keep retrying via timer */ });
// A failure here changes no state — the retry timer keeps trying the current config; only a
// (generation-matched) InitializeSucceeded transitions. It DOES release the single-flight gate, or
// no later tick could ever retry.
Receive<InitializeFailed>(msg =>
{
if (msg.Generation == _initGeneration) CompleteInitAttempt();
});
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
Receive<ForceReconnect>(_ => { /* already reconnecting — no-op */ });
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
@@ -807,12 +983,28 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
private void InitializeAsync(string driverConfigJson, bool teardownFirst = false)
{
_currentConfigJson = driverConfigJson;
var generation = ++_initGeneration;
if (_initInFlight)
{
// SINGLE-FLIGHT (02/H2). The previous attempt is still inside the driver; starting another would run
// its teardown against the transport that attempt is building. Skip — the next tick after the
// result retries, and a config handed over meanwhile is already stored for it.
_log.Debug("DriverInstance {Id}: initialise attempt {Gen} still in flight; skipping this one",
_driverInstanceId, _initGeneration);
return;
}
_initInFlight = true;
var generation = Interlocked.Increment(ref s_initGenerationSource);
_initGeneration = generation;
var initCts = new CancellationTokenSource();
_initCts = initCts;
var self = Self;
var log = _log;
var driverInstanceId = _driverInstanceId;
var driver = _driver;
var invoker = _invoker;
var shutdownTimeout = _shutdownTimeout;
var initializeTimeout = _initializeTimeout;
_ = Task.Run(async () =>
{
try
@@ -823,7 +1015,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
{
// Deliberately UNWRAPPED: teardown is best-effort cleanup of an already-dead
// transport — a shutdown failure must not count toward the Initialize breaker.
await driver.ShutdownAsync(CancellationToken.None);
// BOUNDED: a teardown wedged on the dead transport must not hold the retry loop
// (and this thread) for longer than the bound; it is abandoned and the init proceeds.
using var teardownCts = new CancellationTokenSource(shutdownTimeout);
await driver.ShutdownAsync(teardownCts.Token).WaitAsync(shutdownTimeout);
}
catch (TimeoutException)
{
log.Warning(
"DriverInstance {Id}: ShutdownAsync did not complete within {Bound} before a reconnect " +
"attempt; abandoning it and initialising anyway", driverInstanceId, shutdownTimeout);
}
catch (Exception ex)
{
@@ -841,33 +1042,103 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
// attempt with BrokenCircuitException, which lands in the catch below exactly like a
// real connect failure: InitializeFailed → keep retrying on the timer. The 15 s break
// window is shorter than two 10 s timer ticks, so recovery costs at most one extra tick.
await invoker.ExecuteAsync(
// The outer token is this attempt's own: PostStop cancels it so a stop/restart does not run its
// shutdown underneath an attempt still inside the driver.
// PESSIMISTIC bound on top of the invoker's token-only timeout: a driver that ignores its token
// would otherwise hold the single-flight gate forever. An overrun is abandoned (the task keeps
// running, its eventual outcome is observed and discarded), reported as a failure so the retry
// loop continues, and its late Tell is dropped by the generation guard anyway.
var attempt = invoker.ExecuteAsync(
DriverCapability.Initialize,
driverInstanceId,
async ct => await driver.InitializeAsync(driverConfigJson, ct),
CancellationToken.None);
initCts.Token).AsTask();
try
{
await attempt.WaitAsync(initializeTimeout);
}
catch (TimeoutException)
{
log.Warning(
"DriverInstance {Id}: initialise attempt {Gen} did not complete within {Bound} (the driver ignored " +
"its cancellation token); attempt abandoned, the retry loop continues",
driverInstanceId, generation, initializeTimeout);
try { initCts.Cancel(); } catch (ObjectDisposedException) { }
_ = attempt.ContinueWith(t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted);
self.Tell(new InitializeFailed($"initialise did not complete within {initializeTimeout}", generation));
return;
}
self.Tell(new InitializeSucceeded(generation));
}
catch (Exception ex)
{
self.Tell(new InitializeFailed(ex.Message, generation));
}
finally
{
// The CTS outlives an abandoned attempt (it may still be inside the driver); dispose only once
// nothing can cancel it any more — the actor nulls _initCts when it honours the result.
if (!initCts.IsCancellationRequested) initCts.Dispose();
}
});
}
private async Task HandleWriteAsync(WriteAttribute msg)
/// <summary>Releases the single-flight gate once the in-flight attempt's result has been honoured.</summary>
private void CompleteInitAttempt()
{
_initInFlight = false;
_initCts = null;
}
/// <summary>
/// Launches a write against the driver off the mailbox, or queues it when
/// <see cref="_maxWritesInFlight"/> writes are already inside the driver. Runs on the actor thread; the
/// only actor state it touches is the in-flight count and the queue. The device round-trip itself
/// (<see cref="RunWriteAsync"/>) replies to the captured sender from the thread pool and then
/// self-sends <see cref="WriteCompleted"/> to free its slot.
/// </summary>
private void StartOrQueueWrite(WriteAttribute msg, IActorRef replyTo)
{
var replyTo = Sender;
if (_driver is not IWritable writable)
{
replyTo.Tell(new WriteAttributeResult(false, "Driver does not implement IWritable"));
return;
}
if (_writesInFlight >= _maxWritesInFlight || _tagsInFlight.Contains(msg.TagId))
{
_pendingWrites.AddLast((msg, replyTo, DateTime.UtcNow));
return;
}
LaunchWrite(writable, msg, replyTo);
}
private void LaunchWrite(IWritable writable, WriteAttribute msg, IActorRef replyTo)
{
_writesInFlight++;
_tagsInFlight.Add(msg.TagId);
// Everything the task needs is captured HERE, on the actor thread — ResolveHost reads the driver,
// and Self must not be touched off-context.
var self = Self;
var hostKey = ResolveHost(msg.TagId);
var invoker = _invoker;
_ = RunWriteAsync(writable, invoker, hostKey, msg, replyTo, self, _writeTimeout, _incarnation, _log);
}
private static async Task RunWriteAsync(
IWritable writable, IDriverCapabilityInvoker invoker, string hostKey,
WriteAttribute msg, IActorRef replyTo, IActorRef self, TimeSpan writeTimeout, int incarnation,
ILoggingAdapter log)
{
// Off the actor context from here on: the first await yields, and nothing below reads actor state.
await Task.Yield();
var request = new[] { new WriteRequest(msg.TagId, msg.Value) };
// Bound the write so a hung backend can't pin this actor forever — retry stays
// off by default, but a stalled call still needs an answer.
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
// Bound the write so a hung backend can't hold a slot forever — retry stays
// off by default, but a stalled call still needs an answer. Disposed below, except when the write is
// abandoned (the orphan may still be holding the token).
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var abandoned = false;
try
{
// Route through the resilience invoker (Write pipeline: timeout, per-host breaker). Writes
@@ -877,11 +1148,31 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
// therefore authoritative regardless of any configured Write retry. Per-tag opt-in to retries via
// WriteIdempotentAttribute is the unshipped forward path (see WriteIdempotentAttribute). The outer
// cts is retained as a hard backstop above the (typically tighter) tier timeout.
var results = await _invoker.ExecuteWriteAsync(
ResolveHost(msg.TagId),
// PESSIMISTIC bound on top of the token-based cap: a driver that ignores its token would otherwise
// hold the slot and the tag forever, piling every later write to that tag up in the queue.
var attempt = invoker.ExecuteWriteAsync(
hostKey,
isIdempotent: false,
async ct => await writable.WriteAsync(request, ct),
cts.Token);
cts.Token).AsTask();
IReadOnlyList<WriteResult> results;
try
{
results = await attempt.WaitAsync(writeTimeout);
}
catch (TimeoutException)
{
abandoned = true;
log.Warning(
"DriverInstance write of {Tag} did not complete within {Bound} (the driver ignored its cancellation " +
"token); abandoned — its slot and tag are released, its eventual outcome is discarded",
msg.TagId, writeTimeout);
try { cts.Cancel(); } catch (ObjectDisposedException) { }
_ = attempt.ContinueWith(t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted);
replyTo.Tell(new WriteAttributeResult(false,
$"StatusCode=0x800A0000 (BadTimeout: write did not complete within {writeTimeout.TotalSeconds:F0}s)"));
return;
}
if (results is { Count: 1 } && IsGoodStatus(results[0].StatusCode))
{
replyTo.Tell(new WriteAttributeResult(true, null));
@@ -898,6 +1189,62 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
{
replyTo.Tell(new WriteAttributeResult(false, ex.Message));
}
finally
{
if (!abandoned) cts.Dispose();
self.Tell(new WriteCompleted(msg.TagId, incarnation));
}
}
/// <summary>Frees the write's slot and tag and drains the queue: the oldest entries whose tag is free are
/// launched (in arrival order) while slots remain; entries older than the budget are failed instead. A
/// completion stamped with another incarnation (a write launched before a supervision restart) is ignored —
/// it never held a slot or tag of THIS incarnation.</summary>
private void OnWriteCompleted(WriteCompleted msg)
{
if (msg.Incarnation != _incarnation) return;
if (_writesInFlight > 0) _writesInFlight--;
_tagsInFlight.Remove(msg.TagId);
DrainWriteQueue();
}
private void DrainWriteQueue()
{
if (_driver is not IWritable writable) return;
var now = DateTime.UtcNow;
var node = _pendingWrites.First;
while (node is not null && _writesInFlight < _maxWritesInFlight)
{
var next = node.Next;
var (write, replyTo, enqueuedUtc) = node.Value;
if (now - enqueuedUtc > _queuedWriteBudget)
{
// Already reported as a timeout upstream (the host's Ask is the budget) and the node reverted —
// writing it now would push a stale value to the device. BadTimeout, never written.
_pendingWrites.Remove(node);
replyTo.Tell(new WriteAttributeResult(false,
$"StatusCode=0x800A0000 (BadTimeout: queued for {(now - enqueuedUtc).TotalSeconds:F1}s, over the {_queuedWriteBudget.TotalSeconds:F0}s write budget)"));
}
else if (!_tagsInFlight.Contains(write.TagId))
{
_pendingWrites.Remove(node);
LaunchWrite(writable, write, replyTo);
}
node = next;
}
}
/// <summary>Replies to every queued write with <paramref name="reason"/>; called on each exit from
/// <c>Connected</c>. In-flight writes are left to finish — they already hold the device and reply with its
/// real outcome.</summary>
private void FailQueuedWrites(string reason)
{
while (_pendingWrites.First is { } node)
{
_pendingWrites.RemoveFirst();
node.Value.ReplyTo.Tell(new WriteAttributeResult(false, reason));
}
}
/// <summary>
@@ -1145,6 +1492,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
{
_log.Warning("DriverInstance {Id}: disconnect observed ({Reason}); reconnecting", _driverInstanceId, reason);
DetachSubscription();
FailQueuedWrites("driver not connected");
RecordFault();
Become(Reconnecting);
PublishHealthSnapshot();
@@ -1542,15 +1890,91 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
}
DetachSubscription();
FailQueuedWrites("driver stopped");
// MUST happen: the IDriver instance can outlive this actor (the host respawns a child around the
// same driver object), so a missing unsubscribe accumulates a handler per respawn holding a dead Self.
DetachRediscoverySource();
DetachHostStatusSource();
DetachConnectionLossSource();
try { _driver.ShutdownAsync(CancellationToken.None).GetAwaiter().GetResult(); }
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
// An attempt still inside the driver is cancelled before the shutdown below runs underneath it (on a
// restart the next incarnation re-initialises the same object). Best-effort: the task owns the CTS
// and may have disposed it already.
try { _initCts?.Cancel(); }
catch (ObjectDisposedException) { /* the attempt already finished */ }
// BOUNDED by ONE shared budget across shutdown + dispose. PostStop runs on a dispatcher thread and this
// blocks it, so an unbounded wait on a driver whose shutdown hangs against a dead socket / COM proxy pinned
// that thread for the whole wait, and a few such stops starved the dispatcher. A shutdown that overruns is
// abandoned, and the dispose gets whatever is left of the same budget (never a second full one).
var budget = Stopwatch.StartNew();
using (var cts = new CancellationTokenSource(_shutdownTimeout))
{
try { _driver.ShutdownAsync(cts.Token).WaitAsync(_shutdownTimeout).GetAwaiter().GetResult(); }
catch (TimeoutException)
{
_log.Warning("DriverInstance {Id}: ShutdownAsync did not complete within {Bound} on PostStop; abandoning it",
_driverInstanceId, _shutdownTimeout);
}
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
}
// Dispose ONLY on a real stop. A supervision restart (PreRestart → this PostStop) re-uses the same driver
// object for the next incarnation, which the host re-initialises. Everything else that stops a child —
// deploy removal, operator Restart, respawn on a config change, host shutdown — is final, and the driver
// is never touched again: before this, nothing in the runtime ever disposed an IDriver, so a Calculation
// driver's compiled-script AssemblyLoadContexts and an OpcUaClient driver's LoggerFactory / gate / CTS
// leaked once per respawn (02/L1). Bounded like the shutdown, and runs even after a timed-out shutdown —
// leaking the object is strictly worse than disposing one whose shutdown is still wedged.
if (!_restarting)
{
var remaining = _shutdownTimeout - budget.Elapsed;
DisposeDriver(remaining > MinDisposeBudget ? remaining : MinDisposeBudget);
}
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
new KeyValuePair<string, object?>("event", "stop"),
new KeyValuePair<string, object?>("driver_type", _driver.DriverType));
}
/// <summary>Floor on the dispose's share of the PostStop budget once a timed-out shutdown has used it up —
/// enough for an idempotent dispose to run, not enough to double the stop's thread time.</summary>
private static readonly TimeSpan MinDisposeBudget = TimeSpan.FromMilliseconds(250);
/// <summary>Final teardown of the wrapped driver: <see cref="IAsyncDisposable"/> preferred, else
/// <see cref="IDisposable"/>, never both. A driver implementing neither needs nothing here. Runs even after
/// a timed-out shutdown, so a driver's dispose must be safe CONCURRENTLY with its own still-running
/// <see cref="IDriver.ShutdownAsync"/>.</summary>
/// <param name="bound">The remaining PostStop budget available to an async dispose.</param>
private void DisposeDriver(TimeSpan bound)
{
try
{
switch (_driver)
{
case IAsyncDisposable asyncDisposable:
asyncDisposable.DisposeAsync().AsTask().WaitAsync(bound).GetAwaiter().GetResult();
break;
case IDisposable disposable:
disposable.Dispose();
break;
default:
return;
}
_log.Info("DriverInstance {Id}: disposed {Type} driver on stop", _driverInstanceId, _driver.DriverType);
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
new KeyValuePair<string, object?>("event", "dispose"),
new KeyValuePair<string, object?>("driver_type", _driver.DriverType));
}
catch (TimeoutException)
{
_log.Warning("DriverInstance {Id}: DisposeAsync did not complete within {Bound} on PostStop; abandoning it",
_driverInstanceId, bound);
}
catch (Exception ex)
{
_log.Warning(ex, "DriverInstance {Id}: disposing the driver threw on PostStop", _driverInstanceId);
}
}
}
@@ -7,21 +7,37 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Health;
/// <summary>
/// Single-flight cached health probe against the ConfigDb. Reads cached state via
/// <c>Ask&lt;DbHealthStatus&gt;</c>; a single SELECT 1 runs at most every <c>RefreshInterval</c>.
/// <c>Ask&lt;DbHealthStatus&gt;</c>; a single SELECT runs at most every <c>RefreshInterval</c>.
/// Consumed by both the host's <c>/health/ready</c> endpoint and
/// <c>RedundancyStateActor</c>'s stale calc.
/// </summary>
/// <remarks>
/// The probe runs OFF the mailbox (a thread-pool task that pipes its outcome back), with a bounded
/// command timeout. It used to run synchronously on the actor thread, relying on SqlClient's implicit
/// 15 s connect / 30 s command timeouts — so a hanging SQL Server pinned a dispatcher thread for up to
/// 30 s per tick, later ticks queued behind it, and the cached <see cref="GetStatus"/> answer was
/// unreachable for exactly as long as the database was. Single-flight: a tick that lands while a probe
/// is still running is skipped rather than stacked.
/// </remarks>
public sealed class DbHealthProbeActor : ReceiveActor, IWithTimers
{
public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(5);
/// <summary>Bound on the probe's SQL command, relational providers only (the in-memory provider the
/// actor tests run against has no command timeout).</summary>
public static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(2);
public sealed class GetStatus { public static readonly GetStatus Instance = new(); private GetStatus() { } }
public sealed record DbHealthStatus(bool Reachable, DateTime AsOfUtc, string? LastError);
public sealed class Tick { public static readonly Tick Instance = new(); private Tick() { } }
/// <summary>Piped back by the off-thread probe with its outcome.</summary>
private sealed record ProbeCompleted(DbHealthStatus Status);
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
private readonly ILoggingAdapter _log = Context.GetLogger();
private DbHealthStatus _last = new(false, DateTime.MinValue, "not probed yet");
private bool _probeInFlight;
/// <summary>Gets or sets the timer scheduler for periodic health probes.</summary>
public ITimerScheduler Timers { get; set; } = null!;
@@ -38,28 +54,54 @@ public sealed class DbHealthProbeActor : ReceiveActor, IWithTimers
{
_dbFactory = dbFactory;
Receive<GetStatus>(_ => Sender.Tell(_last));
Receive<Tick>(_ => RunProbe());
Receive<Tick>(_ => StartProbe());
Receive<ProbeCompleted>(msg =>
{
_probeInFlight = false;
_last = msg.Status;
if (!msg.Status.Reachable)
_log.Warning("DbHealthProbe: probe failed: {Error}", msg.Status.LastError);
});
}
/// <inheritdoc />
protected override void PreStart()
{
RunProbe();
StartProbe();
Timers.StartPeriodicTimer("probe", Tick.Instance, RefreshInterval);
}
private void RunProbe()
private void StartProbe()
{
if (_probeInFlight)
{
_log.Debug("DbHealthProbe: previous probe still running; skipping this tick");
return;
}
_probeInFlight = true;
var factory = _dbFactory;
// Never touches actor state; the outcome comes back as a message.
Task.Run(() => Probe(factory)).PipeTo(
Self,
success: status => new ProbeCompleted(status),
failure: ex => new ProbeCompleted(new DbHealthStatus(false, DateTime.UtcNow, ex.Message)));
}
private static DbHealthStatus Probe(IDbContextFactory<OtOpcUaConfigDbContext> factory)
{
try
{
using var db = _dbFactory.CreateDbContext();
using var db = factory.CreateDbContext();
// Local to this lease — the pool resets it on return (see DriverHostActor.LoadArtifactBlob).
if (db.Database.IsRelational())
db.Database.SetCommandTimeout(CommandTimeout);
_ = db.Deployments.AsNoTracking().Take(1).ToList();
_last = new DbHealthStatus(true, DateTime.UtcNow, null);
return new DbHealthStatus(true, DateTime.UtcNow, null);
}
catch (Exception ex)
{
_last = new DbHealthStatus(false, DateTime.UtcNow, ex.Message);
_log.Warning(ex, "DbHealthProbe: probe failed");
return new DbHealthStatus(false, DateTime.UtcNow, ex.Message);
}
}
}
@@ -154,6 +154,42 @@ public sealed class ServerHistorianOptions
/// </summary>
public int MaxTimeWeightedRawRows { get; init; } = 1_000_000;
/// <summary>The default of <see cref="MaxRawValuesPerRead"/>: 10 000 values per raw page / backend read.</summary>
public const int DefaultMaxRawValuesPerRead = 10_000;
/// <summary>
/// The smallest usable <see cref="MaxRawValuesPerRead"/>: a resumed page's read must carry at least one value, the
/// boundary tie it re-reads and trims, AND the lookahead row that tells the server more data follows. At 2 a resumed
/// page has no room at all and every value costs an empty page with a continuation point (the #692 defect, re-created
/// by configuration), so 1 and 2 are raised to 3 (<see cref="EffectiveMaxRawValuesPerRead"/>).
/// </summary>
public const int MinMaxRawValuesPerRead = 3;
/// <summary>
/// Every backend: the most raw values the server returns on ONE HistoryRead-Raw page, and the most rows one
/// backend raw read may materialise — the server's own row ceiling (review R08:1.1, #736). A client's
/// <c>NumValuesPerNode = 0</c> ("all values") no longer reads a whole window into memory: it is served as pages of at
/// most this many values, each with a continuation point (Part 11 allows fewer values than requested with one), through
/// the same tie-safe paging a capped read uses (<c>RawPageLimit</c>: page + boundary ties + lookahead ≤ cap, so a first
/// page holds cap − 1 values). A <c>NumValuesPerNode</c> above it is paged at it too. Advertised as
/// <c>HistoryServerCapabilities/MaxReturnDataValues</c>. The backends' own <c>TOP</c> / <c>LIMIT</c> / <c>max_values</c>
/// are bounded by it independently: a raw read asking for more than this (or for everything) reads at most cap + 1 rows
/// and is REFUSED (<c>BadTooManyOperations</c>) when the window holds more — never answered truncated — so a caller that
/// bypasses the node manager's paging cannot bypass the bound. On the Gateway backend a strict gateway's learned row
/// ceiling (less its boundary-row allowance) still applies when lower. The tie-cluster bound of a resumed page is also
/// at most cap − 1 (as under a gateway ceiling; <see cref="MaxTieClusterOverfetch"/> above that is inert), and a
/// page whose boundary tie cluster is that large steps past it with one empty page. Default 10 000. Non-positive =
/// unbounded, the pre-#736 behaviour (a <see cref="Validate"/> warning); below <see cref="MinMaxRawValuesPerRead"/>
/// (3) is raised to it.
/// </summary>
public int MaxRawValuesPerRead { get; init; } = DefaultMaxRawValuesPerRead;
/// <summary>
/// <see cref="MaxRawValuesPerRead"/> as the backends and the node manager apply it: <c>0</c> (unbounded) when
/// non-positive, otherwise at least <see cref="MinMaxRawValuesPerRead"/>. The one place the rule lives.
/// </summary>
public int EffectiveMaxRawValuesPerRead => MaxRawValuesPerRead <= 0 ? 0 : Math.Max(MinMaxRawValuesPerRead, MaxRawValuesPerRead);
/// <summary>
/// how far into the future an OPC UA HistoryUpdate (Insert / Replace / Update) item's
/// <c>SourceTimestamp</c> may lie, measured from the server clock. An item beyond <c>now + skew</c> is refused
@@ -208,6 +244,10 @@ public sealed class ServerHistorianOptions
warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled.");
if (MaxTimeWeightedRawRows <= 0)
warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples.");
if (MaxRawValuesPerRead <= 0)
warnings.Add($"ServerHistorian:MaxRawValuesPerRead is {MaxRawValuesPerRead} — must be > 0; HistoryRead-Raw pages and backend raw reads are unbounded, so a client asking for a whole window (NumValuesPerNode = 0) materialises every sample in memory.");
else if (MaxRawValuesPerRead < MinMaxRawValuesPerRead)
warnings.Add($"ServerHistorian:MaxRawValuesPerRead is {MaxRawValuesPerRead} — must be at least {MinMaxRawValuesPerRead} (a value, its boundary tie and the lookahead row); treated as {MinMaxRawValuesPerRead}.");
if (EventReadHorizon < TimeSpan.Zero)
warnings.Add($"ServerHistorian:EventReadHorizon is {EventReadHorizon} — negative is treated as no horizon (00:00:00); HistoryReadEvents reads every window in full.");
if (HistoryReadDeadline <= TimeSpan.Zero)
@@ -27,7 +27,27 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
/// </remarks>
public sealed class TelemetryLocalHub : ITelemetryLocalHub
{
/// <summary>
/// How long an eviction tombstone (see <see cref="_evicted"/>) is kept. It only has to outlive the
/// removed child's last late write — an in-flight capability call that completes after
/// <c>StopChild</c>, bounded by the longest capability timeout (seconds) plus the child's own stop —
/// so ten minutes is a wide margin. Without a sweep a driver removed for good kept its id forever.
/// </summary>
public static readonly TimeSpan EvictionTombstoneTtl = TimeSpan.FromMinutes(10);
private readonly object _gate = new();
private readonly TimeProvider _time;
/// <summary>Initializes a hub on the system clock.</summary>
public TelemetryLocalHub() : this(TimeProvider.System) { }
/// <summary>Initializes a hub on <paramref name="time"/> — the tombstone-sweep test seam.</summary>
/// <param name="time">The clock the eviction tombstones are stamped and swept by.</param>
public TelemetryLocalHub(TimeProvider time)
{
ArgumentNullException.ThrowIfNull(time);
_time = time;
}
// Plain Dictionary, NOT ConcurrentDictionary: every access is already under _gate, and
// ConcurrentDictionary.Values would materialize a fresh List (and take its own internal lock) on
@@ -42,9 +62,10 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
private readonly ConcurrentDictionary<string, TelemetryItem.Health> _healthCache = new();
private readonly ConcurrentDictionary<(string InstanceId, string HostName), TelemetryItem.Resilience> _resilienceCache = new();
// Instances evicted and not since readmitted. Their late Health/Resilience items are dropped, so a
// removed driver's still-stopping child cannot re-prime the cache it was just evicted from. Under _gate.
private readonly HashSet<string> _evicted = new(StringComparer.Ordinal);
// Instances evicted and not since readmitted, stamped with the eviction time. Their late Health/Resilience
// items are dropped, so a removed driver's still-stopping child cannot re-prime the cache it was just
// evicted from. Swept past EvictionTombstoneTtl on the next Evict (never on the Emit hot path). Under _gate.
private readonly Dictionary<string, DateTimeOffset> _evicted = new(StringComparer.Ordinal);
/// <inheritdoc />
public void Emit(TelemetryItem item)
@@ -55,9 +76,9 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
{
switch (item)
{
case TelemetryItem.Health h when _evicted.Contains(h.E.DriverInstanceId):
case TelemetryItem.Health h when _evicted.ContainsKey(h.E.DriverInstanceId):
return;
case TelemetryItem.Resilience r when _evicted.Contains(r.E.DriverInstanceId):
case TelemetryItem.Resilience r when _evicted.ContainsKey(r.E.DriverInstanceId):
return;
case TelemetryItem.Health h:
_healthCache[h.E.DriverInstanceId] = h;
@@ -80,7 +101,9 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
lock (_gate)
{
_evicted.Add(driverInstanceId);
var now = _time.GetUtcNow();
SweepExpiredTombstones(now);
_evicted[driverInstanceId] = now;
_healthCache.TryRemove(driverInstanceId, out _);
foreach (var key in _resilienceCache.Keys)
if (string.Equals(key.InstanceId, driverInstanceId, StringComparison.Ordinal))
@@ -97,6 +120,24 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
_evicted.Remove(driverInstanceId);
}
/// <summary>Drops tombstones older than <see cref="EvictionTombstoneTtl"/>. Called under <see cref="_gate"/>
/// from <see cref="Evict"/> only — the set is bounded by distinct removed ids, so a linear pass per
/// eviction is cheap, and Emit never pays for it.</summary>
private void SweepExpiredTombstones(DateTimeOffset now)
{
if (_evicted.Count == 0) return;
List<string>? expired = null;
foreach (var (id, evictedAt) in _evicted)
{
if (now - evictedAt >= EvictionTombstoneTtl)
(expired ??= new List<string>()).Add(id);
}
if (expired is null) return;
foreach (var id in expired)
_evicted.Remove(id);
}
/// <inheritdoc />
public ITelemetrySubscription Subscribe(int boundedCapacity)
{
@@ -0,0 +1,125 @@
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Browsing;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Commons.Tests.Browsing;
/// <summary>
/// Review 02 L1 (host context, 06/G-1): the browser's throwaway capture driver must be <b>disposed</b>, not
/// only shut down. A Galaxy driver's <c>ShutdownAsync</c> left its whole gRPC runtime alive until
/// <c>DisposeAsync</c>, which nothing called — so every <c>/raw</c> browse leaked two channels, an MX session
/// and two stream loops.
/// </summary>
public sealed class DiscoveryDriverBrowserDisposeTests
{
private static readonly TimeSpan ShortOpen = TimeSpan.FromSeconds(5);
private static readonly TimeSpan ShortShutdown = TimeSpan.FromMilliseconds(300);
private static DiscoveryDriverBrowser Browser(IDriverFactory factory) =>
new(factory, NullLogger<DiscoveryDriverBrowser>.Instance, ShortOpen, ShortShutdown);
[Fact]
public async Task OpenAsync_disposes_the_capture_driver_after_shutdown()
{
var driver = new DisposableFakeDiscoveryDriver();
var browser = Browser(new FakeDriverFactory((_, _, _) => driver));
var session = await browser.OpenAsync("Fake", "{}", CancellationToken.None);
session.ShouldNotBeNull();
driver.Inner.Trace.ShouldBe(new[] { "init", "discover", "shutdown" });
driver.DisposeCalls.ShouldBe(1);
driver.DisposedAfterShutdown.ShouldBeTrue("dispose is the final teardown, after the shutdown");
}
[Fact]
public async Task OpenAsync_disposes_the_capture_driver_even_when_discovery_throws()
{
var driver = new DisposableFakeDiscoveryDriver(new FakeDiscoveryDriver
{
OnDiscover = (_, _) => throw new InvalidOperationException("boom-discovery"),
});
var browser = Browser(new FakeDriverFactory((_, _, _) => driver));
await Should.ThrowAsync<InvalidOperationException>(
() => browser.OpenAsync("Fake", "{}", CancellationToken.None));
driver.DisposeCalls.ShouldBe(1);
}
[Fact]
public async Task OpenAsync_disposes_the_capture_driver_even_when_its_shutdown_hangs()
{
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var driver = new DisposableFakeDiscoveryDriver(new FakeDiscoveryDriver { OnShutdown = _ => never.Task });
var browser = Browser(new FakeDriverFactory((_, _, _) => driver));
await browser.OpenAsync("Fake", "{}", CancellationToken.None);
driver.DisposeCalls.ShouldBe(1, "a shutdown that timed out must not skip the dispose");
}
[Fact]
public async Task CanBrowse_disposes_its_probe_driver()
{
var driver = new DisposableFakeDiscoveryDriver();
var browser = Browser(new FakeDriverFactory((_, _, _) => driver));
browser.CanBrowse("Fake", "{}").ShouldBeTrue();
// The CanBrowse teardown is fire-and-forget; wait for it.
await driver.Disposed.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
driver.DisposeCalls.ShouldBe(1);
}
/// <summary>A <see cref="FakeDiscoveryDriver"/> wrapped with <see cref="IAsyncDisposable"/> + <see cref="IDisposable"/>
/// (both, so the "prefer the async one, never both" rule is observable: exactly ONE dispose call lands).</summary>
private sealed class DisposableFakeDiscoveryDriver : IDriver, ITagDiscovery, IAsyncDisposable, IDisposable
{
private int _disposeCalls;
public DisposableFakeDiscoveryDriver(FakeDiscoveryDriver? inner = null) => Inner = inner ?? new FakeDiscoveryDriver();
public FakeDiscoveryDriver Inner { get; }
public int DisposeCalls => Volatile.Read(ref _disposeCalls);
public bool DisposedAfterShutdown { get; private set; }
public TaskCompletionSource Disposed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public string DriverInstanceId => Inner.DriverInstanceId;
public string DriverType => Inner.DriverType;
public bool SupportsOnlineDiscovery => Inner.SupportsOnlineDiscovery;
public DiscoveryRediscoverPolicy RediscoverPolicy => Inner.RediscoverPolicy;
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) =>
Inner.InitializeAsync(driverConfigJson, cancellationToken);
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) =>
Inner.ReinitializeAsync(driverConfigJson, cancellationToken);
public Task ShutdownAsync(CancellationToken cancellationToken) => Inner.ShutdownAsync(cancellationToken);
public DriverHealth GetHealth() => Inner.GetHealth();
#pragma warning disable OTOPCUA0001 // test double delegating to a test double — no resilience pipeline applies
public Task DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken) =>
Inner.DiscoverAsync(builder, cancellationToken);
#pragma warning restore OTOPCUA0001
public ValueTask DisposeAsync()
{
Record();
return ValueTask.CompletedTask;
}
public void Dispose() => Record();
private void Record()
{
Interlocked.Increment(ref _disposeCalls);
DisposedAfterShutdown = Inner.ShutdownCalls > 0;
Disposed.TrySetResult();
}
}
}
@@ -22,9 +22,14 @@ public interface IConformanceBackend : IAsyncDisposable
/// <param name="clock">Its "now"; null means the system clock.</param>
IHistorianDataSource CreateDataSource(TimeProvider? clock = null);
/// <summary>A fresh data source whose time-weighted raw-row cap (<c>ServerHistorian:MaxTimeWeightedRawRows</c>) is <paramref name="maxTimeWeightedRawRows"/>.</summary>
/// <param name="maxTimeWeightedRawRows">The cap (&lt;= 0 = unbounded).</param>
IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows);
/// <summary>
/// A fresh data source whose time-weighted raw-row cap (<c>ServerHistorian:MaxTimeWeightedRawRows</c>) is
/// <paramref name="maxTimeWeightedRawRows"/> and whose server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>,
/// R08:1.1 / #736) is <paramref name="maxRawValuesPerRead"/>.
/// </summary>
/// <param name="maxTimeWeightedRawRows">The time-weighted cap (&lt;= 0 = unbounded).</param>
/// <param name="maxRawValuesPerRead">The raw cap (&lt;= 0 = unbounded, the default).</param>
IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0);
/// <summary>A fresh live-value writer over this backend's store.</summary>
/// <param name="utcNow">The clock used to stamp values that carry no timestamp; null means the system clock.</param>
@@ -25,6 +25,43 @@ public abstract class RawReadConformance : ConformanceBase
r.ContinuationPoint.ShouldBeNull();
}
/// <summary>
/// R08:1.1 (#736): the server raw cap bounds the backend's own <c>TOP</c> / <c>LIMIT</c>. A read asking for everything
/// (or for more than the cap) reads at most cap + 1 rows and is refused when the window holds more — never answered
/// truncated (a short answer is what the server's paging reads as "the last page") — and is served in full when it fits.
/// A read at or under the cap is unchanged. The refusal is a client-request limit, not a backend fault.
/// </summary>
[Fact]
public async Task The_server_raw_cap_bounds_every_read_and_refuses_an_over_cap_window_rather_than_truncating()
{
await ConformanceData.SeedAAsync(B); // five samples at +0, +10, +20, +30, +40
var src = B.CreateDataSource(maxTimeWeightedRawRows: 1_000_000, maxRawValuesPerRead: 3);
src.MaxRawValuesPerRead.ShouldBe(3);
// Five rows under a cap of three: everything / above the cap ⇒ refused; at the cap ⇒ the first three; under ⇒ as asked.
var refused = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), 0, Ct));
refused.Limit.ShouldBe(3);
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), 4, Ct));
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), uint.MaxValue, Ct));
Values(await src.ReadRawAsync("A", At(-1), At(100), 3, Ct)).ShouldBe([10d, 20d, 30d]);
Values(await src.ReadRawAsync("A", At(-1), At(100), 2, Ct)).ShouldBe([10d, 20d]);
// Three rows fit: everything / above the cap are served in full.
Values(await src.ReadRawAsync("A", At(10), At(40), 0, Ct)).ShouldBe([20d, 30d, 40d]);
Values(await src.ReadRawAsync("A", At(10), At(40), 4, Ct)).ShouldBe([20d, 30d, 40d]);
// Reverse reads are bounded the same way, keeping the LATEST rows.
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(100), At(-1), 0, Ct));
Values(await src.ReadRawAsync("A", At(100), At(-1), 3, Ct)).ShouldBe([50d, 40d, 30d]);
Values(await src.ReadRawAsync("A", At(40), At(10), 0, Ct)).ShouldBe([50d, 40d, 30d]);
// A point read and an empty window are unaffected.
Values(await src.ReadRawAsync("A", At(10), At(10), 0, Ct)).ShouldBe([20d]);
(await src.ReadRawAsync("A", At(50), At(60), 0, Ct)).Samples.ShouldBeEmpty();
src.GetHealthSnapshot().TotalFailures.ShouldBe(0, "a refused over-cap read is a client-request limit, not a backend fault");
}
[Fact]
public async Task Forward_read_honours_the_limit()
{
@@ -0,0 +1,305 @@
using Google.Protobuf.WellKnownTypes;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Opc.Ua;
using Opc.Ua.Server;
using Shouldly;
using Xunit;
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
/// <summary>
/// Review R08:1.1 (#736): the server's raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>) bounds the Gateway
/// backend's <c>max_values</c> independently of the node manager. An uncapped raw read (<c>NumValuesPerNode = 0</c>)
/// used to be sent as <see cref="int.MaxValue"/> and materialise the whole window twice (the protobuf list and the
/// snapshot list); a read above the cap now asks for cap + 1 rows and is REFUSED (<see cref="HistoryReadLimitExceededException"/>,
/// <c>BadTooManyOperations</c>) when the window holds more — never answered truncated, because a short answer is what the
/// server's paging reads as "the last page". The data source reports the cap as its
/// <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> (or a strict gateway's lower learned ceiling), so the node
/// manager's existing ceiling paging serves <c>NumValuesPerNode = 0</c> as pages of at most the cap, each with a
/// continuation point.
/// </summary>
public sealed class GatewayRawCapTests : IDisposable
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static readonly DateTime T0 = new(2026, 10, 22, 10, 0, 0, DateTimeKind.Utc);
private const int Cap = 1000;
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-gwrawcap-{Guid.NewGuid():N}");
private static DateTime At(int seconds) => T0.AddSeconds(seconds);
private static HistorianSample Row(DateTime t, double v, uint quality = 0) =>
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = v, OpcQuality = 192, Quality = quality };
/// <summary>AVEVA-shaped: a boundary row at the query start, <c>[start, end]</c> inclusive, capped at <c>max</c>.</summary>
private static FakeHistorianGatewayClient Store(int count, int? ceiling = null)
{
var stored = Enumerable.Range(0, count).Select(i => (T: At(i), V: (double)i)).ToList();
return new FakeHistorianGatewayClient
{
MaxValuesCeiling = ceiling,
RawSource = (start, end, max) =>
{
if (max <= 0) throw new ArgumentOutOfRangeException(nameof(max), "max_values must be positive"); // the client package
var rows = new List<HistorianSample>();
if (stored.All(s => s.T != start))
{
var prior = stored.LastOrDefault(s => s.T < start);
rows.Add(prior.T == default
? new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 }
: Row(start, prior.V, 133));
}
rows.AddRange(stored.Where(s => s.T >= start && s.T <= end).Select(s => Row(s.T, s.V)));
return rows.Take(max);
},
};
}
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake, int cap = Cap) =>
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, maxRawValuesPerRead: cap);
private static int Probe => Cap + 1 + RawAggregateReader.PreWindowAllowance; // cap + 1 rows + the boundary-row allowance
// ---- the data source alone ----------------------------------------------------------------------------------------
[Fact]
public void Without_a_cap_the_source_is_unchanged()
{
new GatewayHistorianDataSource(Store(10), NullLogger<GatewayHistorianDataSource>.Instance).MaxRawValuesPerRead.ShouldBe(0);
Source(Store(10), cap: 0).MaxRawValuesPerRead.ShouldBe(0);
}
[Fact]
public async Task The_cap_is_the_page_limit_until_a_lower_gateway_ceiling_is_learned()
{
Source(Store(3600)).MaxRawValuesPerRead.ShouldBe(Cap);
// A strict gateway whose ceiling is ABOVE the cap: the cap wins, nothing is refused by the gateway.
var higher = Store(3600, ceiling: 2000);
var ds = Source(higher);
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(3600), 0, Ct));
higher.RawCalls.ShouldAllBe(c => c.MaxValues <= 2000);
ds.MaxRawValuesPerRead.ShouldBe(Cap);
// A strict gateway whose ceiling is BELOW the cap: the learned ceiling (less the allowance) wins once known.
var lower = Store(3600, ceiling: 500);
var ds2 = Source(lower);
ds2.MaxRawValuesPerRead.ShouldBe(Cap);
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds2.ReadRawAsync("T", At(0), At(3600), 0, Ct));
ds2.MaxRawValuesPerRead.ShouldBe(500 - RawAggregateReader.PreWindowAllowance);
}
[Fact]
public async Task An_uncapped_read_over_the_cap_is_refused_never_truncated_and_never_asks_for_everything()
{
var fake = Store(3600);
var ds = Source(fake);
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(3600), 0, Ct));
ex.Limit.ShouldBe(Cap);
ex.Message.ShouldContain("MaxRawValuesPerRead");
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([Probe]); // never int.MaxValue
fake.RawStreamsCompleted.ShouldBe(1, "the bounded stream is drained, never abandoned");
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0, "a client-request limit is not a backend fault");
}
[Theory]
[InlineData(10)]
[InlineData(Cap - 1)]
[InlineData(Cap)]
public async Task An_uncapped_read_that_fits_the_cap_is_served_in_full(int count)
{
var fake = Store(count);
var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), 0, Ct);
r.Samples.Select(s => s.Value).ShouldBe(Enumerable.Range(0, count).Select(i => (object?)(double)i));
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([Probe]);
}
[Theory]
[InlineData(Cap + 1u)]
[InlineData(5000u)]
[InlineData(uint.MaxValue)]
public async Task A_client_cap_above_the_server_cap_is_bounded_by_it(uint n)
{
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(3600)).ReadRawAsync("T", At(0), At(3600), n, Ct));
var fake = Store(900);
var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), n, Ct);
r.Samples.Count.ShouldBe(900);
fake.RawCalls.ShouldAllBe(c => c.MaxValues == Probe);
}
[Theory]
[InlineData(Cap)]
[InlineData(Cap - 1)]
[InlineData(1)]
public async Task A_client_cap_at_or_under_the_server_cap_reads_as_before(int n)
{
var fake = Store(3600);
var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), (uint)n, Ct);
r.Samples.Count.ShouldBe(n);
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([n + RawAggregateReader.PreWindowAllowance]);
}
[Fact]
public async Task A_point_read_and_an_empty_window_are_unaffected()
{
var fake = Store(3600);
var ds = Source(fake);
(await ds.ReadRawAsync("T", At(5), At(5), 0, Ct)).Samples.Single().Value.ShouldBe(5d);
(await ds.ReadRawAsync("T", At(5000), At(6000), 0, Ct)).Samples.ShouldBeEmpty();
fake.RawCalls.ShouldAllBe(c => c.MaxValues == Probe);
}
// ---- the factory binding --------------------------------------------------------------------------------------------
[Theory]
[InlineData(777, 777)]
[InlineData(0, 0)]
[InlineData(-1, 0)]
[InlineData(1, ServerHistorianOptions.MinMaxRawValuesPerRead)]
public void The_factory_binds_the_effective_option(int configured, int expected)
{
var options = new ServerHistorianOptions
{
Endpoint = "http://localhost:5222", UseTls = false, ApiKey = "histgw_test_secret", MaxRawValuesPerRead = configured,
};
using var ds = GatewayHistorian.CreateDataSource(options, new ServiceCollection().BuildServiceProvider());
ds.ShouldBeOfType<GatewayHistorianDataSource>().MaxRawValuesPerRead.ShouldBe(expected);
}
[Fact]
public void The_default_option_caps_the_source_at_ten_thousand()
{
var options = new ServerHistorianOptions { Endpoint = "http://localhost:5222", UseTls = false, ApiKey = "histgw_test_secret" };
using var ds = new GatewayHistorianProvider(options).CreateDataSource(new ServiceCollection().BuildServiceProvider());
ds.MaxRawValuesPerRead.ShouldBe(ServerHistorianOptions.DefaultMaxRawValuesPerRead);
}
// ---- through the node manager ---------------------------------------------------------------------------------------
[Theory]
[InlineData(0u, false)]
[InlineData(0u, true)]
[InlineData(5000u, false)]
[InlineData(uint.MaxValue, false)]
public async Task An_uncapped_HistoryRead_pages_at_the_cap_with_continuation_points(uint n, bool reverse)
{
var fake = Store(3600);
var (host, nm, nodeId) = await BootAsync(fake);
try
{
var (values, pages, status) = reverse
? DrainFrom(nm, nodeId, At(3600), At(-1), n, null)
: DrainFrom(nm, nodeId, At(0), At(3600), n, null);
status.Code.ShouldBe(StatusCodes.Good);
var expected = Enumerable.Range(0, 3600).Select(i => (double)i);
values.ShouldBe(reverse ? expected.Reverse() : expected);
pages.Count.ShouldBeGreaterThan(1, "a year-sized window at N = 0 is paged, not one shot");
pages.ShouldAllBe(p => p > 0 && p <= Cap);
pages[0].ShouldBe(Cap - 1); // page + lookahead fills a read
fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Cap + RawAggregateReader.PreWindowAllowance);
}
finally { await host.DisposeAsync(); }
}
[Fact]
public async Task A_bounded_uncapped_HistoryRead_pages_within_the_cap()
{
var fake = Store(3600);
var (host, nm, nodeId) = await BootAsync(fake);
try
{
var (values, pages, status) = DrainFrom(nm, nodeId, At(10).AddMilliseconds(500), At(3000).AddMilliseconds(500), 0, null, returnBounds: true);
status.Code.ShouldBe(StatusCodes.Good);
values.ShouldBe(Enumerable.Range(10, 2992).Select(i => (double)i)); // start bound +10, data +11..+3000, end bound +3001
pages.ShouldAllBe(p => p <= Cap);
fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Cap + RawAggregateReader.PreWindowAllowance);
}
finally { await host.DisposeAsync(); }
}
// ---- helpers ---------------------------------------------------------------------------------------------------------
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootAsync(FakeHistorianGatewayClient fake)
{
var started = await OpcUaHostStart.StartAsync(
port => new OpcUaApplicationHost(
new OpcUaApplicationHostOptions
{
ApplicationName = "OtOpcUa.GatewayRawCapTest",
ApplicationUri = $"urn:OtOpcUa.GatewayRawCapTest:{Guid.NewGuid():N}",
OpcUaPort = port,
PublicHostname = "localhost",
PkiStoreRoot = _pkiRoot,
},
NullLogger<OpcUaApplicationHost>.Instance),
() => new OtOpcUaSdkServer(), Ct);
var nm = started.Server.NodeManager!;
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
nm.HistorianDataSource = Source(fake);
nm.EnsureVariable("eq-1/pv", parentFolderNodeId: null, displayName: "PV", dataType: "Double",
writable: false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
return (started.Host, nm, nm.TryGetVariable("eq-1/pv")!.NodeId);
}
private static (List<double> Values, List<int> Pages, StatusCode Status) DrainFrom(
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? cp, int maxPages = 20,
bool returnBounds = false)
{
var values = new List<double>();
var pages = new List<int>();
ServiceResult error;
do
{
(var page, error, cp) = ReadRaw(nm, nodeId, start, end, max, cp, returnBounds);
values.AddRange(page);
pages.Add(page.Count);
pages.Count.ShouldBeLessThanOrEqualTo(maxPages, "paging must terminate");
}
while (cp is not null && StatusCode.IsGood(error.StatusCode));
return (values, pages, error.StatusCode);
}
private static (IReadOnlyList<double> Values, ServiceResult Error, byte[]? Cp) ReadRaw(
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? inboundCp, bool returnBounds = false)
{
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
var details = new ReadRawModifiedDetails
{
ReturnBounds = returnBounds, IsReadModified = false, StartTime = start, EndTime = end, NumValuesPerNode = max,
};
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = nodeId, ContinuationPoint = inboundCp } };
var results = new List<SdkHistoryReadResult> { null! };
var errors = new List<ServiceResult> { null! };
nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
var values = new List<double>();
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
foreach (var dv in ((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues)
values.Add(Convert.ToDouble(dv.Value));
return (values, errors[0], results[0]?.ContinuationPoint);
}
public void Dispose()
{
if (Directory.Exists(_pkiRoot))
{
try { Directory.Delete(_pkiRoot, recursive: true); }
catch { /* best-effort */ }
}
}
}
@@ -110,8 +110,9 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), timeProvider: clock);
/// <inheritdoc/>
public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows) =>
new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), maxTimeWeightedRawRows: maxTimeWeightedRawRows);
public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0) =>
new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), maxTimeWeightedRawRows: maxTimeWeightedRawRows,
maxRawValuesPerRead: maxRawValuesPerRead);
/// <inheritdoc/>
/// <remarks>The T4 live-value writer over a fresh catalog (fresh positive cache) and this instance's WAL wait.</remarks>
@@ -47,6 +47,21 @@ public sealed class QuestDbHistorianProviderTests
p.SupportedKinds.Count.ShouldBe(6);
}
/// <summary>R08:1.1 (#736): the server raw cap the Host binds reaches the data source; without one it is unbounded as before.</summary>
[Fact]
public void The_server_raw_cap_reaches_the_data_source()
{
var sp = new NullSp();
using var plain = Make();
plain.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(0);
using var capped = new QuestDbHistorianProvider(new QuestDbHistorianOptions
{
ConnectionString = "Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading",
}, maxRawValuesPerRead: 2500);
capped.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(2500);
}
[Fact]
public void Creates_all_five_seams_without_touching_questdb()
{
@@ -45,8 +45,9 @@ internal sealed class ReadTestDb
/// <summary>A fresh data source (fresh catalog cache) over this schema; <paramref name="clock"/> is its "now".</summary>
public SqlServerHistorianDataSource Source(TimeProvider? clock = null,
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) =>
new(Conn, Boot, new SqlServerTagCatalog(Conn, Boot), timeProvider: clock, maxTimeWeightedRawRows: maxTimeWeightedRawRows);
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0) =>
new(Conn, Boot, new SqlServerTagCatalog(Conn, Boot), timeProvider: clock, maxTimeWeightedRawRows: maxTimeWeightedRawRows,
maxRawValuesPerRead: maxRawValuesPerRead);
/// <summary>Inserts a tag row and the given samples. ServerTimeUtc = source time + 1 s (so tests can tell them apart).</summary>
public async Task<int> SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples)
@@ -26,7 +26,8 @@ internal sealed class SqlServerConformanceBackend : IConformanceBackend
public IHistorianDataSource CreateDataSource(TimeProvider? clock = null) => _db.Source(clock);
/// <inheritdoc/>
public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows) => _db.Source(null, maxTimeWeightedRawRows);
public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0) =>
_db.Source(null, maxTimeWeightedRawRows, maxRawValuesPerRead);
/// <inheritdoc/>
public IHistorianValueWriter CreateValueWriter(Func<DateTime>? utcNow = null) =>
@@ -47,6 +47,20 @@ public sealed class SqlServerHistorianProviderTests
p.SupportedKinds.Count.ShouldBe(6);
}
/// <summary>R08:1.1 (#736): the server raw cap the Host binds reaches the data source; without one it is unbounded as before.</summary>
[Fact]
public void The_server_raw_cap_reaches_the_data_source()
{
var sp = new NullSp();
Make().CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(0);
var capped = new SqlServerHistorianProvider(new SqlServerHistorianOptions
{
ConnectionString = "Server=localhost;User Id=sa;Password=x;TrustServerCertificate=True",
}, maxRawValuesPerRead: 2500);
capped.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(2500);
}
[Fact]
public void Creates_all_five_seams_without_touching_sql()
{
@@ -0,0 +1,153 @@
using System.Collections.Concurrent;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
using ZB.MOM.WW.OtOpcUa.AdminUI.Telemetry;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Telemetry;
/// <summary>
/// Review 02 U2: <c>TelemetryDial:CallTimeoutSeconds</c> was bound, defaulted and documented, and consumed
/// nowhere. It now bounds a dial's <b>establishment</b>: a dial that has delivered no event by the deadline is
/// cut and re-dialled with a failure streak. A live node answers a fresh subscribe at once (the node-local hub
/// replays its health/resilience snapshots), so the deadline bites only a dial that never connects — a
/// black-holed host, where the underlying socket connect has no timeout of its own. An established stream
/// is deliberately NOT bounded: HTTP/2 keepalive pings own its liveness, and the wire has no heartbeat, so an
/// idle timeout would churn every quiet node.
/// </summary>
public sealed class TelemetryDialDeadlineTests : TestKit
{
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(8);
[Fact]
public void A_dial_that_delivers_no_event_by_the_deadline_is_cancelled_and_redialled()
{
var dial = new FakeDialLoop();
Spawn(dial, callTimeoutSeconds: 1);
var first = dial.WaitForCount(1, Timeout).Single();
first.Ct.IsCancellationRequested.ShouldBeFalse("the deadline has not elapsed yet");
// The stalled dial is cut at the deadline...
AwaitCondition(() => first.Ct.IsCancellationRequested, Timeout);
// ...and the supervisor re-dials (a stalled dial is a transport failure, never a silent end).
var invocations = dial.WaitForCount(2, Timeout);
invocations[1].CorrelationId.ShouldBe("central-node-1-2");
}
[Fact]
public async Task A_dial_that_delivers_an_event_before_the_deadline_is_left_open()
{
var dial = new FakeDialLoop { DeliverOnOpen = true };
var sinks = new Sinks();
Spawn(dial, callTimeoutSeconds: 1, sinks);
var first = dial.WaitForCount(1, Timeout).Single();
AwaitCondition(() => sinks.HealthStore.Upserts.Count == 1, Timeout);
// Positive control: well past the deadline, the established stream is untouched.
await Task.Delay(TimeSpan.FromSeconds(2.5));
first.Ct.IsCancellationRequested.ShouldBeFalse("an established stream must never be cut by the dial deadline");
dial.Invocations.Count.ShouldBe(1);
}
private IActorRef Spawn(FakeDialLoop dial, int callTimeoutSeconds, Sinks? sinks = null)
{
sinks ??= new Sinks();
var options = new TelemetryDialOptions { ContactRefreshSeconds = 3600, CallTimeoutSeconds = callTimeoutSeconds };
return Sys.ActorOf(TelemetryDialSupervisor.Props(
() => Task.FromResult<IReadOnlyList<TelemetryDialTarget>>(
new[] { new TelemetryDialTarget("node-1", "http://h1:5300") }),
dial.Loop,
sinks.AlarmBroadcaster,
sinks.ScriptBroadcaster,
sinks.HealthStore,
sinks.ResilienceStore,
options));
}
private static DriverHealthChanged Health(string id) =>
new("C1", id, "Healthy", DateTime.UtcNow, null, 0, DateTime.UtcNow);
private sealed class DialInvocation
{
private readonly TaskCompletionSource _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
public required string CorrelationId { get; init; }
public required CancellationToken Ct { get; init; }
public Task Completion => _completion.Task;
public void Complete() => _completion.TrySetResult();
}
/// <summary>Records each invocation; optionally delivers one health event on open; ends only when cancelled.</summary>
private sealed class FakeDialLoop
{
private readonly ConcurrentQueue<DialInvocation> _invocations = new();
public bool DeliverOnOpen { get; init; }
public IReadOnlyList<DialInvocation> Invocations => _invocations.ToArray();
public TelemetryDialLoop Loop => async (_, correlationId, onMapped, _, ct) =>
{
var invocation = new DialInvocation { CorrelationId = correlationId, Ct = ct };
_invocations.Enqueue(invocation);
if (DeliverOnOpen) onMapped(Health("drv-1"));
await using var reg = ct.Register(() => invocation.Complete());
await invocation.Completion;
};
public IReadOnlyList<DialInvocation> WaitForCount(int count, TimeSpan timeout)
{
var deadline = DateTime.UtcNow + timeout;
while (DateTime.UtcNow < deadline)
{
var snapshot = _invocations.ToArray();
if (snapshot.Length >= count) return snapshot;
Thread.Sleep(10);
}
throw new Xunit.Sdk.XunitException(
$"Expected at least {count} dial invocations within {timeout}, saw {_invocations.Count}.");
}
}
private sealed class Sinks
{
public NullBroadcaster<AlarmTransitionEvent> AlarmBroadcaster { get; } = new();
public NullBroadcaster<ScriptLogEntry> ScriptBroadcaster { get; } = new();
public RecordingHealthStore HealthStore { get; } = new();
public NullResilienceStore ResilienceStore { get; } = new();
}
private sealed class NullBroadcaster<T> : IInProcessBroadcaster<T>
{
public event Action<T>? Received;
public event Action<bool>? ConnectionStateChanged;
public bool IsConnected { get; private set; }
public void Publish(T item) => Received?.Invoke(item);
public void SetConnected(bool connected) { IsConnected = connected; ConnectionStateChanged?.Invoke(connected); }
}
private sealed class RecordingHealthStore : IDriverStatusSnapshotStore
{
public ConcurrentQueue<DriverHealthChanged> Upserts { get; } = new();
public event Action<DriverHealthChanged>? SnapshotChanged;
public void Upsert(DriverHealthChanged snapshot) { Upserts.Enqueue(snapshot); SnapshotChanged?.Invoke(snapshot); }
public bool TryGet(string driverInstanceId, out DriverHealthChanged snapshot) { snapshot = null!; return false; }
public IReadOnlyCollection<DriverHealthChanged> GetAll() => Upserts.ToArray();
}
private sealed class NullResilienceStore : IDriverResilienceStatusStore
{
public event Action<DriverResilienceStatusChanged>? SnapshotChanged;
public void Upsert(DriverResilienceStatusChanged snapshot) => SnapshotChanged?.Invoke(snapshot);
public IReadOnlyList<DriverResilienceStatusChanged> GetForInstance(string driverInstanceId) => Array.Empty<DriverResilienceStatusChanged>();
public IReadOnlyCollection<DriverResilienceStatusChanged> GetAll() => Array.Empty<DriverResilienceStatusChanged>();
}
}
@@ -0,0 +1,58 @@
using Opc.Ua;
using Opc.Ua.Server;
using Shouldly;
using Xunit;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
/// <summary>
/// R08:1.1 (#736): the server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>) is advertised as
/// <c>HistoryServerCapabilities/MaxReturnDataValues</c> (i=11273) through <see cref="HistoryPaging.AdvertiseMaxReturnDataValues"/>,
/// read back the way a client would — through the SDK master node manager — over a real server.
/// </summary>
public sealed class HistoryServerCapabilitiesAdvertisementTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
{
private async Task<uint> ReadAdvertisedAsync()
{
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.Read, identity: null);
var nodesToRead = new ReadValueIdCollection
{
new ReadValueId { NodeId = VariableIds.HistoryServerCapabilities_MaxReturnDataValues, AttributeId = Attributes.Value },
};
var (values, _) = await fx.Nm.Server.NodeManager.ReadAsync(context, 0, TimestampsToReturn.Neither, nodesToRead, TestContext.Current.CancellationToken);
values.Count.ShouldBe(1);
StatusCode.IsGood(values[0].StatusCode).ShouldBeTrue(values[0].StatusCode.ToString());
return (uint)values[0].Value;
}
[Fact]
public async Task The_cap_is_advertised_as_MaxReturnDataValues_and_a_re_sync_overwrites_it()
{
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
try
{
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 10_000).ShouldBeTrue();
(await ReadAdvertisedAsync()).ShouldBe(10_000u);
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 2_500).ShouldBeTrue();
(await ReadAdvertisedAsync()).ShouldBe(2_500u);
// Unbounded (the opt-out) advertises Part 11's "no limit".
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 0).ShouldBeTrue();
(await ReadAdvertisedAsync()).ShouldBe(0u);
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, -5).ShouldBeTrue();
(await ReadAdvertisedAsync()).ShouldBe(0u);
}
finally
{
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 0);
fx.Gate.Release();
}
}
[Fact]
public void No_server_is_a_quiet_false()
{
HistoryPaging.AdvertiseMaxReturnDataValues(null, 10_000).ShouldBeFalse();
}
}
@@ -0,0 +1,436 @@
using Opc.Ua;
using Opc.Ua.Server;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
/// <summary>
/// R08:1.1 (#736): the paged-vs-unpaged fuzz for the server raw cap. Every paging change in this repo has had boundary
/// bugs that the hand-written cases missed (silent truncation, an endless cursor, an empty page between every value), so
/// the cap is proved the only way that has worked: random series (bursts, gaps, sub-millisecond stamps, tie clusters),
/// random windows (empty, on a tie, one step past a tie, between samples, forward / reverse / end-only, with and without
/// bounds) and random caps — <c>NumValuesPerNode</c> 0 / 1 / 2 / small / huge, the server cap 3 … 1000 or off, the
/// tie-cluster budget 3 / 8 / 65536 — paged through a REAL SDK node manager against a wire-faithful fake of each backend
/// (SqlServer: tick precision, native reverse, <c>TOP</c>; QuestDB: µs truncation of bounds AND stamps, native reverse,
/// <c>LIMIT</c>; Gateway: forward only, <c>[start, end)</c> after its own boundary-row filtering, walked back by the
/// server) that enforces the backend half of the cap exactly as the real sources do (a read asking for 0 or more than
/// the cap reads cap + 1 rows and is refused when the window holds more).
/// <para>
/// The oracle is the same node manager's one-shot answer with the cap OFF (<c>NumValuesPerNode = 0</c>, no limit),
/// which for a plain read is also checked against the series itself. The concatenation of the pages must equal it
/// exactly; the read must terminate within the number of values plus two pages; no page may exceed the cap; the fake
/// must never see a request for 0 or more than the cap (the "bounded by the same cap" invariant); and a
/// <c>BadTooManyOperations</c> is accepted only when a tie cluster (or, on the Gateway shape, one millisecond) is
/// larger than the budget that the cap and <c>MaxTieClusterOverfetch</c> leave — after an exact prefix. The run is
/// non-vacuous by construction: it asserts that empty windows, mid-tie windows, reverse reads, <c>NumValuesPerNode</c>
/// of 1 and 2, the smallest server cap (3) and every shape occurred, and that most iterations were exact.
/// </para>
/// <para>
/// Iterations: <c>OTOPCUA_PAGING_FUZZ_ITERATIONS</c> (default 10 000, the lane gate). Seeds are stable, so a failure
/// names a reproducible seed.
/// </para>
/// </summary>
public sealed class NodeManagerHistoryReadRawCapFuzzTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
{
private static readonly DateTime T0 = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc);
/// <summary>The backend the fake is shaped like.</summary>
public enum Shape { SqlServer, QuestDb, Gateway }
private enum Direction { Forward, Reverse, EndOnly }
private static int Iterations =>
int.TryParse(Environment.GetEnvironmentVariable("OTOPCUA_PAGING_FUZZ_ITERATIONS"), out var n) && n > 0 ? n : 10_000;
[Fact]
public async Task Paged_equals_unpaged_on_random_series_windows_and_caps_and_always_terminates()
{
var iterations = Iterations;
var counts = new Dictionary<string, int>();
void Count(string key) => counts[key] = counts.GetValueOrDefault(key) + 1;
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
try
{
for (var seed = 0; seed < iterations; seed++)
{
var rng = new Random(seed);
var shape = (Shape)(seed % 3);
var resolution = shape == Shape.QuestDb ? 10L : 1L;
var series = RandomSeries(rng, resolution);
var cap = rng.Next(10) switch { 0 => 0, 1 or 2 => 3, 3 => 4, 4 => 5, 5 => 7, 6 => 10, 7 => 16, 8 => 33, _ => 1000 };
var n = rng.Next(9) switch { 0 or 1 => 0u, 2 or 3 => 1u, 4 or 5 => 2u, 6 => (uint)rng.Next(3, 20), 7 => (uint)rng.Next(20, 200), _ => 100_000u };
var tieBudget = rng.Next(4) switch { 0 => 3, 1 => 8, _ => 65536 };
var direction = rng.Next(5) switch { 0 => Direction.Reverse, 1 => Direction.EndOnly, _ => Direction.Forward };
if (direction == Direction.EndOnly && n == 0) direction = Direction.Forward; // Part 11: one bound only with a cap
var bounds = direction != Direction.EndOnly && rng.Next(4) == 0;
var a = PickBoundary(rng, series, resolution);
var b = PickBoundary(rng, series, resolution);
var (lo, hi) = a <= b ? (a, b) : (b, a);
var (start, end) = direction switch
{
Direction.Forward => (lo, hi),
Direction.Reverse => (hi, lo),
_ => (DateTime.MinValue, hi),
};
var tag = $"seed {seed} {shape} cap {cap} N {n} tie {tieBudget} {direction} bounds {bounds} [{start:O} .. {end:O}]";
// The walk-back (Gateway shape only: reverse / end-only reads, and the bound lookups of any bounded read) refuses
// a millisecond holding more samples than its budget — computed up front, over the whole series, because the
// walk and the lookups read around the window.
var walks = shape == Shape.Gateway && (direction != Direction.Forward || bounds);
var msMax = walks ? MillisecondMax(series) : 0;
// The oracle: the one-shot answer with the cap off (and, for a plain read, the series itself).
var truthSource = new CappedFake(series, shape, cap: 0);
fx.Nm.HistoricalSourceForTest(truthSource, tieBudget);
// An end-only read is paged backward from EndTime (the node manager flips it to a reverse read down to the
// beginning of time); its oracle is the explicit reverse read down to the Unix epoch — the walk-back's floor,
// and Part 11 (the SDK checks it) needs both bounds set when NumValuesPerNode = 0.
var (truth, truthError, truthCp) = direction == Direction.EndOnly
? Read(hi, DateTime.UnixEpoch, 0, bounds: false, cp: null)
: Read(start, end, 0, bounds, cp: null);
if (truthError.StatusCode.Code == StatusCodes.BadTooManyOperations)
{
// Only the walk-back can refuse an uncapped one-shot read, and only past its budget (max(1, M) with no cap).
(walks && msMax > Math.Max(1, tieBudget)).ShouldBeTrue($"the oracle read was refused: {tag} (ms max {msMax})");
Count("oracle refused");
continue;
}
truthError.StatusCode.Code.ShouldBe(StatusCodes.Good, $"oracle: {tag}");
truthCp.ShouldBeNull($"oracle (no cap, N = 0) is one page: {tag}");
if (!bounds)
Render(truth).ShouldBe(Render(Oracle(series, start, end, direction, resolution)), $"oracle vs series: {tag}");
truthSource.BadRequests.ShouldBe(0, $"a reverse window reached a forward-only backend on the oracle read: {tag}");
var windowTies = LargestTieCluster(truth);
if (truth.Count == 0) Count("empty");
if (windowTies >= 2) Count("ties in window");
if (EndsOnTie(series, start, end, direction, resolution)) Count("window ends on a tie");
if (direction == Direction.Reverse) Count("reverse");
if (direction == Direction.EndOnly) Count("end-only");
if (n == 1) Count("N=1");
if (n == 2) Count("N=2");
if (n == 0) Count("N=0");
if (cap == 3) Count("cap=3");
if (bounds) Count("bounds");
Count(shape.ToString());
// A boundary tie cluster as large as the cap leaves a resumed page no room: the server steps past it with one
// empty page carrying a continuation point (the documented #694 ceiling behaviour) — the one empty page allowed.
var oversizedClusters = cap > 0
? truth.Where(v => !(v.Value is null && v.StatusCode.Code == StatusCodes.BadBoundNotFound))
.GroupBy(v => v.SourceTimestamp.Ticks).Count(g => g.Count() >= Math.Max(2, cap - 1))
: 0;
// The read under test.
var source = new CappedFake(series, shape, cap);
fx.Nm.HistoricalSourceForTest(source, tieBudget);
var pages = new List<IReadOnlyList<DataValue>>();
var collected = new List<DataValue>();
byte[]? cp = null;
StatusCode status;
do
{
var (values, error, next) = Read(start, end, n, bounds, cp);
status = error.StatusCode;
if (!StatusCode.IsGood(status)) break;
pages.Add(values);
collected.AddRange(values);
cp = next;
pages.Count.ShouldBeLessThanOrEqualTo(truth.Count + 2 + oversizedClusters, $"paging must terminate: {tag} pages {Sizes(pages)}");
}
while (cp is not null);
source.OverCapRequests.ShouldBe(0, $"the server asked the backend for 0 or more than the cap: {tag}");
source.BadRequests.ShouldBe(0, $"a reverse window reached a forward-only backend: {tag}");
if (status.Code == StatusCodes.BadTooManyOperations)
{
// Legitimate only past the budget the cap leaves: a tie cluster larger than min(M, cap − 1), or on the
// Gateway shape a millisecond holding more than min(M, cap − 2) samples (the walk-back's budget).
var overfetchBound = cap > 0 ? Math.Min(Math.Max(1, tieBudget), Math.Max(1, cap - 1)) : Math.Max(1, tieBudget);
var walkBudget = cap > 0 ? Math.Max(1, Math.Min(tieBudget, cap - 2)) : Math.Max(1, tieBudget);
var legit = windowTies > overfetchBound || (walks && msMax > walkBudget);
legit.ShouldBeTrue($"refused with no tie cluster over {overfetchBound} (max {windowTies}; ms max {msMax} vs {walkBudget}): {tag}");
Render(collected).ShouldBe(Render(truth.Take(collected.Count)), $"the prefix before the refusal: {tag}");
Count("refused");
continue;
}
status.Code.ShouldBe(StatusCodes.Good, $"{tag}");
Render(collected).ShouldBe(Render(truth), $"paged ≠ unpaged: {tag} pages {Sizes(pages)}");
if (cap > 0)
{
// A bounded read whose data fits one page carries its start bound, the data and its end bound: the node
// manager fits the end bound against the client's N but not against the server cap (follow-up F2), so that
// one page may hold cap + 1 values; every other page is at most the cap.
var bothBoundsOnOnePage = bounds && pages.Count == 1;
pages.ShouldAllBe(p => p.Count <= cap + (bothBoundsOnOnePage ? 1 : 0), $"a page over the cap: {tag} pages {Sizes(pages)}");
if (bothBoundsOnOnePage && pages[0].Count == cap + 1) Count("bounded single page at cap+1 (F2)");
}
if (n > 0) pages.ShouldAllBe(p => p.Count <= n, $"a page over N: {tag} pages {Sizes(pages)}");
for (var i = 0; i < pages.Count - 1; i++)
{
if (pages[i].Count > 0) continue;
// Allowed only as the step past a boundary tie cluster the cap cannot page within.
var previousLast = i > 0 && pages[i - 1].Count > 0 ? pages[i - 1][^1].SourceTimestamp.Ticks : (long?)null;
var clusterAtBoundary = previousLast is { } t ? truth.Count(v => v.SourceTimestamp.Ticks == t) : 0;
(cap > 0 && clusterAtBoundary >= Math.Max(2, cap - 1)).ShouldBeTrue(
$"an empty page with a continuation point (page {i}, boundary cluster {clusterAtBoundary}): {tag} pages {Sizes(pages)}");
Count("empty step over an oversized tie cluster");
}
if (pages.Count > 1) Count("paged");
if (pages.Count > 1 && n == 0) Count("paged at N=0");
// A cost guard against a read explosion, not a budget. A native shape reads once per page plus the bound
// lookups. The Gateway shape's walk-back has its own cost profile — ~30 reads a page on a dense tag
// (#691/#693), far more on a sparse series under a tiny tie budget (seed 320: 82 reads with NO cap, seed 6428:
// 120+) — so it is judged RELATIVE to the same request with the cap off (the pre-#736 behaviour), per page:
// the cap must not make the walk explode compared to before. The baseline asks for the page size the cap
// imposes (N = cap − 1): an uncapped N = 0 reverse read used to be ONE int.MaxValue forward read — the very
// memory vector the cap removes — so "what N = 0 cost before" is not the comparison.
var allowedReads = 8 * pages.Count + 8;
if (shape == Shape.Gateway)
{
var baseline = new CappedFake(series, shape, cap: 0);
fx.Nm.HistoricalSourceForTest(baseline, tieBudget);
var baseN = cap > 0 && (n == 0 || n > (uint)cap) ? (uint)(cap - 1) : n;
var basePages = 0;
byte[]? baseCp = null;
do
{
var (_, baseError, next) = Read(start, end, baseN, bounds, baseCp);
if (!StatusCode.IsGood(baseError.StatusCode)) break;
basePages++;
baseCp = next;
}
while (baseCp is not null && basePages <= truth.Count + 2 + oversizedClusters);
allowedReads = 2 * baseline.Reads * Math.Max(1, pages.Count) / Math.Max(1, basePages) + 32 * pages.Count + 16;
}
source.Reads.ShouldBeLessThanOrEqualTo(allowedReads,
$"backend reads exploded: {source.Reads} for {pages.Count} pages (allowed {allowedReads}): {tag}");
Count("exact");
}
}
finally
{
fx.Nm.HistoricalSourceForTest(NullHistorianDataSource.Instance, 65536);
fx.Gate.Release();
}
// Non-vacuous: every category the brief names occurred, and most runs were exact.
var summary = string.Join(", ", counts.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key}={kv.Value}"));
foreach (var required in new[] { "empty", "ties in window", "window ends on a tie", "reverse", "end-only", "N=1", "N=2", "N=0", "cap=3", "bounds", "paged", "paged at N=0", "SqlServer", "QuestDb", "Gateway" })
counts.GetValueOrDefault(required).ShouldBeGreaterThan(iterations / 200, $"{required} under-represented: {summary}");
counts.GetValueOrDefault("exact").ShouldBeGreaterThan(iterations * 6 / 10, summary);
TestContext.Current.TestOutputHelper?.WriteLine($"raw-cap fuzz: {iterations} iterations — {summary}");
}
// --- generators -----------------------------------------------------------------------------
private static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue;
private static DateTime Trunc(DateTime t, long resolution) => new(t.Ticks - t.Ticks % resolution, DateTimeKind.Utc);
/// <summary>Random ascending series: bursts, gaps, sub-millisecond stamps and tie clusters; stamps stored at the resolution.</summary>
private static List<DataValueSnapshot> RandomSeries(Random rng, long resolution)
{
var list = new List<DataValueSnapshot>();
var t = T0.Ticks - rng.Next(0, 3) * TimeSpan.TicksPerHour;
var count = rng.Next(1, 120);
var v = 0;
while (list.Count < count)
{
t += rng.Next(6) switch
{
0 => rng.Next(1, 10), // sub-millisecond (sub-µs on QuestDB: collapses onto a tie)
1 => rng.Next(1, 5) * TimeSpan.TicksPerMillisecond, // dense
2 => rng.Next(1, 3) * TimeSpan.TicksPerSecond,
3 => rng.Next(1, 30) * TimeSpan.TicksPerMinute, // a gap
_ => rng.Next(1, 1000) * TimeSpan.TicksPerMillisecond,
};
var ties = rng.Next(7) == 0 ? rng.Next(2, 9) : 1;
for (var k = 0; k < ties; k++)
{
var ts = Trunc(new DateTime(t, DateTimeKind.Utc), resolution);
var status = rng.Next(12) == 0 ? StatusCodes.Bad : StatusCodes.Good;
list.Add(new DataValueSnapshot(status == StatusCodes.Bad ? null : (double)v, status, ts, ts));
v++;
}
}
return list;
}
/// <summary>A window bound: before / after the data, exactly on a sample (a tie, maybe), one step past one, a tick before one, or between two.</summary>
private static DateTime PickBoundary(Random rng, List<DataValueSnapshot> series, long resolution)
{
var first = Ts(series[0]);
var last = Ts(series[^1]);
var i = rng.Next(series.Count);
var t = Ts(series[i]);
switch (rng.Next(9))
{
case 0: return first.AddSeconds(-rng.Next(1, 100));
case 1: return last.AddSeconds(rng.Next(1, 100));
case 2:
case 3: return t;
case 4: return t.AddTicks(resolution);
case 5: return t.AddTicks(-1); // sub-resolution: a microsecond backend truncates it back ONTO the sample
case 6: return t.AddMilliseconds(1);
default:
var next = i + 1 < series.Count ? Ts(series[i + 1]) : t.AddSeconds(10);
return new DateTime(t.Ticks + (long)((next.Ticks - t.Ticks) * rng.NextDouble()), DateTimeKind.Utc);
}
}
/// <summary>The series' own answer for a plain (no bounds) read: forward <c>[start, end)</c>, point, reverse / end-only <c>(end, start]</c> newest first.</summary>
private static List<DataValue> Oracle(List<DataValueSnapshot> series, DateTime start, DateTime end, Direction direction, long resolution)
{
var s = Trunc(start, resolution);
var e = Trunc(end, resolution);
IEnumerable<DataValueSnapshot> window = direction switch
{
// start == end is Part 11's point read whichever way it was generated: the node manager reads it forward.
Direction.Forward or Direction.Reverse when s == e => series.Where(x => Ts(x) == s),
Direction.Forward => series.Where(x => Ts(x) >= s && Ts(x) < e),
Direction.Reverse => series.Where(x => Ts(x) > e && Ts(x) <= s).Reverse(),
_ => series.Where(x => Ts(x) <= e).Reverse(),
};
return window.Select(x => new DataValue(new Variant(x.Value), new StatusCode(x.StatusCode), Ts(x), Ts(x))).ToList();
}
/// <summary>The largest tie cluster among the returned stored values (a bound placeholder is not a stored value).</summary>
private static int LargestTieCluster(IReadOnlyList<DataValue> truth) =>
truth.Where(v => !(v.Value is null && v.StatusCode.Code == StatusCodes.BadBoundNotFound))
.GroupBy(v => v.SourceTimestamp.Ticks).Select(g => g.Count()).DefaultIfEmpty(0).Max();
/// <summary>The most samples within one millisecond anywhere in the series (the walk-back's refusal rule).</summary>
private static int MillisecondMax(List<DataValueSnapshot> series)
{
var msMax = 0;
for (int i = 0, j = 0; i < series.Count; i++)
{
while (Ts(series[i]).Ticks - Ts(series[j]).Ticks >= TimeSpan.TicksPerMillisecond) j++;
msMax = Math.Max(msMax, i - j + 1);
}
return msMax;
}
private static bool EndsOnTie(List<DataValueSnapshot> series, DateTime start, DateTime end, Direction direction, long resolution)
{
var edge = Trunc(direction == Direction.Forward ? end : direction == Direction.Reverse ? start : end, resolution);
return series.Count(x => Ts(x) == edge) >= 2;
}
// --- helpers ----------------------------------------------------------------------------------
private static string Sizes(IEnumerable<IReadOnlyList<DataValue>> pages) => "[" + string.Join(",", pages.Select(p => p.Count)) + "]";
private static string Render(IEnumerable<DataValue> values) =>
string.Join(" | ", values.Select(v => $"{v.SourceTimestamp.Ticks}={v.Value ?? "null"}/{v.StatusCode.Code:X8}"));
private (IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp) Read(DateTime start, DateTime end, uint max, bool bounds, byte[]? cp)
{
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
var details = new ReadRawModifiedDetails
{
StartTime = start, EndTime = end, NumValuesPerNode = max, ReturnBounds = bounds, IsReadModified = false,
};
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = fx.NodeId, ContinuationPoint = cp } };
var results = new List<SdkHistoryReadResult> { null! };
var errors = new List<ServiceResult> { null! };
fx.Nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
var values = new List<DataValue>();
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
values.AddRange(((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues);
return (values, errors[0], results[0]?.ContinuationPoint);
}
/// <summary>
/// A series-backed historian shaped like one backend, enforcing the backend half of the server raw cap exactly as
/// the real data sources do: it reports the cap as <see cref="MaxRawValuesPerRead"/>; a read asking for 0 or more than
/// the cap reads cap + 1 rows and throws <see cref="HistoryReadLimitExceededException"/> when the window holds more
/// (never a truncated answer). It also records every such request (<see cref="OverCapRequests"/>) — the server must
/// never issue one — and every reverse window sent to the forward-only shape (<see cref="BadRequests"/>).
/// </summary>
private sealed class CappedFake(IReadOnlyList<DataValueSnapshot> series, Shape shape, int cap) : IHistorianDataSource
{
private readonly Lock _gate = new();
public int Reads { get; private set; }
public int OverCapRequests { get; private set; }
public int BadRequests { get; private set; }
public bool SupportsReverseRaw => shape != Shape.Gateway;
public long TimestampResolutionTicks => shape == Shape.QuestDb ? 10 : 1;
public int MaxRawValuesPerRead => cap;
private DateTime Trunc(DateTime t) => NodeManagerHistoryReadRawCapFuzzTests.Trunc(t, TimestampResolutionTicks);
public Task<HistorianRead> ReadRawAsync(
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
{
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
lock (_gate)
{
Reads++;
if (probe) OverCapRequests++;
if (shape == Shape.Gateway && startUtc > endUtc) BadRequests++;
}
if (shape == Shape.Gateway && startUtc > endUtc)
throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)");
startUtc = Trunc(startUtc);
endUtc = Trunc(endUtc);
List<DataValueSnapshot> window;
if (startUtc > endUtc) window = series.Where(s => Ts(s) > endUtc && Ts(s) <= startUtc).Reverse().ToList();
else if (startUtc == endUtc) window = series.Where(s => Ts(s) == startUtc).ToList();
else window = series.Where(s => Ts(s) >= startUtc && Ts(s) < endUtc).ToList();
var top = probe ? cap + 1 : maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
var page = window.Take(top).ToList();
if (probe && page.Count > cap)
throw new HistoryReadLimitExceededException(cap, $"more than MaxRawValuesPerRead ({cap}) samples");
return Task.FromResult(new HistorianRead(page, null));
}
public Task<HistorianRead> ReadProcessedAsync(
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
public Task<HistorianRead> ReadAtTimeAsync(
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
public Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose() { }
}
}
/// <summary>Test-only wiring shared by the paging fuzz: swap the source and the tie budget, with a fresh continuation store.</summary>
internal static class NodeManagerFuzzExtensions
{
public static void HistoricalSourceForTest(this OtOpcUaNodeManager nm, IHistorianDataSource source, int tieBudget)
{
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
nm.MaxTieClusterOverfetch = tieBudget;
nm.HistorianDataSource = source;
}
}
@@ -24,6 +24,15 @@ public sealed class RawPageLimitTests
[InlineData(5000u, 983, 984, 0u, 0u, true)] // the ties alone fill a read: never a 0 (uncapped) request
[InlineData(5000u, 5000, 984, 0u, 0u, true)] // never an underflow
[InlineData(5000u, 0, 1, 0u, 0u, true)]
// R08:1.1 (#736): the server raw cap (default 10 000) is the limit every backend reports — an uncapped read pages at it.
[InlineData(0u, 0, 10_000, 9_999u, 10_000u, false)] // NumValuesPerNode = 0: a first page of cap − 1 + the lookahead row
[InlineData(0u, 1, 10_000, 9_998u, 10_000u, false)] // a resumed page re-reads its one boundary tie
[InlineData(100_000u, 0, 10_000, 9_999u, 10_000u, false)] // a client cap above the server cap is bounded by it
[InlineData(10_000u, 0, 10_000, 9_999u, 10_000u, false)] // exactly the cap: still cap − 1 (the lookahead row needs room)
[InlineData(500u, 0, 10_000, 500u, 501u, false)] // under the cap: unchanged
[InlineData(0u, 0, 3, 2u, 3u, false)] // the smallest usable cap (MinMaxRawValuesPerRead = 3): a first page of two
[InlineData(0u, 1, 3, 1u, 3u, false)] // … and a resumed page still carries one value beside its boundary tie
[InlineData(0u, 1, 2, 0u, 0u, true)] // at 2 a resumed page has NO room — every value would cost an empty page (why 3 is the floor)
public void Plan_fits_page_ties_and_lookahead_into_the_limit(uint want, int k, int limit, uint page, uint request, bool noRoom)
{
var plan = RawPageLimit.Plan(want, k, limit);
@@ -64,7 +64,7 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
{
published.Enqueue(e);
var quality = e.Snapshot.StatusCode == 0u ? OpcUaStatus.Good : OpcUaStatus.Bad;
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
mux.Tell(ValuePublishes.Single(
"calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
};
@@ -75,7 +75,7 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
// compute dedups) until the computed value has flowed all the way through.
AwaitAssert(() =>
{
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
mux.Tell(ValuePublishes.Single(
"src-drv", "src/a", 21.0, OpcUaStatus.Good, Ts));
// Leg 1 — dep flow: src/a=21 → A computes 42 and publishes on calc/engine/out.
@@ -118,14 +118,14 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
{
published.Enqueue(e);
var quality = e.Snapshot.StatusCode == 0u ? OpcUaStatus.Good : OpcUaStatus.Bad;
mux.Tell(new DriverInstanceActor.AttributeValuePublished("calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
mux.Tell(ValuePublishes.Single("calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
};
var adapter = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(driver, mux), "dep-adapter2");
// Leg 1 — before redeploy: src/a=2 → keep=20, remove=7.
AwaitAssert(() =>
{
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/a", 2.0, OpcUaStatus.Good, Ts));
mux.Tell(ValuePublishes.Single("src-drv", "src/a", 2.0, OpcUaStatus.Good, Ts));
published.ShouldContain(e => e.FullReference == "calc/keep" && Equals(e.Snapshot.Value, 20.0));
published.ShouldContain(e => e.FullReference == "calc/remove" && Equals(e.Snapshot.Value, 7.0));
}, PresenceBudget, TimeSpan.FromMilliseconds(100));
@@ -150,8 +150,8 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
// new result; the REMOVED tag never publishes a value derived from the post-redeploy input.
AwaitAssert(() =>
{
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/b", 50.0, OpcUaStatus.Good, Ts));
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/a", 3.0, OpcUaStatus.Good, Ts));
mux.Tell(ValuePublishes.Single("src-drv", "src/b", 50.0, OpcUaStatus.Good, Ts));
mux.Tell(ValuePublishes.Single("src-drv", "src/a", 3.0, OpcUaStatus.Good, Ts));
// (a) new dep B flows through the RE-REGISTERED interest → new tag computes: 50 + 100 = 150.
published.ShouldContain(e => e.FullReference == "calc/add" && Equals(e.Snapshot.Value, 150.0));
@@ -0,0 +1,79 @@
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Review 02 U1: <c>GetDiagnostics</c> answered a placeholder — <c>Guid.Empty</c>, "Spawned", zero devices and
/// "now" for every child, in every behaviour — so the snapshot looked live and said nothing. It now carries
/// each child's row id, instance id, last-published health state, device counts and last change time.
/// </summary>
public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
{
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
private static readonly NodeId TestNode = NodeId.Parse("node-1:4053");
private const string DriverId = "DI-1";
[Fact]
public async Task A_connected_child_reports_its_row_id_name_health_state_and_device_count()
{
var db = NewInMemoryDbFactory();
var rowId = Guid.NewGuid();
var before = DateTime.UtcNow.AddSeconds(-1);
var factory = new RecordingSubscribableDriverFactory("Modbus");
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
db, RevA, new[] { (DriverId, "Line1-Modbus", "dev1", "speed", "Float", false) },
Array.Empty<(string, string, string, string, string?)>(),
enabled: true, driverRowIds: new Dictionary<string, Guid> { [DriverId] = rowId });
var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
var snapshot = await AwaitDiagnosticsAsync(host, s => s.Drivers.Any(d => d.State == "Healthy"));
snapshot.NodeId.ShouldBe(TestNode);
snapshot.CurrentRevision.ShouldBe(RevA);
snapshot.RunningFromCache.ShouldBeFalse();
var driver = snapshot.Drivers.ShouldHaveSingleItem();
driver.DriverInstanceId.ShouldBe(rowId, "the real row id, not Guid.Empty");
driver.Name.ShouldBe(DriverId, "the string instance id every other surface keys on");
driver.State.ShouldBe("Healthy");
driver.ConnectedDevices.ShouldBe(1);
driver.FaultedDevices.ShouldBe(0);
driver.LastChangeUtc.ShouldBeGreaterThan(before);
driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow);
}
[Fact]
public async Task A_stubbed_child_reports_Stubbed_with_no_devices()
{
var db = NewInMemoryDbFactory();
// The factory answers for a type the artifact does not use, so the host stubs the child.
var factory = new RecordingSubscribableDriverFactory("NotModbus");
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
db, RevA, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) },
Array.Empty<(string, string, string, string, string?)>(), enabled: true);
var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
var snapshot = await AwaitDiagnosticsAsync(host, s => s.Drivers.Count == 1);
var driver = snapshot.Drivers.Single();
driver.State.ShouldBe("Stubbed");
driver.ConnectedDevices.ShouldBe(0);
driver.FaultedDevices.ShouldBe(0);
}
private async Task<NodeDiagnosticsSnapshot> AwaitDiagnosticsAsync(IActorRef host, Func<NodeDiagnosticsSnapshot, bool> ready)
{
NodeDiagnosticsSnapshot? snapshot = null;
await AwaitAssertAsync(async () =>
{
snapshot = await host.Ask<NodeDiagnosticsSnapshot>(new GetDiagnostics(CorrelationId.NewId()), PresenceBudget);
ready(snapshot).ShouldBeTrue();
}, PresenceBudget);
return snapshot!;
}
}
@@ -0,0 +1,63 @@
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Review 02 L2: <c>DriverHostActor._lastReprovisionUtc</c> was never pruned — a driver removed by a deploy
/// after a supervision crash kept its stamp, so the map grew with every id ever reprovisioned-then-removed,
/// and (the observable half) a driver removed and later re-added inherited the OLD stamp: its first crash
/// after the re-add was deferred by the backoff as if it were crash-looping, although the new incarnation had
/// never crashed. <c>StopChild</c> now drops the stamp.
/// </summary>
public sealed class DriverHostActorReprovisionStampTests : RuntimeActorTestBase
{
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
private static readonly RevisionHash RevC = RevisionHash.Parse(new string('c', 64));
private static readonly NodeId TestNode = NodeId.Parse("node-1:4053");
private const string DriverId = "DI-1";
private const string SpeedPath = "Plant/Modbus/dev1/speed";
[Fact]
public void A_driver_removed_and_re_added_does_not_inherit_the_old_incarnations_reprovision_stamp()
{
var db = NewInMemoryDbFactory();
// Far longer than the presence budget below: an inherited stamp would defer the re-added driver's
// reprovision past it, which is the failure this test reports.
var backoff = TimeSpan.FromSeconds(60);
var factory = new RecordingSubscribableDriverFactory("Modbus");
var withDriver = new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) };
var none = Array.Empty<(string, string, string, string, string?)>();
var revA = V3ArtifactSeeder.SeedV3Deployment(db, RevA, withDriver, none, enabled: true);
var (host, _, _, coordinator) = V3ArtifactSeeder.SpawnHostAndApplyWithCoordinator(
this, db, TestNode, revA, RevA, factory, reprovisionBackoff: backoff);
AwaitAssert(() => factory.Created.Count.ShouldBe(1), PresenceBudget);
AwaitAssert(() => factory.Created[0].SubscribeCount.ShouldBe(1), PresenceBudget);
// Crash 1 on the first incarnation: reprovisioned immediately, which stamps the id.
factory.Created[0].FireMalformedDataChange(SpeedPath);
AwaitAssert(() => factory.Created[0].InitializeCount.ShouldBe(2), PresenceBudget);
// Deploy B removes the driver (StopChild), deploy C re-adds it: a brand-new driver object.
var revB = V3ArtifactSeeder.SeedV3Deployment(
db, RevB, Array.Empty<(string, string, string, string, string, bool)>(), none, enabled: true);
V3ArtifactSeeder.Dispatch(host, coordinator, revB, RevB);
var revC = V3ArtifactSeeder.SeedV3Deployment(db, RevC, withDriver, none, enabled: true);
V3ArtifactSeeder.Dispatch(host, coordinator, revC, RevC);
AwaitAssert(() => factory.Created.Count.ShouldBe(2), PresenceBudget);
var reAdded = factory.Created[1];
AwaitAssert(() => reAdded.SubscribeCount.ShouldBe(1), PresenceBudget);
// Its FIRST crash must reprovision immediately — it has no history of its own.
reAdded.FireMalformedDataChange(SpeedPath);
AwaitAssert(
() => reAdded.InitializeCount.ShouldBe(
2, "the re-added driver's first reprovision was deferred by the REMOVED incarnation's stamp"),
duration: TimeSpan.FromSeconds(5));
}
}
@@ -0,0 +1,66 @@
using Akka.Actor;
using Akka.Event;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Review 02 H5: the <c>Stale</c> behaviour registered no receive for a driver child's value/alarm bursts (nor
/// for a late <c>ArtifactForApply</c> / <c>ApplyWedged</c> pipe-back), so any that landed there hit
/// <c>Unhandled</c> — dead-letter noise and a dropped message with no log naming why. They are now explicit
/// drops. The two private pipe-back records cannot be constructed from a test; the two public child
/// messages are pinned here, and the private pair by the same registration pattern (compile-time).
/// </summary>
public sealed class DriverHostActorStaleDropTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("stale-drop-test");
[Fact]
public void Child_value_and_alarm_bursts_landing_in_Stale_are_dropped_not_unhandled()
{
var unhandled = CreateTestProbe();
Sys.EventStream.Subscribe(unhandled.Ref, typeof(UnhandledMessage));
// ThrowingDbFactory ⇒ Bootstrap's catch ⇒ Become(Stale).
var host = Sys.ActorOf(DriverHostActor.Props(
new ThrowingDbFactory(), TestNode, CreateTestProbe().Ref,
localRoles: new HashSet<string> { "driver" }));
// Barrier: Stale answers RouteNodeWrite synchronously, so a reply proves the host is in Stale.
var asker = CreateTestProbe();
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget).Reason!.ShouldContain("stale");
// Positive control: an unknown message IS reported as unhandled by this host.
host.Tell("not a message the host knows");
unhandled.ExpectMsg<UnhandledMessage>(m => ReferenceEquals(m.Recipient, host) || m.Recipient.Equals(host), PresenceBudget);
host.Tell(new DriverInstanceActor.AttributeValueBatchPublished(
"DI-1", new[] { new DriverInstanceActor.AttributeValueItem("Plant/x/y", 1.0, OpcUaStatus.Good, DateTime.UtcNow) }));
host.Tell(new DriverInstanceActor.AttributeAlarmPublished(
"DI-1", new AlarmEventArgs(new Handle(), "Plant/x/alarm", "Plant/x/alarm", "AlarmCondition", "test",
AlarmSeverity.High, DateTime.UtcNow)));
// Barrier again, then the absence window.
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget);
unhandled.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
}
private sealed class Handle : IAlarmSubscriptionHandle
{
public string DiagnosticId => "stale-test";
}
private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
{
public OtOpcUaConfigDbContext CreateDbContext() => throw new InvalidOperationException("db unreachable");
}
}
@@ -0,0 +1,288 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Review 02 P2: <c>WriteAttribute</c> was a <c>ReceiveAsync</c>, which suspends the mailbox for the whole
/// device round-trip (up to the 5 s bound). Writes therefore serialised — the SECOND of two operator writes to
/// a ~3 s device missed the host's 8 s Ask and was reverted although the device took it — and every data
/// change queued behind the write. Writes now run off the mailbox under a per-driver in-flight cap; the
/// subscribe/unsubscribe pair that must serialise against the subscription handle stays <c>ReceiveAsync</c>.
/// </summary>
public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBase
{
private static readonly TimeSpan WriteLatency = TimeSpan.FromMilliseconds(1200);
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(
IDriver driver, int? maxWritesInFlight = null, TimeSpan? queuedWriteBudget = null, TimeSpan? writeTimeout = null)
{
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, commsLossTimeout: TimeSpan.Zero, maxWritesInFlight: maxWritesInFlight,
queuedWriteBudget: queuedWriteBudget, writeTimeout: writeTimeout));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => m.Connected, PresenceBudget);
return (actor, parent);
}
[Fact]
public async Task Two_concurrent_slow_writes_both_complete_within_one_device_round_trip()
{
var driver = new SlowWritableDriver { Latency = WriteLatency };
var (actor, _) = Connect(driver);
var sw = Stopwatch.StartNew();
var askBudget = TimeSpan.FromSeconds(8); // the host's Ask
var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-1", 1), askBudget);
var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-2", 2), askBudget);
var results = await Task.WhenAll(first, second);
sw.Stop();
results.ShouldAllBe(r => r.Success);
driver.Writes.Select(w => w.FullReference).ShouldBe(new[] { "tag-1", "tag-2" }, ignoreOrder: true);
// Both were inside the driver at once — that is the proof; the wall clock is only a sanity ceiling
// (serialised writes would take two latencies, and the host's Ask is 8 s).
driver.MaxConcurrentWrites.ShouldBe(2, "the second write waited for the first — writes are still serialised on the mailbox");
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(8));
}
[Fact]
public async Task Data_changes_are_delivered_while_a_write_is_pending()
{
var driver = new SlowWritableDriver { HoldWrites = true };
var (actor, parent) = Connect(driver);
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
var pending = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 7), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
driver.FireDataChange("tag-a", 3.5);
var published = parent.ExpectMsg<DriverInstanceActor.AttributeValueBatchPublished>(PresenceBudget);
published.Items.Single().Value.ShouldBe(3.5);
pending.IsCompleted.ShouldBeFalse("the write is still held — the data change did not wait behind it");
// Positive control: releasing the device completes the write with its real outcome.
driver.ReleaseWrites();
(await pending).Success.ShouldBeTrue();
}
[Fact]
public async Task Writes_beyond_the_in_flight_cap_queue_and_drain_in_order()
{
var driver = new SlowWritableDriver { HoldWrites = true };
var (actor, _) = Connect(driver, maxWritesInFlight: 2);
var asks = Enumerable.Range(0, 4)
.Select(i => actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute($"tag-{i}", i), PresenceBudget))
.ToArray();
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
// Settle: a third write entering the driver would show up here.
await Task.Delay(300);
driver.WritesEntered.ShouldBe(2, "the cap let a third write into the driver");
driver.ReleaseWrites();
var results = await Task.WhenAll(asks);
results.ShouldAllBe(r => r.Success);
// Entry order: the first two slots went to the first two writes, the queued pair entered only after a
// slot freed. Within each pair the order is unspecified — distinct tags run concurrently on the pool;
// only SAME-tag order is guaranteed (pinned by the same-tag test below).
var entered = driver.EnteredRefs;
entered.Count.ShouldBe(4);
entered.Take(2).ShouldBe(new[] { "tag-0", "tag-1" }, ignoreOrder: true);
entered.Skip(2).ShouldBe(new[] { "tag-2", "tag-3" }, ignoreOrder: true);
}
/// <summary>
/// Fix round 1: writes to the SAME tag must keep their order — two concurrent writes v1, v2 could land as
/// v1. A same-tag write waits behind its tag while other tags still run concurrently.
/// </summary>
[Fact]
public async Task Writes_to_the_same_tag_are_serialised_in_order_while_other_tags_run_concurrently()
{
var driver = new SlowWritableDriver { HoldWrites = true };
var (actor, _) = Connect(driver, maxWritesInFlight: 4);
var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
// Positive control: a DIFFERENT tag is not held behind tag-a.
var other = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-b", 9), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-b" });
await Task.Delay(300);
driver.WritesEntered.ShouldBe(2, "the second tag-a write entered the driver while the first was still inside it");
second.IsCompleted.ShouldBeFalse();
driver.ReleaseWrites();
(await Task.WhenAll(first, second, other)).ShouldAllBe(r => r.Success);
driver.Writes.Where(w => w.FullReference == "tag-a").Select(w => w.Value).ShouldBe(new object[] { 1, 2 });
}
/// <summary>
/// Fix round 1: a write that sat in the queue longer than the host's Ask budget has already been reported
/// as a timeout (and reverted) upstream — writing it late would push a stale value to the device.
/// </summary>
[Fact]
public async Task A_queued_write_older_than_the_budget_is_failed_BadTimeout_instead_of_written_late()
{
var driver = new SlowWritableDriver { HoldWrites = true };
var (actor, _) = Connect(driver, maxWritesInFlight: 1, queuedWriteBudget: TimeSpan.FromMilliseconds(300));
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
var stale = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
await Task.Delay(600);
driver.ReleaseWrites();
(await inFlight).Success.ShouldBeTrue();
var staleResult = await stale;
staleResult.Success.ShouldBeFalse();
staleResult.Reason!.ShouldContain("800A0000", customMessage: "BadTimeout");
driver.EnteredRefs.ShouldBe(new[] { "tag-0" }, "the stale write must never reach the device");
}
/// <summary>
/// Fix round 2: a token-ignoring hung write held its tag forever, so every later write to that tag piled up
/// in the queue. The write is bounded pessimistically; on overrun it answers BadTimeout and releases its
/// slot and tag, and a later write to the same tag reaches the device.
/// </summary>
[Fact]
public async Task A_hung_token_ignoring_write_is_abandoned_and_releases_its_tag_for_a_later_write()
{
var driver = new SlowWritableDriver { HoldWrites = true, IgnoreCancellation = true };
var (actor, _) = Connect(driver, maxWritesInFlight: 1, writeTimeout: TimeSpan.FromMilliseconds(300));
var hung = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
var hungResult = await hung;
hungResult.Success.ShouldBeFalse();
hungResult.Reason!.ShouldContain("800A0000", customMessage: "BadTimeout");
driver.MaxConcurrentWrites.ShouldBe(1);
// The tag (and the single slot) are free again: a later tag-a write reaches the device even though the
// abandoned one is still inside it.
var later = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-a" });
driver.MaxConcurrentWrites.ShouldBe(2, "positive control: the abandoned write really is still wedged inside the driver");
driver.ReleaseWrites();
(await later).Success.ShouldBeTrue();
}
[Fact]
public async Task Queued_writes_fail_fast_when_the_driver_leaves_Connected()
{
var driver = new SlowWritableDriver { HoldWrites = true };
var (actor, parent) = Connect(driver, maxWritesInFlight: 1);
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
var queued = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
actor.Tell(new DriverInstanceActor.ForceReconnect());
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => !m.Connected, PresenceBudget);
var queuedResult = await queued;
queuedResult.Success.ShouldBeFalse();
queuedResult.Reason.ShouldBe("driver not connected");
driver.WritesEntered.ShouldBe(1, "a queued write must never reach a driver that is reconnecting");
// The in-flight one still answers with the device's real outcome once it is released.
driver.ReleaseWrites();
(await inFlight).Success.ShouldBeTrue();
}
/// <summary>A writable + subscribable driver whose writes either take <see cref="Latency"/> or hold until released.</summary>
private sealed class SlowWritableDriver : IDriver, IWritable, ISubscribable
{
private readonly StubHandle _handle = new();
private readonly ConcurrentQueue<WriteRequest> _writes = new();
private readonly ConcurrentQueue<string> _enteredRefs = new();
private TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _entered;
private int _inside;
private int _maxInside;
public TimeSpan Latency { get; init; }
public bool HoldWrites { get; init; }
/// <summary>When true a held write ignores its cancellation token — a wedged driver.</summary>
public bool IgnoreCancellation { get; init; }
public IReadOnlyList<WriteRequest> Writes => _writes.ToArray();
public IReadOnlyList<string> EnteredRefs => _enteredRefs.ToArray();
public int WritesEntered => Volatile.Read(ref _entered);
public int MaxConcurrentWrites => Volatile.Read(ref _maxInside);
public string DriverInstanceId => "slow-writer-1";
public string DriverType => "SlowWriter";
public void ReleaseWrites() => _release.TrySetResult();
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
public async Task<IReadOnlyList<WriteResult>> WriteAsync(IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
{
Interlocked.Increment(ref _entered);
foreach (var w in writes) _enteredRefs.Enqueue(w.FullReference);
var inside = Interlocked.Increment(ref _inside);
int seen;
while (inside > (seen = Volatile.Read(ref _maxInside)))
Interlocked.CompareExchange(ref _maxInside, inside, seen);
try
{
if (HoldWrites && IgnoreCancellation) await _release.Task;
else if (HoldWrites) await _release.Task.WaitAsync(cancellationToken);
else if (Latency > TimeSpan.Zero) await Task.Delay(Latency, cancellationToken);
foreach (var w in writes) _writes.Enqueue(w);
return writes.Select(_ => new WriteResult(OpcUaStatus.Good)).ToList();
}
finally
{
Interlocked.Decrement(ref _inside);
}
}
public event EventHandler<DataChangeEventArgs>? OnDataChange;
public Task<ISubscriptionHandle> SubscribeAsync(
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) =>
Task.FromResult<ISubscriptionHandle>(_handle);
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
public void FireDataChange(string fullRef, object? value) =>
OnDataChange?.Invoke(this, new DataChangeEventArgs(
_handle, fullRef, new DataValueSnapshot(value, OpcUaStatus.Good, DateTime.UtcNow, DateTime.UtcNow)));
}
}
@@ -0,0 +1,173 @@
using System.Diagnostics;
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Review 02 L1 / P4: a driver child that stops for good must <b>dispose</b> its <see cref="IDriver"/>
/// (the runtime only ever called <c>ShutdownAsync</c>, so a Calculation driver's compiled-script ALCs and an
/// OpcUaClient driver's LoggerFactory + gate + CTS leaked on every respawn), its <c>ShutdownAsync</c> must be
/// bounded (it used to block a dispatcher thread for as long as a dead socket took to answer), and a
/// <b>supervision restart</b> must NOT dispose — the Props closure re-uses the same driver object for the
/// restarted incarnation, which then has to serve through it.
/// </summary>
public sealed class DriverInstanceActorDisposeTests : RuntimeActorTestBase
{
[Fact]
public void Stopping_the_child_shuts_the_driver_down_and_then_disposes_it_exactly_once()
{
var driver = new DisposableRecordingDriver();
var parent = CreateTestProbe();
parent.IgnoreMessages(m => m is DriverInstanceActor.ConnectivityChanged);
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
AwaitCondition(() => driver.InitializeCount == 1, PresenceBudget);
Watch(actor);
Sys.Stop(actor);
ExpectTerminated(actor, PresenceBudget);
driver.ShutdownCount.ShouldBe(1);
driver.DisposeCount.ShouldBe(1, "a child stopped for good must dispose the driver it wrapped");
driver.Trace.ToArray().ShouldBe(new[] { "init", "shutdown", "dispose" }, "shutdown must land before dispose");
}
[Fact]
public async Task A_supervision_restart_keeps_the_driver_undisposed_and_the_restarted_child_still_serves()
{
var driver = new DisposableRecordingDriver();
var parent = CreateTestProbe();
// The probe parent's own strategy STOPS a failing child; the production host (no strategy anywhere under
// src/Server) gets Akka's default decider, which RESTARTS it — the route under test.
var actor = parent.ChildActorOf(
DriverInstanceActor.Props(driver), new OneForOneStrategy(_ => Directive.Restart));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
// Barrier: Subscribe is only handled once Connected (it is dropped in Connecting).
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => m.Connected, PresenceBudget);
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
// The crash seam (see DriverHostActorRestartTests): a null snapshot NREs on the actor thread, and the
// probe parent's default decider restarts the child in place around the SAME driver object.
driver.FireMalformedDataChange("tag-a");
parent.FishForMessage(m => m is DriverInstanceActor.ReprovisionRequested, PresenceBudget);
driver.ShutdownCount.ShouldBe(1, "PreRestart runs PostStop, which shuts the driver down");
driver.DisposeCount.ShouldBe(0, "a restart re-uses the driver object — disposing it would kill the new incarnation");
// Positive control: the restarted incarnation serves through the undisposed driver. The host would
// re-provision it exactly like this (InitializeRequested + the desired set).
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
driver.FireDataChange("tag-a", 42.0);
var published = (DriverInstanceActor.AttributeValueBatchPublished)parent.FishForMessage(
m => m is DriverInstanceActor.AttributeValueBatchPublished, PresenceBudget);
published.Items.Single().FullReference.ShouldBe("tag-a");
published.Items.Single().Value.ShouldBe(42.0);
// ...and the eventual real stop still disposes it — once.
Watch(actor);
Sys.Stop(actor);
ExpectTerminated(actor, PresenceBudget);
driver.DisposeCount.ShouldBe(1);
}
[Fact]
public void A_hung_ShutdownAsync_is_abandoned_at_the_bound_and_the_driver_is_still_disposed()
{
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
// Ignores its token on purpose: a wedged socket/COM proxy does not honour cancellation either.
var driver = new DisposableRecordingDriver { ShutdownBehavior = _ => never.Task };
var bound = TimeSpan.FromMilliseconds(300);
var parent = CreateTestProbe();
parent.IgnoreMessages(m => m is DriverInstanceActor.ConnectivityChanged);
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver, shutdownTimeout: bound));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
AwaitCondition(() => driver.InitializeCount == 1, PresenceBudget);
Watch(actor);
var sw = Stopwatch.StartNew();
Sys.Stop(actor);
ExpectTerminated(actor, PresenceBudget);
sw.Stop();
driver.ShutdownCount.ShouldBe(1, "the shutdown was attempted (positive control for the bound)");
never.Task.IsCompleted.ShouldBeFalse("the shutdown is still hung — the stop did not wait for it");
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the stop must not be pinned by the hung shutdown");
driver.DisposeCount.ShouldBe(1, "a shutdown that timed out must not skip the dispose");
}
/// <summary>
/// A subscribable driver that records its lifecycle and is <see cref="IAsyncDisposable"/>. Deliberately
/// NOT also <see cref="IDisposable"/>: the actor prefers the async dispose when both exist, and a double
/// implementation would hide a double-dispose.
/// </summary>
private sealed class DisposableRecordingDriver : IDriver, ISubscribable, IAsyncDisposable
{
private readonly StubHandle _handle = new();
private int _initializeCount;
private int _shutdownCount;
private int _disposeCount;
public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }
public List<string> Trace { get; } = new();
public int InitializeCount => Volatile.Read(ref _initializeCount);
public int ShutdownCount => Volatile.Read(ref _shutdownCount);
public int DisposeCount => Volatile.Read(ref _disposeCount);
public string DriverInstanceId => "disposable-1";
public string DriverType => "Disposable";
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
{
Interlocked.Increment(ref _initializeCount);
lock (Trace) Trace.Add("init");
return Task.CompletedTask;
}
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
public Task ShutdownAsync(CancellationToken cancellationToken)
{
Interlocked.Increment(ref _shutdownCount);
lock (Trace) Trace.Add("shutdown");
return ShutdownBehavior is { } behavior ? behavior(cancellationToken) : Task.CompletedTask;
}
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
public ValueTask DisposeAsync()
{
Interlocked.Increment(ref _disposeCount);
lock (Trace) Trace.Add("dispose");
return ValueTask.CompletedTask;
}
public event EventHandler<DataChangeEventArgs>? OnDataChange;
public Task<ISubscriptionHandle> SubscribeAsync(
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) =>
Task.FromResult<ISubscriptionHandle>(_handle);
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
public void FireDataChange(string fullRef, object? value) =>
OnDataChange?.Invoke(this, new DataChangeEventArgs(
_handle, fullRef, new DataValueSnapshot(value, OpcUaStatus.Good, DateTime.UtcNow, DateTime.UtcNow)));
/// <summary>The supervision crash seam — see <see cref="RecordingSubscribableDriver.FireMalformedDataChange"/>.</summary>
public void FireMalformedDataChange(string fullRef) =>
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, null!));
}
}
@@ -0,0 +1,160 @@
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// Review 02 H2: the <c>Reconnecting</c> retry timer fired every <c>reconnectInterval</c> whether or not the
/// previous attempt was still running — <c>_initGeneration</c> only discarded the older attempt's RESULT. With an
/// Initialize bound longer than the cadence (Tier C's 15 s vs. the 10 s timer, or the test harness's unbounded
/// invoker) attempt N+1's <c>ShutdownAsync</c> ran while attempt N's <c>InitializeAsync</c> was still inside the
/// driver: the dispose-before-overwrite race the drivers' rule file forbids, once per overlap, for as long as
/// the device stayed down. The retry is now single-flight.
/// </summary>
public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActorTestBase
{
[Fact]
public void Shutdown_and_Initialize_never_overlap_while_the_init_outlasts_the_retry_cadence()
{
// A 400 ms failing init against a 50 ms retry cadence: without the gate the timer starts a new
// Shutdown+Initialize pair every 50 ms on top of the one still running (eight-deep overlap).
var driver = new InterleavingDriver(initDuration: TimeSpan.FromMilliseconds(400), failUntilAttempt: 4);
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
// Positive control: the gate skips ticks, it does not stop retrying — the 4th attempt succeeds and the
// actor connects.
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
driver.Attempts.ShouldBeGreaterThanOrEqualTo(4);
driver.Overlaps.ShouldBe(0,
$"ShutdownAsync entered while an InitializeAsync was still running. Interleaving: {driver.Interleaving}");
}
[Fact]
public void A_pre_reconnect_teardown_that_hangs_is_abandoned_at_the_bound_and_the_init_still_runs()
{
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 2)
{
// Only the SECOND shutdown (the reconnect teardown) hangs; the first attempt has none.
ShutdownBehavior = _ => never.Task,
};
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
shutdownTimeout: TimeSpan.FromMilliseconds(200)));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
never.Task.IsCompleted.ShouldBeFalse("the teardown is still hung — the attempt did not wait for it");
driver.Attempts.ShouldBe(2);
}
/// <summary>
/// Fix round 1: the single-flight gate only cleared on the attempt's RESULT, and the invoker's timeout is
/// token-only — a driver whose InitializeAsync ignores its token wedged the reconnect loop forever. The
/// attempt is now bounded pessimistically (<c>initializeTimeout</c>); an overrun is abandoned, reported
/// as a failure, and the next tick retries.
/// </summary>
[Fact]
public void A_token_ignoring_never_completing_init_is_abandoned_and_a_later_attempt_connects()
{
var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 1)
{
// Attempt 1 hangs forever and ignores cancellation; attempt 2 completes.
HangOnAttempt = 1,
};
var parent = CreateTestProbe();
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
initializeTimeout: TimeSpan.FromMilliseconds(300)));
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
driver.Attempts.ShouldBe(2);
driver.HungAttemptStillRunning.ShouldBeTrue("positive control: the first attempt really was wedged");
}
/// <summary>Records every Shutdown/Initialize entry and exit so an overlap is a fact, not an inference.</summary>
private sealed class InterleavingDriver : IDriver
{
private readonly TimeSpan _initDuration;
private readonly int _failUntilAttempt;
private readonly Lock _gate = new();
private readonly List<string> _events = new();
private int _attempts;
private int _initsInProgress;
private int _overlaps;
public InterleavingDriver(TimeSpan initDuration, int failUntilAttempt)
{
_initDuration = initDuration;
_failUntilAttempt = failUntilAttempt;
}
public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }
/// <summary>The 1-based attempt that never completes and ignores its token (0 = none).</summary>
public int HangOnAttempt { get; init; }
private readonly TaskCompletionSource _never = new(TaskCreationOptions.RunContinuationsAsynchronously);
public bool HungAttemptStillRunning => !_never.Task.IsCompleted && Volatile.Read(ref _initsInProgress) > 0;
public int Attempts => Volatile.Read(ref _attempts);
public int Overlaps => Volatile.Read(ref _overlaps);
public string Interleaving { get { lock (_gate) return string.Join(" ", _events); } }
public string DriverInstanceId => "interleaving-1";
public string DriverType => "Interleaving";
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
{
var attempt = Interlocked.Increment(ref _attempts);
Interlocked.Increment(ref _initsInProgress);
Record($"init{attempt}>");
try
{
if (attempt == HangOnAttempt) await _never.Task;
if (_initDuration > TimeSpan.Zero) await Task.Delay(_initDuration, CancellationToken.None);
if (attempt < _failUntilAttempt) throw new InvalidOperationException($"init attempt {attempt} refused");
}
finally
{
Record($"<init{attempt}");
Interlocked.Decrement(ref _initsInProgress);
}
}
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
public async Task ShutdownAsync(CancellationToken cancellationToken)
{
if (Volatile.Read(ref _initsInProgress) > 0) Interlocked.Increment(ref _overlaps);
Record("shutdown>");
try
{
if (ShutdownBehavior is { } behavior) await behavior(cancellationToken);
}
finally
{
Record("<shutdown");
}
}
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
private void Record(string e) { lock (_gate) _events.Add(e); }
}
}
@@ -47,6 +47,8 @@ public static class V3ArtifactSeeder
/// is parsed out of <c>TagConfig.alarm</c> by <c>TagConfigIntent</c>), which the alarm-ref half of the
/// restart suite needs — an alarm-bearing raw tag is excluded from the VALUE subscription set and carried
/// in the alarm one instead, so the two travel through different child code paths.</param>
/// <param name="driverRowIds">Optional <c>DriverInstanceRowId</c> (the DB row Guid) per driver id; absent
/// ids get <see cref="Guid.Empty"/>, which is what an artifact without the property parses to.</param>
/// <param name="taglessDrivers">Extra <c>DriverInstance</c> ids carrying NO devices and NO tags. The
/// only way to express a driver that is a running child yet absent from both the value- and alarm-ref
/// maps — the <c>_children.Keys</c> arm of <c>ApplyDesiredSubscriptions</c>'s union, which must still
@@ -58,14 +60,16 @@ public static class V3ArtifactSeeder
(string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs,
bool enabled = false,
IReadOnlyDictionary<string, string>? tagConfigByTag = null,
string[]? taglessDrivers = null)
string[]? taglessDrivers = null,
IReadOnlyDictionary<string, Guid>? driverRowIds = null)
{
Guid RowId(string id) => driverRowIds is not null && driverRowIds.TryGetValue(id, out var g) ? g : Guid.Empty;
var drivers = rawTags
.Select(t => (t.Driver, t.DriverName))
.Distinct()
.Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })
.Select(d => new { DriverInstanceRowId = RowId(d.Driver), DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })
.Concat((taglessDrivers ?? Array.Empty<string>())
.Select(id => new { DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }))
.Select(id => new { DriverInstanceRowId = RowId(id), DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }))
.ToArray();
var devices = rawTags
.Select(t => (t.Driver, t.Device))
@@ -201,7 +205,37 @@ public static class V3ArtifactSeeder
return (actor, mux, coordinator);
}
private static void Dispatch(IActorRef actor, TestProbe coordinator, DeploymentId deploymentId, RevisionHash rev)
/// <summary>The same arrangement as <see cref="SpawnHostAndApply"/>, also returning the coordinator probe so a
/// test can <see cref="Dispatch"/> FURTHER deployments to the same host (remove a driver, re-add it).</summary>
/// <param name="kit">The calling TestKit, supplying the ActorSystem + probes.</param>
/// <param name="db">The factory whose deployment was seeded by <see cref="SeedV3Deployment"/>.</param>
/// <param name="localNode">The host's local node id.</param>
/// <param name="deploymentId">The deployment to dispatch.</param>
/// <param name="rev">The revision hash the deployment was sealed under.</param>
/// <param name="driverFactory">Optional driver factory forwarded to <c>DriverHostActor.Props</c>.</param>
/// <param name="reprovisionBackoff">Optional per-driver supervision-reprovision rate limit.</param>
/// <returns>The host actor plus the publish, mux and coordinator probes wired into it.</returns>
public static (IActorRef Actor, TestProbe Publish, TestProbe Mux, TestProbe Coordinator) SpawnHostAndApplyWithCoordinator(
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
NodeId localNode, DeploymentId deploymentId, RevisionHash rev,
IDriverFactory? driverFactory = null,
TimeSpan? reprovisionBackoff = null)
{
var publish = kit.CreateCleanOutcomePublishProbe();
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory, reprovisionBackoff);
Dispatch(actor, coordinator, deploymentId, rev);
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(RuntimeActorTestBase.PresenceBudget);
return (actor, publish, mux, coordinator);
}
/// <summary>Dispatches <paramref name="deploymentId"/> to <paramref name="actor"/> and waits for its Applied ack.</summary>
/// <param name="actor">The host.</param>
/// <param name="coordinator">The coordinator probe the host acks to.</param>
/// <param name="deploymentId">The deployment to dispatch.</param>
/// <param name="rev">Its revision hash.</param>
public static void Dispatch(IActorRef actor, TestProbe coordinator, DeploymentId deploymentId, RevisionHash rev)
{
actor.Tell(new DispatchDeployment(deploymentId, rev, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(RuntimeActorTestBase.PresenceBudget)
@@ -0,0 +1,23 @@
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
/// <summary>
/// Builds the ONE message shape a driver child sends for its values — a
/// <see cref="DriverInstanceActor.AttributeValueBatchPublished"/> — for tests that push a single value into the
/// dependency mux by hand. The per-value <see cref="DriverInstanceActor.AttributeValuePublished"/> has no
/// production sender (02/U4); tests should not keep one alive either.
/// </summary>
public static class ValuePublishes
{
/// <summary>A one-item batch for <paramref name="fullReference"/>.</summary>
/// <param name="driverInstanceId">The publishing driver's id.</param>
/// <param name="fullReference">The value's reference (RawPath).</param>
/// <param name="value">The value.</param>
/// <param name="statusCode">The full OPC UA status code.</param>
/// <param name="timestampUtc">The source timestamp.</param>
/// <returns>The batch message.</returns>
public static DriverInstanceActor.AttributeValueBatchPublished Single(
string driverInstanceId, string fullReference, object? value, uint statusCode, DateTime timestampUtc) =>
new(driverInstanceId, new[] { new DriverInstanceActor.AttributeValueItem(fullReference, value, statusCode, timestampUtc) });
}
@@ -0,0 +1,73 @@
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
/// <summary>
/// Review 02 P7: <see cref="DbHealthProbeActor"/> ran its SELECT synchronously on the actor thread, so a
/// hanging SQL connect/command held the dispatcher thread for the implicit 15 s / 30 s SqlClient timeouts and
/// queued every later tick (and every <c>GetStatus</c> Ask) behind it. The probe now runs off the mailbox
/// and pipes its outcome back, so the cached answer stays available while a probe hangs.
/// </summary>
public sealed class DbHealthProbeOffThreadTests : RuntimeActorTestBase
{
[Fact]
public async Task GetStatus_answers_from_cache_while_a_probe_is_hung()
{
var factory = new HangingDbFactory();
try
{
var actor = Sys.ActorOf(DbHealthProbeActor.Props(factory));
AwaitCondition(() => factory.Entered > 0, PresenceBudget);
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(3));
status.Reachable.ShouldBeFalse();
status.LastError.ShouldBe("not probed yet");
factory.Entered.ShouldBe(1, "a second tick must not stack a second probe behind a hung one");
}
finally
{
factory.Release();
}
}
[Fact]
public async Task A_probe_that_completes_updates_the_cached_status()
{
// Positive control for the off-thread path: the piped-back outcome lands in the cache.
var db = NewInMemoryDbFactory();
var actor = Sys.ActorOf(DbHealthProbeActor.Props(db));
await AwaitAssertAsync(async () =>
{
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(3));
status.Reachable.ShouldBeTrue();
}, PresenceBudget);
}
/// <summary>Blocks every <c>CreateDbContext</c> until released — a connect that never answers.</summary>
private sealed class HangingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
{
private readonly ManualResetEventSlim _release = new(false);
private int _entered;
public int Entered => Volatile.Read(ref _entered);
public void Release() => _release.Set();
public OtOpcUaConfigDbContext CreateDbContext()
{
Interlocked.Increment(ref _entered);
_release.Wait();
throw new InvalidOperationException("released without a database");
}
}
}
@@ -15,18 +15,22 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
public sealed class HealthProbeActorTests : RuntimeActorTestBase
{
/// <summary>Verifies that the DB health probe actor returns reachable status against an in-memory database.</summary>
/// <summary>Verifies that the DB health probe actor returns reachable status against an in-memory database.
/// The probe runs off the mailbox (02/P7), so the first Ask may still see "not probed yet" — poll.</summary>
[Fact]
public async Task DbHealthProbeActor_returns_reachable_against_in_memory_db()
{
var db = NewInMemoryDbFactory();
var actor = Sys.ActorOf(DbHealthProbeActor.Props(db));
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
DbHealthProbeActor.GetStatus.Instance, PresenceBudget);
await AwaitAssertAsync(async () =>
{
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
DbHealthProbeActor.GetStatus.Instance, PresenceBudget);
status.Reachable.ShouldBeTrue();
status.LastError.ShouldBeNull();
status.Reachable.ShouldBeTrue();
status.LastError.ShouldBeNull();
}, PresenceBudget);
}
/// <summary>Verifies that the peer OPC UA probe actor reports Ok true against a live listener.</summary>
@@ -168,4 +168,68 @@ public sealed class ServerHistorianOptionsTests
}.Validate();
Assert.Empty(w);
}
// ── MaxRawValuesPerRead (R08:1.1, #736) ─────────────────────────────────────────────────────
[Fact]
public void MaxRawValuesPerRead_defaults_to_ten_thousand_and_binds()
{
var o = new ServerHistorianOptions();
Assert.Equal(10_000, o.MaxRawValuesPerRead);
Assert.Equal(10_000, ServerHistorianOptions.DefaultMaxRawValuesPerRead);
Assert.Equal(10_000, o.EffectiveMaxRawValuesPerRead);
var bound = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?> { ["ServerHistorian:MaxRawValuesPerRead"] = "2500" })
.Build()
.GetSection(ServerHistorianOptions.SectionName)
.Get<ServerHistorianOptions>()!;
Assert.Equal(2500, bound.MaxRawValuesPerRead);
Assert.Equal(2500, bound.EffectiveMaxRawValuesPerRead);
Assert.Empty(new ServerHistorianOptions
{
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = 2500,
}.Validate());
}
[Theory]
[InlineData(0)]
[InlineData(-5)]
public void Nonpositive_MaxRawValuesPerRead_is_unbounded_and_warns_when_enabled(int value)
{
var o = new ServerHistorianOptions
{
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = value,
};
Assert.Equal(0, o.EffectiveMaxRawValuesPerRead);
var w = o.Validate();
var m = Assert.Single(w);
Assert.Contains("MaxRawValuesPerRead", m);
Assert.Contains("unbounded", m);
}
[Theory]
[InlineData(1)]
[InlineData(2)]
public void A_MaxRawValuesPerRead_below_three_is_raised_to_three_and_warns_when_enabled(int value)
{
// A resumed page's read must carry a value, the boundary tie it re-reads and the lookahead row that says "more
// follows": at 2 it has no room and every value costs an empty page with a continuation point (the #692 defect).
var o = new ServerHistorianOptions
{
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = value,
};
Assert.Equal(3, o.EffectiveMaxRawValuesPerRead);
Assert.Equal(ServerHistorianOptions.MinMaxRawValuesPerRead, o.EffectiveMaxRawValuesPerRead);
var m = Assert.Single(o.Validate());
Assert.Contains("MaxRawValuesPerRead", m);
Assert.Equal(3, new ServerHistorianOptions { MaxRawValuesPerRead = 3 }.EffectiveMaxRawValuesPerRead);
}
[Fact]
public void Disabled_with_an_odd_MaxRawValuesPerRead_is_silent()
{
Assert.Empty(new ServerHistorianOptions { Enabled = false, MaxRawValuesPerRead = 0 }.Validate());
Assert.Empty(new ServerHistorianOptions { Enabled = false, MaxRawValuesPerRead = 1 }.Validate());
}
}
@@ -0,0 +1,69 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Telemetry;
/// <summary>
/// Review 02 L4: <see cref="TelemetryLocalHub.Evict"/>'s tombstone set only ever shrank on a re-spawn of the
/// SAME id, so a driver removed for good kept its id forever. The tombstone is load-bearing (#609 — it holds
/// back the removed child's late writes), so it cannot simply be cleared; it is now timestamped and swept
/// once older than <see cref="TelemetryLocalHub.EvictionTombstoneTtl"/>, which outlives any in-flight
/// capability call by a wide margin.
/// </summary>
public sealed class TelemetryLocalHubTombstoneSweepTests
{
private static DriverHealthChanged Health(string instanceId) =>
new("cluster-a", instanceId, "Healthy", null, null, 0, DateTime.UtcNow);
private static List<TelemetryItem> Drain(ITelemetrySubscription sub)
{
var items = new List<TelemetryItem>();
while (sub.Reader.TryRead(out var item)) items.Add(item);
return items;
}
[Fact]
public void A_tombstone_holds_late_writes_until_the_ttl_then_is_swept()
{
var clock = new ManualTime { Now = new DateTimeOffset(2026, 10, 22, 12, 0, 0, TimeSpan.Zero) };
var hub = new TelemetryLocalHub(clock);
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
hub.Evict("drv-a");
// Inside the TTL: the late write is dropped (the #609 contract) — and another eviction's sweep does
// not lift a tombstone that is still young.
clock.Now += TelemetryLocalHub.EvictionTombstoneTtl / 2;
hub.Evict("drv-b");
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
using (var sub = hub.Subscribe(16))
Drain(sub).ShouldBeEmpty("a late write inside the TTL must still be held back");
// Past the TTL: the next eviction sweeps drv-a's tombstone and its writes land again.
clock.Now += TelemetryLocalHub.EvictionTombstoneTtl;
hub.Evict("drv-c");
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
using (var sub = hub.Subscribe(16))
Drain(sub).OfType<TelemetryItem.Health>().Single().E.DriverInstanceId.ShouldBe("drv-a");
}
[Fact]
public void Readmit_lifts_a_tombstone_before_its_ttl()
{
var clock = new ManualTime { Now = DateTimeOffset.UtcNow };
var hub = new TelemetryLocalHub(clock);
hub.Evict("drv-a");
hub.Readmit("drv-a");
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
using var sub = hub.Subscribe(16);
Drain(sub).Count.ShouldBe(1);
}
private sealed class ManualTime : TimeProvider
{
public DateTimeOffset Now { get; set; }
public override DateTimeOffset GetUtcNow() => Now;
}
}