Merge branch 'master' into remed/lane-E
This commit is contained in:
@@ -40,6 +40,17 @@
|
||||
<PackageVersion Include="Microsoft.AspNetCore.DataProtection.EntityFrameworkCore" Version="10.0.7" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.0" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="10.0.0" />
|
||||
<!--
|
||||
Surgical transitive pin #4. GHSA-334f-vvp4-7g6p (CVE-2026-70159, NU1903 high) flags
|
||||
Microsoft.AspNetCore.SignalR.Common 10.0.0..10.0.12, which SignalR.Client 10.0.0 pulls in via
|
||||
SignalR.Client.Core. 10.0.13 is the first fixed version, pinned by direct PackageReference in
|
||||
AdminUI (the one project that references SignalR.Client). Bumping SignalR.Client itself to
|
||||
10.0.13 was tried and rejected: SignalR.Client.Core 10.0.13 floors Microsoft.Extensions.Logging /
|
||||
DependencyInjection at 10.0.13, which the AdminUI shared framework absorbs but non-Web consumers
|
||||
downstream (OpcUaServer.IntegrationTests, which references Logging.Abstractions 10.0.7 directly)
|
||||
do not, producing an NU1605 downgrade.
|
||||
-->
|
||||
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Common" Version="10.0.13" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Core" Version="1.2.0" />
|
||||
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="10.0.7" />
|
||||
<!--
|
||||
@@ -149,8 +160,28 @@
|
||||
Microsoft.Extensions.* and, through the EFCore adapter, Microsoft.EntityFrameworkCore at
|
||||
10.0.10, forcing a family-wide servicing bump (NU1605 downgrade cascade) that deserves its own
|
||||
reviewed change.
|
||||
|
||||
2026-10-22: raised 10.0.10 -> 10.0.13. Three more NU1903 advisories (GHSA-7hqp-7c5m-4vq6,
|
||||
GHSA-c399-9853-6jmj, GHSA-r955-8q5j-3h83; vulnerable <= 10.0.12) landed against 10.0.10, again
|
||||
red only on a fresh restore (~67 projects). 10.0.13 is the first fixed version. Same pin site,
|
||||
same reasoning against the DataProtection parent bump.
|
||||
-->
|
||||
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.10" />
|
||||
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.13" />
|
||||
<!--
|
||||
Surgical transitive pin #3. GHSA-846f-f764-76mx (NU1903 high) + GHSA-2fmv-rv2c-59x7 (NU1902
|
||||
moderate) flag System.Security.Cryptography.Pkcs <= 10.0.12 and 9.0.0..9.0.20. Two chains bring
|
||||
it in, both entering at Core.Configuration:
|
||||
- System.Security.Cryptography.Xml 10.0.x -> Pkcs 10.0.x (same version as Xml), and
|
||||
- Microsoft.EntityFrameworkCore.SqlServer 10.0.7 -> Microsoft.Data.SqlClient 6.1.1 -> Pkcs 9.0.4.
|
||||
In Web-SDK consumers (AdminUI, ControlPlane, Host, Security, ...) the ASP.NET Core shared
|
||||
framework prunes the Xml chain, so the SqlClient 9.0.4 floor is what resolves there (~9
|
||||
projects); everywhere else the Xml chain's 10.0.10 won (~67 projects). Raising the Xml pin alone
|
||||
only fixes the second group, so Pkcs is pinned explicitly at 10.0.13 (first fixed 10.x) by a
|
||||
direct PackageReference in Core.Configuration. Its net10.0 dependency group is empty, so the pin
|
||||
floors nothing else. Bumping Microsoft.Data.SqlClient / EFCore.SqlServer instead would be the
|
||||
family-wide servicing bump the Xml note above declines.
|
||||
-->
|
||||
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="10.0.13" />
|
||||
<PackageVersion Include="System.CommandLine" Version="2.0.5" />
|
||||
<PackageVersion Include="System.Data.SqlClient" Version="4.9.0" />
|
||||
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="8.11.0" />
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
# Baseline 2026-10-22 @ ad2b5c40 — Release, filter Category!=E2E&Category!=LiveIntegration
|
||||
# Result: all assemblies passed (0 failed). Run concurrently with 6 Wave-1 worktree builds.
|
||||
Passed! - Failed: 0, Passed: 126, Skipped: 0, Total: 126, Duration: 251 ms - ZB.MOM.WW.OtOpcUa.Client.UI.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 202, Skipped: 0, Total: 202, Duration: 1 s - ZB.MOM.WW.OtOpcUa.Client.Shared.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 16, Skipped: 0, Total: 16, Duration: 91 ms - ZB.MOM.WW.OtOpcUa.Core.VirtualTags.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 138, Skipped: 0, Total: 138, Duration: 2 s - ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 51, Skipped: 0, Total: 51, Duration: 5 s - ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 359, Skipped: 0, Total: 359, Duration: 7 s - ZB.MOM.WW.OtOpcUa.Commons.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 95, Skipped: 0, Total: 95, Duration: 8 s - ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 455, Skipped: 0, Total: 455, Duration: 6 s - ZB.MOM.WW.OtOpcUa.Driver.Ab.Transport.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 437, Skipped: 0, Total: 437, Duration: 8 s - ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 359, Skipped: 1, Total: 360, Duration: 7 s - ZB.MOM.WW.OtOpcUa.Core.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 21, Skipped: 0, Total: 21, Duration: 196 ms - ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browser.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 66, Skipped: 0, Total: 66, Duration: 4 s - ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 1, Skipped: 217, Total: 218, Duration: 249 ms[xUnit.net 00:00:10.55] ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.IntegrationTests.AbLegacyIndirectAddressingTests.Slc500_indirect_word_read_and_write_resolve_through_the_index [SKIP]
|
||||
Passed! - Failed: 0, Passed: 353, Skipped: 8, Total: 361, Duration: 780 ms - ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 752, Skipped: 0, Total: 752, Duration: 9 s - ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 194, Skipped: 0, Total: 194, Duration: 11 s - ZB.MOM.WW.OtOpcUa.Configuration.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 139, Skipped: 0, Total: 139, Duration: 12 s - ZB.MOM.WW.OtOpcUa.Client.CLI.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 597, Skipped: 0, Total: 597, Duration: 9 s - ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 81, Skipped: 0, Total: 81, Duration: 346 ms - ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 14, Skipped: 0, Total: 14, Duration: 4 s - ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 269, Skipped: 0, Total: 269, Duration: 667 ms - ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 16, Skipped: 10, Total: 26, Duration: 10 s - ZB.MOM.WW.OtOpcUa.Driver.AbCip.IntegrationTests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 511, Skipped: 0, Total: 511, Duration: 5 s - ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 1, Skipped: 39, Total: 40, Duration: 96 ms - ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 161, Skipped: 0, Total: 161, Duration: 193 ms - ZB.MOM.WW.OtOpcUa.Driver.Modbus.Addressing.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 42, Skipped: 0, Total: 42, Duration: 4 s - ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 10, Skipped: 0, Total: 10, Duration: 1 s - ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Browser.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 465, Skipped: 0, Total: 465, Duration: 3 s - ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 16, Skipped: 0, Total: 16, Duration: 5 s - ZB.MOM.WW.OtOpcUa.Driver.MTConnect.IntegrationTests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 254, Skipped: 0, Total: 254, Duration: 3 s - ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 47, Skipped: 0, Total: 47, Duration: 149 ms - ZB.MOM.WW.OtOpcUa.Driver.AbCip.Cli.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 48, Skipped: 0, Total: 48, Duration: 106 ms - ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Cli.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 84, Skipped: 0, Total: 84, Duration: 3 s - ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 363, Skipped: 0, Total: 363, Duration: 1 s - ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 49, Skipped: 0, Total: 49, Duration: 449 ms - ZB.MOM.WW.OtOpcUa.Driver.Cli.Common.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 52, Skipped: 0, Total: 52, Duration: 394 ms - ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 563, Skipped: 0, Total: 563, Duration: 6 s - ZB.MOM.WW.OtOpcUa.Driver.S7.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 72, Skipped: 0, Total: 72, Duration: 245 ms - ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 59, Skipped: 0, Total: 59, Duration: 88 ms - ZB.MOM.WW.OtOpcUa.Driver.S7.Cli.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 70, Skipped: 0, Total: 70, Duration: 263 ms - ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 635, Skipped: 0, Total: 635, Duration: 21 s - ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 89, Skipped: 0, Total: 89, Duration: 6 s - ZB.MOM.WW.OtOpcUa.Security.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 420, Skipped: 0, Total: 420, Duration: 20 s - ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 22, Skipped: 0, Total: 22, Duration: 7 s - ZB.MOM.WW.OtOpcUa.Analyzers.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 250, Skipped: 0, Total: 250, Duration: 47 s - ZB.MOM.WW.OtOpcUa.Cluster.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 199, Skipped: 0, Total: 199, Duration: 16 s - ZB.MOM.WW.OtOpcUa.Host.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 1674, Skipped: 0, Total: 1674, Duration: 25 s - ZB.MOM.WW.OtOpcUa.AdminUI.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 825, Skipped: 0, Total: 825, Duration: 1 m 20 s - ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 453, Skipped: 0, Total: 453, Duration: 1 m 30 s - ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 837, Skipped: 3, Total: 840, Duration: 1 m 31 s - ZB.MOM.WW.OtOpcUa.Runtime.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 278, Skipped: 0, Total: 278, Duration: 1 m 37 s - ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 267, Skipped: 14, Total: 281, Duration: 3 m 39 s - ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 992, Skipped: 1, Total: 993, Duration: 4 m 17 s - ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests.dll (net10.0)
|
||||
Passed! - Failed: 0, Passed: 124, Skipped: 0, Total: 124, Duration: 4 m 28 s - ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.dll (net10.0)
|
||||
@@ -72,7 +72,13 @@ public sealed class TelemetryDialOptions
|
||||
/// <summary>How often central refreshes its set of dialable node contacts, in seconds.</summary>
|
||||
public int ContactRefreshSeconds { get; set; } = 60;
|
||||
|
||||
/// <summary>Per-call deadline for a gRPC telemetry dial, in seconds.</summary>
|
||||
/// <summary>
|
||||
/// Deadline, in seconds, for a gRPC telemetry dial to deliver its first event. A live node answers a
|
||||
/// fresh subscribe at once (it replays its health/resilience snapshots), so a dial still silent at the
|
||||
/// deadline is one that never connected — it is cancelled and re-dialled. Bounds establishment only;
|
||||
/// an established stream is kept alive by HTTP/2 keepalive pings, not by this value. Values below 1
|
||||
/// are treated as 1.
|
||||
/// </summary>
|
||||
public int CallTimeoutSeconds { get; set; } = 30;
|
||||
}
|
||||
|
||||
|
||||
@@ -188,6 +188,13 @@ public sealed class DiscoveryDriverBrowser : IUniversalDriverBrowser
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Final teardown of a throwaway capture/probe driver: a bounded <see cref="IDriver.ShutdownAsync"/>
|
||||
/// followed by a bounded dispose. The dispose is NOT optional and runs even after a failed or timed-out
|
||||
/// shutdown: a Galaxy driver's <c>ShutdownAsync</c> releases none of its gRPC runtime (channels, MX
|
||||
/// session, stream loops) — only <c>DisposeAsync</c> does — so a browse that stopped at shutdown leaked a
|
||||
/// full runtime per <c>/raw</c> browse and per <see cref="CanBrowse"/> probe (02/L1, 06/G-1).
|
||||
/// </summary>
|
||||
private async Task BoundedShutdownAsync(IDriver driver, string driverType)
|
||||
{
|
||||
try
|
||||
@@ -199,7 +206,25 @@ public sealed class DiscoveryDriverBrowser : IUniversalDriverBrowser
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Browse-capture shutdown for {DriverType} failed/timed out; instance abandoned", driverType);
|
||||
"Browse-capture shutdown for {DriverType} failed/timed out; disposing the instance anyway", driverType);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
switch (driver)
|
||||
{
|
||||
case IAsyncDisposable asyncDisposable:
|
||||
await asyncDisposable.DisposeAsync().AsTask().WaitAsync(_shutdownTimeout).ConfigureAwait(false);
|
||||
break;
|
||||
case IDisposable disposable:
|
||||
disposable.Dispose();
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Browse-capture dispose for {DriverType} failed/timed out; instance abandoned", driverType);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,10 @@
|
||||
Directory.Packages.props. This is the one project where the DataProtection chain
|
||||
enters the repo; every other consumer reaches it through a ProjectReference here. -->
|
||||
<PackageReference Include="System.Security.Cryptography.Xml"/>
|
||||
<!-- Surgical transitive pin — see the System.Security.Cryptography.Pkcs block in
|
||||
Directory.Packages.props. Microsoft.Data.SqlClient (via EFCore.SqlServer) enters here
|
||||
too and pulls a vulnerable Pkcs 9.0.4 wherever the shared framework prunes the Xml chain. -->
|
||||
<PackageReference Include="System.Security.Cryptography.Pkcs"/>
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions"/>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions"/>
|
||||
<PackageReference Include="Polly.Core"/>
|
||||
|
||||
+68
-21
@@ -67,6 +67,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <summary>The tie-cluster limit the crowding re-reads honour — exposed so the factory wiring is testable.</summary>
|
||||
internal int MaxPreWindowRows => _maxPreWindowRows;
|
||||
|
||||
/// <summary>
|
||||
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one raw read may
|
||||
/// return; 0 = none. See <see cref="MaxRawValuesPerRead"/> and <see cref="ReadRawWindowAsync"/>.
|
||||
/// </summary>
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <summary>A strict gateway's row ceiling, learned from its first refusal.</summary>
|
||||
private readonly GatewayRowCeiling _ceiling;
|
||||
private readonly GatewayTagPresenceCache _tags;
|
||||
@@ -96,11 +102,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// The most rows before a raw window a crowding re-read makes room for (<c>ServerHistorian:MaxTieClusterOverfetch</c>,
|
||||
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
|
||||
/// </param>
|
||||
/// <param name="maxRawValuesPerRead">
|
||||
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
|
||||
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/> (bounded further by a strict
|
||||
/// gateway's learned ceiling), so the server pages at it. A read asking for more (or for everything) asks the gateway
|
||||
/// for cap + 1 rows and is refused when the window holds more, never truncated. <= 0 = unbounded (the pre-#736
|
||||
/// behaviour: an uncapped read is sent as <see cref="int.MaxValue"/>).
|
||||
/// </param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
|
||||
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
|
||||
_atTimeFromRaw = atTimeFromRaw;
|
||||
ArgumentNullException.ThrowIfNull(client);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
@@ -203,6 +217,13 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// a cancelled gateway stream evicts its pooled historian session). <c>ReturnBounds</c> is not plumbed to the
|
||||
/// data source on any backend, so bounding values are never synthesised here.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The server raw cap</b> (R08:1.1, #736). With a cap, a read asking for everything (<c>0</c>) or for more than
|
||||
/// the cap runs as a capped read of cap + 1 — one more than it may return — so an over-cap window is DETECTED and
|
||||
/// refused (<see cref="HistoryReadLimitExceededException"/>) rather than answered truncated (the server's paging
|
||||
/// reads a short answer as the last page) and never sent as <see cref="int.MaxValue"/>. The server itself never asks
|
||||
/// for more than the cap (it pages at <see cref="MaxRawValuesPerRead"/>); this bounds a caller that bypasses it.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<List<HistorianSample>> ReadRawWindowAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
@@ -210,30 +231,39 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
if (cap == 0)
|
||||
var serverCap = _maxRawValuesPerRead;
|
||||
// Under the server raw cap, a read asking for everything or for more than the cap runs as a capped read of
|
||||
// cap + 1 (budgetCap); `cap` stays what the client asked for, and a refusal names the cap that applied (limit).
|
||||
var probe = serverCap > 0 && (cap == 0 || cap > serverCap);
|
||||
var budgetCap = probe ? (int)Math.Min(int.MaxValue, (long)serverCap + 1) : cap;
|
||||
var limit = probe ? serverCap : cap;
|
||||
var advice = probe
|
||||
? $"set NumValuesPerNode at or below ServerHistorian:MaxRawValuesPerRead ({serverCap}) to page"
|
||||
: cap == 0 ? "set NumValuesPerNode to page" : "lower NumValuesPerNode";
|
||||
if (budgetCap == 0)
|
||||
{
|
||||
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
|
||||
if (all.Result.Complete) return all.Result.Samples;
|
||||
// Only reachable on a strict gateway: the ceiling re-read delivered the whole ceiling without reaching
|
||||
// the window end — the window may hold more, and "all values" is never answered from a truncated read.
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
throw new HistoryReadLimitExceededException(limit,
|
||||
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(all.Result.PreWindowRows) };
|
||||
}
|
||||
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)budgetCap + RawAggregateReader.PreWindowAllowance);
|
||||
var pass = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
if (pass.Result.Complete) return Bounded(pass.Result.Samples);
|
||||
|
||||
// A strict gateway's ceiling already bounded that read: a larger budget would only be refused again. The
|
||||
// refusal carries the pre-window rows it saw: the server re-reads the page that many values smaller.
|
||||
if (pass.Budget < asked)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
throw new HistoryReadLimitExceededException(limit,
|
||||
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.")
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); {advice}, " +
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(pass.Result.PreWindowRows) };
|
||||
|
||||
// The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
|
||||
@@ -245,25 +275,25 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows);
|
||||
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
|
||||
{
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)budgetCap + room + RawAggregateReader.PreWindowAllowance);
|
||||
// doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
|
||||
if (_ceiling.Learned is var known and > 0 && budget > known)
|
||||
{
|
||||
var available = known - cap - RawAggregateReader.PreWindowAllowance;
|
||||
var available = known - budgetCap - RawAggregateReader.PreWindowAllowance;
|
||||
if (available <= 0) break;
|
||||
room = Math.Min(room, available);
|
||||
budget = cap + room + RawAggregateReader.PreWindowAllowance;
|
||||
budget = budgetCap + room + RawAggregateReader.PreWindowAllowance;
|
||||
}
|
||||
if (budget <= pass.Budget) break; // no more room, never retry the same full read
|
||||
pass = await PassAsync(budget).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
if (pass.Result.Complete) return Bounded(pass.Result.Samples);
|
||||
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
|
||||
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
|
||||
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
|
||||
room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room));
|
||||
}
|
||||
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
throw new HistoryReadLimitExceededException(limit,
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
|
||||
{ PreWindowRows = preWindow };
|
||||
@@ -271,10 +301,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause.
|
||||
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
|
||||
|
||||
// The cap + 1 probe came back with one sample too many: the window holds more than the server raw cap.
|
||||
List<HistorianSample> Bounded(List<HistorianSample> samples)
|
||||
{
|
||||
if (probe && samples.Count > serverCap)
|
||||
throw new HistoryReadLimitExceededException(serverCap,
|
||||
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({serverCap}) " +
|
||||
"samples; set NumValuesPerNode at or below it to page, or narrow the window.");
|
||||
return samples;
|
||||
}
|
||||
|
||||
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names.
|
||||
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
|
||||
_ceiling.ReadAsync(budget,
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken),
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, budgetCap, b, cancellationToken),
|
||||
(r, _) => !r.Complete);
|
||||
}
|
||||
|
||||
@@ -531,13 +571,20 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
public bool SupportsReverseRaw => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
// 0 until a strict gateway (AllowStreamingBeyondCeiling=false) has refused a budget and
|
||||
// named its ceiling (GatewayRowCeiling); then that ceiling less the
|
||||
// RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row — so a page of this
|
||||
// size asks for exactly the ceiling. Back to 0 when a re-probe finds the gateway streaming again.
|
||||
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
|
||||
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
|
||||
: 0;
|
||||
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none), bounded further by a strict gateway's
|
||||
// (AllowStreamingBeyondCeiling=false) learned ceiling once a refusal has named it (GatewayRowCeiling) — that
|
||||
// ceiling less the RawAggregateReader.PreWindowAllowance a capped read adds for AVEVA's boundary row, so a page of
|
||||
// this size asks for exactly the ceiling. The learned part is forgotten when a re-probe finds the gateway streaming
|
||||
// again; the cap stays. 0 only with no cap and no learned ceiling.
|
||||
public int MaxRawValuesPerRead
|
||||
{
|
||||
get
|
||||
{
|
||||
var learned = _ceiling.Learned is var c and > 0 ? Math.Max(1, c - RawAggregateReader.PreWindowAllowance) : 0;
|
||||
var cap = _maxRawValuesPerRead;
|
||||
return cap > 0 && learned > 0 ? Math.Min(cap, learned) : cap > 0 ? cap : learned;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
// false: the gateway's ReadEvents streams ascending only. The server never sends it a
|
||||
|
||||
+3
-1
@@ -44,7 +44,9 @@ public static class GatewayHistorian
|
||||
// AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch).
|
||||
atTimeFromRaw: options.GatewayAtTimeFromRaw,
|
||||
// A crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch);
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch,
|
||||
// The server raw cap (R08:1.1): the page limit the server reads at, and the bound on every raw read.
|
||||
maxRawValuesPerRead: options.EffectiveMaxRawValuesPerRead);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+34
-4
@@ -66,6 +66,7 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
private readonly TimeProvider _time;
|
||||
private readonly HistorianHealthCounters _health = new();
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
@@ -74,11 +75,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <param name="log">Optional logger.</param>
|
||||
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">
|
||||
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
|
||||
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/>, so the server pages at it.
|
||||
/// A read asking for more (or for everything) is refused when the window holds more, never truncated.
|
||||
/// <= 0 = unbounded (the pre-#736 behaviour).
|
||||
/// </param>
|
||||
internal QuestDbHistorianDataSource(
|
||||
QuestDbConnection conn, QuestDbSchemaBootstrapper bootstrap, QuestDbTagCatalog catalog, ILogger? log = null,
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows,
|
||||
int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
|
||||
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
|
||||
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
|
||||
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
|
||||
@@ -93,12 +102,21 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
// (int.MaxValue); larger values clamp. At most top samples, ContinuationPoint always null: the node
|
||||
// manager pages by re-reading from the last returned timestamp inclusive, which the (ts, tag) dedup
|
||||
// key makes exact.
|
||||
// The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads
|
||||
// cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused
|
||||
// (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging
|
||||
// reads a short answer as the last page, so a silent LIMIT would lose data. The server itself never asks for more
|
||||
// than the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it.
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoryReadResult> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fullReference);
|
||||
var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var cap = _maxRawValuesPerRead;
|
||||
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
|
||||
var top = probe
|
||||
? (int)Math.Min(int.MaxValue, (long)cap + 1)
|
||||
: maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
try
|
||||
{
|
||||
var samples = new List<DataValueSnapshot>();
|
||||
@@ -129,9 +147,19 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
}
|
||||
}
|
||||
|
||||
if (probe && samples.Count > cap)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " +
|
||||
"set NumValuesPerNode at or below it to page, or narrow the window.");
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(samples, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the server raw cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
@@ -360,9 +388,11 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
|
||||
/// <inheritdoc />
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
// 0: a capped raw read is one bounded statement - no backend row ceiling.
|
||||
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a
|
||||
// client's NumValuesPerNode = 0 is served as pages of at most this many values. QuestDB itself has no row ceiling -
|
||||
// a capped raw read is one bounded statement.
|
||||
/// <inheritdoc />
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
public int MaxRawValuesPerRead => _maxRawValuesPerRead;
|
||||
|
||||
// Reverse events reads are served natively (ORDER BY ... DESC).
|
||||
/// <inheritdoc />
|
||||
|
||||
@@ -35,14 +35,17 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
|
||||
/// <summary>Creates the provider over the <c>ServerHistorian:QuestDb</c> options (validated elsewhere).</summary>
|
||||
/// <param name="options">The <c>ServerHistorian:QuestDb</c> options.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">The shared <c>ServerHistorian:MaxRawValuesPerRead</c> raw cap (<= 0 = unbounded; R08:1.1).</param>
|
||||
public QuestDbHistorianProvider(QuestDbHistorianOptions options,
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_maxRawValuesPerRead = maxRawValuesPerRead;
|
||||
}
|
||||
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "QuestDb";
|
||||
@@ -75,7 +78,8 @@ public sealed class QuestDbHistorianProvider : IHistorianBackendProvider, IAsync
|
||||
{
|
||||
var s = Get(sp);
|
||||
return new QuestDbHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
|
||||
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
|
||||
Loggers(sp).CreateLogger<QuestDbHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows,
|
||||
maxRawValuesPerRead: _maxRawValuesPerRead);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
+36
-4
@@ -61,6 +61,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
|
||||
private readonly HistorianHealthCounters _health = new();
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <summary>Creates the data source over the provider's shared connection, bootstrapper and tag catalog.</summary>
|
||||
/// <param name="conn">Connection helper.</param>
|
||||
@@ -69,11 +70,19 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// <param name="log">Optional logger.</param>
|
||||
/// <param name="timeProvider">Clock for "now" (AtTime beyond the last sample, server timestamps); system by default.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">Raw in-window row cap of one time-weighted read (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">
|
||||
/// The server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>, R08:1.1): the most samples one
|
||||
/// <see cref="ReadRawAsync"/> may return — reported as <see cref="MaxRawValuesPerRead"/>, so the server pages at it.
|
||||
/// A read asking for more (or for everything) is refused when the window holds more, never truncated.
|
||||
/// <= 0 = unbounded (the pre-#736 behaviour).
|
||||
/// </param>
|
||||
internal SqlServerHistorianDataSource(
|
||||
SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap, SqlServerTagCatalog catalog, ILogger? log = null,
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
TimeProvider? timeProvider = null, int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows,
|
||||
int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
|
||||
_conn = conn ?? throw new ArgumentNullException(nameof(conn));
|
||||
_bootstrap = bootstrap ?? throw new ArgumentNullException(nameof(bootstrap));
|
||||
_catalog = catalog ?? throw new ArgumentNullException(nameof(catalog));
|
||||
@@ -90,13 +99,22 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
// <c>0</c> is the node manager's "unbounded" sentinel; values above <see cref="int.MaxValue"/> clamp.
|
||||
// The node manager pages by re-reading from the last returned timestamp inclusive and trimming it; the
|
||||
// <c>(TagId, SourceTimeUtc)</c> primary key rules out tie clusters, so that is exact.
|
||||
// The server raw cap (R08:1.1, #736): with a cap, a read asking for everything or for more than the cap reads
|
||||
// cap + 1 rows — one more than it may return — so an over-cap window is DETECTED and refused
|
||||
// (HistoryReadLimitExceededException ⇒ BadTooManyOperations) rather than answered truncated: the server's paging
|
||||
// reads a short answer as the last page, so a silent TOP would lose data. The server itself never asks for more than
|
||||
// the cap (it pages at MaxRawValuesPerRead); this bounds a caller that bypasses it.
|
||||
// </remarks>
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoryReadResult> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(fullReference);
|
||||
var top = maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var cap = _maxRawValuesPerRead;
|
||||
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
|
||||
var top = probe
|
||||
? (int)Math.Min(int.MaxValue, (long)cap + 1)
|
||||
: maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
try
|
||||
{
|
||||
var tagId = await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
|
||||
@@ -126,9 +144,19 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
}
|
||||
}
|
||||
|
||||
if (probe && samples.Count > cap)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than ServerHistorian:MaxRawValuesPerRead ({cap}) samples; " +
|
||||
"set NumValuesPerNode at or below it to page, or narrow the window.");
|
||||
|
||||
RecordSuccess();
|
||||
return new HistoryReadResult(samples, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the server raw cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordFailure(ex);
|
||||
@@ -319,9 +347,13 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
|
||||
/// <inheritdoc />
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
// <remarks><c>0</c>: a capped raw read is one bounded statement — no backend row ceiling.</remarks>
|
||||
// <remarks>
|
||||
// The server raw cap (ServerHistorian:MaxRawValuesPerRead; 0 = none): the server pages every raw read at it, so a
|
||||
// client's NumValuesPerNode = 0 is served as pages of at most this many values. SQL Server itself has no row
|
||||
// ceiling — a capped raw read is one bounded statement.
|
||||
// </remarks>
|
||||
/// <inheritdoc />
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
public int MaxRawValuesPerRead => _maxRawValuesPerRead;
|
||||
|
||||
// <remarks>Reverse events reads are served natively (<c>ORDER BY … DESC</c>).</remarks>
|
||||
/// <inheritdoc />
|
||||
|
||||
+6
-2
@@ -31,14 +31,17 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
|
||||
/// <summary>Creates the provider over the <c>ServerHistorian:SqlServer</c> options (validated elsewhere).</summary>
|
||||
/// <param name="options">The <c>ServerHistorian:SqlServer</c> options.</param>
|
||||
/// <param name="maxTimeWeightedRawRows">The shared <c>ServerHistorian:MaxTimeWeightedRawRows</c> cap (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">The shared <c>ServerHistorian:MaxRawValuesPerRead</c> raw cap (<= 0 = unbounded; R08:1.1).</param>
|
||||
public SqlServerHistorianProvider(SqlServerHistorianOptions options,
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows)
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_maxTimeWeightedRawRows = maxTimeWeightedRawRows;
|
||||
_maxRawValuesPerRead = maxRawValuesPerRead;
|
||||
}
|
||||
|
||||
private readonly int _maxTimeWeightedRawRows;
|
||||
private readonly int _maxRawValuesPerRead;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "SqlServer";
|
||||
@@ -69,7 +72,8 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
|
||||
{
|
||||
var s = Get(sp);
|
||||
return new SqlServerHistorianDataSource(s.Conn, s.Bootstrap, s.Catalog,
|
||||
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows);
|
||||
Loggers(sp).CreateLogger<SqlServerHistorianDataSource>(), maxTimeWeightedRawRows: _maxTimeWeightedRawRows,
|
||||
maxRawValuesPerRead: _maxRawValuesPerRead);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
|
||||
+4
@@ -26,6 +26,10 @@
|
||||
provider's DbProviderFactory (NpgsqlFactory / OracleClientFactory); both packages are fully
|
||||
managed ADO.NET drivers, so adding them costs no native client install anywhere. -->
|
||||
<PackageReference Include="Microsoft.Data.SqlClient" />
|
||||
<!-- Surgical transitive pin — see the System.Security.Cryptography.Pkcs block in
|
||||
Directory.Packages.props. SqlClient 6.1.1 pulls a vulnerable Pkcs 9.0.4; this is the
|
||||
second place the SqlClient chain enters the repo (Core.Configuration is the first). -->
|
||||
<PackageReference Include="System.Security.Cryptography.Pkcs" />
|
||||
<PackageReference Include="Npgsql" />
|
||||
<PackageReference Include="Oracle.ManagedDataAccess.Core" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
@@ -13,7 +15,9 @@ namespace ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
|
||||
/// Sends <see cref="GetDiagnostics"/>; expects a <see cref="NodeDiagnosticsSnapshot"/> reply.
|
||||
///
|
||||
/// On timeout or any other failure (peer down, GetDiagnostics handler missing) returns an
|
||||
/// empty snapshot so the UI degrades to "no data" instead of throwing.
|
||||
/// empty snapshot so the UI degrades to "no data" instead of throwing. The failure is logged at
|
||||
/// Warning, since the snapshot shape cannot carry it: an empty snapshot reads the same as a node with
|
||||
/// no drivers. Consumed by the <c>/fleet/diagnostics</c> page.
|
||||
/// </summary>
|
||||
public sealed class FleetDiagnosticsClient : IFleetDiagnosticsClient
|
||||
{
|
||||
@@ -21,29 +25,38 @@ public sealed class FleetDiagnosticsClient : IFleetDiagnosticsClient
|
||||
|
||||
private readonly ActorSystem _system;
|
||||
private readonly string _systemName;
|
||||
private readonly ILogger<FleetDiagnosticsClient> _log;
|
||||
|
||||
/// <summary>Initializes a new FleetDiagnosticsClient with the given actor system and cluster options.</summary>
|
||||
/// <param name="system">The Akka actor system.</param>
|
||||
/// <param name="options">Cluster configuration options.</param>
|
||||
public FleetDiagnosticsClient(ActorSystem system, IOptions<AkkaClusterOptions> options)
|
||||
/// <param name="log">Logger for the degraded-to-empty path; defaults to a no-op logger.</param>
|
||||
public FleetDiagnosticsClient(
|
||||
ActorSystem system, IOptions<AkkaClusterOptions> options, ILogger<FleetDiagnosticsClient>? log = null)
|
||||
{
|
||||
_system = system;
|
||||
_systemName = options.Value.SystemName;
|
||||
_log = log ?? NullLogger<FleetDiagnosticsClient>.Instance;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<NodeDiagnosticsSnapshot> GetDiagnosticsAsync(NodeId nodeId, CancellationToken ct)
|
||||
{
|
||||
var selection = _system.ActorSelection($"akka.tcp://{_systemName}@{nodeId.Value}/user/driver-host");
|
||||
try
|
||||
{
|
||||
// Inside the try: a malformed node value makes ActorSelection throw, and that must degrade to
|
||||
// "unavailable" like a dead node does, not surface through the page's error boundary.
|
||||
var selection = _system.ActorSelection($"akka.tcp://{_systemName}@{nodeId.Value}/user/driver-host");
|
||||
using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
linked.CancelAfter(AskTimeout);
|
||||
return await selection.Ask<NodeDiagnosticsSnapshot>(
|
||||
new GetDiagnostics(CorrelationId.NewId()), AskTimeout, linked.Token);
|
||||
}
|
||||
catch (Exception)
|
||||
catch (Exception ex) when (!ct.IsCancellationRequested)
|
||||
{
|
||||
_log.LogWarning(ex,
|
||||
"Fleet diagnostics for node {NodeId} unavailable (no answer within {Timeout}s); returning an empty snapshot",
|
||||
nodeId.Value, AskTimeout.TotalSeconds);
|
||||
return new NodeDiagnosticsSnapshot(
|
||||
nodeId,
|
||||
CurrentRevision: null,
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
@page "/fleet/diagnostics"
|
||||
@* Per-node driver diagnostics, fetched on demand from the selected driver node's DriverHostActor over
|
||||
Akka (IFleetDiagnosticsClient → GetDiagnostics → NodeDiagnosticsSnapshot). The node answers from
|
||||
its live child set + each child's last-published health; the "running from cache" flag is the one
|
||||
fact /hosts cannot show (a node booted from its artifact cache looks healthy but is frozen).
|
||||
Reach it as /fleet/diagnostics?node=<host:port> (the NodeId shown on /fleet and /hosts), or pick
|
||||
a driver-role member from the selector. *@
|
||||
@attribute [Authorize(Policy = AdminUiPolicies.AuthenticatedRead)]
|
||||
@rendermode RenderMode.InteractiveServer
|
||||
@using ZB.MOM.WW.OtOpcUa.Cluster
|
||||
@using ZB.MOM.WW.OtOpcUa.Commons.Interfaces
|
||||
@using ZB.MOM.WW.OtOpcUa.Commons.Types
|
||||
@inject IClusterRoleInfo Cluster
|
||||
@inject IFleetDiagnosticsClient Diagnostics
|
||||
@inject NavigationManager Nav
|
||||
@implements IDisposable
|
||||
|
||||
<PageErrorBoundary>
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<h4 class="mb-0">Node diagnostics</h4>
|
||||
<a class="btn btn-sm btn-outline-secondary" href="/fleet">Back to fleet status</a>
|
||||
</div>
|
||||
|
||||
<div class="d-flex align-items-center mb-3 gap-2 flex-wrap">
|
||||
<label class="small text-muted mb-0" for="diag-node">Node</label>
|
||||
<select id="diag-node" class="form-select form-select-sm" style="width:auto"
|
||||
value="@_selectedNode" @onchange="OnNodeSelected">
|
||||
<option value="">— select a driver node —</option>
|
||||
@foreach (var member in _members)
|
||||
{
|
||||
<option value="@member">@member</option>
|
||||
}
|
||||
@if (!string.IsNullOrEmpty(_selectedNode) && !_members.Contains(_selectedNode))
|
||||
{
|
||||
<option value="@_selectedNode">@_selectedNode (not an Up member)</option>
|
||||
}
|
||||
</select>
|
||||
<input type="text" class="form-control form-control-sm" style="width:220px"
|
||||
placeholder="or type host:port…" @bind="_typedNode" @bind:event="oninput"
|
||||
@onkeydown="OnTypedKeyDown" />
|
||||
<button class="btn btn-sm btn-outline-primary" @onclick="UseTypedNodeAsync"
|
||||
disabled="@(string.IsNullOrWhiteSpace(_typedNode))">Go</button>
|
||||
<button class="btn btn-sm btn-outline-primary" @onclick="RefreshAsync"
|
||||
disabled="@(_loading || string.IsNullOrEmpty(_selectedNode))">
|
||||
@if (_loading) { <span class="spinner-border spinner-border-sm me-1" /> }
|
||||
Refresh
|
||||
</button>
|
||||
<span class="text-muted small">
|
||||
Last fetched: @(_fetchedUtc?.ToString("HH:mm:ss 'UTC'") ?? "—")
|
||||
</span>
|
||||
</div>
|
||||
|
||||
@if (string.IsNullOrEmpty(_selectedNode))
|
||||
{
|
||||
<section class="panel notice rise" style="animation-delay:.02s">
|
||||
Pick a driver-role node to fetch its driver diagnostics. The node answers from its own
|
||||
DriverHostActor, so this works for any node reachable over Akka remoting from this admin node.
|
||||
</section>
|
||||
}
|
||||
else if (_snapshot is null)
|
||||
{
|
||||
<p>@(_loading ? "Fetching…" : "No snapshot yet.")</p>
|
||||
}
|
||||
else
|
||||
{
|
||||
<section class="agg-grid rise" style="animation-delay:.02s">
|
||||
<div class="agg-card">
|
||||
<div class="agg-label">Node</div>
|
||||
<div class="agg-value mono">@_snapshot.NodeId.Value</div>
|
||||
</div>
|
||||
<div class="agg-card">
|
||||
<div class="agg-label">Revision</div>
|
||||
<div class="agg-value mono" title="@(_snapshot.CurrentRevision?.Value ?? "")">
|
||||
@(_snapshot.CurrentRevision is { } rev ? DisplayText.Abbreviate(rev.Value, 12) : "—")
|
||||
</div>
|
||||
</div>
|
||||
<div class="agg-card @(_snapshot.RunningFromCache ? "caution" : "")">
|
||||
<div class="agg-label">Config source</div>
|
||||
<div class="agg-value">@(_snapshot.RunningFromCache ? "cache (frozen)" : "live")</div>
|
||||
</div>
|
||||
<div class="agg-card">
|
||||
<div class="agg-label">Drivers</div>
|
||||
<div class="agg-value numeric">@_snapshot.Drivers.Count</div>
|
||||
</div>
|
||||
<div class="agg-card alert">
|
||||
<div class="agg-label">Faulted devices</div>
|
||||
<div class="agg-value numeric">@_snapshot.Drivers.Sum(d => d.FaultedDevices)</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@if (_snapshot.RunningFromCache)
|
||||
{
|
||||
<section class="panel notice rise" style="animation-delay:.05s">
|
||||
This node booted its configuration from its local artifact cache because central was unreachable.
|
||||
It serves its last-known-good address space, but no deployment can reach it until central is back.
|
||||
</section>
|
||||
}
|
||||
|
||||
<section class="panel rise" style="animation-delay:.08s">
|
||||
<div class="panel-head">
|
||||
Drivers
|
||||
<span class="text-muted small ms-2">as of @_snapshot.AsOfUtc.ToString("u")</span>
|
||||
</div>
|
||||
@if (_snapshot.Drivers.Count == 0)
|
||||
{
|
||||
<div class="p-3 text-muted">
|
||||
Empty snapshot: the node has no driver children — or it did not answer within the client's
|
||||
3 s budget (an unreachable node degrades to an empty snapshot; see the AdminUI log for the
|
||||
FleetDiagnosticsClient warning).
|
||||
</div>
|
||||
}
|
||||
else
|
||||
{
|
||||
<div class="table-wrap">
|
||||
<table class="data-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Driver instance</th>
|
||||
<th>Row id</th>
|
||||
<th>State</th>
|
||||
<th class="numeric">Connected devices</th>
|
||||
<th class="numeric">Faulted devices</th>
|
||||
<th title="When the driver child last published a changed health snapshot (its spawn time until the first)">Last health publish</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
@foreach (var d in _snapshot.Drivers.OrderBy(d => d.Name, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
<tr>
|
||||
<td>@d.Name</td>
|
||||
<td><span class="mono" title="@d.DriverInstanceId">@(d.DriverInstanceId == Guid.Empty ? "—" : DisplayText.Abbreviate(d.DriverInstanceId.ToString(), 13))</span></td>
|
||||
<td><span class="chip @StateChipClass(d.State)">@d.State</span></td>
|
||||
<td class="numeric">@d.ConnectedDevices</td>
|
||||
<td class="numeric">@d.FaultedDevices</td>
|
||||
<td>@d.LastChangeUtc.ToString("u")</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
}
|
||||
</section>
|
||||
}
|
||||
</PageErrorBoundary>
|
||||
|
||||
@code {
|
||||
[SupplyParameterFromQuery(Name = "node")]
|
||||
public string? Node { get; set; }
|
||||
|
||||
private IReadOnlyList<string> _members = Array.Empty<string>();
|
||||
private string _selectedNode = "";
|
||||
private string _typedNode = "";
|
||||
private NodeDiagnosticsSnapshot? _snapshot;
|
||||
private bool _loading;
|
||||
private DateTime? _fetchedUtc;
|
||||
|
||||
// Page-lifetime cancellation: a fetch in flight when the circuit goes away is abandoned.
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private volatile bool _disposed;
|
||||
|
||||
protected override async Task OnParametersSetAsync()
|
||||
{
|
||||
_members = Cluster.MembersWithRole(RoleParser.Driver)
|
||||
.Select(n => n.Value)
|
||||
.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
var requested = Node?.Trim() ?? "";
|
||||
if (!string.Equals(requested, _selectedNode, StringComparison.Ordinal))
|
||||
{
|
||||
_selectedNode = requested;
|
||||
_snapshot = null;
|
||||
if (!string.IsNullOrEmpty(_selectedNode)) await LoadAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private Task OnNodeSelected(ChangeEventArgs e)
|
||||
{
|
||||
var value = e.Value?.ToString() ?? "";
|
||||
Navigate(value);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private Task UseTypedNodeAsync()
|
||||
{
|
||||
Navigate(_typedNode.Trim());
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private Task OnTypedKeyDown(KeyboardEventArgs e)
|
||||
{
|
||||
if (e.Key == "Enter") Navigate(_typedNode.Trim());
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
// The node id lives in the URL so a diagnostics view is linkable from /fleet and survives a reload.
|
||||
private void Navigate(string node) =>
|
||||
Nav.NavigateTo(string.IsNullOrEmpty(node)
|
||||
? "/fleet/diagnostics"
|
||||
: "/fleet/diagnostics?node=" + Uri.EscapeDataString(node));
|
||||
|
||||
private async Task RefreshAsync() => await LoadAsync();
|
||||
|
||||
private async Task LoadAsync()
|
||||
{
|
||||
if (_loading || string.IsNullOrEmpty(_selectedNode)) return;
|
||||
_loading = true;
|
||||
try
|
||||
{
|
||||
// The client is fail-safe by contract: a node that does not answer yields an empty snapshot.
|
||||
_snapshot = await Diagnostics.GetDiagnosticsAsync(NodeId.Parse(_selectedNode), _cts.Token);
|
||||
_fetchedUtc = DateTime.UtcNow;
|
||||
}
|
||||
catch (Exception ex) when (ex is OperationCanceledException or ObjectDisposedException)
|
||||
{
|
||||
// Circuit gone mid-fetch — a clean exit, not a fault.
|
||||
}
|
||||
finally
|
||||
{
|
||||
_loading = false;
|
||||
if (!_disposed) StateHasChanged();
|
||||
}
|
||||
}
|
||||
|
||||
private static string StateChipClass(string state) => state switch
|
||||
{
|
||||
"Healthy" => "chip-ok",
|
||||
"Degraded" or "Connecting" or "Reconnecting" => "chip-caution",
|
||||
"Faulted" => "chip-alert",
|
||||
_ => "chip-idle",
|
||||
};
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_disposed = true;
|
||||
_cts.Cancel();
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -129,6 +129,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
|
||||
Receive<TelemetryReceived>(HandleTelemetryReceived);
|
||||
Receive<StreamStopped>(HandleStreamStopped);
|
||||
Receive<Reconnect>(HandleReconnect);
|
||||
Receive<DialDeadline>(HandleDialDeadline);
|
||||
|
||||
// A faulted node-source task pipes here. Log and wait for the next periodic refresh — a stale
|
||||
// node set is far better than crashing the supervisor and dropping every live dialer.
|
||||
@@ -251,6 +252,9 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
|
||||
return;
|
||||
}
|
||||
|
||||
// The stream is over; a still-armed establishment deadline for it must not fire into the reconnect.
|
||||
Timers.Cancel(DeadlineTimerKey(msg.NodeId));
|
||||
|
||||
if (dialer.Connected)
|
||||
{
|
||||
dialer.Connected = false;
|
||||
@@ -294,6 +298,34 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>TelemetryDial:CallTimeoutSeconds</c>: a dial that has delivered no event by the deadline is cut and
|
||||
/// treated as a transport failure (reconnect with a failure streak). <c>Connected</c> flips on the first
|
||||
/// routed event, on this thread, so a dial that connected in time is left alone — the deadline bounds
|
||||
/// ESTABLISHMENT only. A live node answers a fresh subscribe at once (the node-local hub replays its
|
||||
/// health/resilience snapshots), so what this catches is a dial that never connects: a black-holed host,
|
||||
/// where the socket connect itself has no timeout and the stream would otherwise hang forever without a
|
||||
/// reconnect. An established stream is never bounded here — the wire has no heartbeat, keepalive pings own
|
||||
/// its liveness, and an idle cut would churn every quiet node.
|
||||
/// </summary>
|
||||
private void HandleDialDeadline(DialDeadline msg)
|
||||
{
|
||||
if (!_dialers.TryGetValue(msg.NodeId, out var dialer) || msg.Generation != dialer.Generation || dialer.Connected)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Warning(
|
||||
"Telemetry dial to node {NodeId} ({Endpoint}) delivered nothing within {Deadline}s; cancelling it and re-dialing",
|
||||
msg.NodeId, dialer.Target.Endpoint, DialDeadlineFor().TotalSeconds);
|
||||
dialer.Cancel();
|
||||
HandleStreamStopped(new StreamStopped(
|
||||
msg.NodeId, msg.Generation,
|
||||
new TimeoutException($"no telemetry event within {DialDeadlineFor().TotalSeconds:F0}s of dialing")));
|
||||
}
|
||||
|
||||
private TimeSpan DialDeadlineFor() => TimeSpan.FromSeconds(Math.Max(1, _options.CallTimeoutSeconds));
|
||||
|
||||
private void StartDialer(string nodeId)
|
||||
{
|
||||
var dialer = _dialers[nodeId];
|
||||
@@ -306,6 +338,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
|
||||
var cts = new CancellationTokenSource();
|
||||
dialer.Cts = cts;
|
||||
dialer.Connected = false;
|
||||
// One deadline per generation; a superseded tick is dropped by the generation guard in the handler.
|
||||
Timers.StartSingleTimer(DeadlineTimerKey(nodeId), new DialDeadline(nodeId, generation), DialDeadlineFor());
|
||||
|
||||
var self = Self;
|
||||
var target = dialer.Target;
|
||||
@@ -357,6 +391,7 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
|
||||
{
|
||||
dialer.Cancel();
|
||||
Timers.Cancel(ReconnectTimerKey(nodeId));
|
||||
Timers.Cancel(DeadlineTimerKey(nodeId));
|
||||
RecomputePill();
|
||||
}
|
||||
}
|
||||
@@ -433,6 +468,8 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
|
||||
|
||||
private static string ReconnectTimerKey(string nodeId) => $"telemetry-reconnect-{nodeId}";
|
||||
|
||||
private static string DeadlineTimerKey(string nodeId) => $"telemetry-dial-deadline-{nodeId}";
|
||||
|
||||
/// <summary>Per-node dialer bookkeeping. Mutated only on the actor thread.</summary>
|
||||
private sealed class NodeDialer(TelemetryDialTarget target)
|
||||
{
|
||||
@@ -496,4 +533,10 @@ public sealed class TelemetryDialSupervisor : ReceiveActor, IWithTimers
|
||||
/// <summary>Backoff timer tick: start a fresh dialer for the node.</summary>
|
||||
/// <param name="NodeId">The node id to reconnect.</param>
|
||||
public sealed record Reconnect(string NodeId);
|
||||
|
||||
/// <summary>Dial-establishment deadline tick (<c>TelemetryDial:CallTimeoutSeconds</c>); see
|
||||
/// <see cref="HandleDialDeadline"/>.</summary>
|
||||
/// <param name="NodeId">The node whose dial is being checked.</param>
|
||||
/// <param name="Generation">The dialer generation the deadline was armed for.</param>
|
||||
public sealed record DialDeadline(string NodeId, int Generation);
|
||||
}
|
||||
|
||||
@@ -9,6 +9,12 @@
|
||||
<ItemGroup>
|
||||
<FrameworkReference Include="Microsoft.AspNetCore.App"/>
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client"/>
|
||||
<!-- Surgical transitive pin — see the Microsoft.AspNetCore.SignalR.Common block in
|
||||
Directory.Packages.props (GHSA-334f-vvp4-7g6p on SignalR.Common 10.0.0). Inside AdminUI
|
||||
the shared framework already supplies it (hence NU1510), but the PackageReference flows
|
||||
to the non-Web consumers (Host.Tests, AdminUI.Tests, OpcUaServer.IntegrationTests) where
|
||||
SignalR.Client 10.0.0 would otherwise resolve the vulnerable 10.0.0. -->
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Common" NoWarn="NU1510"/>
|
||||
<PackageReference Include="ZB.MOM.WW.Theme"/>
|
||||
<PackageReference Include="ZB.MOM.WW.Secrets.Ui"/>
|
||||
</ItemGroup>
|
||||
|
||||
@@ -45,8 +45,8 @@ public static class HistorianBackendRegistry
|
||||
new Dictionary<string, Func<ServerHistorianOptions, IConfiguration, IHistorianBackendProvider>>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
[ServerHistorianOptions.GatewayBackendName] = (o, _) => new GatewayHistorianProvider(o),
|
||||
[SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows),
|
||||
[QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows),
|
||||
[SqlServerBackendName] = (o, c) => new SqlServerHistorianProvider(BindSqlServer(c), o.MaxTimeWeightedRawRows, o.EffectiveMaxRawValuesPerRead),
|
||||
[QuestDbBackendName] = (o, c) => new QuestDbHistorianProvider(BindQuestDb(c), o.MaxTimeWeightedRawRows, o.EffectiveMaxRawValuesPerRead),
|
||||
};
|
||||
|
||||
/// <summary>The registered backend names, for validation messages.</summary>
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
@@ -110,24 +112,62 @@ internal enum HistoryCursorKind
|
||||
/// <c>NumValuesPerNode</c> samples with a null continuation point — so paging is synthesised here,
|
||||
/// time-based:
|
||||
/// <list type="bullet">
|
||||
/// <item>A page that returns EXACTLY the requested cap (<c>NumValuesPerNode > 0</c>) MAY have
|
||||
/// more behind it ⇒ emit a continuation point.</item>
|
||||
/// <item>A short page (fewer than the cap) is the last page ⇒ no continuation point.</item>
|
||||
/// <item><c>NumValuesPerNode == 0</c> means "all values, no limit" (OPC UA Part 11) ⇒ never page;
|
||||
/// return everything in one shot.</item>
|
||||
/// <item>A page whose read returned the lookahead row past it MAY have more behind it ⇒ emit a continuation
|
||||
/// point (<see cref="IsFullPage"/> judges a read against what was asked for).</item>
|
||||
/// <item>A short read (no lookahead row) is the last page ⇒ no continuation point.</item>
|
||||
/// <item><c>NumValuesPerNode == 0</c> means "all values" (OPC UA Part 11) — but not "in one shot": the page is
|
||||
/// sized to the backend's <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> by <see cref="RawPageLimit"/>
|
||||
/// (the server raw cap <c>ServerHistorian:MaxRawValuesPerRead</c>, default 10 000, and/or a strict gateway's
|
||||
/// learned ceiling; review R08:1.1, #736), and a full page carries a continuation point like any other. Only
|
||||
/// with no limit at all (<c>0</c>, the opt-out) is everything returned in one shot.</item>
|
||||
/// </list>
|
||||
/// All methods are static + pure so they unit-test without a server, a session, or the SDK.
|
||||
/// All paging methods are static + pure so they unit-test without a server, a session, or the SDK;
|
||||
/// <see cref="AdvertiseMaxReturnDataValues"/> is the one SDK-facing member.
|
||||
/// </summary>
|
||||
internal static class HistoryPaging
|
||||
{
|
||||
/// <summary>
|
||||
/// Decide whether a just-returned page is "full" and therefore MAY be followed by more data —
|
||||
/// the signal to emit a continuation point. A page is full when the client asked for a finite
|
||||
/// cap (<paramref name="numValuesPerNode"/> > 0) and the backend returned exactly that many
|
||||
/// samples. A short page (or an unlimited <c>0</c> request) is terminal.
|
||||
/// Advertise the server raw cap as <c>HistoryServerCapabilities/MaxReturnDataValues</c> (i=11273; Part 11 §5.4.2:
|
||||
/// "the maximum number of values that can be returned by the Server for each Node", 0 = no limit). The cap is
|
||||
/// the most values one HistoryRead-Raw page returns (<c>ServerHistorian:MaxRawValuesPerRead</c>), so a client can
|
||||
/// size its <c>NumValuesPerNode</c> and expect a continuation point past it. Idempotent; a re-sync with a
|
||||
/// different cap overwrites. Never throws: a missing diagnostics manager or node, or any SDK failure writing the
|
||||
/// property, leaves the SDK default (0) and returns <c>false</c> for the caller to log — HistoryRead itself still
|
||||
/// enforces the cap.
|
||||
/// </summary>
|
||||
/// <param name="returnedCount">The number of samples the backend returned for this page.</param>
|
||||
/// <param name="numValuesPerNode">The client's per-page cap; <c>0</c> means "no limit".</param>
|
||||
/// <param name="server">The running server.</param>
|
||||
/// <param name="maxRawValuesPerRead">The cap (<= 0 advertises 0 = no limit).</param>
|
||||
/// <returns><c>true</c> when the property was written.</returns>
|
||||
public static bool AdvertiseMaxReturnDataValues(IServerInternal? server, int maxRawValuesPerRead)
|
||||
{
|
||||
if (server?.DiagnosticsNodeManager is not DiagnosticsNodeManager diagnostics) return false;
|
||||
try
|
||||
{
|
||||
lock (diagnostics.Lock)
|
||||
{
|
||||
var capabilities = diagnostics.GetDefaultHistoryCapabilities();
|
||||
if (capabilities?.MaxReturnDataValues is not { } property) return false;
|
||||
property.Value = (uint)Math.Max(0, maxRawValuesPerRead);
|
||||
property.ClearChangeMasks(diagnostics.SystemContext, false);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return false; // advisory only: the caller logs; the cap is enforced by HistoryRead regardless
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decide whether a just-returned read is "full" and therefore MAY be followed by more data —
|
||||
/// the signal to emit a continuation point. A read is full when it was issued with a finite request
|
||||
/// cap (<paramref name="numValuesPerNode"/> > 0 — the <c>RawPagePlan.RequestCap</c>: page + boundary ties +
|
||||
/// lookahead, which under a backend limit or the server raw cap is always finite, uncapped client reads included)
|
||||
/// and the backend returned at least that many samples. A short read is terminal; so is a read issued with a
|
||||
/// request cap of <c>0</c>, which only happens with no limit at all (one shot of the whole window).
|
||||
/// </summary>
|
||||
/// <param name="returnedCount">The number of samples the backend returned for this read.</param>
|
||||
/// <param name="numValuesPerNode">The request cap the read was issued with; <c>0</c> means "no limit".</param>
|
||||
/// <returns><c>true</c> when a continuation point should be emitted; otherwise <c>false</c>.</returns>
|
||||
public static bool IsFullPage(int returnedCount, uint numValuesPerNode) =>
|
||||
numValuesPerNode > 0 && returnedCount >= numValuesPerNode;
|
||||
|
||||
@@ -11,17 +11,22 @@ internal readonly record struct RawPagePlan(int Limit, uint PageCap, uint Reques
|
||||
|
||||
/// <summary>
|
||||
/// sizes a HistoryRead-Raw page to what the backend can return in one read
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — a strict HistorianGateway's learned row ceiling, less
|
||||
/// its boundary-row allowance). With a limit the read — page P + boundary ties k + the lookahead row — is at most L,
|
||||
/// so P = min(want, L − k − 1) (all values: P = L − k − 1); a full page carries a continuation point like any other, as
|
||||
/// Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Previously a cap above a strict gateway's ceiling
|
||||
/// (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>.
|
||||
/// (<see cref="IHistorianDataSource.MaxRawValuesPerRead"/> = L — the server raw cap
|
||||
/// <c>ServerHistorian:MaxRawValuesPerRead</c>, default 10 000, which every backend reports (review R08:1.1, #736),
|
||||
/// and/or a strict HistorianGateway's learned row ceiling less its boundary-row allowance, whichever is lower). With
|
||||
/// a limit the read — page P + boundary ties k + the lookahead row — is at most L, so P = min(want, L − k − 1) (all
|
||||
/// values: P = L − k − 1, i.e. a first page of 9 999 at the default cap); a full page carries a continuation point like
|
||||
/// any other, as Part 11 allows fewer values than <c>NumValuesPerNode</c> with one. Previously a cap above a strict
|
||||
/// gateway's ceiling (or an uncapped read of a window holding more) failed <c>BadTooManyOperations</c>, and before
|
||||
/// #736 an uncapped read on a backend without a ceiling read the whole window in one shot.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Never a silent truncation.</b> The backend still answers a read in full or refuses it; only the server's page is
|
||||
/// smaller, and a page is judged full against what was asked for. A read the backend cannot serve at all (a tie
|
||||
/// cluster larger than one read) stays <c>BadTooManyOperations</c>.
|
||||
/// cluster larger than one read) stays <c>BadTooManyOperations</c>. The backends enforce the same cap themselves
|
||||
/// (a read asking for more than L, or for everything, is refused when the window holds more than L rows), so this
|
||||
/// planner is what keeps the server's own reads inside it: with L > 0 no read it plans ever asks for 0 or more than L.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>One re-read, planned again.</b> A refused read (<see cref="HistoryReadLimitExceededException"/>) is re-planned and
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
using System.Collections.Concurrent;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
|
||||
/// <summary>The last health snapshot a driver child published, as kept by <see cref="ChildHealthLedger"/>.</summary>
|
||||
/// <param name="State">The driver's reported state.</param>
|
||||
/// <param name="LastSuccessfulReadUtc">When the driver last read successfully, if ever.</param>
|
||||
/// <param name="LastError">The driver's last error, if any.</param>
|
||||
/// <param name="HostStatuses">Per-host connectivity for a multi-host driver, or null when the driver has none.</param>
|
||||
/// <param name="PublishedUtc">When the snapshot was published — the child's dedup means this is the last CHANGE.</param>
|
||||
public sealed record ChildHealthRecord(
|
||||
DriverState State,
|
||||
DateTime? LastSuccessfulReadUtc,
|
||||
string? LastError,
|
||||
IReadOnlyList<HostConnectivityStatus>? HostStatuses,
|
||||
DateTime PublishedUtc);
|
||||
|
||||
/// <summary>
|
||||
/// An <see cref="IDriverHealthPublisher"/> decorator that remembers the last snapshot per driver instance
|
||||
/// and forwards every publish to the real publisher. <see cref="DriverHostActor"/> wraps the publisher it
|
||||
/// hands its children in one of these so <c>GetDiagnostics</c> can answer from the children's own health
|
||||
/// (state, host counts, last change) instead of a placeholder — the children publish on their own actor
|
||||
/// threads, hence the concurrent map.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately NOT a message to the host: the children already publish through this seam on every
|
||||
/// change, so a parallel <c>Tell</c> would double every health transition's traffic for one consumer.
|
||||
/// Entries are removed by the host when it stops a child; a late publish from a stopping child may re-add
|
||||
/// one, which the host filters against its live child set when it answers.
|
||||
/// </remarks>
|
||||
public sealed class ChildHealthLedger : IDriverHealthPublisher
|
||||
{
|
||||
private readonly IDriverHealthPublisher _inner;
|
||||
private readonly ConcurrentDictionary<string, ChildHealthRecord> _last = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>Wraps <paramref name="inner"/>.</summary>
|
||||
/// <param name="inner">The publisher every snapshot is forwarded to.</param>
|
||||
public ChildHealthLedger(IDriverHealthPublisher inner)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inner);
|
||||
_inner = inner;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Publish(
|
||||
string clusterId,
|
||||
string driverInstanceId,
|
||||
DriverHealth health,
|
||||
int errorCount5Min,
|
||||
DateTime? rediscoveryNeededUtc = null,
|
||||
string? rediscoveryReason = null,
|
||||
IReadOnlyList<HostConnectivityStatus>? hostStatuses = null,
|
||||
SkippedTagReport? skippedTags = null,
|
||||
IReadOnlyDictionary<string, string>? diagnostics = null)
|
||||
{
|
||||
_last[driverInstanceId] = new ChildHealthRecord(
|
||||
health.State, health.LastSuccessfulRead, health.LastError, hostStatuses, DateTime.UtcNow);
|
||||
_inner.Publish(
|
||||
clusterId, driverInstanceId, health, errorCount5Min, rediscoveryNeededUtc, rediscoveryReason,
|
||||
hostStatuses, skippedTags, diagnostics);
|
||||
}
|
||||
|
||||
/// <summary>The last snapshot published for <paramref name="driverInstanceId"/>, or null if none yet.</summary>
|
||||
/// <param name="driverInstanceId">The driver instance.</param>
|
||||
/// <returns>The record, or null.</returns>
|
||||
public ChildHealthRecord? TryGet(string driverInstanceId) =>
|
||||
_last.TryGetValue(driverInstanceId, out var record) ? record : null;
|
||||
|
||||
/// <summary>Forgets <paramref name="driverInstanceId"/>'s snapshot (the child was stopped).</summary>
|
||||
/// <param name="driverInstanceId">The driver instance.</param>
|
||||
public void Forget(string driverInstanceId) => _last.TryRemove(driverInstanceId, out _);
|
||||
}
|
||||
@@ -195,7 +195,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
private readonly IReadOnlySet<string> _localRoles;
|
||||
private readonly IActorRef? _dependencyMux;
|
||||
private readonly IActorRef? _opcUaPublishActor;
|
||||
private readonly IDriverHealthPublisher _healthPublisher;
|
||||
/// <summary>The publisher handed to every child — a <see cref="ChildHealthLedger"/> over the configured one, so
|
||||
/// <see cref="HandleGetDiagnostics"/> answers from the children's own last-published health.</summary>
|
||||
private readonly ChildHealthLedger _healthPublisher;
|
||||
private readonly IVirtualTagEvaluator _virtualTagEvaluator;
|
||||
private readonly IHistoryWriter _historyWriter;
|
||||
private readonly IActorRef? _virtualTagHostOverride;
|
||||
@@ -594,6 +596,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
|
||||
private sealed record ChildEntry(IActorRef Actor, DriverInstanceSpec Spec, bool Stubbed)
|
||||
{
|
||||
/// <summary>When this child was spawned — the diagnostics' last-change time until it publishes health.</summary>
|
||||
public DateTime SpawnedUtc { get; } = DateTime.UtcNow;
|
||||
|
||||
// Convenience accessors for sites that don't need the full spec.
|
||||
/// <summary>Gets the driver type name from the child's spec.</summary>
|
||||
public string DriverType => Spec.DriverType;
|
||||
@@ -886,7 +891,7 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
_localRoles = localRoles ?? new HashSet<string>(StringComparer.Ordinal);
|
||||
_dependencyMux = dependencyMux;
|
||||
_opcUaPublishActor = opcUaPublishActor;
|
||||
_healthPublisher = healthPublisher ?? NullDriverHealthPublisher.Instance;
|
||||
_healthPublisher = new ChildHealthLedger(healthPublisher ?? NullDriverHealthPublisher.Instance);
|
||||
_virtualTagEvaluator = virtualTagEvaluator ?? NullVirtualTagEvaluator.Instance;
|
||||
_historyWriter = historyWriter ?? NullHistoryWriter.Instance;
|
||||
_virtualTagHostOverride = virtualTagHostOverride;
|
||||
@@ -2489,6 +2494,24 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// A child connectivity transition while the host is Stale has no live address space to annotate — drop
|
||||
// it (the post-recovery rebuild re-materialises conditions, and the child re-announces on its next entry).
|
||||
Receive<DriverInstanceActor.ConnectivityChanged>(_ => { });
|
||||
// Likewise a child's value / alarm burst: Stale is entered from Bootstrap's catch before any child is
|
||||
// spawned, so none should arrive — but the orphan-replay BeginDirectApply in that same try block has
|
||||
// already started a pipe-back before the catch can run, so "cannot happen" rests on ordering. Explicit
|
||||
// drops, so a burst that does land is a logged drop rather than an Unhandled dead-letter.
|
||||
Receive<DriverInstanceActor.AttributeValueBatchPublished>(m =>
|
||||
_log.Debug("DriverHost {Node}: dropping a {Count}-item value burst from {Id} while Stale (no address space)",
|
||||
_localNode, m.Items.Count, m.DriverInstanceId));
|
||||
Receive<DriverInstanceActor.AttributeAlarmPublished>(m =>
|
||||
_log.Debug("DriverHost {Node}: dropping a native alarm transition ({Cond}) from {Id} while Stale",
|
||||
_localNode, m.Args.ConditionId, m.DriverInstanceId));
|
||||
// A late artifact pipe-back from the apply that was in flight when the host fell Stale: that apply is
|
||||
// lost (the coordinator's deadline and the boot-time orphan replay recover it). Warning, not Debug —
|
||||
// an apply outcome is being discarded.
|
||||
Receive<ArtifactForApply>(m =>
|
||||
_log.Warning("DriverHost {Node}: dropping the artifact for {Id} (rev {Rev}) that arrived while Stale; the apply is abandoned",
|
||||
_localNode, m.DeploymentId, m.Revision));
|
||||
Receive<ApplyWedged>(m =>
|
||||
_log.Debug("DriverHost {Node}: late apply-wedge tick for {Id} while Stale; dropping", _localNode, m.DeploymentId));
|
||||
// Likewise the 03/S15 recovery audit: a timer armed before the host went Stale has nothing to report
|
||||
// on (no latch was set while Stale, and the rebuild re-establishes every node's status anyway).
|
||||
Receive<CommsRecoveryAudit>(_ => { });
|
||||
@@ -2496,16 +2519,16 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Timers.StartPeriodicTimer("retry-db", RetryConfigDbConnection.Instance, ReconnectInterval);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Answers <see cref="GetDiagnostics"/> from the live child set and each child's last-published health
|
||||
/// (<see cref="ChildHealthLedger"/>). Used to be a placeholder — <c>Guid.Empty</c>, "Spawned", zero
|
||||
/// devices, "now" — that looked live in every behaviour and was read by nobody (02/U1); the AdminUI
|
||||
/// fleet-diagnostics page is its consumer.
|
||||
/// </summary>
|
||||
private void HandleGetDiagnostics(GetDiagnostics msg)
|
||||
{
|
||||
var drivers = _children
|
||||
.Select(kv => new DriverInstanceDiagnostics(
|
||||
DriverInstanceId: Guid.Empty,
|
||||
Name: kv.Key,
|
||||
State: kv.Value.Stubbed ? "Stubbed" : "Spawned",
|
||||
ConnectedDevices: 0,
|
||||
FaultedDevices: 0,
|
||||
LastChangeUtc: DateTime.UtcNow))
|
||||
.Select(kv => ToDiagnostics(kv.Key, kv.Value, _healthPublisher.TryGet(kv.Key)))
|
||||
.ToArray();
|
||||
var snapshot = new NodeDiagnosticsSnapshot(
|
||||
NodeId: _localNode,
|
||||
@@ -2516,6 +2539,39 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
Sender.Tell(snapshot);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One child's diagnostics row. <c>Name</c> carries the string driver-instance id — the identity every
|
||||
/// other surface (/hosts, telemetry, the artifact) keys on — and <c>DriverInstanceId</c> the DB row Guid;
|
||||
/// the wire record has no slot for the authored display name. Device counts come from the driver's host
|
||||
/// statuses when it is an <c>IHostConnectivityProbe</c> (Running = connected; Stopped/Faulted = faulted;
|
||||
/// Unknown = neither); a single-host driver counts as one device, connected while it reports Healthy or
|
||||
/// Degraded and faulted while it reports Faulted. A child that has not published yet reports
|
||||
/// "Connecting" (or "Stubbed") with its spawn time as the last change.
|
||||
/// </summary>
|
||||
private static DriverInstanceDiagnostics ToDiagnostics(string driverInstanceId, ChildEntry entry, ChildHealthRecord? health)
|
||||
{
|
||||
var name = driverInstanceId;
|
||||
if (entry.Stubbed)
|
||||
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Stubbed", 0, 0, entry.SpawnedUtc);
|
||||
if (health is null)
|
||||
return new DriverInstanceDiagnostics(entry.Spec.DriverInstanceRowId, name, "Connecting", 0, 0, entry.SpawnedUtc);
|
||||
|
||||
int connected, faulted;
|
||||
if (health.HostStatuses is { Count: > 0 } hosts)
|
||||
{
|
||||
connected = hosts.Count(h => h.State == HostState.Running);
|
||||
faulted = hosts.Count(h => h.State is HostState.Stopped or HostState.Faulted);
|
||||
}
|
||||
else
|
||||
{
|
||||
connected = health.State is DriverState.Healthy or DriverState.Degraded ? 1 : 0;
|
||||
faulted = health.State == DriverState.Faulted ? 1 : 0;
|
||||
}
|
||||
|
||||
return new DriverInstanceDiagnostics(
|
||||
entry.Spec.DriverInstanceRowId, name, health.State.ToString(), connected, faulted, health.PublishedUtc);
|
||||
}
|
||||
|
||||
private void HandleDispatchFromSteady(DispatchDeployment msg)
|
||||
{
|
||||
// The short-circuit claims a CLEAN Applied, so it is only honest while the current revision's
|
||||
@@ -3771,6 +3827,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
// id lifts it (the tracker via _invokerFactory.Create, the hub via Readmit).
|
||||
_invokerFactory.Release(driverInstanceId);
|
||||
_telemetryHub?.Evict(driverInstanceId);
|
||||
// The reprovision rate-limit stamp belongs to the incarnation history that just ended. Kept, it grew the
|
||||
// map by one entry per id ever reprovisioned-then-removed AND made a later re-add of the same id inherit a
|
||||
// backoff it never earned (its first crash was deferred as a "crash loop").
|
||||
_lastReprovisionUtc.Remove(driverInstanceId);
|
||||
Timers.Cancel(ReprovisionTimerKey(driverInstanceId));
|
||||
_healthPublisher.Forget(driverInstanceId);
|
||||
_log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
|
||||
}
|
||||
|
||||
@@ -3821,6 +3883,8 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
|
||||
StopDependencyAdapter(msg.DriverInstanceId);
|
||||
Context.Stop(entry.Actor);
|
||||
_children.Remove(msg.DriverInstanceId);
|
||||
// The old incarnation's health must not read as the new one's until it publishes its own.
|
||||
_healthPublisher.Forget(msg.DriverInstanceId);
|
||||
|
||||
// Respawn from the same spec the last reconcile used — preserves RowId, Name, ClusterId.
|
||||
SpawnChild(entry.Spec);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Akka.Event;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
@@ -184,6 +185,52 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// </remarks>
|
||||
public static readonly TimeSpan DefaultCommsLossTimeout = TimeSpan.FromSeconds(90);
|
||||
|
||||
/// <summary>
|
||||
/// Bound on the driver's <see cref="IDriver.ShutdownAsync"/> (and its dispose) when the child stops or
|
||||
/// restarts, and on the teardown that precedes every reconnect attempt. <c>PostStop</c> runs on a
|
||||
/// dispatcher thread and blocks on it, so a driver whose shutdown waits on a dead socket / COM proxy used
|
||||
/// to pin that thread for as long as the wait took — a few such stops starved the default dispatcher and
|
||||
/// wedged <c>CoordinatedShutdown</c> and the host's own restart cascade. A shutdown that overruns is
|
||||
/// abandoned (logged), never awaited further.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan DefaultShutdownTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Default cap on the writes this actor runs against its driver at once. Writes run OFF the mailbox
|
||||
/// (a <c>ReceiveAsync</c> used to suspend it for the whole device round-trip, so writes serialised and
|
||||
/// every data change queued behind them — the second of two operator writes to a slow device missed the
|
||||
/// host's 8 s Ask and was reverted although the device took it). The cap keeps a burst from flooding a
|
||||
/// device with concurrent commands; writes beyond it queue in order and start as slots free.
|
||||
/// </summary>
|
||||
public const int DefaultMaxWritesInFlight = 4;
|
||||
|
||||
/// <summary>
|
||||
/// How long a queued write may wait for a slot before it is failed <c>BadTimeout</c> instead of being
|
||||
/// written. The host's write Ask is 8 s: a write older than that has already been reported as a timeout
|
||||
/// (and its node reverted) upstream, so writing it late would push a stale value to the device.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan DefaultQueuedWriteBudget = TimeSpan.FromSeconds(8);
|
||||
|
||||
/// <summary>
|
||||
/// Pessimistic bound on one <see cref="IDriver.InitializeAsync"/> attempt. The invoker's tier timeout is
|
||||
/// token-only (optimistic), so a driver whose initialise ignores its token would otherwise hold the
|
||||
/// single-flight gate forever and wedge the reconnect loop. 2× the longest tier (Tier C: 15 s).
|
||||
/// </summary>
|
||||
public static readonly TimeSpan DefaultInitializeTimeout = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Pessimistic bound on one write round-trip, on top of the 5 s token-based write cap. A driver whose
|
||||
/// <see cref="IWritable.WriteAsync"/> ignores its token would otherwise hold its slot AND its tag forever,
|
||||
/// piling every later write to that tag up in the queue. 2× the write cap.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan DefaultWriteTimeout = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>Self-sent by a write task once it has replied, releasing its slot and its tag.</summary>
|
||||
/// <param name="TagId">The tag the write held.</param>
|
||||
/// <param name="Incarnation">The actor incarnation that launched it — a late completion from a previous
|
||||
/// incarnation (after a supervision restart) must not free the new incarnation's slot or tag.</param>
|
||||
private sealed record WriteCompleted(string TagId, int Incarnation);
|
||||
|
||||
private readonly IDriver _driver;
|
||||
|
||||
/// <summary>Phase 6.1 resilience seam: every guarded driver-capability call this actor makes is
|
||||
@@ -204,6 +251,47 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// <summary>Comms-loss watchdog threshold; <see cref="TimeSpan.Zero"/> or negative disables it.</summary>
|
||||
private readonly TimeSpan _commsLossTimeout;
|
||||
|
||||
/// <summary>Bound on the driver shutdown / dispose; see <see cref="DefaultShutdownTimeout"/>.</summary>
|
||||
private readonly TimeSpan _shutdownTimeout;
|
||||
|
||||
/// <summary>Cap on concurrent writes; see <see cref="DefaultMaxWritesInFlight"/>.</summary>
|
||||
private readonly int _maxWritesInFlight;
|
||||
|
||||
/// <summary>Writes currently inside the driver. Actor-thread only (incremented on launch, decremented on
|
||||
/// the self-sent <see cref="WriteCompleted"/>).</summary>
|
||||
private int _writesInFlight;
|
||||
|
||||
/// <summary>Writes waiting for a slot (or for their tag), in arrival order, stamped with arrival time.
|
||||
/// Failed fast on every exit from <c>Connected</c>, so a queued write never reaches a driver that is
|
||||
/// reconnecting; failed <c>BadTimeout</c> at dequeue once older than <see cref="_queuedWriteBudget"/>.</summary>
|
||||
private readonly LinkedList<(WriteAttribute Write, IActorRef ReplyTo, DateTime EnqueuedUtc)> _pendingWrites = new();
|
||||
|
||||
/// <summary>Tags with a write currently inside the driver. A second write to the same tag waits in the
|
||||
/// queue behind it — two concurrent writes v1, v2 to one tag could otherwise land as v1.</summary>
|
||||
private readonly HashSet<string> _tagsInFlight = new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>See <see cref="DefaultQueuedWriteBudget"/>.</summary>
|
||||
private readonly TimeSpan _queuedWriteBudget;
|
||||
|
||||
/// <summary>See <see cref="DefaultInitializeTimeout"/>.</summary>
|
||||
private readonly TimeSpan _initializeTimeout;
|
||||
|
||||
/// <summary>See <see cref="DefaultWriteTimeout"/>.</summary>
|
||||
private readonly TimeSpan _writeTimeout;
|
||||
|
||||
/// <summary>This actor object's identity for its own off-mailbox completions (drawn from the process-wide
|
||||
/// generation source, so it is unique across supervision restarts of the same child).</summary>
|
||||
private readonly int _incarnation;
|
||||
|
||||
/// <summary>
|
||||
/// Set by <see cref="PreRestart"/> so the <see cref="PostStop"/> it triggers skips the dispose. A
|
||||
/// supervision restart recreates the actor object around the SAME <see cref="IDriver"/> (the Props closure
|
||||
/// captured it) and the host then re-initialises that driver — disposing it here would hand the new
|
||||
/// incarnation a dead object. Only a real stop (deploy removal, operator Restart, respawn on config change,
|
||||
/// host shutdown) disposes.
|
||||
/// </summary>
|
||||
private bool _restarting;
|
||||
|
||||
/// <summary>When the actor last entered <c>Connected</c>. The comms-loss watchdog's baseline for a
|
||||
/// driver that has never reported a successful read, so "connected but never read anything" is
|
||||
/// measurable rather than indefinitely undefined.</summary>
|
||||
@@ -218,6 +306,26 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// on the actor thread, so no lock is needed.</summary>
|
||||
private int _initGeneration;
|
||||
|
||||
/// <summary>Process-wide source of attempt generations. A supervision restart recreates the actor object
|
||||
/// (fresh fields) while a prior incarnation's attempt may still be running on the same driver; a per-actor
|
||||
/// counter restarting at 0 would let that late result collide with the new incarnation's first attempt.</summary>
|
||||
private static int s_initGenerationSource;
|
||||
|
||||
/// <summary>
|
||||
/// True while an <see cref="InitializeAsync"/> attempt is running (set when it is launched, cleared by its
|
||||
/// piped <see cref="InitializeSucceeded"/>/<see cref="InitializeFailed"/>). The <c>Reconnecting</c> retry
|
||||
/// timer fires on its cadence regardless of whether the previous attempt finished, and the generation
|
||||
/// token only discards an older attempt's RESULT — so with an Initialize bound longer than the cadence
|
||||
/// (Tier C: 15 s vs. 10 s) attempt N+1's teardown ran while attempt N was still inside the driver: the
|
||||
/// dispose-before-overwrite race the drivers' rule file forbids, once per overlap. A tick that lands while
|
||||
/// an attempt is in flight is skipped; the next tick after the result retries. Actor-thread only.
|
||||
/// </summary>
|
||||
private bool _initInFlight;
|
||||
|
||||
/// <summary>Cancellation for the in-flight attempt, so <see cref="PostStop"/> can cancel it before the
|
||||
/// shutdown rather than let it race the teardown on a restart.</summary>
|
||||
private CancellationTokenSource? _initCts;
|
||||
|
||||
/// <summary>
|
||||
/// Timestamps of recent Faulted-state transitions; used to compute the 5-minute error count.
|
||||
/// No lock needed — every read/write site runs inside an Akka message handler, which is
|
||||
@@ -307,6 +415,17 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// <param name="commsLossTimeout">Optional comms-loss watchdog threshold; defaults to
|
||||
/// <see cref="DefaultCommsLossTimeout"/>. <see cref="TimeSpan.Zero"/> disables the watchdog — used by
|
||||
/// tests that need a Connected driver to stay Connected while reporting non-Healthy.</param>
|
||||
/// <param name="shutdownTimeout">Optional bound on the driver's shutdown / dispose on stop and on the
|
||||
/// pre-reconnect teardown; defaults to <see cref="DefaultShutdownTimeout"/>. Exists so a test can prove a
|
||||
/// hung shutdown is abandoned without waiting 5 s.</param>
|
||||
/// <param name="maxWritesInFlight">Optional cap on concurrent writes against the driver; defaults to
|
||||
/// <see cref="DefaultMaxWritesInFlight"/>. Writes beyond it queue in arrival order.</param>
|
||||
/// <param name="queuedWriteBudget">Optional bound on how long a queued write may wait before it is failed
|
||||
/// <c>BadTimeout</c>; defaults to <see cref="DefaultQueuedWriteBudget"/>.</param>
|
||||
/// <param name="initializeTimeout">Optional pessimistic bound on one initialise attempt; defaults to
|
||||
/// <see cref="DefaultInitializeTimeout"/>.</param>
|
||||
/// <param name="writeTimeout">Optional pessimistic bound on one write round-trip; defaults to
|
||||
/// <see cref="DefaultWriteTimeout"/>.</param>
|
||||
/// <returns>A <see cref="Akka.Actor.Props"/> instance configured to create the wrapped <see cref="DriverInstanceActor"/>.</returns>
|
||||
public static Props Props(
|
||||
IDriver driver,
|
||||
@@ -316,7 +435,12 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
string? clusterId = null,
|
||||
IDriverCapabilityInvoker? invoker = null,
|
||||
TimeSpan? healthPollInterval = null,
|
||||
TimeSpan? commsLossTimeout = null) =>
|
||||
TimeSpan? commsLossTimeout = null,
|
||||
TimeSpan? shutdownTimeout = null,
|
||||
int? maxWritesInFlight = null,
|
||||
TimeSpan? queuedWriteBudget = null,
|
||||
TimeSpan? initializeTimeout = null,
|
||||
TimeSpan? writeTimeout = null) =>
|
||||
Akka.Actor.Props.Create(() => new DriverInstanceActor(
|
||||
driver,
|
||||
reconnectInterval ?? DefaultReconnectInterval,
|
||||
@@ -325,7 +449,12 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
clusterId ?? string.Empty,
|
||||
invoker,
|
||||
healthPollInterval,
|
||||
commsLossTimeout));
|
||||
commsLossTimeout,
|
||||
shutdownTimeout,
|
||||
maxWritesInFlight,
|
||||
queuedWriteBudget,
|
||||
initializeTimeout,
|
||||
writeTimeout));
|
||||
|
||||
/// <summary>
|
||||
/// Returns true when the driver should boot in DEV-STUB mode based on host platform and
|
||||
@@ -362,6 +491,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
/// subscription reconcile; defaults to <see cref="HealthPollInterval"/> when null.</param>
|
||||
/// <param name="commsLossTimeout">Comms-loss watchdog threshold; defaults to
|
||||
/// <see cref="DefaultCommsLossTimeout"/> when null, and is disabled when zero or negative.</param>
|
||||
/// <param name="shutdownTimeout">Bound on the driver's shutdown / dispose; defaults to
|
||||
/// <see cref="DefaultShutdownTimeout"/> when null.</param>
|
||||
/// <param name="maxWritesInFlight">Cap on concurrent writes; defaults to
|
||||
/// <see cref="DefaultMaxWritesInFlight"/> when null or non-positive.</param>
|
||||
/// <param name="queuedWriteBudget">Bound on a queued write's wait; defaults to
|
||||
/// <see cref="DefaultQueuedWriteBudget"/> when null or non-positive.</param>
|
||||
/// <param name="initializeTimeout">Pessimistic bound on one initialise attempt; defaults to
|
||||
/// <see cref="DefaultInitializeTimeout"/> when null or non-positive.</param>
|
||||
/// <param name="writeTimeout">Pessimistic bound on one write round-trip; defaults to
|
||||
/// <see cref="DefaultWriteTimeout"/> when null or non-positive.</param>
|
||||
public DriverInstanceActor(
|
||||
IDriver driver,
|
||||
TimeSpan reconnectInterval,
|
||||
@@ -370,8 +509,18 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
string? clusterId = null,
|
||||
IDriverCapabilityInvoker? invoker = null,
|
||||
TimeSpan? healthPollInterval = null,
|
||||
TimeSpan? commsLossTimeout = null)
|
||||
TimeSpan? commsLossTimeout = null,
|
||||
TimeSpan? shutdownTimeout = null,
|
||||
int? maxWritesInFlight = null,
|
||||
TimeSpan? queuedWriteBudget = null,
|
||||
TimeSpan? initializeTimeout = null,
|
||||
TimeSpan? writeTimeout = null)
|
||||
{
|
||||
_writeTimeout = writeTimeout is { } wt && wt > TimeSpan.Zero ? wt : DefaultWriteTimeout;
|
||||
_incarnation = Interlocked.Increment(ref s_initGenerationSource);
|
||||
_maxWritesInFlight = maxWritesInFlight is > 0 ? maxWritesInFlight.Value : DefaultMaxWritesInFlight;
|
||||
_queuedWriteBudget = queuedWriteBudget is { } qb && qb > TimeSpan.Zero ? qb : DefaultQueuedWriteBudget;
|
||||
_initializeTimeout = initializeTimeout is { } it && it > TimeSpan.Zero ? it : DefaultInitializeTimeout;
|
||||
_driver = driver;
|
||||
_invoker = invoker ?? NullDriverCapabilityInvoker.Instance;
|
||||
_driverInstanceId = driver.DriverInstanceId;
|
||||
@@ -380,6 +529,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
_reconnectInterval = reconnectInterval;
|
||||
_healthPollInterval = healthPollInterval ?? HealthPollInterval;
|
||||
_commsLossTimeout = commsLossTimeout ?? DefaultCommsLossTimeout;
|
||||
_shutdownTimeout = shutdownTimeout is { } bound && bound > TimeSpan.Zero ? bound : DefaultShutdownTimeout;
|
||||
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
|
||||
new KeyValuePair<string, object?>("event", startStubbed ? "spawn_stub" : "spawn"),
|
||||
new KeyValuePair<string, object?>("driver_type", driver.DriverType));
|
||||
@@ -421,6 +571,18 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Context.Parent.Tell(new ReprovisionRequested(_driverInstanceId, reason.GetType().Name));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flags the restart before the base implementation runs <see cref="PostStop"/>, so that PostStop
|
||||
/// shuts the driver down (the host re-initialises it on <see cref="ReprovisionRequested"/>) but does
|
||||
/// NOT dispose it — see <see cref="_restarting"/>.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
protected override void PreRestart(Exception reason, object message)
|
||||
{
|
||||
_restarting = true;
|
||||
base.PreRestart(reason, message);
|
||||
}
|
||||
|
||||
private void Stubbed()
|
||||
{
|
||||
// Stubbed drivers accept the standard message contracts but return deterministic
|
||||
@@ -448,6 +610,8 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// "write timeout". Synchronous Receive: Sender.Tell on the actor thread is safe.
|
||||
Receive<WriteAttribute>(_ =>
|
||||
Sender.Tell(new WriteAttributeResult(false, "driver not connected")));
|
||||
// A write launched while Connected can finish after the transition out of it; its slot still frees.
|
||||
Receive<WriteCompleted>(OnWriteCompleted);
|
||||
// An ack arriving while still connecting can't reach the upstream alarm system; drop it (the ack is
|
||||
// fire-and-forget — no reply to surface — and the OPC UA condition state already committed locally).
|
||||
Receive<RouteAlarmAck>(_ =>
|
||||
@@ -455,6 +619,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Receive<InitializeSucceeded>(msg =>
|
||||
{
|
||||
if (msg.Generation != _initGeneration) return;
|
||||
CompleteInitAttempt();
|
||||
_log.Info("DriverInstance {Id}: connected", _driverInstanceId);
|
||||
Become(Connected);
|
||||
PublishHealthSnapshot();
|
||||
@@ -465,6 +630,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Receive<InitializeFailed>(msg =>
|
||||
{
|
||||
if (msg.Generation != _initGeneration) return;
|
||||
CompleteInitAttempt();
|
||||
_log.Warning("DriverInstance {Id}: initialize failed: {Reason}", _driverInstanceId, msg.Reason);
|
||||
RecordFault();
|
||||
Become(Reconnecting);
|
||||
@@ -514,10 +680,14 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
_log.Info("DriverInstance {Id}: ForceReconnect requested by admin; re-entering Reconnecting", _driverInstanceId);
|
||||
DetachSubscription();
|
||||
FailQueuedWrites("driver not connected");
|
||||
Become(Reconnecting);
|
||||
PublishHealthSnapshot();
|
||||
});
|
||||
ReceiveAsync<WriteAttribute>(HandleWriteAsync);
|
||||
// Synchronous Receive: the device call runs off the mailbox (see StartOrQueueWrite), so data changes
|
||||
// and further writes are never head-of-line blocked behind a slow device.
|
||||
Receive<WriteAttribute>(msg => StartOrQueueWrite(msg, Sender));
|
||||
Receive<WriteCompleted>(OnWriteCompleted);
|
||||
ReceiveAsync<RouteAlarmAck>(HandleAcknowledgeAsync);
|
||||
ReceiveAsync<Subscribe>(HandleSubscribeAsync);
|
||||
ReceiveAsync<Unsubscribe>(_ => UnsubscribeAsync());
|
||||
@@ -745,6 +915,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// timeout on an inbound write to a transiently-down driver). Synchronous Receive.
|
||||
Receive<WriteAttribute>(_ =>
|
||||
Sender.Tell(new WriteAttributeResult(false, "driver not connected")));
|
||||
Receive<WriteCompleted>(OnWriteCompleted);
|
||||
// An ack arriving while reconnecting can't reach the upstream alarm system; drop it (fire-and-forget,
|
||||
// no reply — the OPC UA condition state already committed locally on the Part 9 ack).
|
||||
Receive<RouteAlarmAck>(_ =>
|
||||
@@ -752,6 +923,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
Receive<InitializeSucceeded>(msg =>
|
||||
{
|
||||
if (msg.Generation != _initGeneration) return;
|
||||
CompleteInitAttempt();
|
||||
Timers.Cancel("retry-connect");
|
||||
_log.Info("DriverInstance {Id}: reconnected", _driverInstanceId);
|
||||
Become(Connected);
|
||||
@@ -760,9 +932,13 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
AttachAlarmSource();
|
||||
SubscribeDesiredAlarms();
|
||||
});
|
||||
// A failure here is a no-op regardless of generation — the retry timer keeps trying the
|
||||
// current config; only a (generation-matched) InitializeSucceeded transitions state.
|
||||
Receive<InitializeFailed>(_ => { /* keep retrying via timer */ });
|
||||
// A failure here changes no state — the retry timer keeps trying the current config; only a
|
||||
// (generation-matched) InitializeSucceeded transitions. It DOES release the single-flight gate, or
|
||||
// no later tick could ever retry.
|
||||
Receive<InitializeFailed>(msg =>
|
||||
{
|
||||
if (msg.Generation == _initGeneration) CompleteInitAttempt();
|
||||
});
|
||||
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
|
||||
Receive<ForceReconnect>(_ => { /* already reconnecting — no-op */ });
|
||||
Receive<ConnectionLossRaised>(IgnoreConnectionLoss);
|
||||
@@ -807,12 +983,28 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
private void InitializeAsync(string driverConfigJson, bool teardownFirst = false)
|
||||
{
|
||||
_currentConfigJson = driverConfigJson;
|
||||
var generation = ++_initGeneration;
|
||||
if (_initInFlight)
|
||||
{
|
||||
// SINGLE-FLIGHT (02/H2). The previous attempt is still inside the driver; starting another would run
|
||||
// its teardown against the transport that attempt is building. Skip — the next tick after the
|
||||
// result retries, and a config handed over meanwhile is already stored for it.
|
||||
_log.Debug("DriverInstance {Id}: initialise attempt {Gen} still in flight; skipping this one",
|
||||
_driverInstanceId, _initGeneration);
|
||||
return;
|
||||
}
|
||||
|
||||
_initInFlight = true;
|
||||
var generation = Interlocked.Increment(ref s_initGenerationSource);
|
||||
_initGeneration = generation;
|
||||
var initCts = new CancellationTokenSource();
|
||||
_initCts = initCts;
|
||||
var self = Self;
|
||||
var log = _log;
|
||||
var driverInstanceId = _driverInstanceId;
|
||||
var driver = _driver;
|
||||
var invoker = _invoker;
|
||||
var shutdownTimeout = _shutdownTimeout;
|
||||
var initializeTimeout = _initializeTimeout;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
@@ -823,7 +1015,16 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
// Deliberately UNWRAPPED: teardown is best-effort cleanup of an already-dead
|
||||
// transport — a shutdown failure must not count toward the Initialize breaker.
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
// BOUNDED: a teardown wedged on the dead transport must not hold the retry loop
|
||||
// (and this thread) for longer than the bound; it is abandoned and the init proceeds.
|
||||
using var teardownCts = new CancellationTokenSource(shutdownTimeout);
|
||||
await driver.ShutdownAsync(teardownCts.Token).WaitAsync(shutdownTimeout);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
log.Warning(
|
||||
"DriverInstance {Id}: ShutdownAsync did not complete within {Bound} before a reconnect " +
|
||||
"attempt; abandoning it and initialising anyway", driverInstanceId, shutdownTimeout);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -841,33 +1042,103 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// attempt with BrokenCircuitException, which lands in the catch below exactly like a
|
||||
// real connect failure: InitializeFailed → keep retrying on the timer. The 15 s break
|
||||
// window is shorter than two 10 s timer ticks, so recovery costs at most one extra tick.
|
||||
await invoker.ExecuteAsync(
|
||||
// The outer token is this attempt's own: PostStop cancels it so a stop/restart does not run its
|
||||
// shutdown underneath an attempt still inside the driver.
|
||||
// PESSIMISTIC bound on top of the invoker's token-only timeout: a driver that ignores its token
|
||||
// would otherwise hold the single-flight gate forever. An overrun is abandoned (the task keeps
|
||||
// running, its eventual outcome is observed and discarded), reported as a failure so the retry
|
||||
// loop continues, and its late Tell is dropped by the generation guard anyway.
|
||||
var attempt = invoker.ExecuteAsync(
|
||||
DriverCapability.Initialize,
|
||||
driverInstanceId,
|
||||
async ct => await driver.InitializeAsync(driverConfigJson, ct),
|
||||
CancellationToken.None);
|
||||
initCts.Token).AsTask();
|
||||
try
|
||||
{
|
||||
await attempt.WaitAsync(initializeTimeout);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
log.Warning(
|
||||
"DriverInstance {Id}: initialise attempt {Gen} did not complete within {Bound} (the driver ignored " +
|
||||
"its cancellation token); attempt abandoned, the retry loop continues",
|
||||
driverInstanceId, generation, initializeTimeout);
|
||||
try { initCts.Cancel(); } catch (ObjectDisposedException) { }
|
||||
_ = attempt.ContinueWith(t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted);
|
||||
self.Tell(new InitializeFailed($"initialise did not complete within {initializeTimeout}", generation));
|
||||
return;
|
||||
}
|
||||
self.Tell(new InitializeSucceeded(generation));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
self.Tell(new InitializeFailed(ex.Message, generation));
|
||||
}
|
||||
finally
|
||||
{
|
||||
// The CTS outlives an abandoned attempt (it may still be inside the driver); dispose only once
|
||||
// nothing can cancel it any more — the actor nulls _initCts when it honours the result.
|
||||
if (!initCts.IsCancellationRequested) initCts.Dispose();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async Task HandleWriteAsync(WriteAttribute msg)
|
||||
/// <summary>Releases the single-flight gate once the in-flight attempt's result has been honoured.</summary>
|
||||
private void CompleteInitAttempt()
|
||||
{
|
||||
_initInFlight = false;
|
||||
_initCts = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Launches a write against the driver off the mailbox, or queues it when
|
||||
/// <see cref="_maxWritesInFlight"/> writes are already inside the driver. Runs on the actor thread; the
|
||||
/// only actor state it touches is the in-flight count and the queue. The device round-trip itself
|
||||
/// (<see cref="RunWriteAsync"/>) replies to the captured sender from the thread pool and then
|
||||
/// self-sends <see cref="WriteCompleted"/> to free its slot.
|
||||
/// </summary>
|
||||
private void StartOrQueueWrite(WriteAttribute msg, IActorRef replyTo)
|
||||
{
|
||||
var replyTo = Sender;
|
||||
if (_driver is not IWritable writable)
|
||||
{
|
||||
replyTo.Tell(new WriteAttributeResult(false, "Driver does not implement IWritable"));
|
||||
return;
|
||||
}
|
||||
|
||||
if (_writesInFlight >= _maxWritesInFlight || _tagsInFlight.Contains(msg.TagId))
|
||||
{
|
||||
_pendingWrites.AddLast((msg, replyTo, DateTime.UtcNow));
|
||||
return;
|
||||
}
|
||||
|
||||
LaunchWrite(writable, msg, replyTo);
|
||||
}
|
||||
|
||||
private void LaunchWrite(IWritable writable, WriteAttribute msg, IActorRef replyTo)
|
||||
{
|
||||
_writesInFlight++;
|
||||
_tagsInFlight.Add(msg.TagId);
|
||||
// Everything the task needs is captured HERE, on the actor thread — ResolveHost reads the driver,
|
||||
// and Self must not be touched off-context.
|
||||
var self = Self;
|
||||
var hostKey = ResolveHost(msg.TagId);
|
||||
var invoker = _invoker;
|
||||
_ = RunWriteAsync(writable, invoker, hostKey, msg, replyTo, self, _writeTimeout, _incarnation, _log);
|
||||
}
|
||||
|
||||
private static async Task RunWriteAsync(
|
||||
IWritable writable, IDriverCapabilityInvoker invoker, string hostKey,
|
||||
WriteAttribute msg, IActorRef replyTo, IActorRef self, TimeSpan writeTimeout, int incarnation,
|
||||
ILoggingAdapter log)
|
||||
{
|
||||
// Off the actor context from here on: the first await yields, and nothing below reads actor state.
|
||||
await Task.Yield();
|
||||
var request = new[] { new WriteRequest(msg.TagId, msg.Value) };
|
||||
// Bound the write so a hung backend can't pin this actor forever — retry stays
|
||||
// off by default, but a stalled call still needs an answer.
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
// Bound the write so a hung backend can't hold a slot forever — retry stays
|
||||
// off by default, but a stalled call still needs an answer. Disposed below, except when the write is
|
||||
// abandoned (the orphan may still be holding the token).
|
||||
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
var abandoned = false;
|
||||
try
|
||||
{
|
||||
// Route through the resilience invoker (Write pipeline: timeout, per-host breaker). Writes
|
||||
@@ -877,11 +1148,31 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
// therefore authoritative regardless of any configured Write retry. Per-tag opt-in to retries via
|
||||
// WriteIdempotentAttribute is the unshipped forward path (see WriteIdempotentAttribute). The outer
|
||||
// cts is retained as a hard backstop above the (typically tighter) tier timeout.
|
||||
var results = await _invoker.ExecuteWriteAsync(
|
||||
ResolveHost(msg.TagId),
|
||||
// PESSIMISTIC bound on top of the token-based cap: a driver that ignores its token would otherwise
|
||||
// hold the slot and the tag forever, piling every later write to that tag up in the queue.
|
||||
var attempt = invoker.ExecuteWriteAsync(
|
||||
hostKey,
|
||||
isIdempotent: false,
|
||||
async ct => await writable.WriteAsync(request, ct),
|
||||
cts.Token);
|
||||
cts.Token).AsTask();
|
||||
IReadOnlyList<WriteResult> results;
|
||||
try
|
||||
{
|
||||
results = await attempt.WaitAsync(writeTimeout);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
abandoned = true;
|
||||
log.Warning(
|
||||
"DriverInstance write of {Tag} did not complete within {Bound} (the driver ignored its cancellation " +
|
||||
"token); abandoned — its slot and tag are released, its eventual outcome is discarded",
|
||||
msg.TagId, writeTimeout);
|
||||
try { cts.Cancel(); } catch (ObjectDisposedException) { }
|
||||
_ = attempt.ContinueWith(t => _ = t.Exception, TaskContinuationOptions.OnlyOnFaulted);
|
||||
replyTo.Tell(new WriteAttributeResult(false,
|
||||
$"StatusCode=0x800A0000 (BadTimeout: write did not complete within {writeTimeout.TotalSeconds:F0}s)"));
|
||||
return;
|
||||
}
|
||||
if (results is { Count: 1 } && IsGoodStatus(results[0].StatusCode))
|
||||
{
|
||||
replyTo.Tell(new WriteAttributeResult(true, null));
|
||||
@@ -898,6 +1189,62 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
replyTo.Tell(new WriteAttributeResult(false, ex.Message));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!abandoned) cts.Dispose();
|
||||
self.Tell(new WriteCompleted(msg.TagId, incarnation));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Frees the write's slot and tag and drains the queue: the oldest entries whose tag is free are
|
||||
/// launched (in arrival order) while slots remain; entries older than the budget are failed instead. A
|
||||
/// completion stamped with another incarnation (a write launched before a supervision restart) is ignored —
|
||||
/// it never held a slot or tag of THIS incarnation.</summary>
|
||||
private void OnWriteCompleted(WriteCompleted msg)
|
||||
{
|
||||
if (msg.Incarnation != _incarnation) return;
|
||||
if (_writesInFlight > 0) _writesInFlight--;
|
||||
_tagsInFlight.Remove(msg.TagId);
|
||||
DrainWriteQueue();
|
||||
}
|
||||
|
||||
private void DrainWriteQueue()
|
||||
{
|
||||
if (_driver is not IWritable writable) return;
|
||||
var now = DateTime.UtcNow;
|
||||
var node = _pendingWrites.First;
|
||||
while (node is not null && _writesInFlight < _maxWritesInFlight)
|
||||
{
|
||||
var next = node.Next;
|
||||
var (write, replyTo, enqueuedUtc) = node.Value;
|
||||
if (now - enqueuedUtc > _queuedWriteBudget)
|
||||
{
|
||||
// Already reported as a timeout upstream (the host's Ask is the budget) and the node reverted —
|
||||
// writing it now would push a stale value to the device. BadTimeout, never written.
|
||||
_pendingWrites.Remove(node);
|
||||
replyTo.Tell(new WriteAttributeResult(false,
|
||||
$"StatusCode=0x800A0000 (BadTimeout: queued for {(now - enqueuedUtc).TotalSeconds:F1}s, over the {_queuedWriteBudget.TotalSeconds:F0}s write budget)"));
|
||||
}
|
||||
else if (!_tagsInFlight.Contains(write.TagId))
|
||||
{
|
||||
_pendingWrites.Remove(node);
|
||||
LaunchWrite(writable, write, replyTo);
|
||||
}
|
||||
|
||||
node = next;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Replies to every queued write with <paramref name="reason"/>; called on each exit from
|
||||
/// <c>Connected</c>. In-flight writes are left to finish — they already hold the device and reply with its
|
||||
/// real outcome.</summary>
|
||||
private void FailQueuedWrites(string reason)
|
||||
{
|
||||
while (_pendingWrites.First is { } node)
|
||||
{
|
||||
_pendingWrites.RemoveFirst();
|
||||
node.Value.ReplyTo.Tell(new WriteAttributeResult(false, reason));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1145,6 +1492,7 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: disconnect observed ({Reason}); reconnecting", _driverInstanceId, reason);
|
||||
DetachSubscription();
|
||||
FailQueuedWrites("driver not connected");
|
||||
RecordFault();
|
||||
Become(Reconnecting);
|
||||
PublishHealthSnapshot();
|
||||
@@ -1542,15 +1890,91 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
|
||||
DetachSubscription();
|
||||
FailQueuedWrites("driver stopped");
|
||||
// MUST happen: the IDriver instance can outlive this actor (the host respawns a child around the
|
||||
// same driver object), so a missing unsubscribe accumulates a handler per respawn holding a dead Self.
|
||||
DetachRediscoverySource();
|
||||
DetachHostStatusSource();
|
||||
DetachConnectionLossSource();
|
||||
try { _driver.ShutdownAsync(CancellationToken.None).GetAwaiter().GetResult(); }
|
||||
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
|
||||
|
||||
// An attempt still inside the driver is cancelled before the shutdown below runs underneath it (on a
|
||||
// restart the next incarnation re-initialises the same object). Best-effort: the task owns the CTS
|
||||
// and may have disposed it already.
|
||||
try { _initCts?.Cancel(); }
|
||||
catch (ObjectDisposedException) { /* the attempt already finished */ }
|
||||
|
||||
// BOUNDED by ONE shared budget across shutdown + dispose. PostStop runs on a dispatcher thread and this
|
||||
// blocks it, so an unbounded wait on a driver whose shutdown hangs against a dead socket / COM proxy pinned
|
||||
// that thread for the whole wait, and a few such stops starved the dispatcher. A shutdown that overruns is
|
||||
// abandoned, and the dispose gets whatever is left of the same budget (never a second full one).
|
||||
var budget = Stopwatch.StartNew();
|
||||
using (var cts = new CancellationTokenSource(_shutdownTimeout))
|
||||
{
|
||||
try { _driver.ShutdownAsync(cts.Token).WaitAsync(_shutdownTimeout).GetAwaiter().GetResult(); }
|
||||
catch (TimeoutException)
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: ShutdownAsync did not complete within {Bound} on PostStop; abandoning it",
|
||||
_driverInstanceId, _shutdownTimeout);
|
||||
}
|
||||
catch (Exception ex) { _log.Warning(ex, "DriverInstance {Id}: ShutdownAsync threw on PostStop", _driverInstanceId); }
|
||||
}
|
||||
|
||||
// Dispose ONLY on a real stop. A supervision restart (PreRestart → this PostStop) re-uses the same driver
|
||||
// object for the next incarnation, which the host re-initialises. Everything else that stops a child —
|
||||
// deploy removal, operator Restart, respawn on a config change, host shutdown — is final, and the driver
|
||||
// is never touched again: before this, nothing in the runtime ever disposed an IDriver, so a Calculation
|
||||
// driver's compiled-script AssemblyLoadContexts and an OpcUaClient driver's LoggerFactory / gate / CTS
|
||||
// leaked once per respawn (02/L1). Bounded like the shutdown, and runs even after a timed-out shutdown —
|
||||
// leaking the object is strictly worse than disposing one whose shutdown is still wedged.
|
||||
if (!_restarting)
|
||||
{
|
||||
var remaining = _shutdownTimeout - budget.Elapsed;
|
||||
DisposeDriver(remaining > MinDisposeBudget ? remaining : MinDisposeBudget);
|
||||
}
|
||||
|
||||
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
|
||||
new KeyValuePair<string, object?>("event", "stop"),
|
||||
new KeyValuePair<string, object?>("driver_type", _driver.DriverType));
|
||||
}
|
||||
|
||||
/// <summary>Floor on the dispose's share of the PostStop budget once a timed-out shutdown has used it up —
|
||||
/// enough for an idempotent dispose to run, not enough to double the stop's thread time.</summary>
|
||||
private static readonly TimeSpan MinDisposeBudget = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>Final teardown of the wrapped driver: <see cref="IAsyncDisposable"/> preferred, else
|
||||
/// <see cref="IDisposable"/>, never both. A driver implementing neither needs nothing here. Runs even after
|
||||
/// a timed-out shutdown, so a driver's dispose must be safe CONCURRENTLY with its own still-running
|
||||
/// <see cref="IDriver.ShutdownAsync"/>.</summary>
|
||||
/// <param name="bound">The remaining PostStop budget available to an async dispose.</param>
|
||||
private void DisposeDriver(TimeSpan bound)
|
||||
{
|
||||
try
|
||||
{
|
||||
switch (_driver)
|
||||
{
|
||||
case IAsyncDisposable asyncDisposable:
|
||||
asyncDisposable.DisposeAsync().AsTask().WaitAsync(bound).GetAwaiter().GetResult();
|
||||
break;
|
||||
case IDisposable disposable:
|
||||
disposable.Dispose();
|
||||
break;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
|
||||
_log.Info("DriverInstance {Id}: disposed {Type} driver on stop", _driverInstanceId, _driver.DriverType);
|
||||
OtOpcUaTelemetry.DriverInstanceLifecycle.Add(1,
|
||||
new KeyValuePair<string, object?>("event", "dispose"),
|
||||
new KeyValuePair<string, object?>("driver_type", _driver.DriverType));
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
_log.Warning("DriverInstance {Id}: DisposeAsync did not complete within {Bound} on PostStop; abandoning it",
|
||||
_driverInstanceId, bound);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_log.Warning(ex, "DriverInstance {Id}: disposing the driver threw on PostStop", _driverInstanceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,21 +7,37 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Single-flight cached health probe against the ConfigDb. Reads cached state via
|
||||
/// <c>Ask<DbHealthStatus></c>; a single SELECT 1 runs at most every <c>RefreshInterval</c>.
|
||||
/// <c>Ask<DbHealthStatus></c>; a single SELECT runs at most every <c>RefreshInterval</c>.
|
||||
/// Consumed by both the host's <c>/health/ready</c> endpoint and
|
||||
/// <c>RedundancyStateActor</c>'s stale calc.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The probe runs OFF the mailbox (a thread-pool task that pipes its outcome back), with a bounded
|
||||
/// command timeout. It used to run synchronously on the actor thread, relying on SqlClient's implicit
|
||||
/// 15 s connect / 30 s command timeouts — so a hanging SQL Server pinned a dispatcher thread for up to
|
||||
/// 30 s per tick, later ticks queued behind it, and the cached <see cref="GetStatus"/> answer was
|
||||
/// unreachable for exactly as long as the database was. Single-flight: a tick that lands while a probe
|
||||
/// is still running is skipped rather than stacked.
|
||||
/// </remarks>
|
||||
public sealed class DbHealthProbeActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
public static readonly TimeSpan RefreshInterval = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Bound on the probe's SQL command, relational providers only (the in-memory provider the
|
||||
/// actor tests run against has no command timeout).</summary>
|
||||
public static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(2);
|
||||
|
||||
public sealed class GetStatus { public static readonly GetStatus Instance = new(); private GetStatus() { } }
|
||||
public sealed record DbHealthStatus(bool Reachable, DateTime AsOfUtc, string? LastError);
|
||||
public sealed class Tick { public static readonly Tick Instance = new(); private Tick() { } }
|
||||
|
||||
/// <summary>Piped back by the off-thread probe with its outcome.</summary>
|
||||
private sealed record ProbeCompleted(DbHealthStatus Status);
|
||||
|
||||
private readonly IDbContextFactory<OtOpcUaConfigDbContext> _dbFactory;
|
||||
private readonly ILoggingAdapter _log = Context.GetLogger();
|
||||
private DbHealthStatus _last = new(false, DateTime.MinValue, "not probed yet");
|
||||
private bool _probeInFlight;
|
||||
|
||||
/// <summary>Gets or sets the timer scheduler for periodic health probes.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
@@ -38,28 +54,54 @@ public sealed class DbHealthProbeActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
_dbFactory = dbFactory;
|
||||
Receive<GetStatus>(_ => Sender.Tell(_last));
|
||||
Receive<Tick>(_ => RunProbe());
|
||||
Receive<Tick>(_ => StartProbe());
|
||||
Receive<ProbeCompleted>(msg =>
|
||||
{
|
||||
_probeInFlight = false;
|
||||
_last = msg.Status;
|
||||
if (!msg.Status.Reachable)
|
||||
_log.Warning("DbHealthProbe: probe failed: {Error}", msg.Status.LastError);
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void PreStart()
|
||||
{
|
||||
RunProbe();
|
||||
StartProbe();
|
||||
Timers.StartPeriodicTimer("probe", Tick.Instance, RefreshInterval);
|
||||
}
|
||||
|
||||
private void RunProbe()
|
||||
private void StartProbe()
|
||||
{
|
||||
if (_probeInFlight)
|
||||
{
|
||||
_log.Debug("DbHealthProbe: previous probe still running; skipping this tick");
|
||||
return;
|
||||
}
|
||||
|
||||
_probeInFlight = true;
|
||||
var factory = _dbFactory;
|
||||
// Never touches actor state; the outcome comes back as a message.
|
||||
Task.Run(() => Probe(factory)).PipeTo(
|
||||
Self,
|
||||
success: status => new ProbeCompleted(status),
|
||||
failure: ex => new ProbeCompleted(new DbHealthStatus(false, DateTime.UtcNow, ex.Message)));
|
||||
}
|
||||
|
||||
private static DbHealthStatus Probe(IDbContextFactory<OtOpcUaConfigDbContext> factory)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var db = _dbFactory.CreateDbContext();
|
||||
using var db = factory.CreateDbContext();
|
||||
// Local to this lease — the pool resets it on return (see DriverHostActor.LoadArtifactBlob).
|
||||
if (db.Database.IsRelational())
|
||||
db.Database.SetCommandTimeout(CommandTimeout);
|
||||
_ = db.Deployments.AsNoTracking().Take(1).ToList();
|
||||
_last = new DbHealthStatus(true, DateTime.UtcNow, null);
|
||||
return new DbHealthStatus(true, DateTime.UtcNow, null);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_last = new DbHealthStatus(false, DateTime.UtcNow, ex.Message);
|
||||
_log.Warning(ex, "DbHealthProbe: probe failed");
|
||||
return new DbHealthStatus(false, DateTime.UtcNow, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,6 +154,42 @@ public sealed class ServerHistorianOptions
|
||||
/// </summary>
|
||||
public int MaxTimeWeightedRawRows { get; init; } = 1_000_000;
|
||||
|
||||
/// <summary>The default of <see cref="MaxRawValuesPerRead"/>: 10 000 values per raw page / backend read.</summary>
|
||||
public const int DefaultMaxRawValuesPerRead = 10_000;
|
||||
|
||||
/// <summary>
|
||||
/// The smallest usable <see cref="MaxRawValuesPerRead"/>: a resumed page's read must carry at least one value, the
|
||||
/// boundary tie it re-reads and trims, AND the lookahead row that tells the server more data follows. At 2 a resumed
|
||||
/// page has no room at all and every value costs an empty page with a continuation point (the #692 defect, re-created
|
||||
/// by configuration), so 1 and 2 are raised to 3 (<see cref="EffectiveMaxRawValuesPerRead"/>).
|
||||
/// </summary>
|
||||
public const int MinMaxRawValuesPerRead = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Every backend: the most raw values the server returns on ONE HistoryRead-Raw page, and the most rows one
|
||||
/// backend raw read may materialise — the server's own row ceiling (review R08:1.1, #736). A client's
|
||||
/// <c>NumValuesPerNode = 0</c> ("all values") no longer reads a whole window into memory: it is served as pages of at
|
||||
/// most this many values, each with a continuation point (Part 11 allows fewer values than requested with one), through
|
||||
/// the same tie-safe paging a capped read uses (<c>RawPageLimit</c>: page + boundary ties + lookahead ≤ cap, so a first
|
||||
/// page holds cap − 1 values). A <c>NumValuesPerNode</c> above it is paged at it too. Advertised as
|
||||
/// <c>HistoryServerCapabilities/MaxReturnDataValues</c>. The backends' own <c>TOP</c> / <c>LIMIT</c> / <c>max_values</c>
|
||||
/// are bounded by it independently: a raw read asking for more than this (or for everything) reads at most cap + 1 rows
|
||||
/// and is REFUSED (<c>BadTooManyOperations</c>) when the window holds more — never answered truncated — so a caller that
|
||||
/// bypasses the node manager's paging cannot bypass the bound. On the Gateway backend a strict gateway's learned row
|
||||
/// ceiling (less its boundary-row allowance) still applies when lower. The tie-cluster bound of a resumed page is also
|
||||
/// at most cap − 1 (as under a gateway ceiling; <see cref="MaxTieClusterOverfetch"/> above that is inert), and a
|
||||
/// page whose boundary tie cluster is that large steps past it with one empty page. Default 10 000. Non-positive =
|
||||
/// unbounded, the pre-#736 behaviour (a <see cref="Validate"/> warning); below <see cref="MinMaxRawValuesPerRead"/>
|
||||
/// (3) is raised to it.
|
||||
/// </summary>
|
||||
public int MaxRawValuesPerRead { get; init; } = DefaultMaxRawValuesPerRead;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="MaxRawValuesPerRead"/> as the backends and the node manager apply it: <c>0</c> (unbounded) when
|
||||
/// non-positive, otherwise at least <see cref="MinMaxRawValuesPerRead"/>. The one place the rule lives.
|
||||
/// </summary>
|
||||
public int EffectiveMaxRawValuesPerRead => MaxRawValuesPerRead <= 0 ? 0 : Math.Max(MinMaxRawValuesPerRead, MaxRawValuesPerRead);
|
||||
|
||||
/// <summary>
|
||||
/// how far into the future an OPC UA HistoryUpdate (Insert / Replace / Update) item's
|
||||
/// <c>SourceTimestamp</c> may lie, measured from the server clock. An item beyond <c>now + skew</c> is refused
|
||||
@@ -208,6 +244,10 @@ public sealed class ServerHistorianOptions
|
||||
warnings.Add($"ServerHistorian:MaxConcurrentHistoryReadBatches is {MaxConcurrentHistoryReadBatches} — must be > 0; the process-wide HistoryRead batch limiter is disabled.");
|
||||
if (MaxTimeWeightedRawRows <= 0)
|
||||
warnings.Add($"ServerHistorian:MaxTimeWeightedRawRows is {MaxTimeWeightedRawRows} — must be > 0; time-weighted aggregate reads on the SqlServer/QuestDb backends stream an unbounded number of raw samples.");
|
||||
if (MaxRawValuesPerRead <= 0)
|
||||
warnings.Add($"ServerHistorian:MaxRawValuesPerRead is {MaxRawValuesPerRead} — must be > 0; HistoryRead-Raw pages and backend raw reads are unbounded, so a client asking for a whole window (NumValuesPerNode = 0) materialises every sample in memory.");
|
||||
else if (MaxRawValuesPerRead < MinMaxRawValuesPerRead)
|
||||
warnings.Add($"ServerHistorian:MaxRawValuesPerRead is {MaxRawValuesPerRead} — must be at least {MinMaxRawValuesPerRead} (a value, its boundary tie and the lookahead row); treated as {MinMaxRawValuesPerRead}.");
|
||||
if (EventReadHorizon < TimeSpan.Zero)
|
||||
warnings.Add($"ServerHistorian:EventReadHorizon is {EventReadHorizon} — negative is treated as no horizon (00:00:00); HistoryReadEvents reads every window in full.");
|
||||
if (HistoryReadDeadline <= TimeSpan.Zero)
|
||||
|
||||
@@ -27,7 +27,27 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
/// </remarks>
|
||||
public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
{
|
||||
/// <summary>
|
||||
/// How long an eviction tombstone (see <see cref="_evicted"/>) is kept. It only has to outlive the
|
||||
/// removed child's last late write — an in-flight capability call that completes after
|
||||
/// <c>StopChild</c>, bounded by the longest capability timeout (seconds) plus the child's own stop —
|
||||
/// so ten minutes is a wide margin. Without a sweep a driver removed for good kept its id forever.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan EvictionTombstoneTtl = TimeSpan.FromMinutes(10);
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly TimeProvider _time;
|
||||
|
||||
/// <summary>Initializes a hub on the system clock.</summary>
|
||||
public TelemetryLocalHub() : this(TimeProvider.System) { }
|
||||
|
||||
/// <summary>Initializes a hub on <paramref name="time"/> — the tombstone-sweep test seam.</summary>
|
||||
/// <param name="time">The clock the eviction tombstones are stamped and swept by.</param>
|
||||
public TelemetryLocalHub(TimeProvider time)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(time);
|
||||
_time = time;
|
||||
}
|
||||
|
||||
// Plain Dictionary, NOT ConcurrentDictionary: every access is already under _gate, and
|
||||
// ConcurrentDictionary.Values would materialize a fresh List (and take its own internal lock) on
|
||||
@@ -42,9 +62,10 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
private readonly ConcurrentDictionary<string, TelemetryItem.Health> _healthCache = new();
|
||||
private readonly ConcurrentDictionary<(string InstanceId, string HostName), TelemetryItem.Resilience> _resilienceCache = new();
|
||||
|
||||
// Instances evicted and not since readmitted. Their late Health/Resilience items are dropped, so a
|
||||
// removed driver's still-stopping child cannot re-prime the cache it was just evicted from. Under _gate.
|
||||
private readonly HashSet<string> _evicted = new(StringComparer.Ordinal);
|
||||
// Instances evicted and not since readmitted, stamped with the eviction time. Their late Health/Resilience
|
||||
// items are dropped, so a removed driver's still-stopping child cannot re-prime the cache it was just
|
||||
// evicted from. Swept past EvictionTombstoneTtl on the next Evict (never on the Emit hot path). Under _gate.
|
||||
private readonly Dictionary<string, DateTimeOffset> _evicted = new(StringComparer.Ordinal);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Emit(TelemetryItem item)
|
||||
@@ -55,9 +76,9 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
{
|
||||
switch (item)
|
||||
{
|
||||
case TelemetryItem.Health h when _evicted.Contains(h.E.DriverInstanceId):
|
||||
case TelemetryItem.Health h when _evicted.ContainsKey(h.E.DriverInstanceId):
|
||||
return;
|
||||
case TelemetryItem.Resilience r when _evicted.Contains(r.E.DriverInstanceId):
|
||||
case TelemetryItem.Resilience r when _evicted.ContainsKey(r.E.DriverInstanceId):
|
||||
return;
|
||||
case TelemetryItem.Health h:
|
||||
_healthCache[h.E.DriverInstanceId] = h;
|
||||
@@ -80,7 +101,9 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
_evicted.Add(driverInstanceId);
|
||||
var now = _time.GetUtcNow();
|
||||
SweepExpiredTombstones(now);
|
||||
_evicted[driverInstanceId] = now;
|
||||
_healthCache.TryRemove(driverInstanceId, out _);
|
||||
foreach (var key in _resilienceCache.Keys)
|
||||
if (string.Equals(key.InstanceId, driverInstanceId, StringComparison.Ordinal))
|
||||
@@ -97,6 +120,24 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
_evicted.Remove(driverInstanceId);
|
||||
}
|
||||
|
||||
/// <summary>Drops tombstones older than <see cref="EvictionTombstoneTtl"/>. Called under <see cref="_gate"/>
|
||||
/// from <see cref="Evict"/> only — the set is bounded by distinct removed ids, so a linear pass per
|
||||
/// eviction is cheap, and Emit never pays for it.</summary>
|
||||
private void SweepExpiredTombstones(DateTimeOffset now)
|
||||
{
|
||||
if (_evicted.Count == 0) return;
|
||||
List<string>? expired = null;
|
||||
foreach (var (id, evictedAt) in _evicted)
|
||||
{
|
||||
if (now - evictedAt >= EvictionTombstoneTtl)
|
||||
(expired ??= new List<string>()).Add(id);
|
||||
}
|
||||
|
||||
if (expired is null) return;
|
||||
foreach (var id in expired)
|
||||
_evicted.Remove(id);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ITelemetrySubscription Subscribe(int boundedCapacity)
|
||||
{
|
||||
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Browsing;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Commons.Tests.Browsing;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L1 (host context, 06/G-1): the browser's throwaway capture driver must be <b>disposed</b>, not
|
||||
/// only shut down. A Galaxy driver's <c>ShutdownAsync</c> left its whole gRPC runtime alive until
|
||||
/// <c>DisposeAsync</c>, which nothing called — so every <c>/raw</c> browse leaked two channels, an MX session
|
||||
/// and two stream loops.
|
||||
/// </summary>
|
||||
public sealed class DiscoveryDriverBrowserDisposeTests
|
||||
{
|
||||
private static readonly TimeSpan ShortOpen = TimeSpan.FromSeconds(5);
|
||||
private static readonly TimeSpan ShortShutdown = TimeSpan.FromMilliseconds(300);
|
||||
|
||||
private static DiscoveryDriverBrowser Browser(IDriverFactory factory) =>
|
||||
new(factory, NullLogger<DiscoveryDriverBrowser>.Instance, ShortOpen, ShortShutdown);
|
||||
|
||||
[Fact]
|
||||
public async Task OpenAsync_disposes_the_capture_driver_after_shutdown()
|
||||
{
|
||||
var driver = new DisposableFakeDiscoveryDriver();
|
||||
var browser = Browser(new FakeDriverFactory((_, _, _) => driver));
|
||||
|
||||
var session = await browser.OpenAsync("Fake", "{}", CancellationToken.None);
|
||||
|
||||
session.ShouldNotBeNull();
|
||||
driver.Inner.Trace.ShouldBe(new[] { "init", "discover", "shutdown" });
|
||||
driver.DisposeCalls.ShouldBe(1);
|
||||
driver.DisposedAfterShutdown.ShouldBeTrue("dispose is the final teardown, after the shutdown");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OpenAsync_disposes_the_capture_driver_even_when_discovery_throws()
|
||||
{
|
||||
var driver = new DisposableFakeDiscoveryDriver(new FakeDiscoveryDriver
|
||||
{
|
||||
OnDiscover = (_, _) => throw new InvalidOperationException("boom-discovery"),
|
||||
});
|
||||
var browser = Browser(new FakeDriverFactory((_, _, _) => driver));
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(
|
||||
() => browser.OpenAsync("Fake", "{}", CancellationToken.None));
|
||||
|
||||
driver.DisposeCalls.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OpenAsync_disposes_the_capture_driver_even_when_its_shutdown_hangs()
|
||||
{
|
||||
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var driver = new DisposableFakeDiscoveryDriver(new FakeDiscoveryDriver { OnShutdown = _ => never.Task });
|
||||
var browser = Browser(new FakeDriverFactory((_, _, _) => driver));
|
||||
|
||||
await browser.OpenAsync("Fake", "{}", CancellationToken.None);
|
||||
|
||||
driver.DisposeCalls.ShouldBe(1, "a shutdown that timed out must not skip the dispose");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task CanBrowse_disposes_its_probe_driver()
|
||||
{
|
||||
var driver = new DisposableFakeDiscoveryDriver();
|
||||
var browser = Browser(new FakeDriverFactory((_, _, _) => driver));
|
||||
|
||||
browser.CanBrowse("Fake", "{}").ShouldBeTrue();
|
||||
|
||||
// The CanBrowse teardown is fire-and-forget; wait for it.
|
||||
await driver.Disposed.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken);
|
||||
driver.DisposeCalls.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>A <see cref="FakeDiscoveryDriver"/> wrapped with <see cref="IAsyncDisposable"/> + <see cref="IDisposable"/>
|
||||
/// (both, so the "prefer the async one, never both" rule is observable: exactly ONE dispose call lands).</summary>
|
||||
private sealed class DisposableFakeDiscoveryDriver : IDriver, ITagDiscovery, IAsyncDisposable, IDisposable
|
||||
{
|
||||
private int _disposeCalls;
|
||||
|
||||
public DisposableFakeDiscoveryDriver(FakeDiscoveryDriver? inner = null) => Inner = inner ?? new FakeDiscoveryDriver();
|
||||
|
||||
public FakeDiscoveryDriver Inner { get; }
|
||||
public int DisposeCalls => Volatile.Read(ref _disposeCalls);
|
||||
public bool DisposedAfterShutdown { get; private set; }
|
||||
public TaskCompletionSource Disposed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public string DriverInstanceId => Inner.DriverInstanceId;
|
||||
public string DriverType => Inner.DriverType;
|
||||
public bool SupportsOnlineDiscovery => Inner.SupportsOnlineDiscovery;
|
||||
public DiscoveryRediscoverPolicy RediscoverPolicy => Inner.RediscoverPolicy;
|
||||
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) =>
|
||||
Inner.InitializeAsync(driverConfigJson, cancellationToken);
|
||||
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) =>
|
||||
Inner.ReinitializeAsync(driverConfigJson, cancellationToken);
|
||||
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken) => Inner.ShutdownAsync(cancellationToken);
|
||||
|
||||
public DriverHealth GetHealth() => Inner.GetHealth();
|
||||
|
||||
#pragma warning disable OTOPCUA0001 // test double delegating to a test double — no resilience pipeline applies
|
||||
public Task DiscoverAsync(IAddressSpaceBuilder builder, CancellationToken cancellationToken) =>
|
||||
Inner.DiscoverAsync(builder, cancellationToken);
|
||||
#pragma warning restore OTOPCUA0001
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Record();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public void Dispose() => Record();
|
||||
|
||||
private void Record()
|
||||
{
|
||||
Interlocked.Increment(ref _disposeCalls);
|
||||
DisposedAfterShutdown = Inner.ShutdownCalls > 0;
|
||||
Disposed.TrySetResult();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,9 +22,14 @@ public interface IConformanceBackend : IAsyncDisposable
|
||||
/// <param name="clock">Its "now"; null means the system clock.</param>
|
||||
IHistorianDataSource CreateDataSource(TimeProvider? clock = null);
|
||||
|
||||
/// <summary>A fresh data source whose time-weighted raw-row cap (<c>ServerHistorian:MaxTimeWeightedRawRows</c>) is <paramref name="maxTimeWeightedRawRows"/>.</summary>
|
||||
/// <param name="maxTimeWeightedRawRows">The cap (<= 0 = unbounded).</param>
|
||||
IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows);
|
||||
/// <summary>
|
||||
/// A fresh data source whose time-weighted raw-row cap (<c>ServerHistorian:MaxTimeWeightedRawRows</c>) is
|
||||
/// <paramref name="maxTimeWeightedRawRows"/> and whose server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>,
|
||||
/// R08:1.1 / #736) is <paramref name="maxRawValuesPerRead"/>.
|
||||
/// </summary>
|
||||
/// <param name="maxTimeWeightedRawRows">The time-weighted cap (<= 0 = unbounded).</param>
|
||||
/// <param name="maxRawValuesPerRead">The raw cap (<= 0 = unbounded, the default).</param>
|
||||
IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0);
|
||||
|
||||
/// <summary>A fresh live-value writer over this backend's store.</summary>
|
||||
/// <param name="utcNow">The clock used to stamp values that carry no timestamp; null means the system clock.</param>
|
||||
|
||||
@@ -25,6 +25,43 @@ public abstract class RawReadConformance : ConformanceBase
|
||||
r.ContinuationPoint.ShouldBeNull();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// R08:1.1 (#736): the server raw cap bounds the backend's own <c>TOP</c> / <c>LIMIT</c>. A read asking for everything
|
||||
/// (or for more than the cap) reads at most cap + 1 rows and is refused when the window holds more — never answered
|
||||
/// truncated (a short answer is what the server's paging reads as "the last page") — and is served in full when it fits.
|
||||
/// A read at or under the cap is unchanged. The refusal is a client-request limit, not a backend fault.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task The_server_raw_cap_bounds_every_read_and_refuses_an_over_cap_window_rather_than_truncating()
|
||||
{
|
||||
await ConformanceData.SeedAAsync(B); // five samples at +0, +10, +20, +30, +40
|
||||
var src = B.CreateDataSource(maxTimeWeightedRawRows: 1_000_000, maxRawValuesPerRead: 3);
|
||||
src.MaxRawValuesPerRead.ShouldBe(3);
|
||||
|
||||
// Five rows under a cap of three: everything / above the cap ⇒ refused; at the cap ⇒ the first three; under ⇒ as asked.
|
||||
var refused = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), 0, Ct));
|
||||
refused.Limit.ShouldBe(3);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), 4, Ct));
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(-1), At(100), uint.MaxValue, Ct));
|
||||
Values(await src.ReadRawAsync("A", At(-1), At(100), 3, Ct)).ShouldBe([10d, 20d, 30d]);
|
||||
Values(await src.ReadRawAsync("A", At(-1), At(100), 2, Ct)).ShouldBe([10d, 20d]);
|
||||
|
||||
// Three rows fit: everything / above the cap are served in full.
|
||||
Values(await src.ReadRawAsync("A", At(10), At(40), 0, Ct)).ShouldBe([20d, 30d, 40d]);
|
||||
Values(await src.ReadRawAsync("A", At(10), At(40), 4, Ct)).ShouldBe([20d, 30d, 40d]);
|
||||
|
||||
// Reverse reads are bounded the same way, keeping the LATEST rows.
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => src.ReadRawAsync("A", At(100), At(-1), 0, Ct));
|
||||
Values(await src.ReadRawAsync("A", At(100), At(-1), 3, Ct)).ShouldBe([50d, 40d, 30d]);
|
||||
Values(await src.ReadRawAsync("A", At(40), At(10), 0, Ct)).ShouldBe([50d, 40d, 30d]);
|
||||
|
||||
// A point read and an empty window are unaffected.
|
||||
Values(await src.ReadRawAsync("A", At(10), At(10), 0, Ct)).ShouldBe([20d]);
|
||||
(await src.ReadRawAsync("A", At(50), At(60), 0, Ct)).Samples.ShouldBeEmpty();
|
||||
|
||||
src.GetHealthSnapshot().TotalFailures.ShouldBe(0, "a refused over-cap read is a client-request limit, not a backend fault");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Forward_read_honours_the_limit()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Review R08:1.1 (#736): the server's raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>) bounds the Gateway
|
||||
/// backend's <c>max_values</c> independently of the node manager. An uncapped raw read (<c>NumValuesPerNode = 0</c>)
|
||||
/// used to be sent as <see cref="int.MaxValue"/> and materialise the whole window twice (the protobuf list and the
|
||||
/// snapshot list); a read above the cap now asks for cap + 1 rows and is REFUSED (<see cref="HistoryReadLimitExceededException"/>,
|
||||
/// <c>BadTooManyOperations</c>) when the window holds more — never answered truncated, because a short answer is what the
|
||||
/// server's paging reads as "the last page". The data source reports the cap as its
|
||||
/// <see cref="IHistorianDataSource.MaxRawValuesPerRead"/> (or a strict gateway's lower learned ceiling), so the node
|
||||
/// manager's existing ceiling paging serves <c>NumValuesPerNode = 0</c> as pages of at most the cap, each with a
|
||||
/// continuation point.
|
||||
/// </summary>
|
||||
public sealed class GatewayRawCapTests : IDisposable
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static readonly DateTime T0 = new(2026, 10, 22, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private const int Cap = 1000;
|
||||
|
||||
private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-gwrawcap-{Guid.NewGuid():N}");
|
||||
|
||||
private static DateTime At(int seconds) => T0.AddSeconds(seconds);
|
||||
|
||||
private static HistorianSample Row(DateTime t, double v, uint quality = 0) =>
|
||||
new() { Tag = "T", Timestamp = Timestamp.FromDateTime(t), NumericValue = v, OpcQuality = 192, Quality = quality };
|
||||
|
||||
/// <summary>AVEVA-shaped: a boundary row at the query start, <c>[start, end]</c> inclusive, capped at <c>max</c>.</summary>
|
||||
private static FakeHistorianGatewayClient Store(int count, int? ceiling = null)
|
||||
{
|
||||
var stored = Enumerable.Range(0, count).Select(i => (T: At(i), V: (double)i)).ToList();
|
||||
return new FakeHistorianGatewayClient
|
||||
{
|
||||
MaxValuesCeiling = ceiling,
|
||||
RawSource = (start, end, max) =>
|
||||
{
|
||||
if (max <= 0) throw new ArgumentOutOfRangeException(nameof(max), "max_values must be positive"); // the client package
|
||||
var rows = new List<HistorianSample>();
|
||||
if (stored.All(s => s.T != start))
|
||||
{
|
||||
var prior = stored.LastOrDefault(s => s.T < start);
|
||||
rows.Add(prior.T == default
|
||||
? new HistorianSample { Tag = "T", Timestamp = Timestamp.FromDateTime(start), NumericValue = 0, OpcQuality = 0, Quality = 1 }
|
||||
: Row(start, prior.V, 133));
|
||||
}
|
||||
rows.AddRange(stored.Where(s => s.T >= start && s.T <= end).Select(s => Row(s.T, s.V)));
|
||||
return rows.Take(max);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake, int cap = Cap) =>
|
||||
new(fake, NullLogger<GatewayHistorianDataSource>.Instance, maxRawValuesPerRead: cap);
|
||||
|
||||
private static int Probe => Cap + 1 + RawAggregateReader.PreWindowAllowance; // cap + 1 rows + the boundary-row allowance
|
||||
|
||||
// ---- the data source alone ----------------------------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void Without_a_cap_the_source_is_unchanged()
|
||||
{
|
||||
new GatewayHistorianDataSource(Store(10), NullLogger<GatewayHistorianDataSource>.Instance).MaxRawValuesPerRead.ShouldBe(0);
|
||||
Source(Store(10), cap: 0).MaxRawValuesPerRead.ShouldBe(0);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_cap_is_the_page_limit_until_a_lower_gateway_ceiling_is_learned()
|
||||
{
|
||||
Source(Store(3600)).MaxRawValuesPerRead.ShouldBe(Cap);
|
||||
|
||||
// A strict gateway whose ceiling is ABOVE the cap: the cap wins, nothing is refused by the gateway.
|
||||
var higher = Store(3600, ceiling: 2000);
|
||||
var ds = Source(higher);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(3600), 0, Ct));
|
||||
higher.RawCalls.ShouldAllBe(c => c.MaxValues <= 2000);
|
||||
ds.MaxRawValuesPerRead.ShouldBe(Cap);
|
||||
|
||||
// A strict gateway whose ceiling is BELOW the cap: the learned ceiling (less the allowance) wins once known.
|
||||
var lower = Store(3600, ceiling: 500);
|
||||
var ds2 = Source(lower);
|
||||
ds2.MaxRawValuesPerRead.ShouldBe(Cap);
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds2.ReadRawAsync("T", At(0), At(3600), 0, Ct));
|
||||
ds2.MaxRawValuesPerRead.ShouldBe(500 - RawAggregateReader.PreWindowAllowance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_uncapped_read_over_the_cap_is_refused_never_truncated_and_never_asks_for_everything()
|
||||
{
|
||||
var fake = Store(3600);
|
||||
var ds = Source(fake);
|
||||
|
||||
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(() => ds.ReadRawAsync("T", At(0), At(3600), 0, Ct));
|
||||
|
||||
ex.Limit.ShouldBe(Cap);
|
||||
ex.Message.ShouldContain("MaxRawValuesPerRead");
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([Probe]); // never int.MaxValue
|
||||
fake.RawStreamsCompleted.ShouldBe(1, "the bounded stream is drained, never abandoned");
|
||||
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0, "a client-request limit is not a backend fault");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(10)]
|
||||
[InlineData(Cap - 1)]
|
||||
[InlineData(Cap)]
|
||||
public async Task An_uncapped_read_that_fits_the_cap_is_served_in_full(int count)
|
||||
{
|
||||
var fake = Store(count);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), 0, Ct);
|
||||
|
||||
r.Samples.Select(s => s.Value).ShouldBe(Enumerable.Range(0, count).Select(i => (object?)(double)i));
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([Probe]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(Cap + 1u)]
|
||||
[InlineData(5000u)]
|
||||
[InlineData(uint.MaxValue)]
|
||||
public async Task A_client_cap_above_the_server_cap_is_bounded_by_it(uint n)
|
||||
{
|
||||
await Should.ThrowAsync<HistoryReadLimitExceededException>(() => Source(Store(3600)).ReadRawAsync("T", At(0), At(3600), n, Ct));
|
||||
|
||||
var fake = Store(900);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), n, Ct);
|
||||
r.Samples.Count.ShouldBe(900);
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues == Probe);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(Cap)]
|
||||
[InlineData(Cap - 1)]
|
||||
[InlineData(1)]
|
||||
public async Task A_client_cap_at_or_under_the_server_cap_reads_as_before(int n)
|
||||
{
|
||||
var fake = Store(3600);
|
||||
var r = await Source(fake).ReadRawAsync("T", At(0), At(3600), (uint)n, Ct);
|
||||
|
||||
r.Samples.Count.ShouldBe(n);
|
||||
fake.RawCalls.Select(c => c.MaxValues).ShouldBe([n + RawAggregateReader.PreWindowAllowance]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_point_read_and_an_empty_window_are_unaffected()
|
||||
{
|
||||
var fake = Store(3600);
|
||||
var ds = Source(fake);
|
||||
(await ds.ReadRawAsync("T", At(5), At(5), 0, Ct)).Samples.Single().Value.ShouldBe(5d);
|
||||
(await ds.ReadRawAsync("T", At(5000), At(6000), 0, Ct)).Samples.ShouldBeEmpty();
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues == Probe);
|
||||
}
|
||||
|
||||
// ---- the factory binding --------------------------------------------------------------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData(777, 777)]
|
||||
[InlineData(0, 0)]
|
||||
[InlineData(-1, 0)]
|
||||
[InlineData(1, ServerHistorianOptions.MinMaxRawValuesPerRead)]
|
||||
public void The_factory_binds_the_effective_option(int configured, int expected)
|
||||
{
|
||||
var options = new ServerHistorianOptions
|
||||
{
|
||||
Endpoint = "http://localhost:5222", UseTls = false, ApiKey = "histgw_test_secret", MaxRawValuesPerRead = configured,
|
||||
};
|
||||
using var ds = GatewayHistorian.CreateDataSource(options, new ServiceCollection().BuildServiceProvider());
|
||||
ds.ShouldBeOfType<GatewayHistorianDataSource>().MaxRawValuesPerRead.ShouldBe(expected);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_default_option_caps_the_source_at_ten_thousand()
|
||||
{
|
||||
var options = new ServerHistorianOptions { Endpoint = "http://localhost:5222", UseTls = false, ApiKey = "histgw_test_secret" };
|
||||
using var ds = new GatewayHistorianProvider(options).CreateDataSource(new ServiceCollection().BuildServiceProvider());
|
||||
ds.MaxRawValuesPerRead.ShouldBe(ServerHistorianOptions.DefaultMaxRawValuesPerRead);
|
||||
}
|
||||
|
||||
// ---- through the node manager ---------------------------------------------------------------------------------------
|
||||
|
||||
[Theory]
|
||||
[InlineData(0u, false)]
|
||||
[InlineData(0u, true)]
|
||||
[InlineData(5000u, false)]
|
||||
[InlineData(uint.MaxValue, false)]
|
||||
public async Task An_uncapped_HistoryRead_pages_at_the_cap_with_continuation_points(uint n, bool reverse)
|
||||
{
|
||||
var fake = Store(3600);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, status) = reverse
|
||||
? DrainFrom(nm, nodeId, At(3600), At(-1), n, null)
|
||||
: DrainFrom(nm, nodeId, At(0), At(3600), n, null);
|
||||
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
var expected = Enumerable.Range(0, 3600).Select(i => (double)i);
|
||||
values.ShouldBe(reverse ? expected.Reverse() : expected);
|
||||
pages.Count.ShouldBeGreaterThan(1, "a year-sized window at N = 0 is paged, not one shot");
|
||||
pages.ShouldAllBe(p => p > 0 && p <= Cap);
|
||||
pages[0].ShouldBe(Cap - 1); // page + lookahead fills a read
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Cap + RawAggregateReader.PreWindowAllowance);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_bounded_uncapped_HistoryRead_pages_within_the_cap()
|
||||
{
|
||||
var fake = Store(3600);
|
||||
var (host, nm, nodeId) = await BootAsync(fake);
|
||||
try
|
||||
{
|
||||
var (values, pages, status) = DrainFrom(nm, nodeId, At(10).AddMilliseconds(500), At(3000).AddMilliseconds(500), 0, null, returnBounds: true);
|
||||
status.Code.ShouldBe(StatusCodes.Good);
|
||||
values.ShouldBe(Enumerable.Range(10, 2992).Select(i => (double)i)); // start bound +10, data +11..+3000, end bound +3001
|
||||
pages.ShouldAllBe(p => p <= Cap);
|
||||
fake.RawCalls.ShouldAllBe(c => c.MaxValues > 0 && c.MaxValues <= Cap + RawAggregateReader.PreWindowAllowance);
|
||||
}
|
||||
finally { await host.DisposeAsync(); }
|
||||
}
|
||||
|
||||
// ---- helpers ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootAsync(FakeHistorianGatewayClient fake)
|
||||
{
|
||||
var started = await OpcUaHostStart.StartAsync(
|
||||
port => new OpcUaApplicationHost(
|
||||
new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.GatewayRawCapTest",
|
||||
ApplicationUri = $"urn:OtOpcUa.GatewayRawCapTest:{Guid.NewGuid():N}",
|
||||
OpcUaPort = port,
|
||||
PublicHostname = "localhost",
|
||||
PkiStoreRoot = _pkiRoot,
|
||||
},
|
||||
NullLogger<OpcUaApplicationHost>.Instance),
|
||||
() => new OtOpcUaSdkServer(), Ct);
|
||||
var nm = started.Server.NodeManager!;
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
nm.HistorianDataSource = Source(fake);
|
||||
nm.EnsureVariable("eq-1/pv", parentFolderNodeId: null, displayName: "PV", dataType: "Double",
|
||||
writable: false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
|
||||
return (started.Host, nm, nm.TryGetVariable("eq-1/pv")!.NodeId);
|
||||
}
|
||||
|
||||
private static (List<double> Values, List<int> Pages, StatusCode Status) DrainFrom(
|
||||
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? cp, int maxPages = 20,
|
||||
bool returnBounds = false)
|
||||
{
|
||||
var values = new List<double>();
|
||||
var pages = new List<int>();
|
||||
ServiceResult error;
|
||||
do
|
||||
{
|
||||
(var page, error, cp) = ReadRaw(nm, nodeId, start, end, max, cp, returnBounds);
|
||||
values.AddRange(page);
|
||||
pages.Add(page.Count);
|
||||
pages.Count.ShouldBeLessThanOrEqualTo(maxPages, "paging must terminate");
|
||||
}
|
||||
while (cp is not null && StatusCode.IsGood(error.StatusCode));
|
||||
return (values, pages, error.StatusCode);
|
||||
}
|
||||
|
||||
private static (IReadOnlyList<double> Values, ServiceResult Error, byte[]? Cp) ReadRaw(
|
||||
OtOpcUaNodeManager nm, NodeId nodeId, DateTime start, DateTime end, uint max, byte[]? inboundCp, bool returnBounds = false)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
ReturnBounds = returnBounds, IsReadModified = false, StartTime = start, EndTime = end, NumValuesPerNode = max,
|
||||
};
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = nodeId, ContinuationPoint = inboundCp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
|
||||
|
||||
var values = new List<double>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
foreach (var dv in ((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues)
|
||||
values.Add(Convert.ToDouble(dv.Value));
|
||||
return (values, errors[0], results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_pkiRoot))
|
||||
{
|
||||
try { Directory.Delete(_pkiRoot, recursive: true); }
|
||||
catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-2
@@ -110,8 +110,9 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
|
||||
new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), timeProvider: clock);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows) =>
|
||||
new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), maxTimeWeightedRawRows: maxTimeWeightedRawRows);
|
||||
public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0) =>
|
||||
new QuestDbHistorianDataSource(Conn, Boot, NewCatalog(), maxTimeWeightedRawRows: maxTimeWeightedRawRows,
|
||||
maxRawValuesPerRead: maxRawValuesPerRead);
|
||||
|
||||
/// <inheritdoc/>
|
||||
/// <remarks>The T4 live-value writer over a fresh catalog (fresh positive cache) and this instance's WAL wait.</remarks>
|
||||
|
||||
+15
@@ -47,6 +47,21 @@ public sealed class QuestDbHistorianProviderTests
|
||||
p.SupportedKinds.Count.ShouldBe(6);
|
||||
}
|
||||
|
||||
/// <summary>R08:1.1 (#736): the server raw cap the Host binds reaches the data source; without one it is unbounded as before.</summary>
|
||||
[Fact]
|
||||
public void The_server_raw_cap_reaches_the_data_source()
|
||||
{
|
||||
var sp = new NullSp();
|
||||
using var plain = Make();
|
||||
plain.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(0);
|
||||
|
||||
using var capped = new QuestDbHistorianProvider(new QuestDbHistorianOptions
|
||||
{
|
||||
ConnectionString = "Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading",
|
||||
}, maxRawValuesPerRead: 2500);
|
||||
capped.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(2500);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Creates_all_five_seams_without_touching_questdb()
|
||||
{
|
||||
|
||||
+3
-2
@@ -45,8 +45,9 @@ internal sealed class ReadTestDb
|
||||
|
||||
/// <summary>A fresh data source (fresh catalog cache) over this schema; <paramref name="clock"/> is its "now".</summary>
|
||||
public SqlServerHistorianDataSource Source(TimeProvider? clock = null,
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows) =>
|
||||
new(Conn, Boot, new SqlServerTagCatalog(Conn, Boot), timeProvider: clock, maxTimeWeightedRawRows: maxTimeWeightedRawRows);
|
||||
int maxTimeWeightedRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, int maxRawValuesPerRead = 0) =>
|
||||
new(Conn, Boot, new SqlServerTagCatalog(Conn, Boot), timeProvider: clock, maxTimeWeightedRawRows: maxTimeWeightedRawRows,
|
||||
maxRawValuesPerRead: maxRawValuesPerRead);
|
||||
|
||||
/// <summary>Inserts a tag row and the given samples. ServerTimeUtc = source time + 1 s (so tests can tell them apart).</summary>
|
||||
public async Task<int> SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples)
|
||||
|
||||
+2
-1
@@ -26,7 +26,8 @@ internal sealed class SqlServerConformanceBackend : IConformanceBackend
|
||||
public IHistorianDataSource CreateDataSource(TimeProvider? clock = null) => _db.Source(clock);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows) => _db.Source(null, maxTimeWeightedRawRows);
|
||||
public IHistorianDataSource CreateDataSource(int maxTimeWeightedRawRows, int maxRawValuesPerRead = 0) =>
|
||||
_db.Source(null, maxTimeWeightedRawRows, maxRawValuesPerRead);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IHistorianValueWriter CreateValueWriter(Func<DateTime>? utcNow = null) =>
|
||||
|
||||
+14
@@ -47,6 +47,20 @@ public sealed class SqlServerHistorianProviderTests
|
||||
p.SupportedKinds.Count.ShouldBe(6);
|
||||
}
|
||||
|
||||
/// <summary>R08:1.1 (#736): the server raw cap the Host binds reaches the data source; without one it is unbounded as before.</summary>
|
||||
[Fact]
|
||||
public void The_server_raw_cap_reaches_the_data_source()
|
||||
{
|
||||
var sp = new NullSp();
|
||||
Make().CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(0);
|
||||
|
||||
var capped = new SqlServerHistorianProvider(new SqlServerHistorianOptions
|
||||
{
|
||||
ConnectionString = "Server=localhost;User Id=sa;Password=x;TrustServerCertificate=True",
|
||||
}, maxRawValuesPerRead: 2500);
|
||||
capped.CreateDataSource(sp).MaxRawValuesPerRead.ShouldBe(2500);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Creates_all_five_seams_without_touching_sql()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Telemetry;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 U2: <c>TelemetryDial:CallTimeoutSeconds</c> was bound, defaulted and documented, and consumed
|
||||
/// nowhere. It now bounds a dial's <b>establishment</b>: a dial that has delivered no event by the deadline is
|
||||
/// cut and re-dialled with a failure streak. A live node answers a fresh subscribe at once (the node-local hub
|
||||
/// replays its health/resilience snapshots), so the deadline bites only a dial that never connects — a
|
||||
/// black-holed host, where the underlying socket connect has no timeout of its own. An established stream
|
||||
/// is deliberately NOT bounded: HTTP/2 keepalive pings own its liveness, and the wire has no heartbeat, so an
|
||||
/// idle timeout would churn every quiet node.
|
||||
/// </summary>
|
||||
public sealed class TelemetryDialDeadlineTests : TestKit
|
||||
{
|
||||
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(8);
|
||||
|
||||
[Fact]
|
||||
public void A_dial_that_delivers_no_event_by_the_deadline_is_cancelled_and_redialled()
|
||||
{
|
||||
var dial = new FakeDialLoop();
|
||||
Spawn(dial, callTimeoutSeconds: 1);
|
||||
|
||||
var first = dial.WaitForCount(1, Timeout).Single();
|
||||
first.Ct.IsCancellationRequested.ShouldBeFalse("the deadline has not elapsed yet");
|
||||
|
||||
// The stalled dial is cut at the deadline...
|
||||
AwaitCondition(() => first.Ct.IsCancellationRequested, Timeout);
|
||||
// ...and the supervisor re-dials (a stalled dial is a transport failure, never a silent end).
|
||||
var invocations = dial.WaitForCount(2, Timeout);
|
||||
invocations[1].CorrelationId.ShouldBe("central-node-1-2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_dial_that_delivers_an_event_before_the_deadline_is_left_open()
|
||||
{
|
||||
var dial = new FakeDialLoop { DeliverOnOpen = true };
|
||||
var sinks = new Sinks();
|
||||
Spawn(dial, callTimeoutSeconds: 1, sinks);
|
||||
|
||||
var first = dial.WaitForCount(1, Timeout).Single();
|
||||
AwaitCondition(() => sinks.HealthStore.Upserts.Count == 1, Timeout);
|
||||
|
||||
// Positive control: well past the deadline, the established stream is untouched.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2.5));
|
||||
first.Ct.IsCancellationRequested.ShouldBeFalse("an established stream must never be cut by the dial deadline");
|
||||
dial.Invocations.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
private IActorRef Spawn(FakeDialLoop dial, int callTimeoutSeconds, Sinks? sinks = null)
|
||||
{
|
||||
sinks ??= new Sinks();
|
||||
var options = new TelemetryDialOptions { ContactRefreshSeconds = 3600, CallTimeoutSeconds = callTimeoutSeconds };
|
||||
return Sys.ActorOf(TelemetryDialSupervisor.Props(
|
||||
() => Task.FromResult<IReadOnlyList<TelemetryDialTarget>>(
|
||||
new[] { new TelemetryDialTarget("node-1", "http://h1:5300") }),
|
||||
dial.Loop,
|
||||
sinks.AlarmBroadcaster,
|
||||
sinks.ScriptBroadcaster,
|
||||
sinks.HealthStore,
|
||||
sinks.ResilienceStore,
|
||||
options));
|
||||
}
|
||||
|
||||
private static DriverHealthChanged Health(string id) =>
|
||||
new("C1", id, "Healthy", DateTime.UtcNow, null, 0, DateTime.UtcNow);
|
||||
|
||||
private sealed class DialInvocation
|
||||
{
|
||||
private readonly TaskCompletionSource _completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
public required string CorrelationId { get; init; }
|
||||
public required CancellationToken Ct { get; init; }
|
||||
public Task Completion => _completion.Task;
|
||||
public void Complete() => _completion.TrySetResult();
|
||||
}
|
||||
|
||||
/// <summary>Records each invocation; optionally delivers one health event on open; ends only when cancelled.</summary>
|
||||
private sealed class FakeDialLoop
|
||||
{
|
||||
private readonly ConcurrentQueue<DialInvocation> _invocations = new();
|
||||
|
||||
public bool DeliverOnOpen { get; init; }
|
||||
|
||||
public IReadOnlyList<DialInvocation> Invocations => _invocations.ToArray();
|
||||
|
||||
public TelemetryDialLoop Loop => async (_, correlationId, onMapped, _, ct) =>
|
||||
{
|
||||
var invocation = new DialInvocation { CorrelationId = correlationId, Ct = ct };
|
||||
_invocations.Enqueue(invocation);
|
||||
if (DeliverOnOpen) onMapped(Health("drv-1"));
|
||||
await using var reg = ct.Register(() => invocation.Complete());
|
||||
await invocation.Completion;
|
||||
};
|
||||
|
||||
public IReadOnlyList<DialInvocation> WaitForCount(int count, TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var snapshot = _invocations.ToArray();
|
||||
if (snapshot.Length >= count) return snapshot;
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
|
||||
throw new Xunit.Sdk.XunitException(
|
||||
$"Expected at least {count} dial invocations within {timeout}, saw {_invocations.Count}.");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Sinks
|
||||
{
|
||||
public NullBroadcaster<AlarmTransitionEvent> AlarmBroadcaster { get; } = new();
|
||||
public NullBroadcaster<ScriptLogEntry> ScriptBroadcaster { get; } = new();
|
||||
public RecordingHealthStore HealthStore { get; } = new();
|
||||
public NullResilienceStore ResilienceStore { get; } = new();
|
||||
}
|
||||
|
||||
private sealed class NullBroadcaster<T> : IInProcessBroadcaster<T>
|
||||
{
|
||||
public event Action<T>? Received;
|
||||
public event Action<bool>? ConnectionStateChanged;
|
||||
public bool IsConnected { get; private set; }
|
||||
public void Publish(T item) => Received?.Invoke(item);
|
||||
public void SetConnected(bool connected) { IsConnected = connected; ConnectionStateChanged?.Invoke(connected); }
|
||||
}
|
||||
|
||||
private sealed class RecordingHealthStore : IDriverStatusSnapshotStore
|
||||
{
|
||||
public ConcurrentQueue<DriverHealthChanged> Upserts { get; } = new();
|
||||
public event Action<DriverHealthChanged>? SnapshotChanged;
|
||||
public void Upsert(DriverHealthChanged snapshot) { Upserts.Enqueue(snapshot); SnapshotChanged?.Invoke(snapshot); }
|
||||
public bool TryGet(string driverInstanceId, out DriverHealthChanged snapshot) { snapshot = null!; return false; }
|
||||
public IReadOnlyCollection<DriverHealthChanged> GetAll() => Upserts.ToArray();
|
||||
}
|
||||
|
||||
private sealed class NullResilienceStore : IDriverResilienceStatusStore
|
||||
{
|
||||
public event Action<DriverResilienceStatusChanged>? SnapshotChanged;
|
||||
public void Upsert(DriverResilienceStatusChanged snapshot) => SnapshotChanged?.Invoke(snapshot);
|
||||
public IReadOnlyList<DriverResilienceStatusChanged> GetForInstance(string driverInstanceId) => Array.Empty<DriverResilienceStatusChanged>();
|
||||
public IReadOnlyCollection<DriverResilienceStatusChanged> GetAll() => Array.Empty<DriverResilienceStatusChanged>();
|
||||
}
|
||||
}
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// R08:1.1 (#736): the server raw cap (<c>ServerHistorian:MaxRawValuesPerRead</c>) is advertised as
|
||||
/// <c>HistoryServerCapabilities/MaxReturnDataValues</c> (i=11273) through <see cref="HistoryPaging.AdvertiseMaxReturnDataValues"/>,
|
||||
/// read back the way a client would — through the SDK master node manager — over a real server.
|
||||
/// </summary>
|
||||
public sealed class HistoryServerCapabilitiesAdvertisementTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
|
||||
{
|
||||
private async Task<uint> ReadAdvertisedAsync()
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.Read, identity: null);
|
||||
var nodesToRead = new ReadValueIdCollection
|
||||
{
|
||||
new ReadValueId { NodeId = VariableIds.HistoryServerCapabilities_MaxReturnDataValues, AttributeId = Attributes.Value },
|
||||
};
|
||||
var (values, _) = await fx.Nm.Server.NodeManager.ReadAsync(context, 0, TimestampsToReturn.Neither, nodesToRead, TestContext.Current.CancellationToken);
|
||||
values.Count.ShouldBe(1);
|
||||
StatusCode.IsGood(values[0].StatusCode).ShouldBeTrue(values[0].StatusCode.ToString());
|
||||
return (uint)values[0].Value;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_cap_is_advertised_as_MaxReturnDataValues_and_a_re_sync_overwrites_it()
|
||||
{
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 10_000).ShouldBeTrue();
|
||||
(await ReadAdvertisedAsync()).ShouldBe(10_000u);
|
||||
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 2_500).ShouldBeTrue();
|
||||
(await ReadAdvertisedAsync()).ShouldBe(2_500u);
|
||||
|
||||
// Unbounded (the opt-out) advertises Part 11's "no limit".
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 0).ShouldBeTrue();
|
||||
(await ReadAdvertisedAsync()).ShouldBe(0u);
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, -5).ShouldBeTrue();
|
||||
(await ReadAdvertisedAsync()).ShouldBe(0u);
|
||||
}
|
||||
finally
|
||||
{
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(fx.Nm.Server, 0);
|
||||
fx.Gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_server_is_a_quiet_false()
|
||||
{
|
||||
HistoryPaging.AdvertiseMaxReturnDataValues(null, 10_000).ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
+436
@@ -0,0 +1,436 @@
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Server;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using HistorianRead = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
|
||||
using SdkHistoryReadResult = Opc.Ua.HistoryReadResult;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// R08:1.1 (#736): the paged-vs-unpaged fuzz for the server raw cap. Every paging change in this repo has had boundary
|
||||
/// bugs that the hand-written cases missed (silent truncation, an endless cursor, an empty page between every value), so
|
||||
/// the cap is proved the only way that has worked: random series (bursts, gaps, sub-millisecond stamps, tie clusters),
|
||||
/// random windows (empty, on a tie, one step past a tie, between samples, forward / reverse / end-only, with and without
|
||||
/// bounds) and random caps — <c>NumValuesPerNode</c> 0 / 1 / 2 / small / huge, the server cap 3 … 1000 or off, the
|
||||
/// tie-cluster budget 3 / 8 / 65536 — paged through a REAL SDK node manager against a wire-faithful fake of each backend
|
||||
/// (SqlServer: tick precision, native reverse, <c>TOP</c>; QuestDB: µs truncation of bounds AND stamps, native reverse,
|
||||
/// <c>LIMIT</c>; Gateway: forward only, <c>[start, end)</c> after its own boundary-row filtering, walked back by the
|
||||
/// server) that enforces the backend half of the cap exactly as the real sources do (a read asking for 0 or more than
|
||||
/// the cap reads cap + 1 rows and is refused when the window holds more).
|
||||
/// <para>
|
||||
/// The oracle is the same node manager's one-shot answer with the cap OFF (<c>NumValuesPerNode = 0</c>, no limit),
|
||||
/// which for a plain read is also checked against the series itself. The concatenation of the pages must equal it
|
||||
/// exactly; the read must terminate within the number of values plus two pages; no page may exceed the cap; the fake
|
||||
/// must never see a request for 0 or more than the cap (the "bounded by the same cap" invariant); and a
|
||||
/// <c>BadTooManyOperations</c> is accepted only when a tie cluster (or, on the Gateway shape, one millisecond) is
|
||||
/// larger than the budget that the cap and <c>MaxTieClusterOverfetch</c> leave — after an exact prefix. The run is
|
||||
/// non-vacuous by construction: it asserts that empty windows, mid-tie windows, reverse reads, <c>NumValuesPerNode</c>
|
||||
/// of 1 and 2, the smallest server cap (3) and every shape occurred, and that most iterations were exact.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Iterations: <c>OTOPCUA_PAGING_FUZZ_ITERATIONS</c> (default 10 000, the lane gate). Seeds are stable, so a failure
|
||||
/// names a reproducible seed.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class NodeManagerHistoryReadRawCapFuzzTests(BoundsServerFixture fx) : IClassFixture<BoundsServerFixture>
|
||||
{
|
||||
private static readonly DateTime T0 = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>The backend the fake is shaped like.</summary>
|
||||
public enum Shape { SqlServer, QuestDb, Gateway }
|
||||
|
||||
private enum Direction { Forward, Reverse, EndOnly }
|
||||
|
||||
private static int Iterations =>
|
||||
int.TryParse(Environment.GetEnvironmentVariable("OTOPCUA_PAGING_FUZZ_ITERATIONS"), out var n) && n > 0 ? n : 10_000;
|
||||
|
||||
[Fact]
|
||||
public async Task Paged_equals_unpaged_on_random_series_windows_and_caps_and_always_terminates()
|
||||
{
|
||||
var iterations = Iterations;
|
||||
var counts = new Dictionary<string, int>();
|
||||
void Count(string key) => counts[key] = counts.GetValueOrDefault(key) + 1;
|
||||
|
||||
await fx.Gate.WaitAsync(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
for (var seed = 0; seed < iterations; seed++)
|
||||
{
|
||||
var rng = new Random(seed);
|
||||
var shape = (Shape)(seed % 3);
|
||||
var resolution = shape == Shape.QuestDb ? 10L : 1L;
|
||||
var series = RandomSeries(rng, resolution);
|
||||
var cap = rng.Next(10) switch { 0 => 0, 1 or 2 => 3, 3 => 4, 4 => 5, 5 => 7, 6 => 10, 7 => 16, 8 => 33, _ => 1000 };
|
||||
var n = rng.Next(9) switch { 0 or 1 => 0u, 2 or 3 => 1u, 4 or 5 => 2u, 6 => (uint)rng.Next(3, 20), 7 => (uint)rng.Next(20, 200), _ => 100_000u };
|
||||
var tieBudget = rng.Next(4) switch { 0 => 3, 1 => 8, _ => 65536 };
|
||||
var direction = rng.Next(5) switch { 0 => Direction.Reverse, 1 => Direction.EndOnly, _ => Direction.Forward };
|
||||
if (direction == Direction.EndOnly && n == 0) direction = Direction.Forward; // Part 11: one bound only with a cap
|
||||
var bounds = direction != Direction.EndOnly && rng.Next(4) == 0;
|
||||
|
||||
var a = PickBoundary(rng, series, resolution);
|
||||
var b = PickBoundary(rng, series, resolution);
|
||||
var (lo, hi) = a <= b ? (a, b) : (b, a);
|
||||
var (start, end) = direction switch
|
||||
{
|
||||
Direction.Forward => (lo, hi),
|
||||
Direction.Reverse => (hi, lo),
|
||||
_ => (DateTime.MinValue, hi),
|
||||
};
|
||||
var tag = $"seed {seed} {shape} cap {cap} N {n} tie {tieBudget} {direction} bounds {bounds} [{start:O} .. {end:O}]";
|
||||
|
||||
// The walk-back (Gateway shape only: reverse / end-only reads, and the bound lookups of any bounded read) refuses
|
||||
// a millisecond holding more samples than its budget — computed up front, over the whole series, because the
|
||||
// walk and the lookups read around the window.
|
||||
var walks = shape == Shape.Gateway && (direction != Direction.Forward || bounds);
|
||||
var msMax = walks ? MillisecondMax(series) : 0;
|
||||
|
||||
// The oracle: the one-shot answer with the cap off (and, for a plain read, the series itself).
|
||||
var truthSource = new CappedFake(series, shape, cap: 0);
|
||||
fx.Nm.HistoricalSourceForTest(truthSource, tieBudget);
|
||||
// An end-only read is paged backward from EndTime (the node manager flips it to a reverse read down to the
|
||||
// beginning of time); its oracle is the explicit reverse read down to the Unix epoch — the walk-back's floor,
|
||||
// and Part 11 (the SDK checks it) needs both bounds set when NumValuesPerNode = 0.
|
||||
var (truth, truthError, truthCp) = direction == Direction.EndOnly
|
||||
? Read(hi, DateTime.UnixEpoch, 0, bounds: false, cp: null)
|
||||
: Read(start, end, 0, bounds, cp: null);
|
||||
if (truthError.StatusCode.Code == StatusCodes.BadTooManyOperations)
|
||||
{
|
||||
// Only the walk-back can refuse an uncapped one-shot read, and only past its budget (max(1, M) with no cap).
|
||||
(walks && msMax > Math.Max(1, tieBudget)).ShouldBeTrue($"the oracle read was refused: {tag} (ms max {msMax})");
|
||||
Count("oracle refused");
|
||||
continue;
|
||||
}
|
||||
truthError.StatusCode.Code.ShouldBe(StatusCodes.Good, $"oracle: {tag}");
|
||||
truthCp.ShouldBeNull($"oracle (no cap, N = 0) is one page: {tag}");
|
||||
if (!bounds)
|
||||
Render(truth).ShouldBe(Render(Oracle(series, start, end, direction, resolution)), $"oracle vs series: {tag}");
|
||||
truthSource.BadRequests.ShouldBe(0, $"a reverse window reached a forward-only backend on the oracle read: {tag}");
|
||||
|
||||
var windowTies = LargestTieCluster(truth);
|
||||
if (truth.Count == 0) Count("empty");
|
||||
if (windowTies >= 2) Count("ties in window");
|
||||
if (EndsOnTie(series, start, end, direction, resolution)) Count("window ends on a tie");
|
||||
if (direction == Direction.Reverse) Count("reverse");
|
||||
if (direction == Direction.EndOnly) Count("end-only");
|
||||
if (n == 1) Count("N=1");
|
||||
if (n == 2) Count("N=2");
|
||||
if (n == 0) Count("N=0");
|
||||
if (cap == 3) Count("cap=3");
|
||||
if (bounds) Count("bounds");
|
||||
Count(shape.ToString());
|
||||
|
||||
// A boundary tie cluster as large as the cap leaves a resumed page no room: the server steps past it with one
|
||||
// empty page carrying a continuation point (the documented #694 ceiling behaviour) — the one empty page allowed.
|
||||
var oversizedClusters = cap > 0
|
||||
? truth.Where(v => !(v.Value is null && v.StatusCode.Code == StatusCodes.BadBoundNotFound))
|
||||
.GroupBy(v => v.SourceTimestamp.Ticks).Count(g => g.Count() >= Math.Max(2, cap - 1))
|
||||
: 0;
|
||||
|
||||
// The read under test.
|
||||
var source = new CappedFake(series, shape, cap);
|
||||
fx.Nm.HistoricalSourceForTest(source, tieBudget);
|
||||
var pages = new List<IReadOnlyList<DataValue>>();
|
||||
var collected = new List<DataValue>();
|
||||
byte[]? cp = null;
|
||||
StatusCode status;
|
||||
do
|
||||
{
|
||||
var (values, error, next) = Read(start, end, n, bounds, cp);
|
||||
status = error.StatusCode;
|
||||
if (!StatusCode.IsGood(status)) break;
|
||||
pages.Add(values);
|
||||
collected.AddRange(values);
|
||||
cp = next;
|
||||
pages.Count.ShouldBeLessThanOrEqualTo(truth.Count + 2 + oversizedClusters, $"paging must terminate: {tag} pages {Sizes(pages)}");
|
||||
}
|
||||
while (cp is not null);
|
||||
|
||||
source.OverCapRequests.ShouldBe(0, $"the server asked the backend for 0 or more than the cap: {tag}");
|
||||
source.BadRequests.ShouldBe(0, $"a reverse window reached a forward-only backend: {tag}");
|
||||
|
||||
if (status.Code == StatusCodes.BadTooManyOperations)
|
||||
{
|
||||
// Legitimate only past the budget the cap leaves: a tie cluster larger than min(M, cap − 1), or on the
|
||||
// Gateway shape a millisecond holding more than min(M, cap − 2) samples (the walk-back's budget).
|
||||
var overfetchBound = cap > 0 ? Math.Min(Math.Max(1, tieBudget), Math.Max(1, cap - 1)) : Math.Max(1, tieBudget);
|
||||
var walkBudget = cap > 0 ? Math.Max(1, Math.Min(tieBudget, cap - 2)) : Math.Max(1, tieBudget);
|
||||
var legit = windowTies > overfetchBound || (walks && msMax > walkBudget);
|
||||
legit.ShouldBeTrue($"refused with no tie cluster over {overfetchBound} (max {windowTies}; ms max {msMax} vs {walkBudget}): {tag}");
|
||||
Render(collected).ShouldBe(Render(truth.Take(collected.Count)), $"the prefix before the refusal: {tag}");
|
||||
Count("refused");
|
||||
continue;
|
||||
}
|
||||
|
||||
status.Code.ShouldBe(StatusCodes.Good, $"{tag}");
|
||||
Render(collected).ShouldBe(Render(truth), $"paged ≠ unpaged: {tag} pages {Sizes(pages)}");
|
||||
if (cap > 0)
|
||||
{
|
||||
// A bounded read whose data fits one page carries its start bound, the data and its end bound: the node
|
||||
// manager fits the end bound against the client's N but not against the server cap (follow-up F2), so that
|
||||
// one page may hold cap + 1 values; every other page is at most the cap.
|
||||
var bothBoundsOnOnePage = bounds && pages.Count == 1;
|
||||
pages.ShouldAllBe(p => p.Count <= cap + (bothBoundsOnOnePage ? 1 : 0), $"a page over the cap: {tag} pages {Sizes(pages)}");
|
||||
if (bothBoundsOnOnePage && pages[0].Count == cap + 1) Count("bounded single page at cap+1 (F2)");
|
||||
}
|
||||
if (n > 0) pages.ShouldAllBe(p => p.Count <= n, $"a page over N: {tag} pages {Sizes(pages)}");
|
||||
for (var i = 0; i < pages.Count - 1; i++)
|
||||
{
|
||||
if (pages[i].Count > 0) continue;
|
||||
// Allowed only as the step past a boundary tie cluster the cap cannot page within.
|
||||
var previousLast = i > 0 && pages[i - 1].Count > 0 ? pages[i - 1][^1].SourceTimestamp.Ticks : (long?)null;
|
||||
var clusterAtBoundary = previousLast is { } t ? truth.Count(v => v.SourceTimestamp.Ticks == t) : 0;
|
||||
(cap > 0 && clusterAtBoundary >= Math.Max(2, cap - 1)).ShouldBeTrue(
|
||||
$"an empty page with a continuation point (page {i}, boundary cluster {clusterAtBoundary}): {tag} pages {Sizes(pages)}");
|
||||
Count("empty step over an oversized tie cluster");
|
||||
}
|
||||
if (pages.Count > 1) Count("paged");
|
||||
if (pages.Count > 1 && n == 0) Count("paged at N=0");
|
||||
// A cost guard against a read explosion, not a budget. A native shape reads once per page plus the bound
|
||||
// lookups. The Gateway shape's walk-back has its own cost profile — ~30 reads a page on a dense tag
|
||||
// (#691/#693), far more on a sparse series under a tiny tie budget (seed 320: 82 reads with NO cap, seed 6428:
|
||||
// 120+) — so it is judged RELATIVE to the same request with the cap off (the pre-#736 behaviour), per page:
|
||||
// the cap must not make the walk explode compared to before. The baseline asks for the page size the cap
|
||||
// imposes (N = cap − 1): an uncapped N = 0 reverse read used to be ONE int.MaxValue forward read — the very
|
||||
// memory vector the cap removes — so "what N = 0 cost before" is not the comparison.
|
||||
var allowedReads = 8 * pages.Count + 8;
|
||||
if (shape == Shape.Gateway)
|
||||
{
|
||||
var baseline = new CappedFake(series, shape, cap: 0);
|
||||
fx.Nm.HistoricalSourceForTest(baseline, tieBudget);
|
||||
var baseN = cap > 0 && (n == 0 || n > (uint)cap) ? (uint)(cap - 1) : n;
|
||||
var basePages = 0;
|
||||
byte[]? baseCp = null;
|
||||
do
|
||||
{
|
||||
var (_, baseError, next) = Read(start, end, baseN, bounds, baseCp);
|
||||
if (!StatusCode.IsGood(baseError.StatusCode)) break;
|
||||
basePages++;
|
||||
baseCp = next;
|
||||
}
|
||||
while (baseCp is not null && basePages <= truth.Count + 2 + oversizedClusters);
|
||||
allowedReads = 2 * baseline.Reads * Math.Max(1, pages.Count) / Math.Max(1, basePages) + 32 * pages.Count + 16;
|
||||
}
|
||||
source.Reads.ShouldBeLessThanOrEqualTo(allowedReads,
|
||||
$"backend reads exploded: {source.Reads} for {pages.Count} pages (allowed {allowedReads}): {tag}");
|
||||
Count("exact");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
fx.Nm.HistoricalSourceForTest(NullHistorianDataSource.Instance, 65536);
|
||||
fx.Gate.Release();
|
||||
}
|
||||
|
||||
// Non-vacuous: every category the brief names occurred, and most runs were exact.
|
||||
var summary = string.Join(", ", counts.OrderBy(kv => kv.Key).Select(kv => $"{kv.Key}={kv.Value}"));
|
||||
foreach (var required in new[] { "empty", "ties in window", "window ends on a tie", "reverse", "end-only", "N=1", "N=2", "N=0", "cap=3", "bounds", "paged", "paged at N=0", "SqlServer", "QuestDb", "Gateway" })
|
||||
counts.GetValueOrDefault(required).ShouldBeGreaterThan(iterations / 200, $"{required} under-represented: {summary}");
|
||||
counts.GetValueOrDefault("exact").ShouldBeGreaterThan(iterations * 6 / 10, summary);
|
||||
TestContext.Current.TestOutputHelper?.WriteLine($"raw-cap fuzz: {iterations} iterations — {summary}");
|
||||
}
|
||||
|
||||
// --- generators -----------------------------------------------------------------------------
|
||||
|
||||
private static DateTime Ts(DataValueSnapshot s) => s.SourceTimestampUtc ?? DateTime.MinValue;
|
||||
|
||||
private static DateTime Trunc(DateTime t, long resolution) => new(t.Ticks - t.Ticks % resolution, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>Random ascending series: bursts, gaps, sub-millisecond stamps and tie clusters; stamps stored at the resolution.</summary>
|
||||
private static List<DataValueSnapshot> RandomSeries(Random rng, long resolution)
|
||||
{
|
||||
var list = new List<DataValueSnapshot>();
|
||||
var t = T0.Ticks - rng.Next(0, 3) * TimeSpan.TicksPerHour;
|
||||
var count = rng.Next(1, 120);
|
||||
var v = 0;
|
||||
while (list.Count < count)
|
||||
{
|
||||
t += rng.Next(6) switch
|
||||
{
|
||||
0 => rng.Next(1, 10), // sub-millisecond (sub-µs on QuestDB: collapses onto a tie)
|
||||
1 => rng.Next(1, 5) * TimeSpan.TicksPerMillisecond, // dense
|
||||
2 => rng.Next(1, 3) * TimeSpan.TicksPerSecond,
|
||||
3 => rng.Next(1, 30) * TimeSpan.TicksPerMinute, // a gap
|
||||
_ => rng.Next(1, 1000) * TimeSpan.TicksPerMillisecond,
|
||||
};
|
||||
var ties = rng.Next(7) == 0 ? rng.Next(2, 9) : 1;
|
||||
for (var k = 0; k < ties; k++)
|
||||
{
|
||||
var ts = Trunc(new DateTime(t, DateTimeKind.Utc), resolution);
|
||||
var status = rng.Next(12) == 0 ? StatusCodes.Bad : StatusCodes.Good;
|
||||
list.Add(new DataValueSnapshot(status == StatusCodes.Bad ? null : (double)v, status, ts, ts));
|
||||
v++;
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>A window bound: before / after the data, exactly on a sample (a tie, maybe), one step past one, a tick before one, or between two.</summary>
|
||||
private static DateTime PickBoundary(Random rng, List<DataValueSnapshot> series, long resolution)
|
||||
{
|
||||
var first = Ts(series[0]);
|
||||
var last = Ts(series[^1]);
|
||||
var i = rng.Next(series.Count);
|
||||
var t = Ts(series[i]);
|
||||
switch (rng.Next(9))
|
||||
{
|
||||
case 0: return first.AddSeconds(-rng.Next(1, 100));
|
||||
case 1: return last.AddSeconds(rng.Next(1, 100));
|
||||
case 2:
|
||||
case 3: return t;
|
||||
case 4: return t.AddTicks(resolution);
|
||||
case 5: return t.AddTicks(-1); // sub-resolution: a microsecond backend truncates it back ONTO the sample
|
||||
case 6: return t.AddMilliseconds(1);
|
||||
default:
|
||||
var next = i + 1 < series.Count ? Ts(series[i + 1]) : t.AddSeconds(10);
|
||||
return new DateTime(t.Ticks + (long)((next.Ticks - t.Ticks) * rng.NextDouble()), DateTimeKind.Utc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The series' own answer for a plain (no bounds) read: forward <c>[start, end)</c>, point, reverse / end-only <c>(end, start]</c> newest first.</summary>
|
||||
private static List<DataValue> Oracle(List<DataValueSnapshot> series, DateTime start, DateTime end, Direction direction, long resolution)
|
||||
{
|
||||
var s = Trunc(start, resolution);
|
||||
var e = Trunc(end, resolution);
|
||||
IEnumerable<DataValueSnapshot> window = direction switch
|
||||
{
|
||||
// start == end is Part 11's point read whichever way it was generated: the node manager reads it forward.
|
||||
Direction.Forward or Direction.Reverse when s == e => series.Where(x => Ts(x) == s),
|
||||
Direction.Forward => series.Where(x => Ts(x) >= s && Ts(x) < e),
|
||||
Direction.Reverse => series.Where(x => Ts(x) > e && Ts(x) <= s).Reverse(),
|
||||
_ => series.Where(x => Ts(x) <= e).Reverse(),
|
||||
};
|
||||
return window.Select(x => new DataValue(new Variant(x.Value), new StatusCode(x.StatusCode), Ts(x), Ts(x))).ToList();
|
||||
}
|
||||
|
||||
/// <summary>The largest tie cluster among the returned stored values (a bound placeholder is not a stored value).</summary>
|
||||
private static int LargestTieCluster(IReadOnlyList<DataValue> truth) =>
|
||||
truth.Where(v => !(v.Value is null && v.StatusCode.Code == StatusCodes.BadBoundNotFound))
|
||||
.GroupBy(v => v.SourceTimestamp.Ticks).Select(g => g.Count()).DefaultIfEmpty(0).Max();
|
||||
|
||||
/// <summary>The most samples within one millisecond anywhere in the series (the walk-back's refusal rule).</summary>
|
||||
private static int MillisecondMax(List<DataValueSnapshot> series)
|
||||
{
|
||||
var msMax = 0;
|
||||
for (int i = 0, j = 0; i < series.Count; i++)
|
||||
{
|
||||
while (Ts(series[i]).Ticks - Ts(series[j]).Ticks >= TimeSpan.TicksPerMillisecond) j++;
|
||||
msMax = Math.Max(msMax, i - j + 1);
|
||||
}
|
||||
|
||||
return msMax;
|
||||
}
|
||||
|
||||
private static bool EndsOnTie(List<DataValueSnapshot> series, DateTime start, DateTime end, Direction direction, long resolution)
|
||||
{
|
||||
var edge = Trunc(direction == Direction.Forward ? end : direction == Direction.Reverse ? start : end, resolution);
|
||||
return series.Count(x => Ts(x) == edge) >= 2;
|
||||
}
|
||||
|
||||
// --- helpers ----------------------------------------------------------------------------------
|
||||
|
||||
private static string Sizes(IEnumerable<IReadOnlyList<DataValue>> pages) => "[" + string.Join(",", pages.Select(p => p.Count)) + "]";
|
||||
|
||||
private static string Render(IEnumerable<DataValue> values) =>
|
||||
string.Join(" | ", values.Select(v => $"{v.SourceTimestamp.Ticks}={v.Value ?? "null"}/{v.StatusCode.Code:X8}"));
|
||||
|
||||
private (IReadOnlyList<DataValue> Values, ServiceResult Error, byte[]? Cp) Read(DateTime start, DateTime end, uint max, bool bounds, byte[]? cp)
|
||||
{
|
||||
var context = new OperationContext(new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
StartTime = start, EndTime = end, NumValuesPerNode = max, ReturnBounds = bounds, IsReadModified = false,
|
||||
};
|
||||
var nodesToRead = new List<HistoryReadValueId> { new() { NodeId = fx.NodeId, ContinuationPoint = cp } };
|
||||
var results = new List<SdkHistoryReadResult> { null! };
|
||||
var errors = new List<ServiceResult> { null! };
|
||||
fx.Nm.HistoryRead(context, details, TimestampsToReturn.Both, releaseContinuationPoints: false, nodesToRead, results, errors);
|
||||
|
||||
var values = new List<DataValue>();
|
||||
if (results[0]?.HistoryData is { } ho && !ExtensionObject.IsNull(ho))
|
||||
values.AddRange(((HistoryData)ExtensionObject.ToEncodeable(ho)).DataValues);
|
||||
return (values, errors[0], results[0]?.ContinuationPoint);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A series-backed historian shaped like one backend, enforcing the backend half of the server raw cap exactly as
|
||||
/// the real data sources do: it reports the cap as <see cref="MaxRawValuesPerRead"/>; a read asking for 0 or more than
|
||||
/// the cap reads cap + 1 rows and throws <see cref="HistoryReadLimitExceededException"/> when the window holds more
|
||||
/// (never a truncated answer). It also records every such request (<see cref="OverCapRequests"/>) — the server must
|
||||
/// never issue one — and every reverse window sent to the forward-only shape (<see cref="BadRequests"/>).
|
||||
/// </summary>
|
||||
private sealed class CappedFake(IReadOnlyList<DataValueSnapshot> series, Shape shape, int cap) : IHistorianDataSource
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
public int Reads { get; private set; }
|
||||
public int OverCapRequests { get; private set; }
|
||||
public int BadRequests { get; private set; }
|
||||
|
||||
public bool SupportsReverseRaw => shape != Shape.Gateway;
|
||||
|
||||
public long TimestampResolutionTicks => shape == Shape.QuestDb ? 10 : 1;
|
||||
|
||||
public int MaxRawValuesPerRead => cap;
|
||||
|
||||
private DateTime Trunc(DateTime t) => NodeManagerHistoryReadRawCapFuzzTests.Trunc(t, TimestampResolutionTicks);
|
||||
|
||||
public Task<HistorianRead> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode, CancellationToken cancellationToken)
|
||||
{
|
||||
var probe = cap > 0 && (maxValuesPerNode == 0 || maxValuesPerNode > (uint)cap);
|
||||
lock (_gate)
|
||||
{
|
||||
Reads++;
|
||||
if (probe) OverCapRequests++;
|
||||
if (shape == Shape.Gateway && startUtc > endUtc) BadRequests++;
|
||||
}
|
||||
|
||||
if (shape == Shape.Gateway && startUtc > endUtc)
|
||||
throw new InvalidOperationException("start_time must be less than or equal to end_time. (InvalidArgument)");
|
||||
|
||||
startUtc = Trunc(startUtc);
|
||||
endUtc = Trunc(endUtc);
|
||||
List<DataValueSnapshot> window;
|
||||
if (startUtc > endUtc) window = series.Where(s => Ts(s) > endUtc && Ts(s) <= startUtc).Reverse().ToList();
|
||||
else if (startUtc == endUtc) window = series.Where(s => Ts(s) == startUtc).ToList();
|
||||
else window = series.Where(s => Ts(s) >= startUtc && Ts(s) < endUtc).ToList();
|
||||
|
||||
var top = probe ? cap + 1 : maxValuesPerNode == 0 ? int.MaxValue : (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var page = window.Take(top).ToList();
|
||||
if (probe && page.Count > cap)
|
||||
throw new HistoryReadLimitExceededException(cap, $"more than MaxRawValuesPerRead ({cap}) samples");
|
||||
return Task.FromResult(new HistorianRead(page, null));
|
||||
}
|
||||
|
||||
public Task<HistorianRead> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistorianRead> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistorianRead(Array.Empty<DataValueSnapshot>(), null));
|
||||
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HistoricalEventsResult(Array.Empty<HistoricalEvent>(), null));
|
||||
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Test-only wiring shared by the paging fuzz: swap the source and the tie budget, with a fresh continuation store.</summary>
|
||||
internal static class NodeManagerFuzzExtensions
|
||||
{
|
||||
public static void HistoricalSourceForTest(this OtOpcUaNodeManager nm, IHistorianDataSource source, int tieBudget)
|
||||
{
|
||||
nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
|
||||
nm.MaxTieClusterOverfetch = tieBudget;
|
||||
nm.HistorianDataSource = source;
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,15 @@ public sealed class RawPageLimitTests
|
||||
[InlineData(5000u, 983, 984, 0u, 0u, true)] // the ties alone fill a read: never a 0 (uncapped) request
|
||||
[InlineData(5000u, 5000, 984, 0u, 0u, true)] // never an underflow
|
||||
[InlineData(5000u, 0, 1, 0u, 0u, true)]
|
||||
// R08:1.1 (#736): the server raw cap (default 10 000) is the limit every backend reports — an uncapped read pages at it.
|
||||
[InlineData(0u, 0, 10_000, 9_999u, 10_000u, false)] // NumValuesPerNode = 0: a first page of cap − 1 + the lookahead row
|
||||
[InlineData(0u, 1, 10_000, 9_998u, 10_000u, false)] // a resumed page re-reads its one boundary tie
|
||||
[InlineData(100_000u, 0, 10_000, 9_999u, 10_000u, false)] // a client cap above the server cap is bounded by it
|
||||
[InlineData(10_000u, 0, 10_000, 9_999u, 10_000u, false)] // exactly the cap: still cap − 1 (the lookahead row needs room)
|
||||
[InlineData(500u, 0, 10_000, 500u, 501u, false)] // under the cap: unchanged
|
||||
[InlineData(0u, 0, 3, 2u, 3u, false)] // the smallest usable cap (MinMaxRawValuesPerRead = 3): a first page of two
|
||||
[InlineData(0u, 1, 3, 1u, 3u, false)] // … and a resumed page still carries one value beside its boundary tie
|
||||
[InlineData(0u, 1, 2, 0u, 0u, true)] // at 2 a resumed page has NO room — every value would cost an empty page (why 3 is the floor)
|
||||
public void Plan_fits_page_ties_and_lookahead_into_the_limit(uint want, int k, int limit, uint page, uint request, bool noRoom)
|
||||
{
|
||||
var plan = RawPageLimit.Plan(want, k, limit);
|
||||
|
||||
+6
-6
@@ -64,7 +64,7 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
{
|
||||
published.Enqueue(e);
|
||||
var quality = e.Snapshot.StatusCode == 0u ? OpcUaStatus.Good : OpcUaStatus.Bad;
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
|
||||
mux.Tell(ValuePublishes.Single(
|
||||
"calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
};
|
||||
|
||||
@@ -75,7 +75,7 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
// compute dedups) until the computed value has flowed all the way through.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished(
|
||||
mux.Tell(ValuePublishes.Single(
|
||||
"src-drv", "src/a", 21.0, OpcUaStatus.Good, Ts));
|
||||
|
||||
// Leg 1 — dep flow: src/a=21 → A computes 42 and publishes on calc/engine/out.
|
||||
@@ -118,14 +118,14 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
{
|
||||
published.Enqueue(e);
|
||||
var quality = e.Snapshot.StatusCode == 0u ? OpcUaStatus.Good : OpcUaStatus.Bad;
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
mux.Tell(ValuePublishes.Single("calc-drv", e.FullReference, e.Snapshot.Value, quality, Ts));
|
||||
};
|
||||
var adapter = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(driver, mux), "dep-adapter2");
|
||||
|
||||
// Leg 1 — before redeploy: src/a=2 → keep=20, remove=7.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/a", 2.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/a", 2.0, OpcUaStatus.Good, Ts));
|
||||
published.ShouldContain(e => e.FullReference == "calc/keep" && Equals(e.Snapshot.Value, 20.0));
|
||||
published.ShouldContain(e => e.FullReference == "calc/remove" && Equals(e.Snapshot.Value, 7.0));
|
||||
}, PresenceBudget, TimeSpan.FromMilliseconds(100));
|
||||
@@ -150,8 +150,8 @@ public sealed class CalculationDependencyFlowTests : RuntimeActorTestBase
|
||||
// new result; the REMOVED tag never publishes a value derived from the post-redeploy input.
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/b", 50.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(new DriverInstanceActor.AttributeValuePublished("src-drv", "src/a", 3.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/b", 50.0, OpcUaStatus.Good, Ts));
|
||||
mux.Tell(ValuePublishes.Single("src-drv", "src/a", 3.0, OpcUaStatus.Good, Ts));
|
||||
|
||||
// (a) new dep B flows through the RE-REGISTERED interest → new tag computes: 50 + 100 = 150.
|
||||
published.ShouldContain(e => e.FullReference == "calc/add" && Equals(e.Snapshot.Value, 150.0));
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 U1: <c>GetDiagnostics</c> answered a placeholder — <c>Guid.Empty</c>, "Spawned", zero devices and
|
||||
/// "now" for every child, in every behaviour — so the snapshot looked live and said nothing. It now carries
|
||||
/// each child's row id, instance id, last-published health state, device counts and last change time.
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly NodeId TestNode = NodeId.Parse("node-1:4053");
|
||||
private const string DriverId = "DI-1";
|
||||
|
||||
[Fact]
|
||||
public async Task A_connected_child_reports_its_row_id_name_health_state_and_device_count()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var rowId = Guid.NewGuid();
|
||||
var before = DateTime.UtcNow.AddSeconds(-1);
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus");
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevA, new[] { (DriverId, "Line1-Modbus", "dev1", "speed", "Float", false) },
|
||||
Array.Empty<(string, string, string, string, string?)>(),
|
||||
enabled: true, driverRowIds: new Dictionary<string, Guid> { [DriverId] = rowId });
|
||||
var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
|
||||
|
||||
var snapshot = await AwaitDiagnosticsAsync(host, s => s.Drivers.Any(d => d.State == "Healthy"));
|
||||
|
||||
snapshot.NodeId.ShouldBe(TestNode);
|
||||
snapshot.CurrentRevision.ShouldBe(RevA);
|
||||
snapshot.RunningFromCache.ShouldBeFalse();
|
||||
var driver = snapshot.Drivers.ShouldHaveSingleItem();
|
||||
driver.DriverInstanceId.ShouldBe(rowId, "the real row id, not Guid.Empty");
|
||||
driver.Name.ShouldBe(DriverId, "the string instance id every other surface keys on");
|
||||
driver.State.ShouldBe("Healthy");
|
||||
driver.ConnectedDevices.ShouldBe(1);
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
driver.LastChangeUtc.ShouldBeGreaterThan(before);
|
||||
driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_stubbed_child_reports_Stubbed_with_no_devices()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
// The factory answers for a type the artifact does not use, so the host stubs the child.
|
||||
var factory = new RecordingSubscribableDriverFactory("NotModbus");
|
||||
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevA, new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) },
|
||||
Array.Empty<(string, string, string, string, string?)>(), enabled: true);
|
||||
var (host, _, _) = V3ArtifactSeeder.SpawnHostAndApply(this, db, TestNode, deploymentId, RevA, factory);
|
||||
|
||||
var snapshot = await AwaitDiagnosticsAsync(host, s => s.Drivers.Count == 1);
|
||||
|
||||
var driver = snapshot.Drivers.Single();
|
||||
driver.State.ShouldBe("Stubbed");
|
||||
driver.ConnectedDevices.ShouldBe(0);
|
||||
driver.FaultedDevices.ShouldBe(0);
|
||||
}
|
||||
|
||||
private async Task<NodeDiagnosticsSnapshot> AwaitDiagnosticsAsync(IActorRef host, Func<NodeDiagnosticsSnapshot, bool> ready)
|
||||
{
|
||||
NodeDiagnosticsSnapshot? snapshot = null;
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
snapshot = await host.Ask<NodeDiagnosticsSnapshot>(new GetDiagnostics(CorrelationId.NewId()), PresenceBudget);
|
||||
ready(snapshot).ShouldBeTrue();
|
||||
}, PresenceBudget);
|
||||
return snapshot!;
|
||||
}
|
||||
}
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L2: <c>DriverHostActor._lastReprovisionUtc</c> was never pruned — a driver removed by a deploy
|
||||
/// after a supervision crash kept its stamp, so the map grew with every id ever reprovisioned-then-removed,
|
||||
/// and (the observable half) a driver removed and later re-added inherited the OLD stamp: its first crash
|
||||
/// after the re-add was deferred by the backoff as if it were crash-looping, although the new incarnation had
|
||||
/// never crashed. <c>StopChild</c> now drops the stamp.
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorReprovisionStampTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
||||
private static readonly RevisionHash RevC = RevisionHash.Parse(new string('c', 64));
|
||||
private static readonly NodeId TestNode = NodeId.Parse("node-1:4053");
|
||||
private const string DriverId = "DI-1";
|
||||
private const string SpeedPath = "Plant/Modbus/dev1/speed";
|
||||
|
||||
[Fact]
|
||||
public void A_driver_removed_and_re_added_does_not_inherit_the_old_incarnations_reprovision_stamp()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
// Far longer than the presence budget below: an inherited stamp would defer the re-added driver's
|
||||
// reprovision past it, which is the failure this test reports.
|
||||
var backoff = TimeSpan.FromSeconds(60);
|
||||
var factory = new RecordingSubscribableDriverFactory("Modbus");
|
||||
var withDriver = new[] { (DriverId, "Modbus", "dev1", "speed", "Float", false) };
|
||||
var none = Array.Empty<(string, string, string, string, string?)>();
|
||||
|
||||
var revA = V3ArtifactSeeder.SeedV3Deployment(db, RevA, withDriver, none, enabled: true);
|
||||
var (host, _, _, coordinator) = V3ArtifactSeeder.SpawnHostAndApplyWithCoordinator(
|
||||
this, db, TestNode, revA, RevA, factory, reprovisionBackoff: backoff);
|
||||
AwaitAssert(() => factory.Created.Count.ShouldBe(1), PresenceBudget);
|
||||
AwaitAssert(() => factory.Created[0].SubscribeCount.ShouldBe(1), PresenceBudget);
|
||||
|
||||
// Crash 1 on the first incarnation: reprovisioned immediately, which stamps the id.
|
||||
factory.Created[0].FireMalformedDataChange(SpeedPath);
|
||||
AwaitAssert(() => factory.Created[0].InitializeCount.ShouldBe(2), PresenceBudget);
|
||||
|
||||
// Deploy B removes the driver (StopChild), deploy C re-adds it: a brand-new driver object.
|
||||
var revB = V3ArtifactSeeder.SeedV3Deployment(
|
||||
db, RevB, Array.Empty<(string, string, string, string, string, bool)>(), none, enabled: true);
|
||||
V3ArtifactSeeder.Dispatch(host, coordinator, revB, RevB);
|
||||
var revC = V3ArtifactSeeder.SeedV3Deployment(db, RevC, withDriver, none, enabled: true);
|
||||
V3ArtifactSeeder.Dispatch(host, coordinator, revC, RevC);
|
||||
AwaitAssert(() => factory.Created.Count.ShouldBe(2), PresenceBudget);
|
||||
var reAdded = factory.Created[1];
|
||||
AwaitAssert(() => reAdded.SubscribeCount.ShouldBe(1), PresenceBudget);
|
||||
|
||||
// Its FIRST crash must reprovision immediately — it has no history of its own.
|
||||
reAdded.FireMalformedDataChange(SpeedPath);
|
||||
AwaitAssert(
|
||||
() => reAdded.InitializeCount.ShouldBe(
|
||||
2, "the re-added driver's first reprovision was deferred by the REMOVED incarnation's stamp"),
|
||||
duration: TimeSpan.FromSeconds(5));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Event;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 H5: the <c>Stale</c> behaviour registered no receive for a driver child's value/alarm bursts (nor
|
||||
/// for a late <c>ArtifactForApply</c> / <c>ApplyWedged</c> pipe-back), so any that landed there hit
|
||||
/// <c>Unhandled</c> — dead-letter noise and a dropped message with no log naming why. They are now explicit
|
||||
/// drops. The two private pipe-back records cannot be constructed from a test; the two public child
|
||||
/// messages are pinned here, and the private pair by the same registration pattern (compile-time).
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorStaleDropTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("stale-drop-test");
|
||||
|
||||
[Fact]
|
||||
public void Child_value_and_alarm_bursts_landing_in_Stale_are_dropped_not_unhandled()
|
||||
{
|
||||
var unhandled = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(unhandled.Ref, typeof(UnhandledMessage));
|
||||
|
||||
// ThrowingDbFactory ⇒ Bootstrap's catch ⇒ Become(Stale).
|
||||
var host = Sys.ActorOf(DriverHostActor.Props(
|
||||
new ThrowingDbFactory(), TestNode, CreateTestProbe().Ref,
|
||||
localRoles: new HashSet<string> { "driver" }));
|
||||
// Barrier: Stale answers RouteNodeWrite synchronously, so a reply proves the host is in Stale.
|
||||
var asker = CreateTestProbe();
|
||||
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget).Reason!.ShouldContain("stale");
|
||||
|
||||
// Positive control: an unknown message IS reported as unhandled by this host.
|
||||
host.Tell("not a message the host knows");
|
||||
unhandled.ExpectMsg<UnhandledMessage>(m => ReferenceEquals(m.Recipient, host) || m.Recipient.Equals(host), PresenceBudget);
|
||||
|
||||
host.Tell(new DriverInstanceActor.AttributeValueBatchPublished(
|
||||
"DI-1", new[] { new DriverInstanceActor.AttributeValueItem("Plant/x/y", 1.0, OpcUaStatus.Good, DateTime.UtcNow) }));
|
||||
host.Tell(new DriverInstanceActor.AttributeAlarmPublished(
|
||||
"DI-1", new AlarmEventArgs(new Handle(), "Plant/x/alarm", "Plant/x/alarm", "AlarmCondition", "test",
|
||||
AlarmSeverity.High, DateTime.UtcNow)));
|
||||
// Barrier again, then the absence window.
|
||||
host.Tell(new DriverHostActor.RouteNodeWrite("eq-x/speed", 0.0, AddressSpaceRealm.Uns), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(PresenceBudget);
|
||||
|
||||
unhandled.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
private sealed class Handle : IAlarmSubscriptionHandle
|
||||
{
|
||||
public string DiagnosticId => "stale-test";
|
||||
}
|
||||
|
||||
private sealed class ThrowingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext() => throw new InvalidOperationException("db unreachable");
|
||||
}
|
||||
}
|
||||
+288
@@ -0,0 +1,288 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 P2: <c>WriteAttribute</c> was a <c>ReceiveAsync</c>, which suspends the mailbox for the whole
|
||||
/// device round-trip (up to the 5 s bound). Writes therefore serialised — the SECOND of two operator writes to
|
||||
/// a ~3 s device missed the host's 8 s Ask and was reverted although the device took it — and every data
|
||||
/// change queued behind the write. Writes now run off the mailbox under a per-driver in-flight cap; the
|
||||
/// subscribe/unsubscribe pair that must serialise against the subscription handle stays <c>ReceiveAsync</c>.
|
||||
/// </summary>
|
||||
public sealed class DriverInstanceActorConcurrentWriteTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly TimeSpan WriteLatency = TimeSpan.FromMilliseconds(1200);
|
||||
|
||||
private (IActorRef Actor, Akka.TestKit.TestProbe Parent) Connect(
|
||||
IDriver driver, int? maxWritesInFlight = null, TimeSpan? queuedWriteBudget = null, TimeSpan? writeTimeout = null)
|
||||
{
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
|
||||
driver, commsLossTimeout: TimeSpan.Zero, maxWritesInFlight: maxWritesInFlight,
|
||||
queuedWriteBudget: queuedWriteBudget, writeTimeout: writeTimeout));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => m.Connected, PresenceBudget);
|
||||
return (actor, parent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Two_concurrent_slow_writes_both_complete_within_one_device_round_trip()
|
||||
{
|
||||
var driver = new SlowWritableDriver { Latency = WriteLatency };
|
||||
var (actor, _) = Connect(driver);
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var askBudget = TimeSpan.FromSeconds(8); // the host's Ask
|
||||
var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-1", 1), askBudget);
|
||||
var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-2", 2), askBudget);
|
||||
var results = await Task.WhenAll(first, second);
|
||||
sw.Stop();
|
||||
|
||||
results.ShouldAllBe(r => r.Success);
|
||||
driver.Writes.Select(w => w.FullReference).ShouldBe(new[] { "tag-1", "tag-2" }, ignoreOrder: true);
|
||||
// Both were inside the driver at once — that is the proof; the wall clock is only a sanity ceiling
|
||||
// (serialised writes would take two latencies, and the host's Ask is 8 s).
|
||||
driver.MaxConcurrentWrites.ShouldBe(2, "the second write waited for the first — writes are still serialised on the mailbox");
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(8));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Data_changes_are_delivered_while_a_write_is_pending()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, parent) = Connect(driver);
|
||||
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
|
||||
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
|
||||
|
||||
var pending = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 7), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
|
||||
driver.FireDataChange("tag-a", 3.5);
|
||||
var published = parent.ExpectMsg<DriverInstanceActor.AttributeValueBatchPublished>(PresenceBudget);
|
||||
published.Items.Single().Value.ShouldBe(3.5);
|
||||
pending.IsCompleted.ShouldBeFalse("the write is still held — the data change did not wait behind it");
|
||||
|
||||
// Positive control: releasing the device completes the write with its real outcome.
|
||||
driver.ReleaseWrites();
|
||||
(await pending).Success.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Writes_beyond_the_in_flight_cap_queue_and_drain_in_order()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, _) = Connect(driver, maxWritesInFlight: 2);
|
||||
|
||||
var asks = Enumerable.Range(0, 4)
|
||||
.Select(i => actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute($"tag-{i}", i), PresenceBudget))
|
||||
.ToArray();
|
||||
|
||||
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
|
||||
// Settle: a third write entering the driver would show up here.
|
||||
await Task.Delay(300);
|
||||
driver.WritesEntered.ShouldBe(2, "the cap let a third write into the driver");
|
||||
|
||||
driver.ReleaseWrites();
|
||||
var results = await Task.WhenAll(asks);
|
||||
results.ShouldAllBe(r => r.Success);
|
||||
// Entry order: the first two slots went to the first two writes, the queued pair entered only after a
|
||||
// slot freed. Within each pair the order is unspecified — distinct tags run concurrently on the pool;
|
||||
// only SAME-tag order is guaranteed (pinned by the same-tag test below).
|
||||
var entered = driver.EnteredRefs;
|
||||
entered.Count.ShouldBe(4);
|
||||
entered.Take(2).ShouldBe(new[] { "tag-0", "tag-1" }, ignoreOrder: true);
|
||||
entered.Skip(2).ShouldBe(new[] { "tag-2", "tag-3" }, ignoreOrder: true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix round 1: writes to the SAME tag must keep their order — two concurrent writes v1, v2 could land as
|
||||
/// v1. A same-tag write waits behind its tag while other tags still run concurrently.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Writes_to_the_same_tag_are_serialised_in_order_while_other_tags_run_concurrently()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, _) = Connect(driver, maxWritesInFlight: 4);
|
||||
|
||||
var first = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
var second = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
|
||||
// Positive control: a DIFFERENT tag is not held behind tag-a.
|
||||
var other = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-b", 9), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
|
||||
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-b" });
|
||||
|
||||
await Task.Delay(300);
|
||||
driver.WritesEntered.ShouldBe(2, "the second tag-a write entered the driver while the first was still inside it");
|
||||
second.IsCompleted.ShouldBeFalse();
|
||||
|
||||
driver.ReleaseWrites();
|
||||
(await Task.WhenAll(first, second, other)).ShouldAllBe(r => r.Success);
|
||||
driver.Writes.Where(w => w.FullReference == "tag-a").Select(w => w.Value).ShouldBe(new object[] { 1, 2 });
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix round 1: a write that sat in the queue longer than the host's Ask budget has already been reported
|
||||
/// as a timeout (and reverted) upstream — writing it late would push a stale value to the device.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_queued_write_older_than_the_budget_is_failed_BadTimeout_instead_of_written_late()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, _) = Connect(driver, maxWritesInFlight: 1, queuedWriteBudget: TimeSpan.FromMilliseconds(300));
|
||||
|
||||
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
var stale = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
|
||||
|
||||
await Task.Delay(600);
|
||||
driver.ReleaseWrites();
|
||||
|
||||
(await inFlight).Success.ShouldBeTrue();
|
||||
var staleResult = await stale;
|
||||
staleResult.Success.ShouldBeFalse();
|
||||
staleResult.Reason!.ShouldContain("800A0000", customMessage: "BadTimeout");
|
||||
driver.EnteredRefs.ShouldBe(new[] { "tag-0" }, "the stale write must never reach the device");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix round 2: a token-ignoring hung write held its tag forever, so every later write to that tag piled up
|
||||
/// in the queue. The write is bounded pessimistically; on overrun it answers BadTimeout and releases its
|
||||
/// slot and tag, and a later write to the same tag reaches the device.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_hung_token_ignoring_write_is_abandoned_and_releases_its_tag_for_a_later_write()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true, IgnoreCancellation = true };
|
||||
var (actor, _) = Connect(driver, maxWritesInFlight: 1, writeTimeout: TimeSpan.FromMilliseconds(300));
|
||||
|
||||
var hung = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 1), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
|
||||
var hungResult = await hung;
|
||||
hungResult.Success.ShouldBeFalse();
|
||||
hungResult.Reason!.ShouldContain("800A0000", customMessage: "BadTimeout");
|
||||
driver.MaxConcurrentWrites.ShouldBe(1);
|
||||
|
||||
// The tag (and the single slot) are free again: a later tag-a write reaches the device even though the
|
||||
// abandoned one is still inside it.
|
||||
var later = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-a", 2), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 2, PresenceBudget);
|
||||
driver.EnteredRefs.ShouldBe(new[] { "tag-a", "tag-a" });
|
||||
driver.MaxConcurrentWrites.ShouldBe(2, "positive control: the abandoned write really is still wedged inside the driver");
|
||||
|
||||
driver.ReleaseWrites();
|
||||
(await later).Success.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Queued_writes_fail_fast_when_the_driver_leaves_Connected()
|
||||
{
|
||||
var driver = new SlowWritableDriver { HoldWrites = true };
|
||||
var (actor, parent) = Connect(driver, maxWritesInFlight: 1);
|
||||
|
||||
var inFlight = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-0", 0), PresenceBudget);
|
||||
AwaitCondition(() => driver.WritesEntered == 1, PresenceBudget);
|
||||
var queued = actor.Ask<DriverInstanceActor.WriteAttributeResult>(
|
||||
new DriverInstanceActor.WriteAttribute("tag-1", 1), PresenceBudget);
|
||||
|
||||
actor.Tell(new DriverInstanceActor.ForceReconnect());
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => !m.Connected, PresenceBudget);
|
||||
|
||||
var queuedResult = await queued;
|
||||
queuedResult.Success.ShouldBeFalse();
|
||||
queuedResult.Reason.ShouldBe("driver not connected");
|
||||
driver.WritesEntered.ShouldBe(1, "a queued write must never reach a driver that is reconnecting");
|
||||
|
||||
// The in-flight one still answers with the device's real outcome once it is released.
|
||||
driver.ReleaseWrites();
|
||||
(await inFlight).Success.ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>A writable + subscribable driver whose writes either take <see cref="Latency"/> or hold until released.</summary>
|
||||
private sealed class SlowWritableDriver : IDriver, IWritable, ISubscribable
|
||||
{
|
||||
private readonly StubHandle _handle = new();
|
||||
private readonly ConcurrentQueue<WriteRequest> _writes = new();
|
||||
private readonly ConcurrentQueue<string> _enteredRefs = new();
|
||||
private TaskCompletionSource _release = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private int _entered;
|
||||
private int _inside;
|
||||
private int _maxInside;
|
||||
|
||||
public TimeSpan Latency { get; init; }
|
||||
public bool HoldWrites { get; init; }
|
||||
/// <summary>When true a held write ignores its cancellation token — a wedged driver.</summary>
|
||||
public bool IgnoreCancellation { get; init; }
|
||||
|
||||
public IReadOnlyList<WriteRequest> Writes => _writes.ToArray();
|
||||
public IReadOnlyList<string> EnteredRefs => _enteredRefs.ToArray();
|
||||
public int WritesEntered => Volatile.Read(ref _entered);
|
||||
public int MaxConcurrentWrites => Volatile.Read(ref _maxInside);
|
||||
|
||||
public string DriverInstanceId => "slow-writer-1";
|
||||
public string DriverType => "SlowWriter";
|
||||
|
||||
public void ReleaseWrites() => _release.TrySetResult();
|
||||
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
||||
|
||||
public async Task<IReadOnlyList<WriteResult>> WriteAsync(IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _entered);
|
||||
foreach (var w in writes) _enteredRefs.Enqueue(w.FullReference);
|
||||
var inside = Interlocked.Increment(ref _inside);
|
||||
int seen;
|
||||
while (inside > (seen = Volatile.Read(ref _maxInside)))
|
||||
Interlocked.CompareExchange(ref _maxInside, inside, seen);
|
||||
try
|
||||
{
|
||||
if (HoldWrites && IgnoreCancellation) await _release.Task;
|
||||
else if (HoldWrites) await _release.Task.WaitAsync(cancellationToken);
|
||||
else if (Latency > TimeSpan.Zero) await Task.Delay(Latency, cancellationToken);
|
||||
foreach (var w in writes) _writes.Enqueue(w);
|
||||
return writes.Select(_ => new WriteResult(OpcUaStatus.Good)).ToList();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Decrement(ref _inside);
|
||||
}
|
||||
}
|
||||
|
||||
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
||||
|
||||
public Task<ISubscriptionHandle> SubscribeAsync(
|
||||
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<ISubscriptionHandle>(_handle);
|
||||
|
||||
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public void FireDataChange(string fullRef, object? value) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(
|
||||
_handle, fullRef, new DataValueSnapshot(value, OpcUaStatus.Good, DateTime.UtcNow, DateTime.UtcNow)));
|
||||
}
|
||||
}
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L1 / P4: a driver child that stops for good must <b>dispose</b> its <see cref="IDriver"/>
|
||||
/// (the runtime only ever called <c>ShutdownAsync</c>, so a Calculation driver's compiled-script ALCs and an
|
||||
/// OpcUaClient driver's LoggerFactory + gate + CTS leaked on every respawn), its <c>ShutdownAsync</c> must be
|
||||
/// bounded (it used to block a dispatcher thread for as long as a dead socket took to answer), and a
|
||||
/// <b>supervision restart</b> must NOT dispose — the Props closure re-uses the same driver object for the
|
||||
/// restarted incarnation, which then has to serve through it.
|
||||
/// </summary>
|
||||
public sealed class DriverInstanceActorDisposeTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Stopping_the_child_shuts_the_driver_down_and_then_disposes_it_exactly_once()
|
||||
{
|
||||
var driver = new DisposableRecordingDriver();
|
||||
var parent = CreateTestProbe();
|
||||
parent.IgnoreMessages(m => m is DriverInstanceActor.ConnectivityChanged);
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
AwaitCondition(() => driver.InitializeCount == 1, PresenceBudget);
|
||||
|
||||
Watch(actor);
|
||||
Sys.Stop(actor);
|
||||
ExpectTerminated(actor, PresenceBudget);
|
||||
|
||||
driver.ShutdownCount.ShouldBe(1);
|
||||
driver.DisposeCount.ShouldBe(1, "a child stopped for good must dispose the driver it wrapped");
|
||||
driver.Trace.ToArray().ShouldBe(new[] { "init", "shutdown", "dispose" }, "shutdown must land before dispose");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_supervision_restart_keeps_the_driver_undisposed_and_the_restarted_child_still_serves()
|
||||
{
|
||||
var driver = new DisposableRecordingDriver();
|
||||
var parent = CreateTestProbe();
|
||||
// The probe parent's own strategy STOPS a failing child; the production host (no strategy anywhere under
|
||||
// src/Server) gets Akka's default decider, which RESTARTS it — the route under test.
|
||||
var actor = parent.ChildActorOf(
|
||||
DriverInstanceActor.Props(driver), new OneForOneStrategy(_ => Directive.Restart));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
// Barrier: Subscribe is only handled once Connected (it is dropped in Connecting).
|
||||
parent.ExpectMsg<DriverInstanceActor.ConnectivityChanged>(m => m.Connected, PresenceBudget);
|
||||
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
|
||||
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
|
||||
|
||||
// The crash seam (see DriverHostActorRestartTests): a null snapshot NREs on the actor thread, and the
|
||||
// probe parent's default decider restarts the child in place around the SAME driver object.
|
||||
driver.FireMalformedDataChange("tag-a");
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ReprovisionRequested, PresenceBudget);
|
||||
|
||||
driver.ShutdownCount.ShouldBe(1, "PreRestart runs PostStop, which shuts the driver down");
|
||||
driver.DisposeCount.ShouldBe(0, "a restart re-uses the driver object — disposing it would kill the new incarnation");
|
||||
|
||||
// Positive control: the restarted incarnation serves through the undisposed driver. The host would
|
||||
// re-provision it exactly like this (InitializeRequested + the desired set).
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
|
||||
await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
|
||||
new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)), PresenceBudget);
|
||||
driver.FireDataChange("tag-a", 42.0);
|
||||
var published = (DriverInstanceActor.AttributeValueBatchPublished)parent.FishForMessage(
|
||||
m => m is DriverInstanceActor.AttributeValueBatchPublished, PresenceBudget);
|
||||
published.Items.Single().FullReference.ShouldBe("tag-a");
|
||||
published.Items.Single().Value.ShouldBe(42.0);
|
||||
|
||||
// ...and the eventual real stop still disposes it — once.
|
||||
Watch(actor);
|
||||
Sys.Stop(actor);
|
||||
ExpectTerminated(actor, PresenceBudget);
|
||||
driver.DisposeCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_hung_ShutdownAsync_is_abandoned_at_the_bound_and_the_driver_is_still_disposed()
|
||||
{
|
||||
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
// Ignores its token on purpose: a wedged socket/COM proxy does not honour cancellation either.
|
||||
var driver = new DisposableRecordingDriver { ShutdownBehavior = _ => never.Task };
|
||||
var bound = TimeSpan.FromMilliseconds(300);
|
||||
var parent = CreateTestProbe();
|
||||
parent.IgnoreMessages(m => m is DriverInstanceActor.ConnectivityChanged);
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(driver, shutdownTimeout: bound));
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
AwaitCondition(() => driver.InitializeCount == 1, PresenceBudget);
|
||||
|
||||
Watch(actor);
|
||||
var sw = Stopwatch.StartNew();
|
||||
Sys.Stop(actor);
|
||||
ExpectTerminated(actor, PresenceBudget);
|
||||
sw.Stop();
|
||||
|
||||
driver.ShutdownCount.ShouldBe(1, "the shutdown was attempted (positive control for the bound)");
|
||||
never.Task.IsCompleted.ShouldBeFalse("the shutdown is still hung — the stop did not wait for it");
|
||||
sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the stop must not be pinned by the hung shutdown");
|
||||
driver.DisposeCount.ShouldBe(1, "a shutdown that timed out must not skip the dispose");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A subscribable driver that records its lifecycle and is <see cref="IAsyncDisposable"/>. Deliberately
|
||||
/// NOT also <see cref="IDisposable"/>: the actor prefers the async dispose when both exist, and a double
|
||||
/// implementation would hide a double-dispose.
|
||||
/// </summary>
|
||||
private sealed class DisposableRecordingDriver : IDriver, ISubscribable, IAsyncDisposable
|
||||
{
|
||||
private readonly StubHandle _handle = new();
|
||||
private int _initializeCount;
|
||||
private int _shutdownCount;
|
||||
private int _disposeCount;
|
||||
|
||||
public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }
|
||||
|
||||
public List<string> Trace { get; } = new();
|
||||
|
||||
public int InitializeCount => Volatile.Read(ref _initializeCount);
|
||||
public int ShutdownCount => Volatile.Read(ref _shutdownCount);
|
||||
public int DisposeCount => Volatile.Read(ref _disposeCount);
|
||||
|
||||
public string DriverInstanceId => "disposable-1";
|
||||
public string DriverType => "Disposable";
|
||||
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _initializeCount);
|
||||
lock (Trace) Trace.Add("init");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _shutdownCount);
|
||||
lock (Trace) Trace.Add("shutdown");
|
||||
return ShutdownBehavior is { } behavior ? behavior(cancellationToken) : Task.CompletedTask;
|
||||
}
|
||||
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
Interlocked.Increment(ref _disposeCount);
|
||||
lock (Trace) Trace.Add("dispose");
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public event EventHandler<DataChangeEventArgs>? OnDataChange;
|
||||
|
||||
public Task<ISubscriptionHandle> SubscribeAsync(
|
||||
IReadOnlyList<string> fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) =>
|
||||
Task.FromResult<ISubscriptionHandle>(_handle);
|
||||
|
||||
public Task UnsubscribeAsync(ISubscriptionHandle handle, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public void FireDataChange(string fullRef, object? value) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(
|
||||
_handle, fullRef, new DataValueSnapshot(value, OpcUaStatus.Good, DateTime.UtcNow, DateTime.UtcNow)));
|
||||
|
||||
/// <summary>The supervision crash seam — see <see cref="RecordingSubscribableDriver.FireMalformedDataChange"/>.</summary>
|
||||
public void FireMalformedDataChange(string fullRef) =>
|
||||
OnDataChange?.Invoke(this, new DataChangeEventArgs(_handle, fullRef, null!));
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
using Akka.Actor;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 H2: the <c>Reconnecting</c> retry timer fired every <c>reconnectInterval</c> whether or not the
|
||||
/// previous attempt was still running — <c>_initGeneration</c> only discarded the older attempt's RESULT. With an
|
||||
/// Initialize bound longer than the cadence (Tier C's 15 s vs. the 10 s timer, or the test harness's unbounded
|
||||
/// invoker) attempt N+1's <c>ShutdownAsync</c> ran while attempt N's <c>InitializeAsync</c> was still inside the
|
||||
/// driver: the dispose-before-overwrite race the drivers' rule file forbids, once per overlap, for as long as
|
||||
/// the device stayed down. The retry is now single-flight.
|
||||
/// </summary>
|
||||
public sealed class DriverInstanceActorReconnectSingleFlightTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Shutdown_and_Initialize_never_overlap_while_the_init_outlasts_the_retry_cadence()
|
||||
{
|
||||
// A 400 ms failing init against a 50 ms retry cadence: without the gate the timer starts a new
|
||||
// Shutdown+Initialize pair every 50 ms on top of the one still running (eight-deep overlap).
|
||||
var driver = new InterleavingDriver(initDuration: TimeSpan.FromMilliseconds(400), failUntilAttempt: 4);
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
|
||||
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero));
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
|
||||
// Positive control: the gate skips ticks, it does not stop retrying — the 4th attempt succeeds and the
|
||||
// actor connects.
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
|
||||
|
||||
driver.Attempts.ShouldBeGreaterThanOrEqualTo(4);
|
||||
driver.Overlaps.ShouldBe(0,
|
||||
$"ShutdownAsync entered while an InitializeAsync was still running. Interleaving: {driver.Interleaving}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_pre_reconnect_teardown_that_hangs_is_abandoned_at_the_bound_and_the_init_still_runs()
|
||||
{
|
||||
var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 2)
|
||||
{
|
||||
// Only the SECOND shutdown (the reconnect teardown) hangs; the first attempt has none.
|
||||
ShutdownBehavior = _ => never.Task,
|
||||
};
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
|
||||
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
|
||||
shutdownTimeout: TimeSpan.FromMilliseconds(200)));
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
|
||||
never.Task.IsCompleted.ShouldBeFalse("the teardown is still hung — the attempt did not wait for it");
|
||||
driver.Attempts.ShouldBe(2);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fix round 1: the single-flight gate only cleared on the attempt's RESULT, and the invoker's timeout is
|
||||
/// token-only — a driver whose InitializeAsync ignores its token wedged the reconnect loop forever. The
|
||||
/// attempt is now bounded pessimistically (<c>initializeTimeout</c>); an overrun is abandoned, reported
|
||||
/// as a failure, and the next tick retries.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void A_token_ignoring_never_completing_init_is_abandoned_and_a_later_attempt_connects()
|
||||
{
|
||||
var driver = new InterleavingDriver(initDuration: TimeSpan.Zero, failUntilAttempt: 1)
|
||||
{
|
||||
// Attempt 1 hangs forever and ignores cancellation; attempt 2 completes.
|
||||
HangOnAttempt = 1,
|
||||
};
|
||||
var parent = CreateTestProbe();
|
||||
var actor = parent.ChildActorOf(DriverInstanceActor.Props(
|
||||
driver, reconnectInterval: TimeSpan.FromMilliseconds(50), commsLossTimeout: TimeSpan.Zero,
|
||||
initializeTimeout: TimeSpan.FromMilliseconds(300)));
|
||||
|
||||
actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
|
||||
|
||||
parent.FishForMessage(m => m is DriverInstanceActor.ConnectivityChanged { Connected: true }, PresenceBudget);
|
||||
driver.Attempts.ShouldBe(2);
|
||||
driver.HungAttemptStillRunning.ShouldBeTrue("positive control: the first attempt really was wedged");
|
||||
}
|
||||
|
||||
/// <summary>Records every Shutdown/Initialize entry and exit so an overlap is a fact, not an inference.</summary>
|
||||
private sealed class InterleavingDriver : IDriver
|
||||
{
|
||||
private readonly TimeSpan _initDuration;
|
||||
private readonly int _failUntilAttempt;
|
||||
private readonly Lock _gate = new();
|
||||
private readonly List<string> _events = new();
|
||||
private int _attempts;
|
||||
private int _initsInProgress;
|
||||
private int _overlaps;
|
||||
|
||||
public InterleavingDriver(TimeSpan initDuration, int failUntilAttempt)
|
||||
{
|
||||
_initDuration = initDuration;
|
||||
_failUntilAttempt = failUntilAttempt;
|
||||
}
|
||||
|
||||
public Func<CancellationToken, Task>? ShutdownBehavior { get; init; }
|
||||
|
||||
/// <summary>The 1-based attempt that never completes and ignores its token (0 = none).</summary>
|
||||
public int HangOnAttempt { get; init; }
|
||||
|
||||
private readonly TaskCompletionSource _never = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public bool HungAttemptStillRunning => !_never.Task.IsCompleted && Volatile.Read(ref _initsInProgress) > 0;
|
||||
|
||||
public int Attempts => Volatile.Read(ref _attempts);
|
||||
public int Overlaps => Volatile.Read(ref _overlaps);
|
||||
public string Interleaving { get { lock (_gate) return string.Join(" ", _events); } }
|
||||
|
||||
public string DriverInstanceId => "interleaving-1";
|
||||
public string DriverType => "Interleaving";
|
||||
|
||||
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
var attempt = Interlocked.Increment(ref _attempts);
|
||||
Interlocked.Increment(ref _initsInProgress);
|
||||
Record($"init{attempt}>");
|
||||
try
|
||||
{
|
||||
if (attempt == HangOnAttempt) await _never.Task;
|
||||
if (_initDuration > TimeSpan.Zero) await Task.Delay(_initDuration, CancellationToken.None);
|
||||
if (attempt < _failUntilAttempt) throw new InvalidOperationException($"init attempt {attempt} refused");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Record($"<init{attempt}");
|
||||
Interlocked.Decrement(ref _initsInProgress);
|
||||
}
|
||||
}
|
||||
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public async Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (Volatile.Read(ref _initsInProgress) > 0) Interlocked.Increment(ref _overlaps);
|
||||
Record("shutdown>");
|
||||
try
|
||||
{
|
||||
if (ShutdownBehavior is { } behavior) await behavior(cancellationToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Record("<shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);
|
||||
|
||||
private void Record(string e) { lock (_gate) _events.Add(e); }
|
||||
}
|
||||
}
|
||||
@@ -47,6 +47,8 @@ public static class V3ArtifactSeeder
|
||||
/// is parsed out of <c>TagConfig.alarm</c> by <c>TagConfigIntent</c>), which the alarm-ref half of the
|
||||
/// restart suite needs — an alarm-bearing raw tag is excluded from the VALUE subscription set and carried
|
||||
/// in the alarm one instead, so the two travel through different child code paths.</param>
|
||||
/// <param name="driverRowIds">Optional <c>DriverInstanceRowId</c> (the DB row Guid) per driver id; absent
|
||||
/// ids get <see cref="Guid.Empty"/>, which is what an artifact without the property parses to.</param>
|
||||
/// <param name="taglessDrivers">Extra <c>DriverInstance</c> ids carrying NO devices and NO tags. The
|
||||
/// only way to express a driver that is a running child yet absent from both the value- and alarm-ref
|
||||
/// maps — the <c>_children.Keys</c> arm of <c>ApplyDesiredSubscriptions</c>'s union, which must still
|
||||
@@ -58,14 +60,16 @@ public static class V3ArtifactSeeder
|
||||
(string Area, string Line, string Equip, string BackingTag, string? Effective)[] unsRefs,
|
||||
bool enabled = false,
|
||||
IReadOnlyDictionary<string, string>? tagConfigByTag = null,
|
||||
string[]? taglessDrivers = null)
|
||||
string[]? taglessDrivers = null,
|
||||
IReadOnlyDictionary<string, Guid>? driverRowIds = null)
|
||||
{
|
||||
Guid RowId(string id) => driverRowIds is not null && driverRowIds.TryGetValue(id, out var g) ? g : Guid.Empty;
|
||||
var drivers = rawTags
|
||||
.Select(t => (t.Driver, t.DriverName))
|
||||
.Distinct()
|
||||
.Select(d => new { DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })
|
||||
.Select(d => new { DriverInstanceRowId = RowId(d.Driver), DriverInstanceId = d.Driver, RawFolderId = "rf-plant", Name = d.DriverName, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled })
|
||||
.Concat((taglessDrivers ?? Array.Empty<string>())
|
||||
.Select(id => new { DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }))
|
||||
.Select(id => new { DriverInstanceRowId = RowId(id), DriverInstanceId = id, RawFolderId = "rf-plant", Name = id, DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = enabled }))
|
||||
.ToArray();
|
||||
var devices = rawTags
|
||||
.Select(t => (t.Driver, t.Device))
|
||||
@@ -201,7 +205,37 @@ public static class V3ArtifactSeeder
|
||||
return (actor, mux, coordinator);
|
||||
}
|
||||
|
||||
private static void Dispatch(IActorRef actor, TestProbe coordinator, DeploymentId deploymentId, RevisionHash rev)
|
||||
/// <summary>The same arrangement as <see cref="SpawnHostAndApply"/>, also returning the coordinator probe so a
|
||||
/// test can <see cref="Dispatch"/> FURTHER deployments to the same host (remove a driver, re-add it).</summary>
|
||||
/// <param name="kit">The calling TestKit, supplying the ActorSystem + probes.</param>
|
||||
/// <param name="db">The factory whose deployment was seeded by <see cref="SeedV3Deployment"/>.</param>
|
||||
/// <param name="localNode">The host's local node id.</param>
|
||||
/// <param name="deploymentId">The deployment to dispatch.</param>
|
||||
/// <param name="rev">The revision hash the deployment was sealed under.</param>
|
||||
/// <param name="driverFactory">Optional driver factory forwarded to <c>DriverHostActor.Props</c>.</param>
|
||||
/// <param name="reprovisionBackoff">Optional per-driver supervision-reprovision rate limit.</param>
|
||||
/// <returns>The host actor plus the publish, mux and coordinator probes wired into it.</returns>
|
||||
public static (IActorRef Actor, TestProbe Publish, TestProbe Mux, TestProbe Coordinator) SpawnHostAndApplyWithCoordinator(
|
||||
TestKitBase kit, IDbContextFactory<OtOpcUaConfigDbContext> db,
|
||||
NodeId localNode, DeploymentId deploymentId, RevisionHash rev,
|
||||
IDriverFactory? driverFactory = null,
|
||||
TimeSpan? reprovisionBackoff = null)
|
||||
{
|
||||
var publish = kit.CreateCleanOutcomePublishProbe();
|
||||
var (actor, mux, coordinator) = Spawn(kit, db, localNode, publish.Ref, driverFactory, reprovisionBackoff);
|
||||
|
||||
Dispatch(actor, coordinator, deploymentId, rev);
|
||||
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(RuntimeActorTestBase.PresenceBudget);
|
||||
|
||||
return (actor, publish, mux, coordinator);
|
||||
}
|
||||
|
||||
/// <summary>Dispatches <paramref name="deploymentId"/> to <paramref name="actor"/> and waits for its Applied ack.</summary>
|
||||
/// <param name="actor">The host.</param>
|
||||
/// <param name="coordinator">The coordinator probe the host acks to.</param>
|
||||
/// <param name="deploymentId">The deployment to dispatch.</param>
|
||||
/// <param name="rev">Its revision hash.</param>
|
||||
public static void Dispatch(IActorRef actor, TestProbe coordinator, DeploymentId deploymentId, RevisionHash rev)
|
||||
{
|
||||
actor.Tell(new DispatchDeployment(deploymentId, rev, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(RuntimeActorTestBase.PresenceBudget)
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Builds the ONE message shape a driver child sends for its values — a
|
||||
/// <see cref="DriverInstanceActor.AttributeValueBatchPublished"/> — for tests that push a single value into the
|
||||
/// dependency mux by hand. The per-value <see cref="DriverInstanceActor.AttributeValuePublished"/> has no
|
||||
/// production sender (02/U4); tests should not keep one alive either.
|
||||
/// </summary>
|
||||
public static class ValuePublishes
|
||||
{
|
||||
/// <summary>A one-item batch for <paramref name="fullReference"/>.</summary>
|
||||
/// <param name="driverInstanceId">The publishing driver's id.</param>
|
||||
/// <param name="fullReference">The value's reference (RawPath).</param>
|
||||
/// <param name="value">The value.</param>
|
||||
/// <param name="statusCode">The full OPC UA status code.</param>
|
||||
/// <param name="timestampUtc">The source timestamp.</param>
|
||||
/// <returns>The batch message.</returns>
|
||||
public static DriverInstanceActor.AttributeValueBatchPublished Single(
|
||||
string driverInstanceId, string fullReference, object? value, uint statusCode, DateTime timestampUtc) =>
|
||||
new(driverInstanceId, new[] { new DriverInstanceActor.AttributeValueItem(fullReference, value, statusCode, timestampUtc) });
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 P7: <see cref="DbHealthProbeActor"/> ran its SELECT synchronously on the actor thread, so a
|
||||
/// hanging SQL connect/command held the dispatcher thread for the implicit 15 s / 30 s SqlClient timeouts and
|
||||
/// queued every later tick (and every <c>GetStatus</c> Ask) behind it. The probe now runs off the mailbox
|
||||
/// and pipes its outcome back, so the cached answer stays available while a probe hangs.
|
||||
/// </summary>
|
||||
public sealed class DbHealthProbeOffThreadTests : RuntimeActorTestBase
|
||||
{
|
||||
[Fact]
|
||||
public async Task GetStatus_answers_from_cache_while_a_probe_is_hung()
|
||||
{
|
||||
var factory = new HangingDbFactory();
|
||||
try
|
||||
{
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(factory));
|
||||
AwaitCondition(() => factory.Entered > 0, PresenceBudget);
|
||||
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(3));
|
||||
|
||||
status.Reachable.ShouldBeFalse();
|
||||
status.LastError.ShouldBe("not probed yet");
|
||||
factory.Entered.ShouldBe(1, "a second tick must not stack a second probe behind a hung one");
|
||||
}
|
||||
finally
|
||||
{
|
||||
factory.Release();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_probe_that_completes_updates_the_cached_status()
|
||||
{
|
||||
// Positive control for the off-thread path: the piped-back outcome lands in the cache.
|
||||
var db = NewInMemoryDbFactory();
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(db));
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, TimeSpan.FromSeconds(3));
|
||||
status.Reachable.ShouldBeTrue();
|
||||
}, PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>Blocks every <c>CreateDbContext</c> until released — a connect that never answers.</summary>
|
||||
private sealed class HangingDbFactory : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
private readonly ManualResetEventSlim _release = new(false);
|
||||
private int _entered;
|
||||
|
||||
public int Entered => Volatile.Read(ref _entered);
|
||||
|
||||
public void Release() => _release.Set();
|
||||
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
{
|
||||
Interlocked.Increment(ref _entered);
|
||||
_release.Wait();
|
||||
throw new InvalidOperationException("released without a database");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,18 +15,22 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Health;
|
||||
|
||||
public sealed class HealthProbeActorTests : RuntimeActorTestBase
|
||||
{
|
||||
/// <summary>Verifies that the DB health probe actor returns reachable status against an in-memory database.</summary>
|
||||
/// <summary>Verifies that the DB health probe actor returns reachable status against an in-memory database.
|
||||
/// The probe runs off the mailbox (02/P7), so the first Ask may still see "not probed yet" — poll.</summary>
|
||||
[Fact]
|
||||
public async Task DbHealthProbeActor_returns_reachable_against_in_memory_db()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var actor = Sys.ActorOf(DbHealthProbeActor.Props(db));
|
||||
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, PresenceBudget);
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
var status = await actor.Ask<DbHealthProbeActor.DbHealthStatus>(
|
||||
DbHealthProbeActor.GetStatus.Instance, PresenceBudget);
|
||||
|
||||
status.Reachable.ShouldBeTrue();
|
||||
status.LastError.ShouldBeNull();
|
||||
status.Reachable.ShouldBeTrue();
|
||||
status.LastError.ShouldBeNull();
|
||||
}, PresenceBudget);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the peer OPC UA probe actor reports Ok true against a live listener.</summary>
|
||||
|
||||
@@ -168,4 +168,68 @@ public sealed class ServerHistorianOptionsTests
|
||||
}.Validate();
|
||||
Assert.Empty(w);
|
||||
}
|
||||
|
||||
// ── MaxRawValuesPerRead (R08:1.1, #736) ─────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void MaxRawValuesPerRead_defaults_to_ten_thousand_and_binds()
|
||||
{
|
||||
var o = new ServerHistorianOptions();
|
||||
Assert.Equal(10_000, o.MaxRawValuesPerRead);
|
||||
Assert.Equal(10_000, ServerHistorianOptions.DefaultMaxRawValuesPerRead);
|
||||
Assert.Equal(10_000, o.EffectiveMaxRawValuesPerRead);
|
||||
|
||||
var bound = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?> { ["ServerHistorian:MaxRawValuesPerRead"] = "2500" })
|
||||
.Build()
|
||||
.GetSection(ServerHistorianOptions.SectionName)
|
||||
.Get<ServerHistorianOptions>()!;
|
||||
Assert.Equal(2500, bound.MaxRawValuesPerRead);
|
||||
Assert.Equal(2500, bound.EffectiveMaxRawValuesPerRead);
|
||||
Assert.Empty(new ServerHistorianOptions
|
||||
{
|
||||
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = 2500,
|
||||
}.Validate());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(-5)]
|
||||
public void Nonpositive_MaxRawValuesPerRead_is_unbounded_and_warns_when_enabled(int value)
|
||||
{
|
||||
var o = new ServerHistorianOptions
|
||||
{
|
||||
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = value,
|
||||
};
|
||||
Assert.Equal(0, o.EffectiveMaxRawValuesPerRead);
|
||||
var w = o.Validate();
|
||||
var m = Assert.Single(w);
|
||||
Assert.Contains("MaxRawValuesPerRead", m);
|
||||
Assert.Contains("unbounded", m);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(1)]
|
||||
[InlineData(2)]
|
||||
public void A_MaxRawValuesPerRead_below_three_is_raised_to_three_and_warns_when_enabled(int value)
|
||||
{
|
||||
// A resumed page's read must carry a value, the boundary tie it re-reads and the lookahead row that says "more
|
||||
// follows": at 2 it has no room and every value costs an empty page with a continuation point (the #692 defect).
|
||||
var o = new ServerHistorianOptions
|
||||
{
|
||||
Enabled = true, Endpoint = "https://h:5222", ApiKey = "histgw_x_y", MaxRawValuesPerRead = value,
|
||||
};
|
||||
Assert.Equal(3, o.EffectiveMaxRawValuesPerRead);
|
||||
Assert.Equal(ServerHistorianOptions.MinMaxRawValuesPerRead, o.EffectiveMaxRawValuesPerRead);
|
||||
var m = Assert.Single(o.Validate());
|
||||
Assert.Contains("MaxRawValuesPerRead", m);
|
||||
Assert.Equal(3, new ServerHistorianOptions { MaxRawValuesPerRead = 3 }.EffectiveMaxRawValuesPerRead);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Disabled_with_an_odd_MaxRawValuesPerRead_is_silent()
|
||||
{
|
||||
Assert.Empty(new ServerHistorianOptions { Enabled = false, MaxRawValuesPerRead = 0 }.Validate());
|
||||
Assert.Empty(new ServerHistorianOptions { Enabled = false, MaxRawValuesPerRead = 1 }.Validate());
|
||||
}
|
||||
}
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Review 02 L4: <see cref="TelemetryLocalHub.Evict"/>'s tombstone set only ever shrank on a re-spawn of the
|
||||
/// SAME id, so a driver removed for good kept its id forever. The tombstone is load-bearing (#609 — it holds
|
||||
/// back the removed child's late writes), so it cannot simply be cleared; it is now timestamped and swept
|
||||
/// once older than <see cref="TelemetryLocalHub.EvictionTombstoneTtl"/>, which outlives any in-flight
|
||||
/// capability call by a wide margin.
|
||||
/// </summary>
|
||||
public sealed class TelemetryLocalHubTombstoneSweepTests
|
||||
{
|
||||
private static DriverHealthChanged Health(string instanceId) =>
|
||||
new("cluster-a", instanceId, "Healthy", null, null, 0, DateTime.UtcNow);
|
||||
|
||||
private static List<TelemetryItem> Drain(ITelemetrySubscription sub)
|
||||
{
|
||||
var items = new List<TelemetryItem>();
|
||||
while (sub.Reader.TryRead(out var item)) items.Add(item);
|
||||
return items;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_tombstone_holds_late_writes_until_the_ttl_then_is_swept()
|
||||
{
|
||||
var clock = new ManualTime { Now = new DateTimeOffset(2026, 10, 22, 12, 0, 0, TimeSpan.Zero) };
|
||||
var hub = new TelemetryLocalHub(clock);
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
hub.Evict("drv-a");
|
||||
|
||||
// Inside the TTL: the late write is dropped (the #609 contract) — and another eviction's sweep does
|
||||
// not lift a tombstone that is still young.
|
||||
clock.Now += TelemetryLocalHub.EvictionTombstoneTtl / 2;
|
||||
hub.Evict("drv-b");
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using (var sub = hub.Subscribe(16))
|
||||
Drain(sub).ShouldBeEmpty("a late write inside the TTL must still be held back");
|
||||
|
||||
// Past the TTL: the next eviction sweeps drv-a's tombstone and its writes land again.
|
||||
clock.Now += TelemetryLocalHub.EvictionTombstoneTtl;
|
||||
hub.Evict("drv-c");
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using (var sub = hub.Subscribe(16))
|
||||
Drain(sub).OfType<TelemetryItem.Health>().Single().E.DriverInstanceId.ShouldBe("drv-a");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Readmit_lifts_a_tombstone_before_its_ttl()
|
||||
{
|
||||
var clock = new ManualTime { Now = DateTimeOffset.UtcNow };
|
||||
var hub = new TelemetryLocalHub(clock);
|
||||
hub.Evict("drv-a");
|
||||
hub.Readmit("drv-a");
|
||||
|
||||
hub.Emit(new TelemetryItem.Health(Health("drv-a")));
|
||||
using var sub = hub.Subscribe(16);
|
||||
Drain(sub).Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
private sealed class ManualTime : TimeProvider
|
||||
{
|
||||
public DateTimeOffset Now { get; set; }
|
||||
public override DateTimeOffset GetUtcNow() => Now;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user