Merge branch 'fix/912-split-connect-budget' (#912)
v2-ci / unit-tests (push) Failing after 3h14m42s
v2-ci / unit-tests (push) Failing after 3h14m42s
This commit is contained in:
@@ -75,10 +75,16 @@ measurements; they are not fresh SDK 2.0.1 interoperability evidence.
|
||||
`EndpointUrls` (a list) takes precedence over the single-URL `EndpointUrl` and
|
||||
provides ordered **failover** — the driver tries each candidate in turn at every
|
||||
`InitializeAsync` (including the comms-loss watchdog's re-initialise), and the first to connect wins (e.g. a hot-standby pair on
|
||||
4840 / 4841). Each candidate gets `PerEndpointConnectTimeout` (default 3 s) for discovery + secure channel + session. That
|
||||
budget includes **client-side** work (certificate load/validation, the asymmetric handshake), so a cold process on a busy
|
||||
host can exceed it against a healthy server; an expiry is reported as `TimeoutException: … PerEndpointConnectTimeout (3 s)`
|
||||
(#910 — the SDK alone says `BadRequestInterrupted`, which reads like a server fault). See
|
||||
4840 / 4841). Each candidate's connect runs in **two separately bounded phases** (#912):
|
||||
`PerEndpointConnectTimeout` (default **3 s**) covers TCP + endpoint discovery, so a dead or unreachable candidate is
|
||||
passed over in about 3 s; `SessionCreateTimeout` (default **10 s**) then covers OpenSecureChannel + CreateSession +
|
||||
ActivateSession on an endpoint that answered discovery. The session phase carries the **client-side** work (certificate
|
||||
load/validation, the asymmetric crypto of a secured handshake), which a cold process on a busy host can stretch past 3 s
|
||||
against a healthy server (#910). The larger budget only costs time when an endpoint answers discovery and then stalls —
|
||||
not when it is down. The defaults keep two such candidates (2 × (3 + 10) s) inside the host's 30 s initialise bound;
|
||||
raise either per driver if your topology needs it, keeping that sum in mind. An expiry is reported by the budget's name,
|
||||
e.g. `TimeoutException: … SessionCreateTimeout (10 s)` (#910 — the SDK alone says `BadRequestInterrupted`, which reads
|
||||
like a server fault). The AdminUI browse picker still uses one combined budget (`PerEndpointConnectTimeout` clamped to 5–30 s). See
|
||||
`src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Contracts/OpcUaClientDriverOptions.cs`
|
||||
for every field (security policy/mode, auth type, session timeout, keep-alive,
|
||||
browse root, node/depth caps). `ReconnectPeriod` was **deleted** (#676 — unused since #670 removed the SDK reconnect
|
||||
|
||||
+15
-4
@@ -48,13 +48,24 @@ public sealed record OpcUaClientDriverOptions
|
||||
public IReadOnlyList<string> EndpointUrls { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Per-endpoint connect-attempt timeout during the failover sweep. Short enough that
|
||||
/// cycling through several dead servers doesn't blow the overall init budget, long
|
||||
/// enough to tolerate a slow TLS handshake on a healthy server. Applied independently
|
||||
/// of <see cref="Timeout"/> which governs steady-state operations.
|
||||
/// Per-endpoint budget for the <b>reachability</b> phase of a connect during the failover sweep: TCP connect +
|
||||
/// endpoint discovery (GetEndpoints). Short so a dead or unreachable candidate is passed over quickly. The
|
||||
/// secure-channel + session phase that follows has its own budget, <see cref="SessionCreateTimeout"/> (#912).
|
||||
/// Applied independently of <see cref="Timeout"/> which governs steady-state operations.
|
||||
/// </summary>
|
||||
public TimeSpan PerEndpointConnectTimeout { get; init; } = TimeSpan.FromSeconds(3);
|
||||
|
||||
/// <summary>
|
||||
/// Per-endpoint budget for the <b>session</b> phase of a connect, which only starts once the endpoint has answered
|
||||
/// discovery: OpenSecureChannel + CreateSession + ActivateSession. Larger than
|
||||
/// <see cref="PerEndpointConnectTimeout"/> because it covers client-side work — loading the application
|
||||
/// certificate, validating the server's, the asymmetric crypto of a secured handshake — that a cold process on a
|
||||
/// busy host can stretch past 3 s against a healthy server (#910/#912). It does not slow failover past a dead
|
||||
/// endpoint (that fails in the reachability phase). Default 10 s keeps two hung candidates
|
||||
/// (2 × (3 + 10) s) inside the host's 30 s initialise bound.
|
||||
/// </summary>
|
||||
public TimeSpan SessionCreateTimeout { get; init; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
/// <summary>
|
||||
/// Security policy to require when selecting an endpoint. Either a
|
||||
/// <see cref="OpcUaSecurityPolicy"/> enum constant or a free-form string (for
|
||||
|
||||
@@ -1003,6 +1003,10 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
throw new InvalidOperationException(
|
||||
"OpcUaClient SubscriptionLifetimeCount must be at least 3x SubscriptionKeepAliveCount " +
|
||||
$"(got lifetime={opts.SubscriptionLifetimeCount}, keep-alive={opts.SubscriptionKeepAliveCount}).");
|
||||
if (opts.PerEndpointConnectTimeout <= TimeSpan.Zero)
|
||||
throw new InvalidOperationException("OpcUaClient PerEndpointConnectTimeout must be positive.");
|
||||
if (opts.SessionCreateTimeout <= TimeSpan.Zero)
|
||||
throw new InvalidOperationException("OpcUaClient SessionCreateTimeout must be positive.");
|
||||
if (opts.SubscriptionPublishingInterval is { } pi && pi <= TimeSpan.Zero)
|
||||
throw new InvalidOperationException("OpcUaClient SubscriptionPublishingInterval must be positive when set.");
|
||||
if (!string.IsNullOrWhiteSpace(opts.DiscoveryUrl)
|
||||
@@ -1037,10 +1041,12 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
OpcUaUserIdentityFactory.Build(options);
|
||||
|
||||
/// <summary>
|
||||
/// Open a session against a single endpoint URL. Bounded by
|
||||
/// <see cref="OpcUaClientDriverOptions.PerEndpointConnectTimeout"/> so the failover
|
||||
/// sweep doesn't spend its full budget on one dead server. Moved out of
|
||||
/// <see cref="InitializeAsync"/> so the failover loop body stays readable.
|
||||
/// Open a session against a single endpoint URL, in two separately bounded phases (#912):
|
||||
/// reachability (TCP + discovery) under <see cref="OpcUaClientDriverOptions.PerEndpointConnectTimeout"/>, so the
|
||||
/// failover sweep passes a dead server quickly, then secure channel + session under
|
||||
/// <see cref="OpcUaClientDriverOptions.SessionCreateTimeout"/>, which only a reachable server reaches and which
|
||||
/// carries the client-side certificate and crypto work. Moved out of <see cref="InitializeAsync"/> so the
|
||||
/// failover loop body stays readable.
|
||||
/// </summary>
|
||||
private async Task<ISession> OpenSessionOnEndpointAsync(
|
||||
ApplicationConfiguration appConfig,
|
||||
@@ -1050,36 +1056,54 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
|
||||
UserIdentity identity,
|
||||
CancellationToken ct)
|
||||
{
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
cts.CancelAfter(_options.PerEndpointConnectTimeout);
|
||||
var selected = await RunWithinBudgetAsync(
|
||||
_options.PerEndpointConnectTimeout, nameof(OpcUaClientDriverOptions.PerEndpointConnectTimeout),
|
||||
budget => SelectMatchingEndpointAsync(appConfig, endpointUrl, policy, mode, budget), ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return await RunWithinBudgetAsync(
|
||||
_options.SessionCreateTimeout, nameof(OpcUaClientDriverOptions.SessionCreateTimeout),
|
||||
budget => CreateSessionAsync(appConfig, selected, identity, budget), ct)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Run one connect phase under its own budget, linked to <paramref name="ct"/>. A phase cut short by the budget
|
||||
/// (and not by the caller) is rethrown as a <see cref="TimeoutException"/> naming the option.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Gitea #910: the SDK reports a wait cut short by the budget as BadRequestInterrupted (or a bare
|
||||
/// TaskCanceledException), which reads like a server or network fault. Name the budget instead.
|
||||
/// </remarks>
|
||||
/// <param name="budget">The phase's time budget.</param>
|
||||
/// <param name="optionName">The option that sets it, for the error text.</param>
|
||||
/// <param name="phase">The phase body; receives the budget-linked token.</param>
|
||||
/// <param name="ct">The caller's token.</param>
|
||||
/// <returns>The phase result.</returns>
|
||||
/// <exception cref="TimeoutException">The budget expired before the phase completed.</exception>
|
||||
internal static async Task<T> RunWithinBudgetAsync<T>(
|
||||
TimeSpan budget, string optionName, Func<CancellationToken, Task<T>> phase, CancellationToken ct)
|
||||
{
|
||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
cts.CancelAfter(budget);
|
||||
try
|
||||
{
|
||||
return await OpenSessionWithinBudgetAsync(appConfig, endpointUrl, policy, mode, identity, cts.Token)
|
||||
.ConfigureAwait(false);
|
||||
return await phase(cts.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (cts.IsCancellationRequested && !ct.IsCancellationRequested)
|
||||
{
|
||||
// Gitea #910: the SDK reports a wait cut short by this budget as BadRequestInterrupted (or a bare
|
||||
// TaskCanceledException), which reads like a server or network fault. Name the budget instead. It also
|
||||
// covers CLIENT-side work (loading the application certificate, validating the server's, the asymmetric
|
||||
// crypto of OpenSecureChannel), so a cold process on a busy host can exceed it with a healthy server.
|
||||
throw new TimeoutException(
|
||||
"The connect did not complete within PerEndpointConnectTimeout " +
|
||||
$"({_options.PerEndpointConnectTimeout.TotalSeconds:0.###} s); the SDK reported: {ex.Message}", ex);
|
||||
$"The connect did not complete within {optionName} ({budget.TotalSeconds:0.###} s); " +
|
||||
$"the SDK reported: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<ISession> OpenSessionWithinBudgetAsync(
|
||||
private async Task<ISession> CreateSessionAsync(
|
||||
ApplicationConfiguration appConfig,
|
||||
string endpointUrl,
|
||||
OpcUaSecurityPolicy policy,
|
||||
OpcUaSecurityMode mode,
|
||||
EndpointDescription selected,
|
||||
UserIdentity identity,
|
||||
CancellationToken budget)
|
||||
{
|
||||
var selected = await SelectMatchingEndpointAsync(
|
||||
appConfig, endpointUrl, policy, mode, budget).ConfigureAwait(false);
|
||||
var endpointConfig = EndpointConfiguration.Create(appConfig);
|
||||
endpointConfig.OperationTimeout = (int)_options.Timeout.TotalMilliseconds;
|
||||
var endpoint = new ConfiguredEndpoint(null, selected, endpointConfig);
|
||||
|
||||
+10
-1
@@ -37,7 +37,13 @@
|
||||
<label for="@FieldId("field-05")" class="form-label">Per-endpoint connect timeout (s)</label>
|
||||
<InputNumber DisplayName="Per-endpoint connect timeout (s)" aria-describedby="@FieldIds("field-05-error", "field-05-hint")" id="@FieldId("field-05")" @bind-Value="_form.OpcUa.PerEndpointConnectTimeoutSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div id="@FieldId("field-05-error")" class="field-error">@if (_fieldEditContext is not null) { <ValidationMessage For="@(() => _form.OpcUa.PerEndpointConnectTimeoutSeconds)" /> }</div>
|
||||
<div id="@FieldId("field-05-hint")" class="form-text">Default 3 s — failover sweep budget.</div>
|
||||
<div id="@FieldId("field-05-hint")" class="form-text">Default 3 s — TCP + discovery; how fast failover passes a dead endpoint.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label for="@FieldId("field-30")" class="form-label">Session create timeout (s)</label>
|
||||
<InputNumber DisplayName="Session create timeout (s)" aria-describedby="@FieldIds("field-30-error", "field-30-hint")" id="@FieldId("field-30")" @bind-Value="_form.OpcUa.SessionCreateTimeoutSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
|
||||
<div id="@FieldId("field-30-error")" class="field-error">@if (_fieldEditContext is not null) { <ValidationMessage For="@(() => _form.OpcUa.SessionCreateTimeoutSeconds)" /> }</div>
|
||||
<div id="@FieldId("field-30-hint")" class="form-text">Default 10 s — secure channel + session on a reachable endpoint.</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<label for="@FieldId("field-06")" class="form-label">Operation timeout (s)</label>
|
||||
@@ -366,6 +372,7 @@
|
||||
public string SessionName { get; set; } = "OtOpcUa-Gateway";
|
||||
public bool AutoAcceptCertificates { get; set; } = false;
|
||||
public int PerEndpointConnectTimeoutSeconds { get; set; } = 3;
|
||||
public int SessionCreateTimeoutSeconds { get; set; } = 10;
|
||||
public int TimeoutSeconds { get; set; } = 10;
|
||||
public int SessionTimeoutSeconds { get; set; } = 120;
|
||||
public int KeepAliveIntervalSeconds { get; set; } = 5;
|
||||
@@ -407,6 +414,7 @@
|
||||
SessionName = r.SessionName,
|
||||
AutoAcceptCertificates = r.AutoAcceptCertificates,
|
||||
PerEndpointConnectTimeoutSeconds = (int)r.PerEndpointConnectTimeout.TotalSeconds,
|
||||
SessionCreateTimeoutSeconds = (int)r.SessionCreateTimeout.TotalSeconds,
|
||||
TimeoutSeconds = (int)r.Timeout.TotalSeconds,
|
||||
SessionTimeoutSeconds = (int)r.SessionTimeout.TotalSeconds,
|
||||
KeepAliveIntervalSeconds = (int)r.KeepAliveInterval.TotalSeconds,
|
||||
@@ -446,6 +454,7 @@
|
||||
SessionName = SessionName,
|
||||
AutoAcceptCertificates = AutoAcceptCertificates,
|
||||
PerEndpointConnectTimeout = TimeSpan.FromSeconds(PerEndpointConnectTimeoutSeconds),
|
||||
SessionCreateTimeout = TimeSpan.FromSeconds(SessionCreateTimeoutSeconds),
|
||||
Timeout = TimeSpan.FromSeconds(TimeoutSeconds),
|
||||
SessionTimeout = TimeSpan.FromSeconds(SessionTimeoutSeconds),
|
||||
KeepAliveInterval = TimeSpan.FromSeconds(KeepAliveIntervalSeconds),
|
||||
|
||||
@@ -109,4 +109,73 @@ public sealed class OpcUaClientFailoverTests
|
||||
ex.Message.ShouldContain("TimeoutException");
|
||||
ex.Message.ShouldContain("PerEndpointConnectTimeout (0.4 s)");
|
||||
}
|
||||
|
||||
/// <summary>#912: the two connect phases default to a short reachability budget and a larger session budget.</summary>
|
||||
[Fact]
|
||||
public void Connect_budgets_default_to_3s_reachability_and_10s_session()
|
||||
{
|
||||
var o = new OpcUaClientDriverOptions();
|
||||
o.PerEndpointConnectTimeout.ShouldBe(TimeSpan.FromSeconds(3));
|
||||
o.SessionCreateTimeout.ShouldBe(TimeSpan.FromSeconds(10));
|
||||
// Two candidates that both hang in the session phase must still fit the host's initialise bound.
|
||||
(2 * (o.PerEndpointConnectTimeout + o.SessionCreateTimeout)).ShouldBeLessThan(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
|
||||
/// <summary>#912: a phase cut short by its own budget is reported as that budget, by option name.</summary>
|
||||
[Fact]
|
||||
public async Task A_phase_budget_expiry_names_its_option()
|
||||
{
|
||||
var ex = await Should.ThrowAsync<TimeoutException>(() => OpcUaClientDriver.RunWithinBudgetAsync<int>(
|
||||
TimeSpan.FromMilliseconds(100), nameof(OpcUaClientDriverOptions.SessionCreateTimeout),
|
||||
async budget => { await Task.Delay(Timeout.Infinite, budget); return 0; },
|
||||
TestContext.Current.CancellationToken));
|
||||
|
||||
ex.Message.ShouldContain("SessionCreateTimeout (0.1 s)");
|
||||
ex.InnerException.ShouldBeAssignableTo<OperationCanceledException>();
|
||||
}
|
||||
|
||||
/// <summary>#912: a caller cancel is not dressed up as a budget expiry.</summary>
|
||||
[Fact]
|
||||
public async Task A_caller_cancel_passes_through_unchanged()
|
||||
{
|
||||
using var caller = new CancellationTokenSource(TimeSpan.FromMilliseconds(100));
|
||||
await Should.ThrowAsync<OperationCanceledException>(() => OpcUaClientDriver.RunWithinBudgetAsync<int>(
|
||||
TimeSpan.FromSeconds(30), "X",
|
||||
async budget => { await Task.Delay(Timeout.Infinite, budget); return 0; },
|
||||
caller.Token));
|
||||
}
|
||||
|
||||
/// <summary>#912: a phase that completes inside its budget returns its result, and a non-timeout fault is not rewritten.</summary>
|
||||
[Fact]
|
||||
public async Task A_phase_inside_its_budget_returns_and_faults_pass_through()
|
||||
{
|
||||
(await OpcUaClientDriver.RunWithinBudgetAsync(TimeSpan.FromSeconds(5), "X",
|
||||
_ => Task.FromResult(42), TestContext.Current.CancellationToken)).ShouldBe(42);
|
||||
|
||||
await Should.ThrowAsync<InvalidOperationException>(() => OpcUaClientDriver.RunWithinBudgetAsync<int>(
|
||||
TimeSpan.FromSeconds(5), "X", _ => throw new InvalidOperationException("boom"),
|
||||
TestContext.Current.CancellationToken));
|
||||
}
|
||||
|
||||
/// <summary>#912: non-positive connect budgets are refused at initialise.</summary>
|
||||
[Fact]
|
||||
public void ValidateTuning_rejects_non_positive_connect_budgets()
|
||||
{
|
||||
Should.Throw<InvalidOperationException>(() => OpcUaClientDriver.ValidateTuning(
|
||||
new OpcUaClientDriverOptions { SessionCreateTimeout = TimeSpan.Zero }))
|
||||
.Message.ShouldContain("SessionCreateTimeout");
|
||||
Should.Throw<InvalidOperationException>(() => OpcUaClientDriver.ValidateTuning(
|
||||
new OpcUaClientDriverOptions { PerEndpointConnectTimeout = TimeSpan.Zero }))
|
||||
.Message.ShouldContain("PerEndpointConnectTimeout");
|
||||
}
|
||||
|
||||
/// <summary>#912: the factory binds <c>sessionCreateTimeout</c>; JSON without it keeps the default.</summary>
|
||||
[Fact]
|
||||
public void Factory_binds_sessionCreateTimeout()
|
||||
{
|
||||
OpcUaClientDriverFactoryExtensions.ParseOptions("d", """{"sessionCreateTimeout":"00:00:20"}""")
|
||||
.SessionCreateTimeout.ShouldBe(TimeSpan.FromSeconds(20));
|
||||
OpcUaClientDriverFactoryExtensions.ParseOptions("d", "{}")
|
||||
.SessionCreateTimeout.ShouldBe(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -100,6 +100,7 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
|
||||
UserCertificatePath = @"C:\certs\user.pfx",
|
||||
UserCertificatePassword = "certpass",
|
||||
PerEndpointConnectTimeout = TimeSpan.FromSeconds(4),
|
||||
SessionCreateTimeout = TimeSpan.FromSeconds(17),
|
||||
Timeout = TimeSpan.FromSeconds(12),
|
||||
SessionTimeout = TimeSpan.FromSeconds(150),
|
||||
KeepAliveInterval = TimeSpan.FromSeconds(7),
|
||||
@@ -130,6 +131,7 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
|
||||
.ToRecord(endpointUrls).UserCertificatePath.ShouldBe("plant1/user.pfx");
|
||||
result.UserCertificatePassword.ShouldBe("certpass");
|
||||
result.PerEndpointConnectTimeout.ShouldBe(TimeSpan.FromSeconds(4));
|
||||
result.SessionCreateTimeout.ShouldBe(TimeSpan.FromSeconds(17)); // #912 — the form must not drop it on save
|
||||
result.Timeout.ShouldBe(TimeSpan.FromSeconds(12));
|
||||
result.SessionTimeout.ShouldBe(TimeSpan.FromSeconds(150));
|
||||
result.KeepAliveInterval.ShouldBe(TimeSpan.FromSeconds(7));
|
||||
|
||||
Reference in New Issue
Block a user