fix(opcuaclient): split the per-endpoint connect budget into reachability + session phases (#912)

PerEndpointConnectTimeout (3 s) now bounds only TCP + discovery, so failover past a
dead endpoint stays fast. The new SessionCreateTimeout (10 s) bounds OpenSecureChannel
+ CreateSession on an endpoint that answered discovery, where the client-side cert and
crypto work lives (#910). 2 x (3 + 10) s stays inside the host's 30 s initialise bound.
Expiries name the budget that ran out; both budgets must be positive; AdminUI form
carries the new field so a save does not drop it; driver docs updated.

Closes #912
This commit is contained in:
Joseph Doherty
2026-11-15 16:55:22 -05:00
parent 8ff424fc5a
commit 678498a4a6
6 changed files with 150 additions and 29 deletions
+10 -4
View File
@@ -75,10 +75,16 @@ measurements; they are not fresh SDK 2.0.1 interoperability evidence.
`EndpointUrls` (a list) takes precedence over the single-URL `EndpointUrl` and
provides ordered **failover** — the driver tries each candidate in turn at every
`InitializeAsync` (including the comms-loss watchdog's re-initialise), and the first to connect wins (e.g. a hot-standby pair on
4840 / 4841). Each candidate gets `PerEndpointConnectTimeout` (default 3 s) for discovery + secure channel + session. That
budget includes **client-side** work (certificate load/validation, the asymmetric handshake), so a cold process on a busy
host can exceed it against a healthy server; an expiry is reported as `TimeoutException: … PerEndpointConnectTimeout (3 s)`
(#910 — the SDK alone says `BadRequestInterrupted`, which reads like a server fault). See
4840 / 4841). Each candidate's connect runs in **two separately bounded phases** (#912):
`PerEndpointConnectTimeout` (default **3 s**) covers TCP + endpoint discovery, so a dead or unreachable candidate is
passed over in about 3 s; `SessionCreateTimeout` (default **10 s**) then covers OpenSecureChannel + CreateSession +
ActivateSession on an endpoint that answered discovery. The session phase carries the **client-side** work (certificate
load/validation, the asymmetric crypto of a secured handshake), which a cold process on a busy host can stretch past 3 s
against a healthy server (#910). The larger budget only costs time when an endpoint answers discovery and then stalls —
not when it is down. The defaults keep two such candidates (2 × (3 + 10) s) inside the host's 30 s initialise bound;
raise either per driver if your topology needs it, keeping that sum in mind. An expiry is reported by the budget's name,
e.g. `TimeoutException: … SessionCreateTimeout (10 s)` (#910 — the SDK alone says `BadRequestInterrupted`, which reads
like a server fault). The AdminUI browse picker still uses one combined budget (`PerEndpointConnectTimeout` clamped to 5–30 s). See
`src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Contracts/OpcUaClientDriverOptions.cs`
for every field (security policy/mode, auth type, session timeout, keep-alive,
browse root, node/depth caps). `ReconnectPeriod` was **deleted** (#676 — unused since #670 removed the SDK reconnect
@@ -48,13 +48,24 @@ public sealed record OpcUaClientDriverOptions
public IReadOnlyList<string> EndpointUrls { get; init; } = [];
/// <summary>
/// Per-endpoint connect-attempt timeout during the failover sweep. Short enough that
/// cycling through several dead servers doesn't blow the overall init budget, long
/// enough to tolerate a slow TLS handshake on a healthy server. Applied independently
/// of <see cref="Timeout"/> which governs steady-state operations.
/// Per-endpoint budget for the <b>reachability</b> phase of a connect during the failover sweep: TCP connect +
/// endpoint discovery (GetEndpoints). Short so a dead or unreachable candidate is passed over quickly. The
/// secure-channel + session phase that follows has its own budget, <see cref="SessionCreateTimeout"/> (#912).
/// Applied independently of <see cref="Timeout"/> which governs steady-state operations.
/// </summary>
public TimeSpan PerEndpointConnectTimeout { get; init; } = TimeSpan.FromSeconds(3);
/// <summary>
/// Per-endpoint budget for the <b>session</b> phase of a connect, which only starts once the endpoint has answered
/// discovery: OpenSecureChannel + CreateSession + ActivateSession. Larger than
/// <see cref="PerEndpointConnectTimeout"/> because it covers client-side work — loading the application
/// certificate, validating the server's, the asymmetric crypto of a secured handshake — that a cold process on a
/// busy host can stretch past 3 s against a healthy server (#910/#912). It does not slow failover past a dead
/// endpoint (that fails in the reachability phase). Default 10 s keeps two hung candidates
/// (2 × (3 + 10) s) inside the host's 30 s initialise bound.
/// </summary>
public TimeSpan SessionCreateTimeout { get; init; } = TimeSpan.FromSeconds(10);
/// <summary>
/// Security policy to require when selecting an endpoint. Either a
/// <see cref="OpcUaSecurityPolicy"/> enum constant or a free-form string (for
@@ -1003,6 +1003,10 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
throw new InvalidOperationException(
"OpcUaClient SubscriptionLifetimeCount must be at least 3x SubscriptionKeepAliveCount " +
$"(got lifetime={opts.SubscriptionLifetimeCount}, keep-alive={opts.SubscriptionKeepAliveCount}).");
if (opts.PerEndpointConnectTimeout <= TimeSpan.Zero)
throw new InvalidOperationException("OpcUaClient PerEndpointConnectTimeout must be positive.");
if (opts.SessionCreateTimeout <= TimeSpan.Zero)
throw new InvalidOperationException("OpcUaClient SessionCreateTimeout must be positive.");
if (opts.SubscriptionPublishingInterval is { } pi && pi <= TimeSpan.Zero)
throw new InvalidOperationException("OpcUaClient SubscriptionPublishingInterval must be positive when set.");
if (!string.IsNullOrWhiteSpace(opts.DiscoveryUrl)
@@ -1037,10 +1041,12 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
OpcUaUserIdentityFactory.Build(options);
/// <summary>
/// Open a session against a single endpoint URL. Bounded by
/// <see cref="OpcUaClientDriverOptions.PerEndpointConnectTimeout"/> so the failover
/// sweep doesn't spend its full budget on one dead server. Moved out of
/// <see cref="InitializeAsync"/> so the failover loop body stays readable.
/// Open a session against a single endpoint URL, in two separately bounded phases (#912):
/// reachability (TCP + discovery) under <see cref="OpcUaClientDriverOptions.PerEndpointConnectTimeout"/>, so the
/// failover sweep passes a dead server quickly, then secure channel + session under
/// <see cref="OpcUaClientDriverOptions.SessionCreateTimeout"/>, which only a reachable server reaches and which
/// carries the client-side certificate and crypto work. Moved out of <see cref="InitializeAsync"/> so the
/// failover loop body stays readable.
/// </summary>
private async Task<ISession> OpenSessionOnEndpointAsync(
ApplicationConfiguration appConfig,
@@ -1050,36 +1056,54 @@ public sealed class OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWrit
UserIdentity identity,
CancellationToken ct)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
cts.CancelAfter(_options.PerEndpointConnectTimeout);
var selected = await RunWithinBudgetAsync(
_options.PerEndpointConnectTimeout, nameof(OpcUaClientDriverOptions.PerEndpointConnectTimeout),
budget => SelectMatchingEndpointAsync(appConfig, endpointUrl, policy, mode, budget), ct)
.ConfigureAwait(false);
return await RunWithinBudgetAsync(
_options.SessionCreateTimeout, nameof(OpcUaClientDriverOptions.SessionCreateTimeout),
budget => CreateSessionAsync(appConfig, selected, identity, budget), ct)
.ConfigureAwait(false);
}
/// <summary>
/// Run one connect phase under its own budget, linked to <paramref name="ct"/>. A phase cut short by the budget
/// (and not by the caller) is rethrown as a <see cref="TimeoutException"/> naming the option.
/// </summary>
/// <remarks>
/// Gitea #910: the SDK reports a wait cut short by the budget as BadRequestInterrupted (or a bare
/// TaskCanceledException), which reads like a server or network fault. Name the budget instead.
/// </remarks>
/// <param name="budget">The phase's time budget.</param>
/// <param name="optionName">The option that sets it, for the error text.</param>
/// <param name="phase">The phase body; receives the budget-linked token.</param>
/// <param name="ct">The caller's token.</param>
/// <returns>The phase result.</returns>
/// <exception cref="TimeoutException">The budget expired before the phase completed.</exception>
internal static async Task<T> RunWithinBudgetAsync<T>(
TimeSpan budget, string optionName, Func<CancellationToken, Task<T>> phase, CancellationToken ct)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
cts.CancelAfter(budget);
try
{
return await OpenSessionWithinBudgetAsync(appConfig, endpointUrl, policy, mode, identity, cts.Token)
.ConfigureAwait(false);
return await phase(cts.Token).ConfigureAwait(false);
}
catch (Exception ex) when (cts.IsCancellationRequested && !ct.IsCancellationRequested)
{
// Gitea #910: the SDK reports a wait cut short by this budget as BadRequestInterrupted (or a bare
// TaskCanceledException), which reads like a server or network fault. Name the budget instead. It also
// covers CLIENT-side work (loading the application certificate, validating the server's, the asymmetric
// crypto of OpenSecureChannel), so a cold process on a busy host can exceed it with a healthy server.
throw new TimeoutException(
"The connect did not complete within PerEndpointConnectTimeout " +
$"({_options.PerEndpointConnectTimeout.TotalSeconds:0.###} s); the SDK reported: {ex.Message}", ex);
$"The connect did not complete within {optionName} ({budget.TotalSeconds:0.###} s); " +
$"the SDK reported: {ex.Message}", ex);
}
}
private async Task<ISession> OpenSessionWithinBudgetAsync(
private async Task<ISession> CreateSessionAsync(
ApplicationConfiguration appConfig,
string endpointUrl,
OpcUaSecurityPolicy policy,
OpcUaSecurityMode mode,
EndpointDescription selected,
UserIdentity identity,
CancellationToken budget)
{
var selected = await SelectMatchingEndpointAsync(
appConfig, endpointUrl, policy, mode, budget).ConfigureAwait(false);
var endpointConfig = EndpointConfiguration.Create(appConfig);
endpointConfig.OperationTimeout = (int)_options.Timeout.TotalMilliseconds;
var endpoint = new ConfiguredEndpoint(null, selected, endpointConfig);
@@ -37,7 +37,13 @@
<label for="@FieldId("field-05")" class="form-label">Per-endpoint connect timeout (s)</label>
<InputNumber DisplayName="Per-endpoint connect timeout (s)" aria-describedby="@FieldIds("field-05-error", "field-05-hint")" id="@FieldId("field-05")" @bind-Value="_form.OpcUa.PerEndpointConnectTimeoutSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
<div id="@FieldId("field-05-error")" class="field-error">@if (_fieldEditContext is not null) { <ValidationMessage For="@(() => _form.OpcUa.PerEndpointConnectTimeoutSeconds)" /> }</div>
<div id="@FieldId("field-05-hint")" class="form-text">Default 3 s — failover sweep budget.</div>
<div id="@FieldId("field-05-hint")" class="form-text">Default 3 s — TCP + discovery; how fast failover passes a dead endpoint.</div>
</div>
<div class="col-md-3">
<label for="@FieldId("field-30")" class="form-label">Session create timeout (s)</label>
<InputNumber DisplayName="Session create timeout (s)" aria-describedby="@FieldIds("field-30-error", "field-30-hint")" id="@FieldId("field-30")" @bind-Value="_form.OpcUa.SessionCreateTimeoutSeconds" @bind-Value:after="EmitAsync" class="form-control form-control-sm" />
<div id="@FieldId("field-30-error")" class="field-error">@if (_fieldEditContext is not null) { <ValidationMessage For="@(() => _form.OpcUa.SessionCreateTimeoutSeconds)" /> }</div>
<div id="@FieldId("field-30-hint")" class="form-text">Default 10 s — secure channel + session on a reachable endpoint.</div>
</div>
<div class="col-md-3">
<label for="@FieldId("field-06")" class="form-label">Operation timeout (s)</label>
@@ -366,6 +372,7 @@
public string SessionName { get; set; } = "OtOpcUa-Gateway";
public bool AutoAcceptCertificates { get; set; } = false;
public int PerEndpointConnectTimeoutSeconds { get; set; } = 3;
public int SessionCreateTimeoutSeconds { get; set; } = 10;
public int TimeoutSeconds { get; set; } = 10;
public int SessionTimeoutSeconds { get; set; } = 120;
public int KeepAliveIntervalSeconds { get; set; } = 5;
@@ -407,6 +414,7 @@
SessionName = r.SessionName,
AutoAcceptCertificates = r.AutoAcceptCertificates,
PerEndpointConnectTimeoutSeconds = (int)r.PerEndpointConnectTimeout.TotalSeconds,
SessionCreateTimeoutSeconds = (int)r.SessionCreateTimeout.TotalSeconds,
TimeoutSeconds = (int)r.Timeout.TotalSeconds,
SessionTimeoutSeconds = (int)r.SessionTimeout.TotalSeconds,
KeepAliveIntervalSeconds = (int)r.KeepAliveInterval.TotalSeconds,
@@ -446,6 +454,7 @@
SessionName = SessionName,
AutoAcceptCertificates = AutoAcceptCertificates,
PerEndpointConnectTimeout = TimeSpan.FromSeconds(PerEndpointConnectTimeoutSeconds),
SessionCreateTimeout = TimeSpan.FromSeconds(SessionCreateTimeoutSeconds),
Timeout = TimeSpan.FromSeconds(TimeoutSeconds),
SessionTimeout = TimeSpan.FromSeconds(SessionTimeoutSeconds),
KeepAliveInterval = TimeSpan.FromSeconds(KeepAliveIntervalSeconds),
@@ -109,4 +109,73 @@ public sealed class OpcUaClientFailoverTests
ex.Message.ShouldContain("TimeoutException");
ex.Message.ShouldContain("PerEndpointConnectTimeout (0.4 s)");
}
/// <summary>#912: the two connect phases default to a short reachability budget and a larger session budget.</summary>
[Fact]
public void Connect_budgets_default_to_3s_reachability_and_10s_session()
{
var o = new OpcUaClientDriverOptions();
o.PerEndpointConnectTimeout.ShouldBe(TimeSpan.FromSeconds(3));
o.SessionCreateTimeout.ShouldBe(TimeSpan.FromSeconds(10));
// Two candidates that both hang in the session phase must still fit the host's initialise bound.
(2 * (o.PerEndpointConnectTimeout + o.SessionCreateTimeout)).ShouldBeLessThan(TimeSpan.FromSeconds(30));
}
/// <summary>#912: a phase cut short by its own budget is reported as that budget, by option name.</summary>
[Fact]
public async Task A_phase_budget_expiry_names_its_option()
{
var ex = await Should.ThrowAsync<TimeoutException>(() => OpcUaClientDriver.RunWithinBudgetAsync<int>(
TimeSpan.FromMilliseconds(100), nameof(OpcUaClientDriverOptions.SessionCreateTimeout),
async budget => { await Task.Delay(Timeout.Infinite, budget); return 0; },
TestContext.Current.CancellationToken));
ex.Message.ShouldContain("SessionCreateTimeout (0.1 s)");
ex.InnerException.ShouldBeAssignableTo<OperationCanceledException>();
}
/// <summary>#912: a caller cancel is not dressed up as a budget expiry.</summary>
[Fact]
public async Task A_caller_cancel_passes_through_unchanged()
{
using var caller = new CancellationTokenSource(TimeSpan.FromMilliseconds(100));
await Should.ThrowAsync<OperationCanceledException>(() => OpcUaClientDriver.RunWithinBudgetAsync<int>(
TimeSpan.FromSeconds(30), "X",
async budget => { await Task.Delay(Timeout.Infinite, budget); return 0; },
caller.Token));
}
/// <summary>#912: a phase that completes inside its budget returns its result, and a non-timeout fault is not rewritten.</summary>
[Fact]
public async Task A_phase_inside_its_budget_returns_and_faults_pass_through()
{
(await OpcUaClientDriver.RunWithinBudgetAsync(TimeSpan.FromSeconds(5), "X",
_ => Task.FromResult(42), TestContext.Current.CancellationToken)).ShouldBe(42);
await Should.ThrowAsync<InvalidOperationException>(() => OpcUaClientDriver.RunWithinBudgetAsync<int>(
TimeSpan.FromSeconds(5), "X", _ => throw new InvalidOperationException("boom"),
TestContext.Current.CancellationToken));
}
/// <summary>#912: non-positive connect budgets are refused at initialise.</summary>
[Fact]
public void ValidateTuning_rejects_non_positive_connect_budgets()
{
Should.Throw<InvalidOperationException>(() => OpcUaClientDriver.ValidateTuning(
new OpcUaClientDriverOptions { SessionCreateTimeout = TimeSpan.Zero }))
.Message.ShouldContain("SessionCreateTimeout");
Should.Throw<InvalidOperationException>(() => OpcUaClientDriver.ValidateTuning(
new OpcUaClientDriverOptions { PerEndpointConnectTimeout = TimeSpan.Zero }))
.Message.ShouldContain("PerEndpointConnectTimeout");
}
/// <summary>#912: the factory binds <c>sessionCreateTimeout</c>; JSON without it keeps the default.</summary>
[Fact]
public void Factory_binds_sessionCreateTimeout()
{
OpcUaClientDriverFactoryExtensions.ParseOptions("d", """{"sessionCreateTimeout":"00:00:20"}""")
.SessionCreateTimeout.ShouldBe(TimeSpan.FromSeconds(20));
OpcUaClientDriverFactoryExtensions.ParseOptions("d", "{}")
.SessionCreateTimeout.ShouldBe(TimeSpan.FromSeconds(10));
}
}
@@ -100,6 +100,7 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
UserCertificatePath = @"C:\certs\user.pfx",
UserCertificatePassword = "certpass",
PerEndpointConnectTimeout = TimeSpan.FromSeconds(4),
SessionCreateTimeout = TimeSpan.FromSeconds(17),
Timeout = TimeSpan.FromSeconds(12),
SessionTimeout = TimeSpan.FromSeconds(150),
KeepAliveInterval = TimeSpan.FromSeconds(7),
@@ -130,6 +131,7 @@ public sealed class OpcUaClientDriverPageFormSerializationTests
.ToRecord(endpointUrls).UserCertificatePath.ShouldBe("plant1/user.pfx");
result.UserCertificatePassword.ShouldBe("certpass");
result.PerEndpointConnectTimeout.ShouldBe(TimeSpan.FromSeconds(4));
result.SessionCreateTimeout.ShouldBe(TimeSpan.FromSeconds(17)); // #912 — the form must not drop it on save
result.Timeout.ShouldBe(TimeSpan.FromSeconds(12));
result.SessionTimeout.ShouldBe(TimeSpan.FromSeconds(150));
result.KeepAliveInterval.ShouldBe(TimeSpan.FromSeconds(7));