feat(dcl): ServiceLevel-aware OPC UA failover for primary/backup pairs (#138)

Prerequisite for OtOpcUa Warm redundancy (lmxopcua#717): a Warm standby keeps a
healthy session but serves BadOutOfService at a standby ServiceLevel (90/50, boot 1),
so a connection that lands there sat on Bad quality forever.

- OPC UA connections with a backup poll Server_ServiceLevel (i=2267) on the connected
  endpoint (IServiceLevelReadable, implemented by OpcUaDataConnection; never raises
  Disconnected). A level < 200 sustained for the degraded window (default 10 s), or
  the first sample after any (re)connect, triggers a probe of the other endpoint
  (short-lived session: connect, read, dispose). Switch only when it reports a
  Healthy (>= 200) higher level, and not within the minimum dwell (default 60 s) of
  the previous ServiceLevel switch. Both degraded, other unreachable or level
  unknown: stay put. 250/240 (today's OtOpcUa) never starts a streak.
- The switch reuses the failover path: SwitchToOtherEndpoint() now shared by all
  three triggers, then BecomeReconnecting(ReconnectCause.ServiceLevelFailover) -
  zero delay, no unstable accounting, ReSubscribeAll on the new endpoint.
- Config: OpcUaEndpointConfig.ServiceLevelFailover {enabled, degradedWindowSeconds,
  minDwellSeconds}, read from the PRIMARY endpoint's JSON only; null = defaults
  (enabled, 10, 60). Inside the encrypted protocol JSON, so no schema change or
  migration. Central UI: fields in the Backup card next to Failover Retry Count.
  Global: ScadaBridge:DataConnection:ServiceLevelPollInterval (2 s) and
  ServiceLevelProbeTimeout (15 s).
- Tests: actor tests per trigger/non-trigger with positive controls, an unsolicited
  promotion RefreshStart/RefreshEnd test at client and actor level, adapter tests,
  and a real in-process OPC UA server pair end to end. All 15 mutants killed.
- Docs: Component-DataConnectionLayer, components/DataConnectionLayer, Component-CLI.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:06:45 -05:00
parent 16209831ca
commit 712c5117ee
23 changed files with 1760 additions and 36 deletions
@@ -0,0 +1,22 @@
namespace ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
/// <summary>
/// Optional capability for an <see cref="IDataConnection"/> whose server publishes a redundancy
/// ServiceLevel (OPC UA <c>Server_ServiceLevel</c>, <c>i=2267</c>). The DataConnectionActor polls it
/// on the connected endpoint, and probes the other endpoint with it, to drive ServiceLevel-aware
/// failover (ScadaBridge#138). An adapter without the capability is never switched on ServiceLevel.
/// </summary>
public interface IServiceLevelReadable
{
/// <summary>
/// Reads the server's current ServiceLevel. Never raises the connection's
/// <see cref="IDataConnection.Disconnected"/> event: a lost session is the reconnect machine's
/// business, not this read's.
/// </summary>
/// <param name="cancellationToken">Bounds the read.</param>
/// <returns>
/// The ServiceLevel (0–255), or null when it is unknown: not connected, a Bad status, a value
/// that is not a byte, or a failed read.
/// </returns>
Task<byte?> ReadServiceLevelAsync(CancellationToken cancellationToken = default);
}
@@ -83,6 +83,15 @@ public readonly record struct OpcUaConfigParseResult
/// </summary>
public static class OpcUaEndpointConfigSerializer
{
/// <summary>Flat-dict key for <see cref="OpcUaServiceLevelFailoverConfig.Enabled"/>.</summary>
public const string ServiceLevelFailoverEnabledKey = "ServiceLevelFailover.Enabled";
/// <summary>Flat-dict key for <see cref="OpcUaServiceLevelFailoverConfig.DegradedWindowSeconds"/>.</summary>
public const string ServiceLevelFailoverWindowKey = "ServiceLevelFailover.DegradedWindowSeconds";
/// <summary>Flat-dict key for <see cref="OpcUaServiceLevelFailoverConfig.MinDwellSeconds"/>.</summary>
public const string ServiceLevelFailoverDwellKey = "ServiceLevelFailover.MinDwellSeconds";
private static readonly JsonSerializerOptions JsonOpts = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
@@ -216,6 +225,12 @@ public static class OpcUaEndpointConfigSerializer
dict["Deadband.Type"] = db.Type.ToString();
dict["Deadband.Value"] = db.Value.ToString(System.Globalization.CultureInfo.InvariantCulture);
}
if (config.ServiceLevelFailover is { } sl)
{
dict[ServiceLevelFailoverEnabledKey] = sl.Enabled.ToString();
dict[ServiceLevelFailoverWindowKey] = sl.DegradedWindowSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture);
dict[ServiceLevelFailoverDwellKey] = sl.MinDwellSeconds.ToString(System.Globalization.CultureInfo.InvariantCulture);
}
return dict;
}
@@ -291,6 +306,18 @@ public static class OpcUaEndpointConfigSerializer
c.Deadband = db;
}
if (dict.ContainsKey(ServiceLevelFailoverEnabledKey)
|| dict.ContainsKey(ServiceLevelFailoverWindowKey)
|| dict.ContainsKey(ServiceLevelFailoverDwellKey))
{
var sl = new OpcUaServiceLevelFailoverConfig();
if (dict.TryGetValue(ServiceLevelFailoverEnabledKey, out var slEn) && bool.TryParse(slEn, out var slEnabled))
sl.Enabled = slEnabled;
TryAssignInt(dict, ServiceLevelFailoverWindowKey, v => sl.DegradedWindowSeconds = v);
TryAssignInt(dict, ServiceLevelFailoverDwellKey, v => sl.MinDwellSeconds = v);
c.ServiceLevelFailover = sl;
}
return c;
}
@@ -96,4 +96,12 @@ public sealed class OpcUaEndpointConfig
/// Heartbeat configuration for connection monitoring.
/// </summary>
public OpcUaHeartbeatConfig? Heartbeat { get; set; }
// Redundancy (optional; null = defaults, see OpcUaServiceLevelFailoverConfig)
/// <summary>
/// ServiceLevel-aware failover settings (ScadaBridge#138). Connection-level: only the primary
/// endpoint's value is read, and only when the connection has a backup. Null means the
/// defaults (enabled, 10 s window, 60 s dwell).
/// </summary>
public OpcUaServiceLevelFailoverConfig? ServiceLevelFailover { get; set; }
}
@@ -0,0 +1,46 @@
namespace ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections;
/// <summary>
/// ServiceLevel-aware failover for an OPC UA connection with a primary AND a backup endpoint
/// (ScadaBridge#138; OtOpcUa Warm redundancy, lmxopcua#717). A connection-level setting: only the
/// <b>primary</b> endpoint's copy is read, and it has no effect on a connection without a backup.
///
/// <para>
/// The connected endpoint's <c>Server_ServiceLevel</c> (<c>i=2267</c>) is polled. When it stays in
/// the Degraded band (below 200, OPC UA Part 4 §6.6.2.4.2) for <see cref="DegradedWindowSeconds"/>
/// and the other endpoint is reachable and reports a Healthy-band level (200 or more) that is also
/// higher, the connection fails over to it. A switch is not repeated within
/// <see cref="MinDwellSeconds"/>. Both endpoints degraded, or the other one unreachable, means the
/// connection stays where it is. On connect, a Degraded endpoint is left immediately (no window) for
/// a Healthy, higher other endpoint, subject to the same dwell.
/// </para>
///
/// <para>
/// A stored configuration without this object behaves as the defaults: enabled, 10 s window,
/// 60 s dwell. With today's OtOpcUa pair (250 Primary / 240 Secondary, both Healthy band) it never
/// triggers.
/// </para>
/// </summary>
public sealed class OpcUaServiceLevelFailoverConfig
{
/// <summary>Default for <see cref="DegradedWindowSeconds"/>.</summary>
public const int DefaultDegradedWindowSeconds = 10;
/// <summary>Default for <see cref="MinDwellSeconds"/>.</summary>
public const int DefaultMinDwellSeconds = 60;
/// <summary>Whether ServiceLevel can trigger a failover. Default true.</summary>
public bool Enabled { get; set; } = true;
/// <summary>
/// How long, in seconds, the connected endpoint must report a Degraded-band ServiceLevel
/// without a break before the other endpoint is checked. Default 10.
/// </summary>
public int DegradedWindowSeconds { get; set; } = DefaultDegradedWindowSeconds;
/// <summary>
/// Minimum time, in seconds, after a ServiceLevel-driven switch before another one may happen
/// (anti-flap). Default 60.
/// </summary>
public int MinDwellSeconds { get; set; } = DefaultMinDwellSeconds;
}
@@ -74,6 +74,14 @@ public static class OpcUaEndpointConfigValidator
if (config.Deadband is { } db && db.Value <= 0)
errors.Add(Err("Deadband.Value", "Must be > 0."));
if (config.ServiceLevelFailover is { } sl)
{
if (sl.DegradedWindowSeconds < 1 || sl.DegradedWindowSeconds > 3600)
errors.Add(Err("ServiceLevelFailover.DegradedWindowSeconds", "Must be between 1 and 3600."));
if (sl.MinDwellSeconds < 0 || sl.MinDwellSeconds > 86400)
errors.Add(Err("ServiceLevelFailover.MinDwellSeconds", "Must be between 0 and 86400."));
}
return errors.Count == 0
? ValidationResult.Success()
: ValidationResult.FromErrors(errors.ToArray());