feat(dcl): ServiceLevel-aware OPC UA failover for primary/backup pairs (#138)
Prerequisite for OtOpcUa Warm redundancy (lmxopcua#717): a Warm standby keeps a
healthy session but serves BadOutOfService at a standby ServiceLevel (90/50, boot 1),
so a connection that lands there sat on Bad quality forever.
- OPC UA connections with a backup poll Server_ServiceLevel (i=2267) on the connected
endpoint (IServiceLevelReadable, implemented by OpcUaDataConnection; never raises
Disconnected). A level < 200 sustained for the degraded window (default 10 s), or
the first sample after any (re)connect, triggers a probe of the other endpoint
(short-lived session: connect, read, dispose). Switch only when it reports a
Healthy (>= 200) higher level, and not within the minimum dwell (default 60 s) of
the previous ServiceLevel switch. Both degraded, other unreachable or level
unknown: stay put. 250/240 (today's OtOpcUa) never starts a streak.
- The switch reuses the failover path: SwitchToOtherEndpoint() now shared by all
three triggers, then BecomeReconnecting(ReconnectCause.ServiceLevelFailover) -
zero delay, no unstable accounting, ReSubscribeAll on the new endpoint.
- Config: OpcUaEndpointConfig.ServiceLevelFailover {enabled, degradedWindowSeconds,
minDwellSeconds}, read from the PRIMARY endpoint's JSON only; null = defaults
(enabled, 10, 60). Inside the encrypted protocol JSON, so no schema change or
migration. Central UI: fields in the Backup card next to Failover Retry Count.
Global: ScadaBridge:DataConnection:ServiceLevelPollInterval (2 s) and
ServiceLevelProbeTimeout (15 s).
- Tests: actor tests per trigger/non-trigger with positive controls, an unsolicited
promotion RefreshStart/RefreshEnd test at client and actor level, adapter tests,
and a real in-process OPC UA server pair end to end. All 15 mutants killed.
- Docs: Component-DataConnectionLayer, components/DataConnectionLayer, Component-CLI.
Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
+112
@@ -0,0 +1,112 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Reflection;
|
||||
using Opc.Ua;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#138 / OtOpcUa Warm design §4.5 + §5 "Alarms are compatible": when a Warm standby
|
||||
/// is promoted, OtOpcUa runs a ConditionRefresh on every live event item ON ITS OWN — the client
|
||||
/// never asked. The existing items therefore receive an unsolicited RefreshStart, the retained
|
||||
/// conditions, RefreshEnd, then new events. ScadaBridge must treat that exactly like the refresh it
|
||||
/// triggers itself on subscribe: one framed snapshot with its own epoch, and live derivation
|
||||
/// before and after it.
|
||||
/// </summary>
|
||||
public class RealOpcUaPromotionRefreshTests
|
||||
{
|
||||
private const string Handle = "h1";
|
||||
private const string Source = "ns=2;s=Tank01";
|
||||
|
||||
private static readonly MethodInfo Handler =
|
||||
typeof(RealOpcUaClient).GetMethod("HandleAlarmEvent", BindingFlags.Instance | BindingFlags.NonPublic)!;
|
||||
|
||||
/// <summary>What <c>CreateAlarmSubscriptionAsync</c> sets up for a live handle: its per-condition state map.</summary>
|
||||
private static void SeedSubscribedHandle(RealOpcUaClient client)
|
||||
{
|
||||
var field = typeof(RealOpcUaClient).GetField("_alarmLastState", BindingFlags.Instance | BindingFlags.NonPublic)!;
|
||||
var map = (ConcurrentDictionary<string, Dictionary<string, (bool Active, bool Acked)>>)field.GetValue(client)!;
|
||||
map[Handle] = new Dictionary<string, (bool Active, bool Acked)>(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
private static void Boundary(RealOpcUaClient client, NodeId type, List<NativeAlarmTransition> sink) =>
|
||||
Handler.Invoke(client, [Handle, Source,
|
||||
new EventFieldList { EventFields = new Variant[] { Variant.From(type), Variant.Null, Variant.Null, Variant.Null, Variant.Null, Variant.Null } },
|
||||
(Action<NativeAlarmTransition>)sink.Add]);
|
||||
|
||||
private static void Condition(RealOpcUaClient client, string condition, bool active, bool acked, List<NativeAlarmTransition> sink)
|
||||
{
|
||||
var fields = new Variant[]
|
||||
{
|
||||
Variant.From(ObjectTypeIds.AlarmConditionType), // 0 EventType
|
||||
Variant.From(new NodeId("Tank01", 2)), // 1 SourceNode
|
||||
Variant.From("Tank01"), // 2 SourceName
|
||||
Variant.From(DateTime.UtcNow), // 3 Time
|
||||
Variant.From(new LocalizedText("msg")), // 4 Message
|
||||
Variant.From((ushort)500), // 5 Severity
|
||||
Variant.From(active), // 6 ActiveState/Id
|
||||
Variant.From(acked), // 7 AckedState/Id
|
||||
Variant.Null, // 8 ConfirmedState/Id
|
||||
Variant.From(false), // 9 SuppressedOrShelved
|
||||
Variant.Null, // 10 ShelvingState
|
||||
Variant.From(condition), // 11 ConditionName
|
||||
};
|
||||
Handler.Invoke(client, [Handle, Source, new EventFieldList { EventFields = fields },
|
||||
(Action<NativeAlarmTransition>)sink.Add]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Subscribe-time refresh, live traffic, then the UNSOLICITED promotion refresh on the same
|
||||
/// item, then live traffic again. The promotion replay is framed SnapshotStart → Snapshot… →
|
||||
/// SnapshotComplete under a fresh epoch (distinct from the subscribe-time one), and live events
|
||||
/// after it are derived live again — the replay does not leave the item stuck in Snapshot mode.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void UnsolicitedPromotionRefresh_IsFramedAsItsOwnSnapshot_AndLiveDerivationResumes()
|
||||
{
|
||||
var client = new RealOpcUaClient();
|
||||
SeedSubscribedHandle(client);
|
||||
var sink = new List<NativeAlarmTransition>();
|
||||
|
||||
// Subscribe-time refresh (the one ScadaBridge triggers): Hi is active.
|
||||
Boundary(client, ObjectTypeIds.RefreshStartEventType, sink);
|
||||
Condition(client, "Hi", active: true, acked: false, sink);
|
||||
Boundary(client, ObjectTypeIds.RefreshEndEventType, sink);
|
||||
|
||||
// Live while the old Active serves: Lo raises.
|
||||
Condition(client, "Lo", active: true, acked: false, sink);
|
||||
var liveBefore = sink[^1];
|
||||
|
||||
// Promotion on the other node: an unsolicited refresh on the existing item.
|
||||
Boundary(client, ObjectTypeIds.RefreshStartEventType, sink);
|
||||
Condition(client, "Hi", active: true, acked: false, sink);
|
||||
Condition(client, "Lo", active: true, acked: false, sink);
|
||||
Boundary(client, ObjectTypeIds.RefreshEndEventType, sink);
|
||||
|
||||
// Live after promotion: Lo is acknowledged.
|
||||
Condition(client, "Lo", active: true, acked: true, sink);
|
||||
var liveAfter = sink[^1];
|
||||
|
||||
Assert.Equal(
|
||||
["SnapshotStart", "Snapshot", "SnapshotComplete", liveBefore.Kind.ToString(),
|
||||
"SnapshotStart", "Snapshot", "Snapshot", "SnapshotComplete", liveAfter.Kind.ToString()],
|
||||
sink.Select(t => t.Kind.ToString()));
|
||||
|
||||
// Live kinds really are live (not Snapshot) on both sides of the promotion refresh.
|
||||
Assert.Equal(AlarmTransitionKind.Raise, liveBefore.Kind);
|
||||
Assert.Equal(AlarmTransitionKind.Acknowledge, liveAfter.Kind);
|
||||
Assert.Null(liveBefore.SnapshotId);
|
||||
Assert.Null(liveAfter.SnapshotId);
|
||||
|
||||
// The promotion replay has its own epoch, shared by its boundaries and its conditions.
|
||||
var subscribeEpoch = sink[0].SnapshotId;
|
||||
var promotionEpoch = sink[4].SnapshotId;
|
||||
Assert.NotNull(subscribeEpoch);
|
||||
Assert.NotNull(promotionEpoch);
|
||||
Assert.NotEqual(subscribeEpoch, promotionEpoch);
|
||||
Assert.All(sink.Skip(4).Take(4), t => Assert.Equal(promotionEpoch, t.SnapshotId));
|
||||
Assert.All(sink.Skip(4).Take(4), t => Assert.Equal(Source, t.SnapshotSourceReference));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user