test(validation): ScadaBridge consumer matrix — failover suite, UNS subscribe, ConditionRefresh after reconnect (ScadaBridge#125)
v2-ci / unit-tests (push) Has been cancelled

Probe 'failover <primary> <backup> <mode> <dir>' drives ScadaBridge's own DataConnectionActor
(production primary/backup machine, real DataConnectionFactory, default DataConnectionOptions,
FailoverRetryCount 3) against two servers; run.py loses the primary (SIGKILL or SIGSTOP) once the
probe reports a steady primary, and the probe requires Bad quality on loss, "Connected to backup",
the backup's own value, a fresh ConditionRefresh replaying the native condition, and a write via the
backup. rig-failover.sh runs the same probe against the local docker-dev central pair.

Matrix legs gain a UNS subscription case and a Server-object alarm subscription on the reconnected
session whose ConditionRefresh must replay the active condition (11 cases). run.py gains --versions,
--suites, --failover-modes and --failover-loss.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-12 09:28:00 -05:00
parent e5a03e3cce
commit 438484fdca
5 changed files with 523 additions and 82 deletions
@@ -34,10 +34,13 @@ if (args.Length == 3 && args[0] == "prepare")
Console.WriteLine($"Consumer identity {publicCert.Thumbprint} {publicCert.Subject}");
return 0;
}
if (args.Length != 3) throw new ArgumentException("Usage: Probe <endpoint> <None|Sign|SignAndEncrypt> <new-result-directory> | Probe prepare <new-pki-directory> <public-der-out>");
var endpoint = args[0];
var mode = Enum.Parse<OpcUaSecurityMode>(args[1]);
var root = Path.GetFullPath(args[2]);
// ScadaBridge#125: "failover <primary> <backup> <mode> <dir>" drives the real DataConnectionActor
// primary/backup machine (Failover.cs); the runner kills the primary once the probe reports it is steady.
var failoverMode = args.Length == 5 && args[0] == "failover";
if (!failoverMode && args.Length != 3) throw new ArgumentException("Usage: Probe <endpoint> <None|Sign|SignAndEncrypt> <new-result-directory> | Probe failover <primary> <backup> <mode> <new-result-directory> | Probe prepare <new-pki-directory> <public-der-out>");
var endpoint = failoverMode ? args[1] : args[0];
var mode = Enum.Parse<OpcUaSecurityMode>(failoverMode ? args[3] : args[1]);
var root = Path.GetFullPath(failoverMode ? args[4] : args[2]);
if (Directory.Exists(root)) throw new IOException("Refusing to reuse an evidence/PKI directory.");
Directory.CreateDirectory(root);
static void Require(bool value, string reason) { if (!value) throw new InvalidOperationException(reason); }
@@ -50,25 +53,32 @@ var pki = Environment.GetEnvironmentVariable("SCADABRIDGE_PROBE_PKI") is { Lengt
var options = IdentityOptions(pki);
var ownBefore = Directory.Exists(Path.Combine(pki, "own", "certs")) ? Directory.GetFiles(Path.Combine(pki, "own", "certs")).Order().ToArray() : [];
// Populate only this probe's trust list; never change the consumer's existing stores.
// Several server certificate directories (a failover pair) are separated by Path.PathSeparator.
var trustSource = Environment.GetEnvironmentVariable("SCADABRIDGE_PROBE_SERVER_CERTS");
if (!string.IsNullOrEmpty(trustSource))
{
var destination = Path.Combine(options.TrustedPeerStorePath, "certs");
Directory.CreateDirectory(destination);
foreach (var cert in Directory.GetFiles(trustSource, "*.der")) File.Copy(cert, Path.Combine(destination, Path.GetFileName(cert)));
foreach (var source in trustSource.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries))
foreach (var cert in Directory.GetFiles(source, "*.der")) File.Copy(cert, Path.Combine(destination, Path.GetFileName(cert)), overwrite: true);
}
// The rig variant (a deployed OtOpcUa pair, OpcUa:AllowAnonymous=true) sets SCADABRIDGE_PROBE_ANONYMOUS=1.
OpcUaEndpointConfig ConfigFor(string url) => new()
{
EndpointUrl = url, SecurityMode = mode, AutoAcceptUntrustedCerts = false,
PublishingIntervalMs = 100, SamplingIntervalMs = 100,
UserIdentity = Environment.GetEnvironmentVariable("SCADABRIDGE_PROBE_ANONYMOUS") == "1"
? new OpcUaUserIdentityConfig { TokenType = OpcUaUserTokenType.Anonymous }
: new OpcUaUserIdentityConfig
{
TokenType = OpcUaUserTokenType.UsernamePassword, Username = "apikey",
Password = Environment.GetEnvironmentVariable("OPCUA_MATRIX_PASSWORD") ?? throw new InvalidOperationException("Missing task credential"),
},
};
if (failoverMode) return await Failover.RunAsync(args[1], args[2], root, options, ConfigFor);
var factory = new RealOpcUaClientFactory(options, NullLoggerFactory.Instance);
await using var adapter = new OpcUaDataConnection(factory, NullLogger<OpcUaDataConnection>.Instance);
var config = new OpcUaEndpointConfig
{
EndpointUrl = endpoint, SecurityMode = mode, AutoAcceptUntrustedCerts = false,
PublishingIntervalMs = 100, SamplingIntervalMs = 100,
UserIdentity = new OpcUaUserIdentityConfig
{
TokenType = OpcUaUserTokenType.UsernamePassword, Username = "apikey",
Password = Environment.GetEnvironmentVariable("OPCUA_MATRIX_PASSWORD") ?? throw new InvalidOperationException("Missing task credential"),
},
};
var config = ConfigFor(endpoint);
async Task Check(string name, Func<Task> action)
{
try { await action(); cases.Add(new { name, status = "passed" }); }
@@ -120,6 +130,16 @@ if (failures == 0)
Require(actual.Quality.ToString() == "Good", "Quality was not Good");
await adapter.UnsubscribeAsync(handle, ct);
});
// ScadaBridge#125: subscriptions on the UNS NodeId too (Eq/Scalar is a separate UNS variable).
await Check("actual-adapter-monitor-uns", async () =>
{
var values = new ConcurrentQueue<TagValue>();
var handle = await adapter.SubscribeAsync(uns + "Scalar", (_, value) => values.Enqueue(value), ct);
Require((await adapter.WriteAsync(uns + "Scalar", 714, ct)).Success, "Monitored UNS write denied");
await Wait(() => values.Any(x => x.Value is int i && i == 714));
Require(values.Last(x => x.Value is int i && i == 714).Quality.ToString() == "Good", "UNS quality was not Good");
await adapter.UnsubscribeAsync(handle, ct);
});
await Check("actual-adapter-alarm-filter-refresh", async () =>
{
var alarms = new ConcurrentQueue<NativeAlarmTransition>();
@@ -133,6 +153,19 @@ if (failures == 0)
await adapter.DisconnectAsync(ct);
await adapter.ConnectAsync(OpcUaEndpointConfigSerializer.ToFlatDict(config), ct);
Require((await adapter.ReadAsync(raw + "Scalar", ct)).Success, "Consumer failed after reconnect");
});
// ScadaBridge#125: the Server-object alarm subscription on the NEW session must get its own
// ConditionRefresh — the active native condition is REPLAYED (Snapshot kind) and the refresh completes.
await Check("actual-adapter-alarm-refresh-after-reconnect", async () =>
{
var alarms = new ConcurrentQueue<NativeAlarmTransition>();
var handle = await adapter.SubscribeAlarmsAsync("i=2253", null, alarms.Enqueue, ct);
await Wait(() => alarms.Any(x => x.Kind == AlarmTransitionKind.SnapshotComplete));
var replayed = alarms.TakeWhile(x => x.Kind != AlarmTransitionKind.SnapshotComplete).ToArray();
Require(replayed.Any(x => x.Kind == AlarmTransitionKind.Snapshot && x.SourceReference.Contains("Alarm", StringComparison.Ordinal)),
"ConditionRefresh on the reconnected session did not replay the active native condition; saw: "
+ string.Join(", ", replayed.Select(x => $"{x.Kind}:{x.SourceReference}:{x.Message}")));
await adapter.UnsubscribeAlarmsAsync(handle, ct);
await adapter.DisconnectAsync(ct);
});
}