test(validation): ScadaBridge consumer matrix — failover suite, UNS subscribe, ConditionRefresh after reconnect (ScadaBridge#125)
v2-ci / unit-tests (push) Has been cancelled
v2-ci / unit-tests (push) Has been cancelled
Probe 'failover <primary> <backup> <mode> <dir>' drives ScadaBridge's own DataConnectionActor (production primary/backup machine, real DataConnectionFactory, default DataConnectionOptions, FailoverRetryCount 3) against two servers; run.py loses the primary (SIGKILL or SIGSTOP) once the probe reports a steady primary, and the probe requires Bad quality on loss, "Connected to backup", the backup's own value, a fresh ConditionRefresh replaying the native condition, and a write via the backup. rig-failover.sh runs the same probe against the local docker-dev central pair. Matrix legs gain a UNS subscription case and a Server-object alarm subscription on the reconnected session whose ConditionRefresh must replay the active condition (11 cases). run.py gains --versions, --suites, --failover-modes and --failover-loss. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -0,0 +1,225 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
|
||||
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#125 redundancy leg. Drives ScadaBridge's own <see cref="DataConnectionActor"/> — the
|
||||
/// production primary/backup state machine, the real <see cref="DataConnectionFactory"/>, its production
|
||||
/// <see cref="DataConnectionOptions"/> defaults and <c>FailoverRetryCount = 3</c> — against two servers.
|
||||
/// Phase 1 proves a steady primary (tag values, write, Server-object alarm subscription + ConditionRefresh).
|
||||
/// The probe then writes <c>primary-ready</c> and waits for <c>primary-killed</c>; whoever runs it (run.py,
|
||||
/// or a person against the docker-dev pair) kills the primary abruptly in between. Phase 2 requires: Bad
|
||||
/// quality pushed on loss, the actor reporting "Connected to backup", Good values from the backup, a fresh
|
||||
/// ConditionRefresh on the backup session, and (optionally) a write through the backup.
|
||||
/// </summary>
|
||||
internal static class Failover
|
||||
{
|
||||
const string Connection = "sdk2-pair";
|
||||
|
||||
static string Env(string name, string fallback) =>
|
||||
Environment.GetEnvironmentVariable(name) is { Length: > 0 } v ? v : fallback;
|
||||
|
||||
static void Require(bool value, string reason) { if (!value) throw new InvalidOperationException(reason); }
|
||||
|
||||
static bool Is(object? value, long expected)
|
||||
{
|
||||
try { return value is not null and not string && Convert.ToInt64(value) == expected; }
|
||||
catch (Exception) { return false; }
|
||||
}
|
||||
|
||||
public static async Task<int> RunAsync(string primary, string backup, string root, OpcUaGlobalOptions options,
|
||||
Func<string, OpcUaEndpointConfig> configFor)
|
||||
{
|
||||
// Defaults fit the opcua-sdk-compatibility ProcessHost fixture; the docker-dev rig overrides them.
|
||||
var tags = Env("SCADABRIDGE_PROBE_FAILOVER_TAGS",
|
||||
"nsu=https://zb.com/otopcua/raw;s=Matrix/dev/Scalar,nsu=https://zb.com/otopcua/uns;s=Eq/Scalar").Split(',');
|
||||
var writeTag = Env("SCADABRIDGE_PROBE_FAILOVER_WRITE_TAG", tags[0]); // "none" = no writes
|
||||
var backupValue = Env("SCADABRIDGE_PROBE_FAILOVER_BACKUP_VALUE", "17"); // "none" = not checked
|
||||
var alarmMessage = Env("SCADABRIDGE_PROBE_FAILOVER_ALARM_PREFIX", "matrix-"); // "none" = refresh only
|
||||
var killWait = TimeSpan.FromSeconds(int.Parse(Env("SCADABRIDGE_PROBE_FAILOVER_KILL_WAIT_S", "90")));
|
||||
var failoverWait = TimeSpan.FromSeconds(int.Parse(Env("SCADABRIDGE_PROBE_FAILOVER_WAIT_S", "240")));
|
||||
|
||||
var cases = new List<object>();
|
||||
var failures = 0;
|
||||
var events = new ConcurrentQueue<(DateTimeOffset At, object Message)>();
|
||||
var health = DispatchProxy.Create<ISiteHealthCollector, HealthRecorder>();
|
||||
var calls = ((HealthRecorder)(object)health).Calls;
|
||||
var dcOptions = new DataConnectionOptions(); // production defaults (ReconnectInterval 5 s, ...)
|
||||
var factory = new DataConnectionFactory(NullLoggerFactory.Instance, Options.Create(options), null, Options.Create(dcOptions));
|
||||
var primaryConfig = OpcUaEndpointConfigSerializer.ToFlatDict(configFor(primary));
|
||||
var backupConfig = OpcUaEndpointConfigSerializer.ToFlatDict(configFor(backup));
|
||||
DateTimeOffset? killedAt = null, backupConnectedAt = null;
|
||||
var timings = new Dictionary<string, double>();
|
||||
|
||||
async Task Check(string name, Func<Task> action)
|
||||
{
|
||||
try { await action(); cases.Add(new { name, status = "passed" }); }
|
||||
catch (Exception e) { failures++; cases.Add(new { name, status = "failed", error = e.ToString() }); }
|
||||
}
|
||||
async Task<DateTimeOffset> Until(Func<DateTimeOffset?> found, TimeSpan limit, string what)
|
||||
{
|
||||
var stop = DateTime.UtcNow + limit;
|
||||
while (DateTime.UtcNow < stop)
|
||||
{
|
||||
if (found() is { } at) return at;
|
||||
await Task.Delay(50);
|
||||
}
|
||||
throw new TimeoutException($"{what}: not observed within {limit.TotalSeconds:F0} s");
|
||||
}
|
||||
DateTimeOffset? Event<T>(DateTimeOffset after, Func<T, bool> match) =>
|
||||
events.Where(e => e.At >= after && e.Message is T t && match(t)).Select(e => (DateTimeOffset?)e.At).FirstOrDefault();
|
||||
DateTimeOffset? Health(DateTimeOffset after, string label) =>
|
||||
calls.Where(c => c.At >= after && c.Method == nameof(ISiteHealthCollector.UpdateConnectionEndpoint) && Equals(c.Args.ElementAtOrDefault(1), label))
|
||||
.Select(c => (DateTimeOffset?)c.At).FirstOrDefault();
|
||||
bool GoodValue(TagValueUpdate u, string tag) => u.TagPath == tag && u.Quality == QualityCode.Good;
|
||||
|
||||
using var system = ActorSystem.Create("probe", "akka.loglevel = INFO\nakka.stdout-loglevel = INFO");
|
||||
var instance = system.ActorOf(Props.Create(() => new Collector(events)), "instance");
|
||||
var adapter = factory.Create("OpcUa", primaryConfig);
|
||||
var actor = system.ActorOf(Props.Create(() => new DataConnectionActor(
|
||||
Connection, adapter, dcOptions, health, factory, "OpcUa", primaryConfig, backupConfig, 3, null)), Connection);
|
||||
var started = DateTimeOffset.UtcNow;
|
||||
actor.Tell(new SubscribeTagsRequest("tags", "probe-instance", Connection, tags, started), instance);
|
||||
actor.Tell(new SubscribeAlarmsRequest("alarms", "probe-instance", Connection, "i=2253", null, started), instance);
|
||||
|
||||
await Check("primary-connected", async () => await Until(() => Health(started, "Connected to primary"), TimeSpan.FromSeconds(60), "Connected to primary"));
|
||||
await Check("primary-subscriptions-accepted", async () =>
|
||||
{
|
||||
await Until(() => Event<SubscribeTagsResponse>(started, r => r.Success), TimeSpan.FromSeconds(30), "SubscribeTagsResponse success");
|
||||
await Until(() => Event<SubscribeAlarmsResponse>(started, r => r.Success), TimeSpan.FromSeconds(30), "SubscribeAlarmsResponse success");
|
||||
});
|
||||
await Check("primary-good-values-all-tags", async () =>
|
||||
{
|
||||
foreach (var tag in tags) await Until(() => Event<TagValueUpdate>(started, u => GoodValue(u, tag)), TimeSpan.FromSeconds(30), "Good value for " + tag);
|
||||
});
|
||||
await Check("primary-alarm-condition-refresh", async () =>
|
||||
{
|
||||
await Until(() => Event<NativeAlarmTransitionUpdate>(started, a => a.Transition.Kind == AlarmTransitionKind.SnapshotComplete), TimeSpan.FromSeconds(30), "SnapshotComplete on primary");
|
||||
if (alarmMessage != "none")
|
||||
await Until(() => Event<NativeAlarmTransitionUpdate>(started, a => a.Transition.Message.StartsWith(alarmMessage, StringComparison.Ordinal)), TimeSpan.FromSeconds(30), "native alarm on primary");
|
||||
});
|
||||
if (writeTag != "none")
|
||||
await Check("primary-write", async () =>
|
||||
{
|
||||
var at = DateTimeOffset.UtcNow;
|
||||
var response = await actor.Ask<WriteTagResponse>(new WriteTagRequest("w1", Connection, writeTag, 412, at), TimeSpan.FromSeconds(30));
|
||||
Require(response.Success, response.ErrorMessage ?? "write refused");
|
||||
await Until(() => Event<TagValueUpdate>(at, u => GoodValue(u, writeTag) && Is(u.Value, 412)), TimeSpan.FromSeconds(30), "412 echoed by the primary");
|
||||
});
|
||||
|
||||
// Loss evidence is matched from the moment the probe hands the primary over to be killed, not from
|
||||
// the recorded kill time: the consumer can see the socket close before the killer timestamps it.
|
||||
var readyAt = DateTimeOffset.UtcNow;
|
||||
if (failures == 0)
|
||||
{
|
||||
File.WriteAllText(Path.Combine(root, "primary-ready"), readyAt.ToString("O"));
|
||||
Console.WriteLine("PRIMARY-READY " + Path.Combine(root, "primary-ready"));
|
||||
var killedFile = Path.Combine(root, "primary-killed");
|
||||
await Check("primary-killed-by-runner", async () =>
|
||||
{
|
||||
// The marker carries the kill time; a half-written marker is simply read again.
|
||||
killedAt = await Until(() => File.Exists(killedFile) && DateTimeOffset.TryParse(File.ReadAllText(killedFile).Trim(), out var at) ? at : null,
|
||||
killWait, "primary-killed marker");
|
||||
});
|
||||
}
|
||||
if (killedAt is { } k)
|
||||
{
|
||||
await Check("loss-pushes-bad-quality", async () =>
|
||||
{
|
||||
var at = await Until(() => Event<TagValueUpdate>(readyAt, u => u.Quality != QualityCode.Good)
|
||||
?? Event<ConnectionQualityChanged>(readyAt, q => q.Quality != QualityCode.Good), failoverWait, "Bad quality after primary loss");
|
||||
timings["killToBadQualitySeconds"] = (at - k).TotalSeconds;
|
||||
});
|
||||
await Check("failover-connected-to-backup", async () =>
|
||||
{
|
||||
backupConnectedAt = await Until(() => Health(readyAt, "Connected to backup"), failoverWait, "Connected to backup");
|
||||
timings["killToBackupConnectedSeconds"] = (backupConnectedAt.Value - k).TotalSeconds;
|
||||
});
|
||||
}
|
||||
if (backupConnectedAt is { } b)
|
||||
{
|
||||
await Check("backup-good-values-all-tags", async () =>
|
||||
{
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
var at = await Until(() => Event<TagValueUpdate>(b, u => GoodValue(u, tag)), TimeSpan.FromSeconds(60), "Good value from backup for " + tag);
|
||||
timings["killToGood:" + tag] = (at - killedAt!.Value).TotalSeconds;
|
||||
}
|
||||
// The fixture's backup never saw the primary's 412, so its own value proves the feed's source.
|
||||
if (backupValue != "none" && writeTag != "none")
|
||||
{
|
||||
var first = events.Where(e => e.At >= b && e.Message is TagValueUpdate u && GoodValue(u, writeTag)).Select(e => (TagValueUpdate)e.Message).First();
|
||||
Require(Is(first.Value, long.Parse(backupValue)), $"First Good value after failover was {first.Value}, expected the backup's own {backupValue}");
|
||||
}
|
||||
});
|
||||
await Check("backup-alarm-condition-refresh", async () =>
|
||||
{
|
||||
var at = await Until(() => Event<NativeAlarmTransitionUpdate>(b, a => a.Transition.Kind == AlarmTransitionKind.SnapshotComplete), TimeSpan.FromSeconds(60), "SnapshotComplete on backup");
|
||||
timings["killToBackupSnapshotCompleteSeconds"] = (at - killedAt!.Value).TotalSeconds;
|
||||
if (alarmMessage != "none")
|
||||
await Until(() => Event<NativeAlarmTransitionUpdate>(b, a => a.Transition.Kind == AlarmTransitionKind.Snapshot
|
||||
&& a.Transition.Message.StartsWith(alarmMessage, StringComparison.Ordinal)), TimeSpan.FromSeconds(30), "native condition replayed by the backup's ConditionRefresh");
|
||||
});
|
||||
if (writeTag != "none")
|
||||
await Check("backup-write", async () =>
|
||||
{
|
||||
var at = DateTimeOffset.UtcNow;
|
||||
var response = await actor.Ask<WriteTagResponse>(new WriteTagRequest("w2", Connection, writeTag, 913, at), TimeSpan.FromSeconds(30));
|
||||
Require(response.Success, response.ErrorMessage ?? "write refused");
|
||||
await Until(() => Event<TagValueUpdate>(at, u => GoodValue(u, writeTag) && Is(u.Value, 913)), TimeSpan.FromSeconds(30), "913 echoed by the backup");
|
||||
});
|
||||
}
|
||||
|
||||
await system.Terminate();
|
||||
var sdkAssembly = typeof(Opc.Ua.NodeId).Assembly;
|
||||
var result = new
|
||||
{
|
||||
leg = "failover",
|
||||
consumer = typeof(DataConnectionActor).Assembly.GetName().Version?.ToString(),
|
||||
sdk = FileVersionInfo.GetVersionInfo(sdkAssembly.Location).ProductVersion,
|
||||
process = Environment.ProcessId, primary, backup, autoAccept = false,
|
||||
failoverRetryCount = 3, reconnectIntervalSeconds = dcOptions.ReconnectInterval.TotalSeconds,
|
||||
killedAt, backupConnectedAt, timings, cases, failures,
|
||||
endpointLabels = calls.Where(c => c.Method == nameof(ISiteHealthCollector.UpdateConnectionEndpoint))
|
||||
.Select(c => new { at = c.At, label = c.Args.ElementAtOrDefault(1)?.ToString() }),
|
||||
alarmTransitions = events.Where(e => e.Message is NativeAlarmTransitionUpdate)
|
||||
.Select(e => new { at = e.At, kind = ((NativeAlarmTransitionUpdate)e.Message).Transition.Kind.ToString(), message = ((NativeAlarmTransitionUpdate)e.Message).Transition.Message, source = ((NativeAlarmTransitionUpdate)e.Message).Transition.SourceReference }),
|
||||
alarmSourceUnavailable = events.Where(e => e.Message is NativeAlarmSourceUnavailable).Select(e => e.At),
|
||||
values = events.Where(e => e.Message is TagValueUpdate).Select(e => new { at = e.At, tag = ((TagValueUpdate)e.Message).TagPath, value = ((TagValueUpdate)e.Message).Value?.ToString(), quality = ((TagValueUpdate)e.Message).Quality.ToString() }),
|
||||
};
|
||||
File.WriteAllText(Path.Combine(root, "result.json"), JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }));
|
||||
Console.WriteLine($"Failover probe: {cases.Count - failures} passed, {failures} failed; {Path.Combine(root, "result.json")}");
|
||||
return failures == 0 && killedAt is not null ? 0 : 1;
|
||||
}
|
||||
|
||||
/// <summary>Stands in for the Instance Actor: records every message the connection actor sends it.</summary>
|
||||
sealed class Collector : ReceiveActor
|
||||
{
|
||||
public Collector(ConcurrentQueue<(DateTimeOffset, object)> sink) => ReceiveAny(m => sink.Enqueue((DateTimeOffset.UtcNow, m)));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Records every <see cref="ISiteHealthCollector"/> call (the actor's own endpoint/health edges).</summary>
|
||||
public class HealthRecorder : DispatchProxy
|
||||
{
|
||||
public ConcurrentQueue<(DateTimeOffset At, string Method, object?[] Args)> Calls { get; } = new();
|
||||
|
||||
protected override object? Invoke(MethodInfo? targetMethod, object?[]? args)
|
||||
{
|
||||
Calls.Enqueue((DateTimeOffset.UtcNow, targetMethod!.Name, args ?? []));
|
||||
var type = targetMethod.ReturnType;
|
||||
return type == typeof(void) || !type.IsValueType ? null : Activator.CreateInstance(type);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user