using System.Collections.Concurrent; using System.Diagnostics; using System.Reflection; using System.Text.Json; using Akka.Actor; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection; using ZB.MOM.WW.ScadaBridge.Commons.Serialization; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections; using ZB.MOM.WW.ScadaBridge.DataConnectionLayer; using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors; /// /// ScadaBridge#125 redundancy leg. Drives ScadaBridge's own — the /// production primary/backup state machine, the real , its production /// defaults and FailoverRetryCount = 3 — against two servers. /// Phase 1 proves a steady primary (tag values, write, Server-object alarm subscription + ConditionRefresh). /// The probe then writes primary-ready and waits for primary-killed; whoever runs it (run.py, /// or a person against the docker-dev pair) kills the primary abruptly in between. Phase 2 requires: Bad /// quality pushed on loss, the actor reporting "Connected to backup", Good values from the backup, a fresh /// ConditionRefresh on the backup session, and (optionally) a write through the backup. /// internal static class Failover { const string Connection = "sdk2-pair"; static string Env(string name, string fallback) => Environment.GetEnvironmentVariable(name) is { Length: > 0 } v ? v : fallback; static void Require(bool value, string reason) { if (!value) throw new InvalidOperationException(reason); } static bool Is(object? value, long expected) { try { return value is not null and not string && Convert.ToInt64(value) == expected; } catch (Exception) { return false; } } public static async Task RunAsync(string primary, string backup, string root, OpcUaGlobalOptions options, Func configFor) { // Defaults fit the opcua-sdk-compatibility ProcessHost fixture; the docker-dev rig overrides them. var tags = Env("SCADABRIDGE_PROBE_FAILOVER_TAGS", "nsu=https://zb.com/otopcua/raw;s=Matrix/dev/Scalar,nsu=https://zb.com/otopcua/uns;s=Eq/Scalar").Split(','); var writeTag = Env("SCADABRIDGE_PROBE_FAILOVER_WRITE_TAG", tags[0]); // "none" = no writes var backupValue = Env("SCADABRIDGE_PROBE_FAILOVER_BACKUP_VALUE", "17"); // "none" = not checked var alarmMessage = Env("SCADABRIDGE_PROBE_FAILOVER_ALARM_PREFIX", "matrix-"); // "none" = refresh only var killWait = TimeSpan.FromSeconds(int.Parse(Env("SCADABRIDGE_PROBE_FAILOVER_KILL_WAIT_S", "90"))); var failoverWait = TimeSpan.FromSeconds(int.Parse(Env("SCADABRIDGE_PROBE_FAILOVER_WAIT_S", "240"))); var cases = new List(); var failures = 0; var events = new ConcurrentQueue<(DateTimeOffset At, object Message)>(); var health = DispatchProxy.Create(); var calls = ((HealthRecorder)(object)health).Calls; var dcOptions = new DataConnectionOptions(); // production defaults (ReconnectInterval 5 s, ...) var factory = new DataConnectionFactory(NullLoggerFactory.Instance, Options.Create(options), null, Options.Create(dcOptions)); var primaryConfig = OpcUaEndpointConfigSerializer.ToFlatDict(configFor(primary)); var backupConfig = OpcUaEndpointConfigSerializer.ToFlatDict(configFor(backup)); DateTimeOffset? killedAt = null, backupConnectedAt = null; var timings = new Dictionary(); async Task Check(string name, Func action) { try { await action(); cases.Add(new { name, status = "passed" }); } catch (Exception e) { failures++; cases.Add(new { name, status = "failed", error = e.ToString() }); } } async Task Until(Func found, TimeSpan limit, string what) { var stop = DateTime.UtcNow + limit; while (DateTime.UtcNow < stop) { if (found() is { } at) return at; await Task.Delay(50); } throw new TimeoutException($"{what}: not observed within {limit.TotalSeconds:F0} s"); } DateTimeOffset? Event(DateTimeOffset after, Func match) => events.Where(e => e.At >= after && e.Message is T t && match(t)).Select(e => (DateTimeOffset?)e.At).FirstOrDefault(); DateTimeOffset? Health(DateTimeOffset after, string label) => calls.Where(c => c.At >= after && c.Method == nameof(ISiteHealthCollector.UpdateConnectionEndpoint) && Equals(c.Args.ElementAtOrDefault(1), label)) .Select(c => (DateTimeOffset?)c.At).FirstOrDefault(); bool GoodValue(TagValueUpdate u, string tag) => u.TagPath == tag && u.Quality == QualityCode.Good; using var system = ActorSystem.Create("probe", "akka.loglevel = INFO\nakka.stdout-loglevel = INFO"); var instance = system.ActorOf(Props.Create(() => new Collector(events)), "instance"); var adapter = factory.Create("OpcUa", primaryConfig); var actor = system.ActorOf(Props.Create(() => new DataConnectionActor( Connection, adapter, dcOptions, health, factory, "OpcUa", primaryConfig, backupConfig, 3, null)), Connection); var started = DateTimeOffset.UtcNow; actor.Tell(new SubscribeTagsRequest("tags", "probe-instance", Connection, tags, started), instance); actor.Tell(new SubscribeAlarmsRequest("alarms", "probe-instance", Connection, "i=2253", null, started), instance); await Check("primary-connected", async () => await Until(() => Health(started, "Connected to primary"), TimeSpan.FromSeconds(60), "Connected to primary")); await Check("primary-subscriptions-accepted", async () => { await Until(() => Event(started, r => r.Success), TimeSpan.FromSeconds(30), "SubscribeTagsResponse success"); await Until(() => Event(started, r => r.Success), TimeSpan.FromSeconds(30), "SubscribeAlarmsResponse success"); }); await Check("primary-good-values-all-tags", async () => { foreach (var tag in tags) await Until(() => Event(started, u => GoodValue(u, tag)), TimeSpan.FromSeconds(30), "Good value for " + tag); }); await Check("primary-alarm-condition-refresh", async () => { await Until(() => Event(started, a => a.Transition.Kind == AlarmTransitionKind.SnapshotComplete), TimeSpan.FromSeconds(30), "SnapshotComplete on primary"); if (alarmMessage != "none") await Until(() => Event(started, a => a.Transition.Message.StartsWith(alarmMessage, StringComparison.Ordinal)), TimeSpan.FromSeconds(30), "native alarm on primary"); }); if (writeTag != "none") await Check("primary-write", async () => { var at = DateTimeOffset.UtcNow; var response = await actor.Ask(new WriteTagRequest("w1", Connection, writeTag, 412, at), TimeSpan.FromSeconds(30)); Require(response.Success, response.ErrorMessage ?? "write refused"); await Until(() => Event(at, u => GoodValue(u, writeTag) && Is(u.Value, 412)), TimeSpan.FromSeconds(30), "412 echoed by the primary"); }); // Loss evidence is matched from the moment the probe hands the primary over to be killed, not from // the recorded kill time: the consumer can see the socket close before the killer timestamps it. var readyAt = DateTimeOffset.UtcNow; if (failures == 0) { File.WriteAllText(Path.Combine(root, "primary-ready"), readyAt.ToString("O")); Console.WriteLine("PRIMARY-READY " + Path.Combine(root, "primary-ready")); var killedFile = Path.Combine(root, "primary-killed"); await Check("primary-killed-by-runner", async () => { // The marker carries the kill time; a half-written marker is simply read again. killedAt = await Until(() => File.Exists(killedFile) && DateTimeOffset.TryParse(File.ReadAllText(killedFile).Trim(), out var at) ? at : null, killWait, "primary-killed marker"); }); } if (killedAt is { } k) { await Check("loss-pushes-bad-quality", async () => { var at = await Until(() => Event(readyAt, u => u.Quality != QualityCode.Good) ?? Event(readyAt, q => q.Quality != QualityCode.Good), failoverWait, "Bad quality after primary loss"); timings["killToBadQualitySeconds"] = (at - k).TotalSeconds; }); await Check("failover-connected-to-backup", async () => { backupConnectedAt = await Until(() => Health(readyAt, "Connected to backup"), failoverWait, "Connected to backup"); timings["killToBackupConnectedSeconds"] = (backupConnectedAt.Value - k).TotalSeconds; }); } if (backupConnectedAt is { } b) { await Check("backup-good-values-all-tags", async () => { foreach (var tag in tags) { var at = await Until(() => Event(b, u => GoodValue(u, tag)), TimeSpan.FromSeconds(60), "Good value from backup for " + tag); timings["killToGood:" + tag] = (at - killedAt!.Value).TotalSeconds; } // The fixture's backup never saw the primary's 412, so its own value proves the feed's source. if (backupValue != "none" && writeTag != "none") { var first = events.Where(e => e.At >= b && e.Message is TagValueUpdate u && GoodValue(u, writeTag)).Select(e => (TagValueUpdate)e.Message).First(); Require(Is(first.Value, long.Parse(backupValue)), $"First Good value after failover was {first.Value}, expected the backup's own {backupValue}"); } }); await Check("backup-alarm-condition-refresh", async () => { var at = await Until(() => Event(b, a => a.Transition.Kind == AlarmTransitionKind.SnapshotComplete), TimeSpan.FromSeconds(60), "SnapshotComplete on backup"); timings["killToBackupSnapshotCompleteSeconds"] = (at - killedAt!.Value).TotalSeconds; if (alarmMessage != "none") await Until(() => Event(b, a => a.Transition.Kind == AlarmTransitionKind.Snapshot && a.Transition.Message.StartsWith(alarmMessage, StringComparison.Ordinal)), TimeSpan.FromSeconds(30), "native condition replayed by the backup's ConditionRefresh"); }); if (writeTag != "none") await Check("backup-write", async () => { var at = DateTimeOffset.UtcNow; var response = await actor.Ask(new WriteTagRequest("w2", Connection, writeTag, 913, at), TimeSpan.FromSeconds(30)); Require(response.Success, response.ErrorMessage ?? "write refused"); await Until(() => Event(at, u => GoodValue(u, writeTag) && Is(u.Value, 913)), TimeSpan.FromSeconds(30), "913 echoed by the backup"); }); } await system.Terminate(); var sdkAssembly = typeof(Opc.Ua.NodeId).Assembly; var result = new { leg = "failover", consumer = typeof(DataConnectionActor).Assembly.GetName().Version?.ToString(), sdk = FileVersionInfo.GetVersionInfo(sdkAssembly.Location).ProductVersion, process = Environment.ProcessId, primary, backup, autoAccept = false, failoverRetryCount = 3, reconnectIntervalSeconds = dcOptions.ReconnectInterval.TotalSeconds, killedAt, backupConnectedAt, timings, cases, failures, endpointLabels = calls.Where(c => c.Method == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)) .Select(c => new { at = c.At, label = c.Args.ElementAtOrDefault(1)?.ToString() }), alarmTransitions = events.Where(e => e.Message is NativeAlarmTransitionUpdate) .Select(e => new { at = e.At, kind = ((NativeAlarmTransitionUpdate)e.Message).Transition.Kind.ToString(), message = ((NativeAlarmTransitionUpdate)e.Message).Transition.Message, source = ((NativeAlarmTransitionUpdate)e.Message).Transition.SourceReference }), alarmSourceUnavailable = events.Where(e => e.Message is NativeAlarmSourceUnavailable).Select(e => e.At), values = events.Where(e => e.Message is TagValueUpdate).Select(e => new { at = e.At, tag = ((TagValueUpdate)e.Message).TagPath, value = ((TagValueUpdate)e.Message).Value?.ToString(), quality = ((TagValueUpdate)e.Message).Quality.ToString() }), }; File.WriteAllText(Path.Combine(root, "result.json"), JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true })); Console.WriteLine($"Failover probe: {cases.Count - failures} passed, {failures} failed; {Path.Combine(root, "result.json")}"); return failures == 0 && killedAt is not null ? 0 : 1; } /// Stands in for the Instance Actor: records every message the connection actor sends it. sealed class Collector : ReceiveActor { public Collector(ConcurrentQueue<(DateTimeOffset, object)> sink) => ReceiveAny(m => sink.Enqueue((DateTimeOffset.UtcNow, m))); } } /// Records every call (the actor's own endpoint/health edges). public class HealthRecorder : DispatchProxy { public ConcurrentQueue<(DateTimeOffset At, string Method, object?[] Args)> Calls { get; } = new(); protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) { Calls.Enqueue((DateTimeOffset.UtcNow, targetMethod!.Name, args ?? [])); var type = targetMethod.ReturnType; return type == typeof(void) || !type.IsValueType ? null : Activator.CreateInstance(type); } }