lmxopcua/scripts/validation/scadabridge-sdk-compatibility/Failover.cs

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Reflection;
using System.Text.Json;
using Akka.Actor;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;

/// <summary>
/// ScadaBridge#125 redundancy leg. Drives ScadaBridge's own <see cref="DataConnectionActor"/> — the
/// production primary/backup state machine, the real <see cref="DataConnectionFactory"/>, its production
/// <see cref="DataConnectionOptions"/> defaults and <c>FailoverRetryCount = 3</c> — against two servers.
/// Phase 1 proves a steady primary (tag values, write, Server-object alarm subscription + ConditionRefresh).
/// The probe then writes <c>primary-ready</c> and waits for <c>primary-killed</c>; whoever runs it (run.py,
/// or a person against the docker-dev pair) kills the primary abruptly in between. Phase 2 requires: Bad
/// quality pushed on loss, the actor reporting "Connected to backup", Good values from the backup, a fresh
/// ConditionRefresh on the backup session, and (optionally) a write through the backup.
/// </summary>
internal static class Failover
{
    const string Connection = "sdk2-pair";

    static string Env(string name, string fallback) =>
        Environment.GetEnvironmentVariable(name) is { Length: > 0 } v ? v : fallback;

    static void Require(bool value, string reason) { if (!value) throw new InvalidOperationException(reason); }

    static bool Is(object? value, long expected)
    {
        try { return value is not null and not string && Convert.ToInt64(value) == expected; }
        catch (Exception) { return false; }
    }

    public static async Task<int> RunAsync(string primary, string backup, string root, OpcUaGlobalOptions options,
        Func<string, OpcUaEndpointConfig> configFor)
    {
        // Defaults fit the opcua-sdk-compatibility ProcessHost fixture; the docker-dev rig overrides them.
        var tags = Env("SCADABRIDGE_PROBE_FAILOVER_TAGS",
            "nsu=https://zb.com/otopcua/raw;s=Matrix/dev/Scalar,nsu=https://zb.com/otopcua/uns;s=Eq/Scalar").Split(',');
        var writeTag = Env("SCADABRIDGE_PROBE_FAILOVER_WRITE_TAG", tags[0]);              // "none" = no writes
        var backupValue = Env("SCADABRIDGE_PROBE_FAILOVER_BACKUP_VALUE", "17");            // "none" = not checked
        var alarmMessage = Env("SCADABRIDGE_PROBE_FAILOVER_ALARM_PREFIX", "matrix-");     // "none" = refresh only
        var killWait = TimeSpan.FromSeconds(int.Parse(Env("SCADABRIDGE_PROBE_FAILOVER_KILL_WAIT_S", "90")));
        var failoverWait = TimeSpan.FromSeconds(int.Parse(Env("SCADABRIDGE_PROBE_FAILOVER_WAIT_S", "240")));

        var cases = new List<object>();
        var failures = 0;
        var events = new ConcurrentQueue<(DateTimeOffset At, object Message)>();
        var health = DispatchProxy.Create<ISiteHealthCollector, HealthRecorder>();
        var calls = ((HealthRecorder)(object)health).Calls;
        var dcOptions = new DataConnectionOptions(); // production defaults (ReconnectInterval 5 s, ...)
        var factory = new DataConnectionFactory(NullLoggerFactory.Instance, Options.Create(options), null, Options.Create(dcOptions));
        var primaryConfig = OpcUaEndpointConfigSerializer.ToFlatDict(configFor(primary));
        var backupConfig = OpcUaEndpointConfigSerializer.ToFlatDict(configFor(backup));
        DateTimeOffset? killedAt = null, backupConnectedAt = null;
        var timings = new Dictionary<string, double>();

        async Task Check(string name, Func<Task> action)
        {
            try { await action(); cases.Add(new { name, status = "passed" }); }
            catch (Exception e) { failures++; cases.Add(new { name, status = "failed", error = e.ToString() }); }
        }
        async Task<DateTimeOffset> Until(Func<DateTimeOffset?> found, TimeSpan limit, string what)
        {
            var stop = DateTime.UtcNow + limit;
            while (DateTime.UtcNow < stop)
            {
                if (found() is { } at) return at;
                await Task.Delay(50);
            }
            throw new TimeoutException($"{what}: not observed within {limit.TotalSeconds:F0} s");
        }
        DateTimeOffset? Event<T>(DateTimeOffset after, Func<T, bool> match) =>
            events.Where(e => e.At >= after && e.Message is T t && match(t)).Select(e => (DateTimeOffset?)e.At).FirstOrDefault();
        DateTimeOffset? Health(DateTimeOffset after, string label) =>
            calls.Where(c => c.At >= after && c.Method == nameof(ISiteHealthCollector.UpdateConnectionEndpoint) && Equals(c.Args.ElementAtOrDefault(1), label))
                .Select(c => (DateTimeOffset?)c.At).FirstOrDefault();
        bool GoodValue(TagValueUpdate u, string tag) => u.TagPath == tag && u.Quality == QualityCode.Good;

        using var system = ActorSystem.Create("probe", "akka.loglevel = INFO\nakka.stdout-loglevel = INFO");
        var instance = system.ActorOf(Props.Create(() => new Collector(events)), "instance");
        var adapter = factory.Create("OpcUa", primaryConfig);
        var actor = system.ActorOf(Props.Create(() => new DataConnectionActor(
            Connection, adapter, dcOptions, health, factory, "OpcUa", primaryConfig, backupConfig, 3, null)), Connection);
        var started = DateTimeOffset.UtcNow;
        actor.Tell(new SubscribeTagsRequest("tags", "probe-instance", Connection, tags, started), instance);
        actor.Tell(new SubscribeAlarmsRequest("alarms", "probe-instance", Connection, "i=2253", null, started), instance);

        await Check("primary-connected", async () => await Until(() => Health(started, "Connected to primary"), TimeSpan.FromSeconds(60), "Connected to primary"));
        await Check("primary-subscriptions-accepted", async () =>
        {
            await Until(() => Event<SubscribeTagsResponse>(started, r => r.Success), TimeSpan.FromSeconds(30), "SubscribeTagsResponse success");
            await Until(() => Event<SubscribeAlarmsResponse>(started, r => r.Success), TimeSpan.FromSeconds(30), "SubscribeAlarmsResponse success");
        });
        await Check("primary-good-values-all-tags", async () =>
        {
            foreach (var tag in tags) await Until(() => Event<TagValueUpdate>(started, u => GoodValue(u, tag)), TimeSpan.FromSeconds(30), "Good value for " + tag);
        });
        await Check("primary-alarm-condition-refresh", async () =>
        {
            await Until(() => Event<NativeAlarmTransitionUpdate>(started, a => a.Transition.Kind == AlarmTransitionKind.SnapshotComplete), TimeSpan.FromSeconds(30), "SnapshotComplete on primary");
            if (alarmMessage != "none")
                await Until(() => Event<NativeAlarmTransitionUpdate>(started, a => a.Transition.Message.StartsWith(alarmMessage, StringComparison.Ordinal)), TimeSpan.FromSeconds(30), "native alarm on primary");
        });
        if (writeTag != "none")
            await Check("primary-write", async () =>
            {
                var at = DateTimeOffset.UtcNow;
                var response = await actor.Ask<WriteTagResponse>(new WriteTagRequest("w1", Connection, writeTag, 412, at), TimeSpan.FromSeconds(30));
                Require(response.Success, response.ErrorMessage ?? "write refused");
                await Until(() => Event<TagValueUpdate>(at, u => GoodValue(u, writeTag) && Is(u.Value, 412)), TimeSpan.FromSeconds(30), "412 echoed by the primary");
            });

        // Loss evidence is matched from the moment the probe hands the primary over to be killed, not from
        // the recorded kill time: the consumer can see the socket close before the killer timestamps it.
        var readyAt = DateTimeOffset.UtcNow;
        if (failures == 0)
        {
            File.WriteAllText(Path.Combine(root, "primary-ready"), readyAt.ToString("O"));
            Console.WriteLine("PRIMARY-READY " + Path.Combine(root, "primary-ready"));
            var killedFile = Path.Combine(root, "primary-killed");
            await Check("primary-killed-by-runner", async () =>
            {
                // The marker carries the kill time; a half-written marker is simply read again.
                killedAt = await Until(() => File.Exists(killedFile) && DateTimeOffset.TryParse(File.ReadAllText(killedFile).Trim(), out var at) ? at : null,
                    killWait, "primary-killed marker");
            });
        }
        if (killedAt is { } k)
        {
            await Check("loss-pushes-bad-quality", async () =>
            {
                var at = await Until(() => Event<TagValueUpdate>(readyAt, u => u.Quality != QualityCode.Good)
                    ?? Event<ConnectionQualityChanged>(readyAt, q => q.Quality != QualityCode.Good), failoverWait, "Bad quality after primary loss");
                timings["killToBadQualitySeconds"] = (at - k).TotalSeconds;
            });
            await Check("failover-connected-to-backup", async () =>
            {
                backupConnectedAt = await Until(() => Health(readyAt, "Connected to backup"), failoverWait, "Connected to backup");
                timings["killToBackupConnectedSeconds"] = (backupConnectedAt.Value - k).TotalSeconds;
            });
        }
        if (backupConnectedAt is { } b)
        {
            await Check("backup-good-values-all-tags", async () =>
            {
                foreach (var tag in tags)
                {
                    var at = await Until(() => Event<TagValueUpdate>(b, u => GoodValue(u, tag)), TimeSpan.FromSeconds(60), "Good value from backup for " + tag);
                    timings["killToGood:" + tag] = (at - killedAt!.Value).TotalSeconds;
                }
                // The fixture's backup never saw the primary's 412, so its own value proves the feed's source.
                if (backupValue != "none" && writeTag != "none")
                {
                    var first = events.Where(e => e.At >= b && e.Message is TagValueUpdate u && GoodValue(u, writeTag)).Select(e => (TagValueUpdate)e.Message).First();
                    Require(Is(first.Value, long.Parse(backupValue)), $"First Good value after failover was {first.Value}, expected the backup's own {backupValue}");
                }
            });
            await Check("backup-alarm-condition-refresh", async () =>
            {
                var at = await Until(() => Event<NativeAlarmTransitionUpdate>(b, a => a.Transition.Kind == AlarmTransitionKind.SnapshotComplete), TimeSpan.FromSeconds(60), "SnapshotComplete on backup");
                timings["killToBackupSnapshotCompleteSeconds"] = (at - killedAt!.Value).TotalSeconds;
                if (alarmMessage != "none")
                    await Until(() => Event<NativeAlarmTransitionUpdate>(b, a => a.Transition.Kind == AlarmTransitionKind.Snapshot
                        && a.Transition.Message.StartsWith(alarmMessage, StringComparison.Ordinal)), TimeSpan.FromSeconds(30), "native condition replayed by the backup's ConditionRefresh");
            });
            if (writeTag != "none")
                await Check("backup-write", async () =>
                {
                    var at = DateTimeOffset.UtcNow;
                    var response = await actor.Ask<WriteTagResponse>(new WriteTagRequest("w2", Connection, writeTag, 913, at), TimeSpan.FromSeconds(30));
                    Require(response.Success, response.ErrorMessage ?? "write refused");
                    await Until(() => Event<TagValueUpdate>(at, u => GoodValue(u, writeTag) && Is(u.Value, 913)), TimeSpan.FromSeconds(30), "913 echoed by the backup");
                });
        }

        await system.Terminate();
        var sdkAssembly = typeof(Opc.Ua.NodeId).Assembly;
        var result = new
        {
            leg = "failover",
            consumer = typeof(DataConnectionActor).Assembly.GetName().Version?.ToString(),
            sdk = FileVersionInfo.GetVersionInfo(sdkAssembly.Location).ProductVersion,
            process = Environment.ProcessId, primary, backup, autoAccept = false,
            failoverRetryCount = 3, reconnectIntervalSeconds = dcOptions.ReconnectInterval.TotalSeconds,
            killedAt, backupConnectedAt, timings, cases, failures,
            endpointLabels = calls.Where(c => c.Method == nameof(ISiteHealthCollector.UpdateConnectionEndpoint))
                .Select(c => new { at = c.At, label = c.Args.ElementAtOrDefault(1)?.ToString() }),
            alarmTransitions = events.Where(e => e.Message is NativeAlarmTransitionUpdate)
                .Select(e => new { at = e.At, kind = ((NativeAlarmTransitionUpdate)e.Message).Transition.Kind.ToString(), message = ((NativeAlarmTransitionUpdate)e.Message).Transition.Message, source = ((NativeAlarmTransitionUpdate)e.Message).Transition.SourceReference }),
            alarmSourceUnavailable = events.Where(e => e.Message is NativeAlarmSourceUnavailable).Select(e => e.At),
            values = events.Where(e => e.Message is TagValueUpdate).Select(e => new { at = e.At, tag = ((TagValueUpdate)e.Message).TagPath, value = ((TagValueUpdate)e.Message).Value?.ToString(), quality = ((TagValueUpdate)e.Message).Quality.ToString() }),
        };
        File.WriteAllText(Path.Combine(root, "result.json"), JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }));
        Console.WriteLine($"Failover probe: {cases.Count - failures} passed, {failures} failed; {Path.Combine(root, "result.json")}");
        return failures == 0 && killedAt is not null ? 0 : 1;
    }

    /// <summary>Stands in for the Instance Actor: records every message the connection actor sends it.</summary>
    sealed class Collector : ReceiveActor
    {
        public Collector(ConcurrentQueue<(DateTimeOffset, object)> sink) => ReceiveAny(m => sink.Enqueue((DateTimeOffset.UtcNow, m)));
    }
}

/// <summary>Records every <see cref="ISiteHealthCollector"/> call (the actor's own endpoint/health edges).</summary>
public class HealthRecorder : DispatchProxy
{
    public ConcurrentQueue<(DateTimeOffset At, string Method, object?[] Args)> Calls { get; } = new();

    protected override object? Invoke(MethodInfo? targetMethod, object?[]? args)
    {
        Calls.Enqueue((DateTimeOffset.UtcNow, targetMethod!.Name, args ?? []));
        var type = targetMethod.ReturnType;
        return type == typeof(void) || !type.IsValueType ? null : Activator.CreateInstance(type);
    }
}