ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/OpcUaServiceLevelTests.cs

using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using NSubstitute.ExceptionExtensions;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Serialization;
using ZB.MOM.WW.ScadaBridge.Commons.Types.DataConnections;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;

namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;

/// <summary>
/// ScadaBridge#138 at the OPC UA adapter: <see cref="OpcUaDataConnection.ReadServiceLevelAsync"/>
/// against a substitute client (status / failure mapping) and against real in-process OPC UA
/// servers (the actual <c>i=2267</c> node), plus the whole failover through the real
/// <see cref="DataConnectionFactory"/> against a real server pair.
/// </summary>
public sealed class OpcUaServiceLevelTests : TestKit
{
    private readonly string _clientPki =
        Path.Combine(Path.GetTempPath(), "sb138-client", Guid.NewGuid().ToString("N"));

    /// <summary>Initializes the test kit.</summary>
    public OpcUaServiceLevelTests() : base(@"akka.loglevel = DEBUG") { }

    /// <inheritdoc />
    protected override void Dispose(bool disposing)
    {
        base.Dispose(disposing);
        try { Directory.Delete(_clientPki, recursive: true); } catch { /* best-effort */ }
    }

    private OpcUaGlobalOptions ClientOptions() => new()
    {
        ApplicationName = "SB138Client",
        ApplicationUri = "urn:localhost:SB138Client",
        OwnCertificateStorePath = Path.Combine(_clientPki, "own"),
        TrustedPeerStorePath = Path.Combine(_clientPki, "trusted"),
        TrustedIssuerStorePath = Path.Combine(_clientPki, "issuers"),
        RejectedCertificateStorePath = Path.Combine(_clientPki, "rejected"),
        AutoCreateApplicationCertificate = true,
    };

    private static IDictionary<string, string> Flat(string url, OpcUaServiceLevelFailoverConfig? sl = null) =>
        OpcUaEndpointConfigSerializer.ToFlatDict(new OpcUaEndpointConfig
        {
            EndpointUrl = url,
            SecurityMode = OpcUaSecurityMode.None,
            ServiceLevelFailover = sl,
        });

    private static (OpcUaDataConnection Adapter, IOpcUaClient Client) WithSubstituteClient()
    {
        var client = Substitute.For<IOpcUaClient>();
        client.IsConnected.Returns(true);
        var factory = Substitute.For<IOpcUaClientFactory>();
        factory.Create().Returns(client);
        return (new OpcUaDataConnection(factory, NullLogger<OpcUaDataConnection>.Instance), client);
    }

    // ── Status / failure mapping (substitute client) ──

    /// <summary>The value comes from <c>i=2267</c>; a Good read of a byte is that byte.</summary>
    [Fact]
    public async Task Read_GoodByte_IsTheLevel_FromTheServiceLevelNode()
    {
        var (adapter, client) = WithSubstituteClient();
        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
            .Returns(((object?)(byte)90, DateTime.UtcNow, 0u));
        await adapter.ConnectAsync(new Dictionary<string, string>());

        Assert.Equal((byte)90, await adapter.ReadServiceLevelAsync());
    }

    /// <summary>A Bad status, or a value that is not a byte-ranged integer, is unknown (null).</summary>
    [Fact]
    public async Task Read_BadStatusOrNonByteValue_IsUnknown()
    {
        var (adapter, client) = WithSubstituteClient();
        await adapter.ConnectAsync(new Dictionary<string, string>());

        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
            .Returns(((object?)(byte)250, DateTime.UtcNow, 0x80340000u)); // BadNodeIdUnknown
        Assert.Null(await adapter.ReadServiceLevelAsync());

        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
            .Returns(((object?)"250", DateTime.UtcNow, 0u));
        Assert.Null(await adapter.ReadServiceLevelAsync());

        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
            .Returns(((object?)300, DateTime.UtcNow, 0u));
        Assert.Null(await adapter.ReadServiceLevelAsync());

        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
            .Returns(((object?)240, DateTime.UtcNow, 0u)); // Int32 in range → accepted
        Assert.Equal((byte)240, await adapter.ReadServiceLevelAsync());
    }

    /// <summary>
    /// A failed read is unknown and — unlike <see cref="OpcUaDataConnection.ReadAsync"/> — never
    /// raises <c>Disconnected</c>: the poll must not drive the reconnect machine.
    /// </summary>
    [Fact]
    public async Task Read_Failure_IsUnknown_AndNeverRaisesDisconnected()
    {
        var (adapter, client) = WithSubstituteClient();
        client.ReadValueAsync("i=2267", Arg.Any<CancellationToken>())
            .ThrowsAsync(new InvalidOperationException("session dropped"));
        var disconnected = 0;
        adapter.Disconnected += () => disconnected++;
        await adapter.ConnectAsync(new Dictionary<string, string>());

        Assert.Null(await adapter.ReadServiceLevelAsync());
        Assert.Equal(0, disconnected);

        // Positive control: the ordinary read path DOES raise it for the same failure.
        client.ReadValueAsync("ns=2;s=x", Arg.Any<CancellationToken>())
            .ThrowsAsync(new InvalidOperationException("session dropped"));
        await Assert.ThrowsAsync<InvalidOperationException>(() => adapter.ReadAsync("ns=2;s=x"));
        Assert.Equal(1, disconnected);
    }

    /// <summary>Before connect (no client) the level is unknown, and no read is attempted.</summary>
    [Fact]
    public async Task Read_NotConnected_IsUnknown()
    {
        var (adapter, client) = WithSubstituteClient();
        Assert.Null(await adapter.ReadServiceLevelAsync());

        await adapter.ConnectAsync(new Dictionary<string, string>());
        client.IsConnected.Returns(false);
        Assert.Null(await adapter.ReadServiceLevelAsync());
        await client.DidNotReceive().ReadValueAsync(Arg.Any<string>(), Arg.Any<CancellationToken>());
    }

    // ── Real OPC UA servers ──

    /// <summary>The real adapter reads the real server's <c>Server_ServiceLevel</c> as it changes.</summary>
    [Fact]
    public async Task RealServer_ReadServiceLevel_TracksTheServersValue()
    {
        await using var server = await SecureOpcUaTestServer.StartAsync();
        await using var adapter = new OpcUaDataConnection(
            new RealOpcUaClientFactory(ClientOptions()), NullLogger<OpcUaDataConnection>.Instance);
        await adapter.ConnectAsync(Flat(server.Url));

        foreach (var level in new byte[] { 250, 240, 90, 50, 1 })
        {
            server.SetServiceLevel(level);
            Assert.Equal(level, await adapter.ReadServiceLevelAsync());
        }
    }

    /// <summary>
    /// End to end on two real servers through the production factory: the connection starts on a
    /// primary that is the Warm standby (90) while the backup is Active (250), and ends up on the
    /// backup. Then the mirror case on a fresh connection — today's 250/240 pair — stays on the
    /// primary.
    /// </summary>
    [Fact]
    public async Task RealServerPair_StandbyPrimary_FailsOverToTheActiveBackup_ActivePair_StaysPut()
    {
        await using var a = await SecureOpcUaTestServer.StartAsync();
        await using var b = await SecureOpcUaTestServer.StartAsync();
        var factory = new DataConnectionFactory(NullLoggerFactory.Instance, Options.Create(ClientOptions()));
        var options = new DataConnectionOptions
        {
            ReconnectInterval = TimeSpan.FromMilliseconds(200),
            ServiceLevelPollInterval = TimeSpan.FromMilliseconds(100),
            ServiceLevelProbeTimeout = TimeSpan.FromSeconds(20),
        };
        var sl = new OpcUaServiceLevelFailoverConfig { DegradedWindowSeconds = 1, MinDwellSeconds = 0 };

        async Task<string> Endpoint(IActorRef actor) =>
            (await actor.Ask<DataConnectionHealthReport>(new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(5)))
            .ActiveEndpoint;

        // Warm: A is the standby, B the Active.
        a.SetServiceLevel(90);
        b.SetServiceLevel(250);
        var health = Substitute.For<ISiteHealthCollector>();
        var primary = Flat(a.Url, sl);
        var warm = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
            "warm", factory.Create("OpcUa", primary), options, health, factory, "OpcUa",
            primary, Flat(b.Url), 3, null)));
        await AwaitConditionAsync(async () => await Endpoint(warm) == "Backup", TimeSpan.FromSeconds(30));
        AwaitCondition(() => health.ReceivedCalls().Any(c =>
            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
            && c.GetArguments() is ["warm", "Connected to backup"]), TimeSpan.FromSeconds(30));
        Sys.Stop(warm);

        // Today: A is the Secondary (240), B the Primary (250) — both Active band.
        a.SetServiceLevel(240);
        var today = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
            "today", factory.Create("OpcUa", primary), options, health, factory, "OpcUa",
            primary, Flat(b.Url), 3, null)));
        AwaitCondition(() => health.ReceivedCalls().Any(c =>
            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
            && c.GetArguments() is ["today", "Connected to primary"]), TimeSpan.FromSeconds(30));
        await Task.Delay(TimeSpan.FromSeconds(2.5)); // > window, ~25 polls
        Assert.Equal("Primary", await Endpoint(today));
        Assert.DoesNotContain(health.ReceivedCalls(), c =>
            c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
            && c.GetArguments() is ["today", "Connected to backup"]);
    }

    /// <summary>
    /// ScadaBridge#141 end to end, the QA Warm gate's step 4a: the connection is on the Warm Primary
    /// (A, 250) with the standby (B, 90) as backup, both on the DEFAULT policy (10 s window, 60 s
    /// dwell) and the default 5 s reconnect interval with three retries. A is stopped gracefully the
    /// way OtOpcUa does it — the peer promotes to 250, A reports ServiceLevel 0 for a moment, then
    /// the server goes away. The connection must be live on B within seconds; the connectivity path
    /// alone (keep-alive + three 5 s retries of the stopped A) needs well over 15 s.
    /// </summary>
    [Fact]
    public async Task RealServerPair_GracefulStopOfTheWarmPrimary_FailsOverWithinSeconds()
    {
        var a = await SecureOpcUaTestServer.StartAsync();
        await using var b = await SecureOpcUaTestServer.StartAsync();
        try
        {
            var factory = new DataConnectionFactory(NullLoggerFactory.Instance, Options.Create(ClientOptions()));
            var options = new DataConnectionOptions(); // production defaults: 5 s reconnect, 2 s poll, 15 s probe
            a.SetServiceLevel(250);
            b.SetServiceLevel(90);
            var health = Substitute.For<ISiteHealthCollector>();
            var primary = Flat(a.Url); // null ServiceLevelFailover = defaults (enabled, 10 s, 60 s)

            bool ConnectedTo(string endpoint) => health.ReceivedCalls().Any(c =>
                c.GetMethodInfo().Name == nameof(ISiteHealthCollector.UpdateConnectionEndpoint)
                && c.GetArguments() is ["stop", var label] && (string)label! == $"Connected to {endpoint}");

            var actor = Sys.ActorOf(Props.Create(() => new DataConnectionActor(
                "stop", factory.Create("OpcUa", primary), options, health, factory, "OpcUa",
                primary, Flat(b.Url), 3, null)));
            AwaitCondition(() => ConnectedTo("primary"), TimeSpan.FromSeconds(30));
            await Task.Delay(TimeSpan.FromSeconds(3)); // a few Healthy polls: no connect-time effect left

            var sw = System.Diagnostics.Stopwatch.StartNew();
            b.SetServiceLevel(250);           // the peer promotes (0.2 s on QA)
            a.SetServiceLevel(0);             // the stopping node reports Maintenance...
            await Task.Delay(TimeSpan.FromSeconds(1));
            await a.DisposeAsync();           // ...and is gone well inside the 10 s window

            AwaitCondition(() => ConnectedTo("backup"), TimeSpan.FromSeconds(30));
            Assert.True(sw.Elapsed < TimeSpan.FromSeconds(10),
                $"failover took {sw.Elapsed.TotalSeconds:F1}s (QA before #141: 66 s)");
            Assert.Equal("Backup", (await actor.Ask<DataConnectionHealthReport>(
                new DataConnectionActor.GetHealthReport(), TimeSpan.FromSeconds(5))).ActiveEndpoint);
        }
        finally
        {
            await a.DisposeAsync();
        }
    }
}