lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverInstanceActorResilienceWiringTests.cs

using System.Collections.Concurrent;
using Akka.Actor;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;

namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;

/// <summary>
///     Runtime guard for the Phase 6.1 resilience wiring (arch-review #10 / RESILIENCE-DISPATCH-GAP):
///     proves the <see cref="DriverInstanceActor"/> routes each driver-capability call through the
///     injected <see cref="IDriverCapabilityInvoker"/> at RUNTIME, with the correct
///     <see cref="DriverCapability"/> and resolved host key — not just that the calls compile wrapped
///     (the <c>OTOPCUA0001</c> analyzer is the compile-time guard; this is the behavioural one).
///     A recording invoker stands in for the real <c>CapabilityInvoker</c> so the test stays in the
///     Polly-free Runtime layer.
/// </summary>
public sealed class DriverInstanceActorResilienceWiringTests : RuntimeActorTestBase
{
    /// <summary>
    ///     A bulk subscribe issued on connect (via <c>ResubscribeDesired</c>) routes through the invoker
    ///     with <see cref="DriverCapability.Subscribe"/> keyed on the driver-instance id (single-host
    ///     fallback for a set that may span hosts). The driver's <c>SubscribeAsync</c> only runs from
    ///     inside the invoker's call-site, so <c>SubscribeCount &gt;= 1</c> already proves the route.
    /// </summary>
    [Fact]
    public async Task Subscribe_routes_through_the_capability_invoker()
    {
        var invoker = new RecordingInvoker();
        var driver = new SubscribableStubDriver();
        var actor = Sys.ActorOf(DriverInstanceActor.Props(driver, invoker: invoker));

        actor.Tell(new DriverInstanceActor.SetDesiredSubscriptions(
            new[] { "tag-a", "tag-b" }, TimeSpan.FromMilliseconds(100)));
        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));
        AwaitCondition(() => driver.SubscribeCount >= 1, TimeSpan.FromSeconds(3));

        // Barrier: round-trip a Connected-only Ask so the async subscribe handler has fully recorded.
        await actor.Ask<DriverInstanceActor.SubscriptionEstablished>(
            new DriverInstanceActor.Subscribe(new[] { "tag-a" }, TimeSpan.FromMilliseconds(100)),
            TimeSpan.FromSeconds(3));

        invoker.Calls.ShouldContain(c => c.Capability == DriverCapability.Subscribe && c.Host == driver.DriverInstanceId);
    }

    /// <summary>
    ///     A write routes through the invoker's <c>ExecuteWriteAsync</c> path (recorded as
    ///     <see cref="DriverCapability.Write"/>) with the host resolved via
    ///     <see cref="IPerCallHostResolver"/> — proving per-host breaker keying is wired, not the
    ///     single-host fallback. A write that arrives before <c>Connected</c> fast-fails WITHOUT
    ///     touching the invoker, so the loop drives to Connected and the recorded call is the real one.
    /// </summary>
    [Fact]
    public async Task Write_routes_through_the_invoker_with_resolved_host()
    {
        var invoker = new RecordingInvoker();
        var driver = new HostResolvingWritableDriver();   // ResolveHost("tag-1") => "plc-7"
        var actor = Sys.ActorOf(DriverInstanceActor.Props(driver, invoker: invoker));
        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));

        // Retry until the write succeeds (i.e. the actor reached Connected). Connecting-state writes
        // fast-fail on a synchronous path that never reaches the invoker, so they don't pollute Calls.
        DriverInstanceActor.WriteAttributeResult? reply = null;
        for (var i = 0; i < 60 && reply is not { Success: true }; i++)
        {
            reply = await actor.Ask<DriverInstanceActor.WriteAttributeResult>(
                new DriverInstanceActor.WriteAttribute("tag-1", 42), TimeSpan.FromSeconds(1));
            if (reply is not { Success: true }) await Task.Delay(25);
        }

        reply.ShouldNotBeNull();
        reply!.Success.ShouldBeTrue();
        driver.Writes.Count.ShouldBeGreaterThanOrEqualTo(1);   // the invoker's call-site actually ran
        invoker.Calls.ShouldContain(c => c.Capability == DriverCapability.Write && c.Host == "plc-7");
        // 03/S12: the dispatch site must default writes to NON-idempotent so the invoker's no-retry arm is
        // authoritative — a command-shaped write (pulse, counter, supervisory) must never replay.
        invoker.WriteCalls.ShouldContain(c => c.Host == "plc-7" && c.IsIdempotent == false);
    }

    /// <summary>
    ///     R2-10 T10 closure: <c>IDriver.InitializeAsync</c> routes through the invoker as
    ///     <see cref="DriverCapability.Initialize"/> keyed on the driver-instance id (init spans the
    ///     whole driver, like a bulk subscribe). Before this wiring the connect path was the ONE
    ///     capability call the invoker never saw — a driver that could not connect contributed zero
    ///     breaker input, so the resilience panel stayed quiet for a dead endpoint.
    /// </summary>
    [Fact]
    public void Initialize_routes_through_the_capability_invoker()
    {
        var invoker = new RecordingInvoker();
        var driver = new HostResolvingWritableDriver();
        var actor = Sys.ActorOf(DriverInstanceActor.Props(driver, invoker: invoker));

        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));

        AwaitCondition(
            () => invoker.Calls.Any(c => c.Capability == DriverCapability.Initialize),
            TimeSpan.FromSeconds(3));
        invoker.Calls.ShouldContain(c =>
            c.Capability == DriverCapability.Initialize && c.Host == driver.DriverInstanceId);
    }

    /// <summary>
    ///     The open-breaker shape end to end at the actor: when the INVOKER itself throws on the
    ///     Initialize route (exactly what an open circuit does — the call site is never reached),
    ///     the actor treats it as an ordinary init failure — lands in Reconnecting, keeps the retry
    ///     timer running, and recovers to Connected once the invoker lets an attempt through. This
    ///     pins two things: the reconnect-RETRY path is wrapped too (not just the initial connect),
    ///     and a breaker fast-fail can never wedge or crash the actor.
    /// </summary>
    [Fact]
    public async Task An_invoker_fast_fail_on_init_keeps_the_actor_retrying_until_recovery()
    {
        var invoker = new FastFailingInitInvoker(failFirst: 2);
        var driver = new HostResolvingWritableDriver();
        var actor = Sys.ActorOf(DriverInstanceActor.Props(
            driver, invoker: invoker, reconnectInterval: TimeSpan.FromMilliseconds(100)));

        actor.Tell(new DriverInstanceActor.InitializeRequested("{}"));

        // The first two init attempts die inside the invoker (open breaker); recovery is proven by a
        // write succeeding — WriteAttribute fast-fails synchronously in Connecting/Reconnecting, so a
        // Success reply is only possible from Connected.
        DriverInstanceActor.WriteAttributeResult? reply = null;
        for (var i = 0; i < 120 && reply is not { Success: true }; i++)
        {
            reply = await actor.Ask<DriverInstanceActor.WriteAttributeResult>(
                new DriverInstanceActor.WriteAttribute("tag-1", 42), TimeSpan.FromSeconds(1));
            if (reply is not { Success: true }) await Task.Delay(25);
        }

        reply.ShouldNotBeNull();
        reply!.Success.ShouldBeTrue("the actor must recover to Connected once the breaker lets an attempt through");
        invoker.InitializeAttempts.ShouldBeGreaterThanOrEqualTo(3,
            "the reconnect-retry path must route through the invoker (initial attempt + at least the two fast-failed retries)");
    }

    /// <summary>Stands in for an OPEN circuit breaker on the Initialize route: the first
    /// <paramref name="failFirst"/> Initialize calls throw WITHOUT invoking the call site (the
    /// driver's connect never runs — exactly Polly's <c>BrokenCircuitException</c> shape), then the
    /// breaker "closes" and calls pass through. All other capabilities always pass through.</summary>
    private sealed class FastFailingInitInvoker : IDriverCapabilityInvoker
    {
        private readonly int _failFirst;
        private int _initializeAttempts;

        public FastFailingInitInvoker(int failFirst) => _failFirst = failFirst;

        public int InitializeAttempts => Volatile.Read(ref _initializeAttempts);

        public ValueTask<TResult> ExecuteAsync<TResult>(
            DriverCapability capability, string hostName,
            Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
            => callSite(cancellationToken);

        public ValueTask ExecuteAsync(
            DriverCapability capability, string hostName,
            Func<CancellationToken, ValueTask> callSite, CancellationToken cancellationToken)
        {
            if (capability == DriverCapability.Initialize
                && Interlocked.Increment(ref _initializeAttempts) <= _failFirst)
            {
                throw new InvalidOperationException("circuit open (simulated fast-fail)");
            }
            return callSite(cancellationToken);
        }

        public ValueTask<TResult> ExecuteWriteAsync<TResult>(
            string hostName, bool isIdempotent,
            Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
            => callSite(cancellationToken);
    }

    /// <summary>Records every capability + resolved-host key routed through it, then delegates to the
    /// real call-site so the driver still runs. Stands in for <c>CapabilityInvoker</c> without pulling
    /// Polly / Core into the Runtime test.</summary>
    private sealed class RecordingInvoker : IDriverCapabilityInvoker
    {
        public ConcurrentQueue<(DriverCapability Capability, string Host)> Calls { get; } = new();
        public ConcurrentQueue<(string Host, bool IsIdempotent)> WriteCalls { get; } = new();

        public ValueTask<TResult> ExecuteAsync<TResult>(
            DriverCapability capability, string hostName,
            Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
        {
            Calls.Enqueue((capability, hostName));
            return callSite(cancellationToken);
        }

        public ValueTask ExecuteAsync(
            DriverCapability capability, string hostName,
            Func<CancellationToken, ValueTask> callSite, CancellationToken cancellationToken)
        {
            Calls.Enqueue((capability, hostName));
            return callSite(cancellationToken);
        }

        public ValueTask<TResult> ExecuteWriteAsync<TResult>(
            string hostName, bool isIdempotent,
            Func<CancellationToken, ValueTask<TResult>> callSite, CancellationToken cancellationToken)
        {
            Calls.Enqueue((DriverCapability.Write, hostName));
            WriteCalls.Enqueue((hostName, isIdempotent));
            return callSite(cancellationToken);
        }
    }

    /// <summary>A writable driver that also resolves a per-tag host, so the write path exercises
    /// <see cref="DriverInstanceActor"/>'s <c>ResolveHost</c> (via <see cref="IPerCallHostResolver"/>).</summary>
    private sealed class HostResolvingWritableDriver : IDriver, IWritable, IPerCallHostResolver
    {
        public List<WriteRequest> Writes { get; } = new();

        public string DriverInstanceId => "host-resolving-driver";
        public string DriverType => "Stub";

        public string ResolveHost(string fullReference) => "plc-7";

        public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
        public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
        public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
        public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, null);

        public Task<IReadOnlyList<WriteResult>> WriteAsync(
            IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
        {
            Writes.AddRange(writes);
            IReadOnlyList<WriteResult> results = writes.Select(_ => new WriteResult(0u)).ToList();
            return Task.FromResult(results);
        }
    }
}