ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.InboundAPI.Tests/StandbyRequestForwardingTests.cs

using System.Net;
using System.Text;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;

namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests;

/// <summary>
/// Standby-node forwarding (2026-08-28): a standby central node must relay
/// <c>POST /api/{methodName}</c> to the active peer instead of refusing with 503,
/// while never executing the method itself and never relaying more than one hop.
/// Covers both the <see cref="InboundApiEndpointFilter"/> decision and the
/// <see cref="HttpStandbyRequestForwarder"/> relay mechanics.
/// </summary>
public class StandbyRequestForwardingTests
{
    // ------------------------------------------------------------------
    // Filter: when does a standby relay, and when does it refuse?
    // ------------------------------------------------------------------

    /// <summary>Verifies that the relay does not negotiate an encoding it cannot forward.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Relay_DoesNotNegotiateAnEncodingItCannotForward()
    {
        var handler = new FakeHandler(request =>
        {
            Assert.False(request.Headers.Contains("Accept-Encoding"));
            return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("plain") });
        });
        var context = BuildRequestContext("{}");
        context.Request.Headers.AcceptEncoding = "gzip, br";
        await CreateForwarder(handler, "http://a").ForwardAsync(context, CancellationToken.None);

    }

    /// <summary>Verifies that a request the peer applied whose response was lost is not replayed.</summary>
    /// <param name="timeout">True to simulate the lost response as a timeout, false to simulate a connection failure.</param>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Theory]
    [InlineData(false)]
    [InlineData(true)]
    public async Task EI4_AppliedThenResponseLost_IsNotReplayed(bool timeout)
    {
        var effects = 0;
        var handler = new FakeHandler(_ =>
        {
            effects++;
            if (timeout) throw new TaskCanceledException("response lost after apply");
            throw new HttpRequestException("response lost after apply");
        });
        var result = await CreateForwarder(handler, "http://a", "http://b")
            .ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
        Assert.Equal(1, effects);
        var ctx = new DefaultHttpContext { RequestServices = new ServiceCollection().AddLogging().BuildServiceProvider() };
        ctx.Response.Body = new MemoryStream();
        await result.ExecuteAsync(ctx);
        ctx.Response.Body.Position = 0;
        Assert.Contains("OUTCOME_UNKNOWN", await new StreamReader(ctx.Response.Body).ReadToEndAsync());
    }

    /// <summary>Verifies that a standby with a forwarder configured relays the request and does not run the handler.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Standby_WithForwarderConfigured_RelaysAndDoesNotRunHandler()
    {
        var forwarder = new StubForwarder(configured: true, result: new HttpStandbyRequestForwarder.RelayedResult(200, "text/plain", "relayed"u8.ToArray()));
        var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder);
        var next = NextSentinel(out var handlerRan);

        var result = await CreateFilter().InvokeAsync(invocation, next);

        Assert.False(handlerRan());
        Assert.Equal(1, forwarder.Calls);
        Assert.Same(forwarder.Result, result);
        Assert.True(ctx.Items.ContainsKey(InboundApiEndpointFilter.RelayedToActiveItemKey));
    }

    /// <summary>Verifies that a standby without a configured forwarder returns 503 STANDBY_NODE marked for peers.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Standby_WithForwarderUnconfigured_Returns503StandbyNode_MarkedForPeers()
    {
        var forwarder = new StubForwarder(configured: false, result: Results.Ok());
        var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder);
        var next = NextSentinel(out var handlerRan);

        var result = await CreateFilter().InvokeAsync(invocation, next);

        Assert.False(handlerRan());
        Assert.Equal(0, forwarder.Calls);
        AssertStatus(result, StatusCodes.Status503ServiceUnavailable);
        Assert.Equal("1", ctx.Response.Headers[HttpStandbyRequestForwarder.StandbyResponseHeader]);
        Assert.False(ctx.Items.ContainsKey(InboundApiEndpointFilter.RelayedToActiveItemKey));
    }

    /// <summary>Verifies that a standby does not relay an already forwarded request again (loop guard).</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Standby_AlreadyForwardedRequest_IsNotRelayedAgain_LoopGuard()
    {
        var forwarder = new StubForwarder(configured: true, result: Results.Ok());
        var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder);
        ctx.Request.Headers[HttpStandbyRequestForwarder.ForwardedRequestHeader] = "central-b";
        var next = NextSentinel(out var handlerRan);

        var result = await CreateFilter().InvokeAsync(invocation, next);

        Assert.False(handlerRan());
        Assert.Equal(0, forwarder.Calls);
        AssertStatus(result, StatusCodes.Status503ServiceUnavailable);
        Assert.Equal("1", ctx.Response.Headers[HttpStandbyRequestForwarder.StandbyResponseHeader]);
    }

    /// <summary>Verifies that a standby returns 413 for an oversized body before relaying.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Standby_OversizedBody_Returns413_BeforeRelaying()
    {
        var options = new InboundApiOptions();
        var forwarder = new StubForwarder(configured: true, result: Results.Ok());
        var (_, invocation) = BuildInvocation(activeNode: false, forwarder,
            contentLength: options.MaxRequestBodyBytes + 1);
        var next = NextSentinel(out var handlerRan);

        var result = await CreateFilter(options).InvokeAsync(invocation, next);

        Assert.False(handlerRan());
        Assert.Equal(0, forwarder.Calls);
        AssertStatus(result, StatusCodes.Status413PayloadTooLarge);
    }

    /// <summary>Verifies that an active node serves a forwarded request locally.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task ActiveNode_ForwardedRequest_IsServedLocally()
    {
        // The active node receives the relay from its standby peer and serves it —
        // the marker header only matters when the receiving node is NOT active.
        var forwarder = new StubForwarder(configured: true, result: Results.Ok());
        var (ctx, invocation) = BuildInvocation(activeNode: true, forwarder);
        ctx.Request.Headers[HttpStandbyRequestForwarder.ForwardedRequestHeader] = "central-b";
        var next = NextSentinel(out var handlerRan);

        await CreateFilter().InvokeAsync(invocation, next);

        Assert.True(handlerRan());
        Assert.Equal(0, forwarder.Calls);
    }

    // ------------------------------------------------------------------
    // Forwarder: relay mechanics against a fake HTTP peer
    // ------------------------------------------------------------------

    /// <summary>Verifies that the forwarder relays the method, path, query, headers and body and replays the peer response.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Forwarder_RelaysMethodPathQueryHeadersAndBody_AndReplaysPeerResponse()
    {
        HttpRequestMessage? seen = null;
        string? seenBody = null;
        var handler = new FakeHandler(async req =>
        {
            seen = req;
            seenBody = await req.Content!.ReadAsStringAsync();
            return new HttpResponseMessage(HttpStatusCode.Created)
            {
                Content = new StringContent("{\"ok\":true}", Encoding.UTF8, "application/json"),
            };
        });
        var forwarder = CreateForwarder(handler, "http://central-b:5000/");
        var ctx = BuildRequestContext("{\"x\":1}");
        ctx.Request.Headers["X-API-Key"] = "sbk_key_secret";
        ctx.Request.Headers["Host"] = "central-a:5000";

        var result = await forwarder.ForwardAsync(ctx, CancellationToken.None);

        var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
        Assert.Equal(201, relayed.StatusCode);
        Assert.StartsWith("application/json", relayed.ContentType);
        Assert.Equal("{\"ok\":true}", Encoding.UTF8.GetString(relayed.Body));

        Assert.NotNull(seen);
        Assert.Equal(HttpMethod.Post, seen!.Method);
        Assert.Equal("http://central-b:5000/api/GetRecipe?v=2", seen.RequestUri!.ToString());
        Assert.Equal("{\"x\":1}", seenBody);
        Assert.Equal("sbk_key_secret", Assert.Single(seen.Headers.GetValues("X-API-Key")));
        Assert.Equal("central-a", Assert.Single(seen.Headers.GetValues(HttpStandbyRequestForwarder.ForwardedRequestHeader)));
        Assert.Null(seen.Headers.Host); // hop-by-hop / origin headers are not relayed
        Assert.Equal("application/json", seen.Content!.Headers.ContentType!.MediaType);
    }

    /// <summary>Verifies that the forwarder replays a peer error status unchanged.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Forwarder_ReplaysPeerErrorStatusUnchanged()
    {
        var handler = new FakeHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.Forbidden)
        {
            Content = new StringContent("{\"code\":\"NOT_APPROVED\"}", Encoding.UTF8, "application/json"),
        }));
        var forwarder = CreateForwarder(handler, "http://central-b:5000");

        var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);

        var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
        Assert.Equal(403, relayed.StatusCode);
        Assert.Contains("NOT_APPROVED", Encoding.UTF8.GetString(relayed.Body));
    }

    /// <summary>Verifies that the forwarder tries the next peer with the same body when a peer is unreachable.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Forwarder_PeerUnreachable_TriesNextPeer_WithSameBody()
    {
        var bodies = new List<string>();
        var handler = new FakeHandler(async req =>
        {
            bodies.Add(await req.Content!.ReadAsStringAsync());
            if (req.RequestUri!.Host == "dead")
                throw new HttpRequestException("connection refused", new System.Net.Sockets.SocketException((int)System.Net.Sockets.SocketError.ConnectionRefused));
            return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") };
        });
        var forwarder = CreateForwarder(handler, "http://dead:5000", "http://alive:5000");

        var result = await forwarder.ForwardAsync(BuildRequestContext("{\"n\":7}"), CancellationToken.None);

        var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
        Assert.Equal(200, relayed.StatusCode);
        Assert.Equal(new[] { "{\"n\":7}", "{\"n\":7}" }, bodies);
    }

    /// <summary>Verifies that the forwarder tries the next peer when a peer is also a standby.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Forwarder_PeerIsAlsoStandby_TriesNextPeer()
    {
        var handler = new FakeHandler(req =>
        {
            if (req.RequestUri!.Host == "standby")
            {
                var refusal = new HttpResponseMessage(HttpStatusCode.ServiceUnavailable);
                refusal.Headers.TryAddWithoutValidation(HttpStandbyRequestForwarder.StandbyResponseHeader, "1");
                return Task.FromResult(refusal);
            }
            return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") });
        });
        var forwarder = CreateForwarder(handler, "http://standby:5000", "http://active:5000");

        var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);

        var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
        Assert.Equal(200, relayed.StatusCode);
        Assert.Equal("served", Encoding.UTF8.GetString(relayed.Body));
    }

    /// <summary>Verifies that a genuine 503 from the active peer is replayed rather than retried.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Forwarder_Genuine503FromActivePeer_IsReplayed_NotRetried()
    {
        // A 503 WITHOUT the standby marker is the active node's own answer (e.g. a
        // method script returning 503) and must be replayed, not treated as a miss.
        var calls = 0;
        var handler = new FakeHandler(_ =>
        {
            calls++;
            return Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)
            {
                Content = new StringContent("busy"),
            });
        });
        var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000");

        var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);

        var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
        Assert.Equal(503, relayed.StatusCode);
        Assert.Equal("busy", Encoding.UTF8.GetString(relayed.Body));
        Assert.Equal(1, calls);
    }

    /// <summary>Verifies that the forwarder returns 503 ACTIVE_NODE_UNREACHABLE when all peers fail.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Forwarder_AllPeersFail_Returns503ActiveNodeUnreachable()
    {
        var handler = new FakeHandler(_ => throw new HttpRequestException(HttpRequestError.NameResolutionError, "down"));
        var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000");

        var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);

        Assert.IsNotType<HttpStandbyRequestForwarder.RelayedResult>(result);
        AssertStatus(result, StatusCodes.Status503ServiceUnavailable);
    }

    /// <summary>Verifies that a peer timeout returns an uncertain result without trying the next peer.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Forwarder_PeerTimesOut_ReturnsUncertainWithoutTryingNextPeer()
    {
        var handler = new FakeHandler(async (req, ct) =>
        {
            if (req.RequestUri!.Host == "slow")
            {
                await Task.Delay(Timeout.InfiniteTimeSpan, ct); // honours the per-attempt deadline
            }
            return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") };
        });
        var forwarder = CreateForwarder(handler, TimeSpan.FromMilliseconds(100), "http://slow:5000", "http://fast:5000");

        var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);

        Assert.IsNotType<HttpStandbyRequestForwarder.RelayedResult>(result);
        AssertStatus(result, StatusCodes.Status502BadGateway);
    }

    /// <summary>Verifies that a caller abort propagates and does not try the next peer.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task Forwarder_CallerAborts_Propagates_DoesNotTryNextPeer()
    {
        var calls = 0;
        using var callerCts = new CancellationTokenSource();
        var handler = new FakeHandler(async (_, ct) =>
        {
            calls++;
            callerCts.Cancel();
            await Task.Delay(Timeout.InfiniteTimeSpan, ct);
            throw new InvalidOperationException("unreachable");
        });
        var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000");

        await Assert.ThrowsAnyAsync<OperationCanceledException>(
            () => forwarder.ForwardAsync(BuildRequestContext("{}"), callerCts.Token));
        Assert.Equal(1, calls);
    }

    /// <summary>Verifies that IsConfigured tracks the peer list.</summary>
    [Fact]
    public void Forwarder_IsConfigured_TracksPeerList()
    {
        Assert.False(CreateForwarder(new FakeHandler(_ => throw new InvalidOperationException())).IsConfigured);
        Assert.True(CreateForwarder(new FakeHandler(_ => throw new InvalidOperationException()), "http://b:5000").IsConfigured);
    }

    /// <summary>Verifies that a relayed result writes the status, content type and body.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
    public async Task RelayedResult_WritesStatusContentTypeAndBody()
    {
        var ctx = new DefaultHttpContext();
        ctx.Response.Body = new MemoryStream();
        var relayed = new HttpStandbyRequestForwarder.RelayedResult(418, "text/plain", Encoding.UTF8.GetBytes("tea"));

        await relayed.ExecuteAsync(ctx);

        Assert.Equal(418, ctx.Response.StatusCode);
        Assert.Equal("text/plain", ctx.Response.ContentType);
        Assert.Equal(3, ctx.Response.ContentLength);
        ctx.Response.Body.Position = 0;
        Assert.Equal("tea", new StreamReader(ctx.Response.Body).ReadToEnd());
    }

    // ------------------------------------------------------------------
    // helpers
    // ------------------------------------------------------------------

    private static InboundApiEndpointFilter CreateFilter(InboundApiOptions? options = null) =>
        new(NullLogger<InboundApiEndpointFilter>.Instance,
            new StaticOptionsMonitor<InboundApiOptions>(options ?? new InboundApiOptions()));

    private static (DefaultHttpContext ctx, DefaultEndpointFilterInvocationContext invocation)
        BuildInvocation(bool activeNode, IStandbyRequestForwarder forwarder, long contentLength = 2)
    {
        var services = new ServiceCollection();
        services.AddSingleton<IActiveNodeGate>(new StubActiveNodeGate(activeNode));
        services.AddSingleton(forwarder);
        var ctx = new DefaultHttpContext { RequestServices = services.BuildServiceProvider() };
        ctx.Request.Method = "POST";
        ctx.Request.ContentLength = contentLength;
        return (ctx, new DefaultEndpointFilterInvocationContext(ctx));
    }

    private static EndpointFilterDelegate NextSentinel(out Func<bool> wasCalled)
    {
        var called = false;
        wasCalled = () => called;
        return _ =>
        {
            called = true;
            return ValueTask.FromResult<object?>(Results.Ok("handler-ran"));
        };
    }

    private static void AssertStatus(object? result, int expected)
    {
        var status = Assert.IsAssignableFrom<IStatusCodeHttpResult>(result);
        Assert.Equal(expected, status.StatusCode);
    }

    private static DefaultHttpContext BuildRequestContext(string body)
    {
        var ctx = new DefaultHttpContext();
        ctx.Request.Method = "POST";
        ctx.Request.Path = "/api/GetRecipe";
        ctx.Request.QueryString = new QueryString("?v=2");
        ctx.Request.ContentType = "application/json";
        ctx.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(body));
        return ctx;
    }

    private static HttpStandbyRequestForwarder CreateForwarder(FakeHandler handler, params string[] peers) =>
        CreateForwarder(handler, TimeSpan.FromSeconds(5), peers);

    private static HttpStandbyRequestForwarder CreateForwarder(FakeHandler handler, TimeSpan timeout, params string[] peers) =>
        new(new FakeHttpClientFactory(handler),
            new StaticOptionsMonitor<InboundApiOptions>(new InboundApiOptions
            {
                StandbyForwardPeerEndpoints = peers.ToList(),
                StandbyForwardTimeout = timeout,
            }),
            NullLogger<HttpStandbyRequestForwarder>.Instance);

    private sealed class StubForwarder : IStandbyRequestForwarder
    {
        /// <summary>Initializes the stub with its configured flag and canned result.</summary>
        /// <param name="configured">Whether the forwarder reports itself as configured.</param>
        /// <param name="result">The result returned from forwarding.</param>
        public StubForwarder(bool configured, IResult result) { IsConfigured = configured; Result = result; }
        /// <inheritdoc />
        public bool IsConfigured { get; }
        /// <summary>Gets the canned result returned from forwarding.</summary>
        public IResult Result { get; }
        /// <summary>Gets the number of times forwarding was invoked.</summary>
        public int Calls { get; private set; }
        /// <inheritdoc />
        public Task<IResult> ForwardAsync(HttpContext context, CancellationToken cancellationToken)
        {
            Calls++;
            return Task.FromResult(Result);
        }
    }

    private sealed class StubActiveNodeGate : IActiveNodeGate
    {
        /// <summary>Initializes the gate with a fixed active-node answer.</summary>
        /// <param name="isActive">Whether this node reports itself as active.</param>
        public StubActiveNodeGate(bool isActive) => IsActiveNode = isActive;
        /// <inheritdoc />
        public bool IsActiveNode { get; }
    }

    private sealed class FakeHandler : HttpMessageHandler
    {
        private readonly Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> _send;
        /// <summary>Initializes the handler with a send delegate that ignores the cancellation token.</summary>
        /// <param name="send">The delegate that produces the response for a request.</param>
        public FakeHandler(Func<HttpRequestMessage, Task<HttpResponseMessage>> send) => _send = (r, _) => send(r);
        /// <summary>Initializes the handler with a send delegate that observes the cancellation token.</summary>
        /// <param name="send">The delegate that produces the response for a request.</param>
        public FakeHandler(Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> send) => _send = send;
        /// <inheritdoc />
        protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct) =>
            _send(request, ct);
    }

    private sealed class FakeHttpClientFactory : IHttpClientFactory
    {
        private readonly HttpMessageHandler _handler;
        /// <summary>Initializes the factory with the handler every client will use.</summary>
        /// <param name="handler">The message handler shared by created clients.</param>
        public FakeHttpClientFactory(HttpMessageHandler handler) => _handler = handler;
        /// <summary>Returns a client over the canned handler.</summary>
        /// <param name="name">The requested client name (ignored).</param>
        /// <returns>An <see cref="HttpClient"/> that uses the fake handler.</returns>
        public HttpClient CreateClient(string name) =>
            new(_handler, disposeHandler: false) { Timeout = Timeout.InfiniteTimeSpan };
    }

    private sealed class StaticOptionsMonitor<T> : IOptionsMonitor<T>
    {
        /// <summary>Initializes the monitor with a fixed value.</summary>
        /// <param name="value">The value returned by the monitor.</param>
        public StaticOptionsMonitor(T value) => CurrentValue = value;
        /// <summary>Gets the canned options value.</summary>
        public T CurrentValue { get; }
        /// <summary>Returns the canned options value regardless of name.</summary>
        /// <param name="name">The requested options name (ignored).</param>
        /// <returns>The canned options value.</returns>
        public T Get(string? name) => CurrentValue;
        /// <summary>Ignores the change listener; the fake never raises changes.</summary>
        /// <param name="listener">The change listener (ignored).</param>
        /// <returns>Always null.</returns>
        public IDisposable? OnChange(Action<T, string?> listener) => null;
    }
}