using System.Net; using System.Text; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests; /// /// Standby-node forwarding (2026-08-28): a standby central node must relay /// POST /api/{methodName} to the active peer instead of refusing with 503, /// while never executing the method itself and never relaying more than one hop. /// Covers both the decision and the /// relay mechanics. /// public class StandbyRequestForwardingTests { // ------------------------------------------------------------------ // Filter: when does a standby relay, and when does it refuse? // ------------------------------------------------------------------ [Fact] public async Task Relay_DoesNotNegotiateAnEncodingItCannotForward() { var handler = new FakeHandler(request => { Assert.False(request.Headers.Contains("Accept-Encoding")); return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("plain") }); }); var context = BuildRequestContext("{}"); context.Request.Headers.AcceptEncoding = "gzip, br"; await CreateForwarder(handler, "http://a").ForwardAsync(context, CancellationToken.None); } [Theory] [InlineData(false)] [InlineData(true)] public async Task EI4_AppliedThenResponseLost_IsNotReplayed(bool timeout) { var effects = 0; var handler = new FakeHandler(_ => { effects++; if (timeout) throw new TaskCanceledException("response lost after apply"); throw new HttpRequestException("response lost after apply"); }); var result = await CreateForwarder(handler, "http://a", "http://b") .ForwardAsync(BuildRequestContext("{}"), CancellationToken.None); Assert.Equal(1, effects); var ctx = new DefaultHttpContext { RequestServices = new ServiceCollection().AddLogging().BuildServiceProvider() }; ctx.Response.Body = new MemoryStream(); await result.ExecuteAsync(ctx); ctx.Response.Body.Position = 0; Assert.Contains("OUTCOME_UNKNOWN", await new StreamReader(ctx.Response.Body).ReadToEndAsync()); } [Fact] public async Task Standby_WithForwarderConfigured_RelaysAndDoesNotRunHandler() { var forwarder = new StubForwarder(configured: true, result: new HttpStandbyRequestForwarder.RelayedResult(200, "text/plain", "relayed"u8.ToArray())); var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder); var next = NextSentinel(out var handlerRan); var result = await CreateFilter().InvokeAsync(invocation, next); Assert.False(handlerRan()); Assert.Equal(1, forwarder.Calls); Assert.Same(forwarder.Result, result); Assert.True(ctx.Items.ContainsKey(InboundApiEndpointFilter.RelayedToActiveItemKey)); } [Fact] public async Task Standby_WithForwarderUnconfigured_Returns503StandbyNode_MarkedForPeers() { var forwarder = new StubForwarder(configured: false, result: Results.Ok()); var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder); var next = NextSentinel(out var handlerRan); var result = await CreateFilter().InvokeAsync(invocation, next); Assert.False(handlerRan()); Assert.Equal(0, forwarder.Calls); AssertStatus(result, StatusCodes.Status503ServiceUnavailable); Assert.Equal("1", ctx.Response.Headers[HttpStandbyRequestForwarder.StandbyResponseHeader]); Assert.False(ctx.Items.ContainsKey(InboundApiEndpointFilter.RelayedToActiveItemKey)); } [Fact] public async Task Standby_AlreadyForwardedRequest_IsNotRelayedAgain_LoopGuard() { var forwarder = new StubForwarder(configured: true, result: Results.Ok()); var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder); ctx.Request.Headers[HttpStandbyRequestForwarder.ForwardedRequestHeader] = "central-b"; var next = NextSentinel(out var handlerRan); var result = await CreateFilter().InvokeAsync(invocation, next); Assert.False(handlerRan()); Assert.Equal(0, forwarder.Calls); AssertStatus(result, StatusCodes.Status503ServiceUnavailable); Assert.Equal("1", ctx.Response.Headers[HttpStandbyRequestForwarder.StandbyResponseHeader]); } [Fact] public async Task Standby_OversizedBody_Returns413_BeforeRelaying() { var options = new InboundApiOptions(); var forwarder = new StubForwarder(configured: true, result: Results.Ok()); var (_, invocation) = BuildInvocation(activeNode: false, forwarder, contentLength: options.MaxRequestBodyBytes + 1); var next = NextSentinel(out var handlerRan); var result = await CreateFilter(options).InvokeAsync(invocation, next); Assert.False(handlerRan()); Assert.Equal(0, forwarder.Calls); AssertStatus(result, StatusCodes.Status413PayloadTooLarge); } [Fact] public async Task ActiveNode_ForwardedRequest_IsServedLocally() { // The active node receives the relay from its standby peer and serves it — // the marker header only matters when the receiving node is NOT active. var forwarder = new StubForwarder(configured: true, result: Results.Ok()); var (ctx, invocation) = BuildInvocation(activeNode: true, forwarder); ctx.Request.Headers[HttpStandbyRequestForwarder.ForwardedRequestHeader] = "central-b"; var next = NextSentinel(out var handlerRan); await CreateFilter().InvokeAsync(invocation, next); Assert.True(handlerRan()); Assert.Equal(0, forwarder.Calls); } // ------------------------------------------------------------------ // Forwarder: relay mechanics against a fake HTTP peer // ------------------------------------------------------------------ [Fact] public async Task Forwarder_RelaysMethodPathQueryHeadersAndBody_AndReplaysPeerResponse() { HttpRequestMessage? seen = null; string? seenBody = null; var handler = new FakeHandler(async req => { seen = req; seenBody = await req.Content!.ReadAsStringAsync(); return new HttpResponseMessage(HttpStatusCode.Created) { Content = new StringContent("{\"ok\":true}", Encoding.UTF8, "application/json"), }; }); var forwarder = CreateForwarder(handler, "http://central-b:5000/"); var ctx = BuildRequestContext("{\"x\":1}"); ctx.Request.Headers["X-API-Key"] = "sbk_key_secret"; ctx.Request.Headers["Host"] = "central-a:5000"; var result = await forwarder.ForwardAsync(ctx, CancellationToken.None); var relayed = Assert.IsType(result); Assert.Equal(201, relayed.StatusCode); Assert.StartsWith("application/json", relayed.ContentType); Assert.Equal("{\"ok\":true}", Encoding.UTF8.GetString(relayed.Body)); Assert.NotNull(seen); Assert.Equal(HttpMethod.Post, seen!.Method); Assert.Equal("http://central-b:5000/api/GetRecipe?v=2", seen.RequestUri!.ToString()); Assert.Equal("{\"x\":1}", seenBody); Assert.Equal("sbk_key_secret", Assert.Single(seen.Headers.GetValues("X-API-Key"))); Assert.Equal("central-a", Assert.Single(seen.Headers.GetValues(HttpStandbyRequestForwarder.ForwardedRequestHeader))); Assert.Null(seen.Headers.Host); // hop-by-hop / origin headers are not relayed Assert.Equal("application/json", seen.Content!.Headers.ContentType!.MediaType); } [Fact] public async Task Forwarder_ReplaysPeerErrorStatusUnchanged() { var handler = new FakeHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.Forbidden) { Content = new StringContent("{\"code\":\"NOT_APPROVED\"}", Encoding.UTF8, "application/json"), })); var forwarder = CreateForwarder(handler, "http://central-b:5000"); var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None); var relayed = Assert.IsType(result); Assert.Equal(403, relayed.StatusCode); Assert.Contains("NOT_APPROVED", Encoding.UTF8.GetString(relayed.Body)); } [Fact] public async Task Forwarder_PeerUnreachable_TriesNextPeer_WithSameBody() { var bodies = new List(); var handler = new FakeHandler(async req => { bodies.Add(await req.Content!.ReadAsStringAsync()); if (req.RequestUri!.Host == "dead") throw new HttpRequestException("connection refused", new System.Net.Sockets.SocketException((int)System.Net.Sockets.SocketError.ConnectionRefused)); return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") }; }); var forwarder = CreateForwarder(handler, "http://dead:5000", "http://alive:5000"); var result = await forwarder.ForwardAsync(BuildRequestContext("{\"n\":7}"), CancellationToken.None); var relayed = Assert.IsType(result); Assert.Equal(200, relayed.StatusCode); Assert.Equal(new[] { "{\"n\":7}", "{\"n\":7}" }, bodies); } [Fact] public async Task Forwarder_PeerIsAlsoStandby_TriesNextPeer() { var handler = new FakeHandler(req => { if (req.RequestUri!.Host == "standby") { var refusal = new HttpResponseMessage(HttpStatusCode.ServiceUnavailable); refusal.Headers.TryAddWithoutValidation(HttpStandbyRequestForwarder.StandbyResponseHeader, "1"); return Task.FromResult(refusal); } return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") }); }); var forwarder = CreateForwarder(handler, "http://standby:5000", "http://active:5000"); var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None); var relayed = Assert.IsType(result); Assert.Equal(200, relayed.StatusCode); Assert.Equal("served", Encoding.UTF8.GetString(relayed.Body)); } [Fact] public async Task Forwarder_Genuine503FromActivePeer_IsReplayed_NotRetried() { // A 503 WITHOUT the standby marker is the active node's own answer (e.g. a // method script returning 503) and must be replayed, not treated as a miss. var calls = 0; var handler = new FakeHandler(_ => { calls++; return Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable) { Content = new StringContent("busy"), }); }); var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000"); var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None); var relayed = Assert.IsType(result); Assert.Equal(503, relayed.StatusCode); Assert.Equal("busy", Encoding.UTF8.GetString(relayed.Body)); Assert.Equal(1, calls); } [Fact] public async Task Forwarder_AllPeersFail_Returns503ActiveNodeUnreachable() { var handler = new FakeHandler(_ => throw new HttpRequestException(HttpRequestError.NameResolutionError, "down")); var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000"); var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None); Assert.IsNotType(result); AssertStatus(result, StatusCodes.Status503ServiceUnavailable); } [Fact] public async Task Forwarder_PeerTimesOut_ReturnsUncertainWithoutTryingNextPeer() { var handler = new FakeHandler(async (req, ct) => { if (req.RequestUri!.Host == "slow") { await Task.Delay(Timeout.InfiniteTimeSpan, ct); // honours the per-attempt deadline } return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") }; }); var forwarder = CreateForwarder(handler, TimeSpan.FromMilliseconds(100), "http://slow:5000", "http://fast:5000"); var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None); Assert.IsNotType(result); AssertStatus(result, StatusCodes.Status502BadGateway); } [Fact] public async Task Forwarder_CallerAborts_Propagates_DoesNotTryNextPeer() { var calls = 0; using var callerCts = new CancellationTokenSource(); var handler = new FakeHandler(async (_, ct) => { calls++; callerCts.Cancel(); await Task.Delay(Timeout.InfiniteTimeSpan, ct); throw new InvalidOperationException("unreachable"); }); var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000"); await Assert.ThrowsAnyAsync( () => forwarder.ForwardAsync(BuildRequestContext("{}"), callerCts.Token)); Assert.Equal(1, calls); } [Fact] public void Forwarder_IsConfigured_TracksPeerList() { Assert.False(CreateForwarder(new FakeHandler(_ => throw new InvalidOperationException())).IsConfigured); Assert.True(CreateForwarder(new FakeHandler(_ => throw new InvalidOperationException()), "http://b:5000").IsConfigured); } [Fact] public async Task RelayedResult_WritesStatusContentTypeAndBody() { var ctx = new DefaultHttpContext(); ctx.Response.Body = new MemoryStream(); var relayed = new HttpStandbyRequestForwarder.RelayedResult(418, "text/plain", Encoding.UTF8.GetBytes("tea")); await relayed.ExecuteAsync(ctx); Assert.Equal(418, ctx.Response.StatusCode); Assert.Equal("text/plain", ctx.Response.ContentType); Assert.Equal(3, ctx.Response.ContentLength); ctx.Response.Body.Position = 0; Assert.Equal("tea", new StreamReader(ctx.Response.Body).ReadToEnd()); } // ------------------------------------------------------------------ // helpers // ------------------------------------------------------------------ private static InboundApiEndpointFilter CreateFilter(InboundApiOptions? options = null) => new(NullLogger.Instance, new StaticOptionsMonitor(options ?? new InboundApiOptions())); private static (DefaultHttpContext ctx, DefaultEndpointFilterInvocationContext invocation) BuildInvocation(bool activeNode, IStandbyRequestForwarder forwarder, long contentLength = 2) { var services = new ServiceCollection(); services.AddSingleton(new StubActiveNodeGate(activeNode)); services.AddSingleton(forwarder); var ctx = new DefaultHttpContext { RequestServices = services.BuildServiceProvider() }; ctx.Request.Method = "POST"; ctx.Request.ContentLength = contentLength; return (ctx, new DefaultEndpointFilterInvocationContext(ctx)); } private static EndpointFilterDelegate NextSentinel(out Func wasCalled) { var called = false; wasCalled = () => called; return _ => { called = true; return ValueTask.FromResult(Results.Ok("handler-ran")); }; } private static void AssertStatus(object? result, int expected) { var status = Assert.IsAssignableFrom(result); Assert.Equal(expected, status.StatusCode); } private static DefaultHttpContext BuildRequestContext(string body) { var ctx = new DefaultHttpContext(); ctx.Request.Method = "POST"; ctx.Request.Path = "/api/GetRecipe"; ctx.Request.QueryString = new QueryString("?v=2"); ctx.Request.ContentType = "application/json"; ctx.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(body)); return ctx; } private static HttpStandbyRequestForwarder CreateForwarder(FakeHandler handler, params string[] peers) => CreateForwarder(handler, TimeSpan.FromSeconds(5), peers); private static HttpStandbyRequestForwarder CreateForwarder(FakeHandler handler, TimeSpan timeout, params string[] peers) => new(new FakeHttpClientFactory(handler), new StaticOptionsMonitor(new InboundApiOptions { StandbyForwardPeerEndpoints = peers.ToList(), StandbyForwardTimeout = timeout, }), NullLogger.Instance); private sealed class StubForwarder : IStandbyRequestForwarder { public StubForwarder(bool configured, IResult result) { IsConfigured = configured; Result = result; } public bool IsConfigured { get; } public IResult Result { get; } public int Calls { get; private set; } public Task ForwardAsync(HttpContext context, CancellationToken cancellationToken) { Calls++; return Task.FromResult(Result); } } private sealed class StubActiveNodeGate : IActiveNodeGate { public StubActiveNodeGate(bool isActive) => IsActiveNode = isActive; public bool IsActiveNode { get; } } private sealed class FakeHandler : HttpMessageHandler { private readonly Func> _send; public FakeHandler(Func> send) => _send = (r, _) => send(r); public FakeHandler(Func> send) => _send = send; protected override Task SendAsync(HttpRequestMessage request, CancellationToken ct) => _send(request, ct); } private sealed class FakeHttpClientFactory : IHttpClientFactory { private readonly HttpMessageHandler _handler; public FakeHttpClientFactory(HttpMessageHandler handler) => _handler = handler; public HttpClient CreateClient(string name) => new(_handler, disposeHandler: false) { Timeout = Timeout.InfiniteTimeSpan }; } private sealed class StaticOptionsMonitor : IOptionsMonitor { public StaticOptionsMonitor(T value) => CurrentValue = value; public T CurrentValue { get; } public T Get(string? name) => CurrentValue; public IDisposable? OnChange(Action listener) => null; } }