Joseph Doherty
437 lines
19 KiB
C#
437 lines
19 KiB
C#
using System.Net;
|
|
using System.Text;
|
|
using Microsoft.AspNetCore.Http;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.InboundAPI.Tests;
|
|
|
|
/// <summary>
|
|
/// Standby-node forwarding (2026-08-28): a standby central node must relay
|
|
/// <c>POST /api/{methodName}</c> to the active peer instead of refusing with 503,
|
|
/// while never executing the method itself and never relaying more than one hop.
|
|
/// Covers both the <see cref="InboundApiEndpointFilter"/> decision and the
|
|
/// <see cref="HttpStandbyRequestForwarder"/> relay mechanics.
|
|
/// </summary>
|
|
public class StandbyRequestForwardingTests
|
|
{
|
|
// ------------------------------------------------------------------
|
|
// Filter: when does a standby relay, and when does it refuse?
|
|
// ------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Relay_DoesNotNegotiateAnEncodingItCannotForward()
|
|
{
|
|
var handler = new FakeHandler(request =>
|
|
{
|
|
Assert.False(request.Headers.Contains("Accept-Encoding"));
|
|
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("plain") });
|
|
});
|
|
var context = BuildRequestContext("{}");
|
|
context.Request.Headers.AcceptEncoding = "gzip, br";
|
|
await CreateForwarder(handler, "http://a").ForwardAsync(context, CancellationToken.None);
|
|
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(false)]
|
|
[InlineData(true)]
|
|
public async Task EI4_AppliedThenResponseLost_IsNotReplayed(bool timeout)
|
|
{
|
|
var effects = 0;
|
|
var handler = new FakeHandler(_ =>
|
|
{
|
|
effects++;
|
|
if (timeout) throw new TaskCanceledException("response lost after apply");
|
|
throw new HttpRequestException("response lost after apply");
|
|
});
|
|
var result = await CreateForwarder(handler, "http://a", "http://b")
|
|
.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
|
|
Assert.Equal(1, effects);
|
|
var ctx = new DefaultHttpContext { RequestServices = new ServiceCollection().AddLogging().BuildServiceProvider() };
|
|
ctx.Response.Body = new MemoryStream();
|
|
await result.ExecuteAsync(ctx);
|
|
ctx.Response.Body.Position = 0;
|
|
Assert.Contains("OUTCOME_UNKNOWN", await new StreamReader(ctx.Response.Body).ReadToEndAsync());
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Standby_WithForwarderConfigured_RelaysAndDoesNotRunHandler()
|
|
{
|
|
var forwarder = new StubForwarder(configured: true, result: new HttpStandbyRequestForwarder.RelayedResult(200, "text/plain", "relayed"u8.ToArray()));
|
|
var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder);
|
|
var next = NextSentinel(out var handlerRan);
|
|
|
|
var result = await CreateFilter().InvokeAsync(invocation, next);
|
|
|
|
Assert.False(handlerRan());
|
|
Assert.Equal(1, forwarder.Calls);
|
|
Assert.Same(forwarder.Result, result);
|
|
Assert.True(ctx.Items.ContainsKey(InboundApiEndpointFilter.RelayedToActiveItemKey));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Standby_WithForwarderUnconfigured_Returns503StandbyNode_MarkedForPeers()
|
|
{
|
|
var forwarder = new StubForwarder(configured: false, result: Results.Ok());
|
|
var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder);
|
|
var next = NextSentinel(out var handlerRan);
|
|
|
|
var result = await CreateFilter().InvokeAsync(invocation, next);
|
|
|
|
Assert.False(handlerRan());
|
|
Assert.Equal(0, forwarder.Calls);
|
|
AssertStatus(result, StatusCodes.Status503ServiceUnavailable);
|
|
Assert.Equal("1", ctx.Response.Headers[HttpStandbyRequestForwarder.StandbyResponseHeader]);
|
|
Assert.False(ctx.Items.ContainsKey(InboundApiEndpointFilter.RelayedToActiveItemKey));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Standby_AlreadyForwardedRequest_IsNotRelayedAgain_LoopGuard()
|
|
{
|
|
var forwarder = new StubForwarder(configured: true, result: Results.Ok());
|
|
var (ctx, invocation) = BuildInvocation(activeNode: false, forwarder);
|
|
ctx.Request.Headers[HttpStandbyRequestForwarder.ForwardedRequestHeader] = "central-b";
|
|
var next = NextSentinel(out var handlerRan);
|
|
|
|
var result = await CreateFilter().InvokeAsync(invocation, next);
|
|
|
|
Assert.False(handlerRan());
|
|
Assert.Equal(0, forwarder.Calls);
|
|
AssertStatus(result, StatusCodes.Status503ServiceUnavailable);
|
|
Assert.Equal("1", ctx.Response.Headers[HttpStandbyRequestForwarder.StandbyResponseHeader]);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Standby_OversizedBody_Returns413_BeforeRelaying()
|
|
{
|
|
var options = new InboundApiOptions();
|
|
var forwarder = new StubForwarder(configured: true, result: Results.Ok());
|
|
var (_, invocation) = BuildInvocation(activeNode: false, forwarder,
|
|
contentLength: options.MaxRequestBodyBytes + 1);
|
|
var next = NextSentinel(out var handlerRan);
|
|
|
|
var result = await CreateFilter(options).InvokeAsync(invocation, next);
|
|
|
|
Assert.False(handlerRan());
|
|
Assert.Equal(0, forwarder.Calls);
|
|
AssertStatus(result, StatusCodes.Status413PayloadTooLarge);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ActiveNode_ForwardedRequest_IsServedLocally()
|
|
{
|
|
// The active node receives the relay from its standby peer and serves it —
|
|
// the marker header only matters when the receiving node is NOT active.
|
|
var forwarder = new StubForwarder(configured: true, result: Results.Ok());
|
|
var (ctx, invocation) = BuildInvocation(activeNode: true, forwarder);
|
|
ctx.Request.Headers[HttpStandbyRequestForwarder.ForwardedRequestHeader] = "central-b";
|
|
var next = NextSentinel(out var handlerRan);
|
|
|
|
await CreateFilter().InvokeAsync(invocation, next);
|
|
|
|
Assert.True(handlerRan());
|
|
Assert.Equal(0, forwarder.Calls);
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Forwarder: relay mechanics against a fake HTTP peer
|
|
// ------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Forwarder_RelaysMethodPathQueryHeadersAndBody_AndReplaysPeerResponse()
|
|
{
|
|
HttpRequestMessage? seen = null;
|
|
string? seenBody = null;
|
|
var handler = new FakeHandler(async req =>
|
|
{
|
|
seen = req;
|
|
seenBody = await req.Content!.ReadAsStringAsync();
|
|
return new HttpResponseMessage(HttpStatusCode.Created)
|
|
{
|
|
Content = new StringContent("{\"ok\":true}", Encoding.UTF8, "application/json"),
|
|
};
|
|
});
|
|
var forwarder = CreateForwarder(handler, "http://central-b:5000/");
|
|
var ctx = BuildRequestContext("{\"x\":1}");
|
|
ctx.Request.Headers["X-API-Key"] = "sbk_key_secret";
|
|
ctx.Request.Headers["Host"] = "central-a:5000";
|
|
|
|
var result = await forwarder.ForwardAsync(ctx, CancellationToken.None);
|
|
|
|
var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
|
|
Assert.Equal(201, relayed.StatusCode);
|
|
Assert.StartsWith("application/json", relayed.ContentType);
|
|
Assert.Equal("{\"ok\":true}", Encoding.UTF8.GetString(relayed.Body));
|
|
|
|
Assert.NotNull(seen);
|
|
Assert.Equal(HttpMethod.Post, seen!.Method);
|
|
Assert.Equal("http://central-b:5000/api/GetRecipe?v=2", seen.RequestUri!.ToString());
|
|
Assert.Equal("{\"x\":1}", seenBody);
|
|
Assert.Equal("sbk_key_secret", Assert.Single(seen.Headers.GetValues("X-API-Key")));
|
|
Assert.Equal("central-a", Assert.Single(seen.Headers.GetValues(HttpStandbyRequestForwarder.ForwardedRequestHeader)));
|
|
Assert.Null(seen.Headers.Host); // hop-by-hop / origin headers are not relayed
|
|
Assert.Equal("application/json", seen.Content!.Headers.ContentType!.MediaType);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Forwarder_ReplaysPeerErrorStatusUnchanged()
|
|
{
|
|
var handler = new FakeHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.Forbidden)
|
|
{
|
|
Content = new StringContent("{\"code\":\"NOT_APPROVED\"}", Encoding.UTF8, "application/json"),
|
|
}));
|
|
var forwarder = CreateForwarder(handler, "http://central-b:5000");
|
|
|
|
var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
|
|
|
|
var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
|
|
Assert.Equal(403, relayed.StatusCode);
|
|
Assert.Contains("NOT_APPROVED", Encoding.UTF8.GetString(relayed.Body));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Forwarder_PeerUnreachable_TriesNextPeer_WithSameBody()
|
|
{
|
|
var bodies = new List<string>();
|
|
var handler = new FakeHandler(async req =>
|
|
{
|
|
bodies.Add(await req.Content!.ReadAsStringAsync());
|
|
if (req.RequestUri!.Host == "dead")
|
|
throw new HttpRequestException("connection refused", new System.Net.Sockets.SocketException((int)System.Net.Sockets.SocketError.ConnectionRefused));
|
|
return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") };
|
|
});
|
|
var forwarder = CreateForwarder(handler, "http://dead:5000", "http://alive:5000");
|
|
|
|
var result = await forwarder.ForwardAsync(BuildRequestContext("{\"n\":7}"), CancellationToken.None);
|
|
|
|
var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
|
|
Assert.Equal(200, relayed.StatusCode);
|
|
Assert.Equal(new[] { "{\"n\":7}", "{\"n\":7}" }, bodies);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Forwarder_PeerIsAlsoStandby_TriesNextPeer()
|
|
{
|
|
var handler = new FakeHandler(req =>
|
|
{
|
|
if (req.RequestUri!.Host == "standby")
|
|
{
|
|
var refusal = new HttpResponseMessage(HttpStatusCode.ServiceUnavailable);
|
|
refusal.Headers.TryAddWithoutValidation(HttpStandbyRequestForwarder.StandbyResponseHeader, "1");
|
|
return Task.FromResult(refusal);
|
|
}
|
|
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") });
|
|
});
|
|
var forwarder = CreateForwarder(handler, "http://standby:5000", "http://active:5000");
|
|
|
|
var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
|
|
|
|
var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
|
|
Assert.Equal(200, relayed.StatusCode);
|
|
Assert.Equal("served", Encoding.UTF8.GetString(relayed.Body));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Forwarder_Genuine503FromActivePeer_IsReplayed_NotRetried()
|
|
{
|
|
// A 503 WITHOUT the standby marker is the active node's own answer (e.g. a
|
|
// method script returning 503) and must be replayed, not treated as a miss.
|
|
var calls = 0;
|
|
var handler = new FakeHandler(_ =>
|
|
{
|
|
calls++;
|
|
return Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)
|
|
{
|
|
Content = new StringContent("busy"),
|
|
});
|
|
});
|
|
var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000");
|
|
|
|
var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
|
|
|
|
var relayed = Assert.IsType<HttpStandbyRequestForwarder.RelayedResult>(result);
|
|
Assert.Equal(503, relayed.StatusCode);
|
|
Assert.Equal("busy", Encoding.UTF8.GetString(relayed.Body));
|
|
Assert.Equal(1, calls);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Forwarder_AllPeersFail_Returns503ActiveNodeUnreachable()
|
|
{
|
|
var handler = new FakeHandler(_ => throw new HttpRequestException(HttpRequestError.NameResolutionError, "down"));
|
|
var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000");
|
|
|
|
var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
|
|
|
|
Assert.IsNotType<HttpStandbyRequestForwarder.RelayedResult>(result);
|
|
AssertStatus(result, StatusCodes.Status503ServiceUnavailable);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Forwarder_PeerTimesOut_ReturnsUncertainWithoutTryingNextPeer()
|
|
{
|
|
var handler = new FakeHandler(async (req, ct) =>
|
|
{
|
|
if (req.RequestUri!.Host == "slow")
|
|
{
|
|
await Task.Delay(Timeout.InfiniteTimeSpan, ct); // honours the per-attempt deadline
|
|
}
|
|
return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("served") };
|
|
});
|
|
var forwarder = CreateForwarder(handler, TimeSpan.FromMilliseconds(100), "http://slow:5000", "http://fast:5000");
|
|
|
|
var result = await forwarder.ForwardAsync(BuildRequestContext("{}"), CancellationToken.None);
|
|
|
|
Assert.IsNotType<HttpStandbyRequestForwarder.RelayedResult>(result);
|
|
AssertStatus(result, StatusCodes.Status502BadGateway);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Forwarder_CallerAborts_Propagates_DoesNotTryNextPeer()
|
|
{
|
|
var calls = 0;
|
|
using var callerCts = new CancellationTokenSource();
|
|
var handler = new FakeHandler(async (_, ct) =>
|
|
{
|
|
calls++;
|
|
callerCts.Cancel();
|
|
await Task.Delay(Timeout.InfiniteTimeSpan, ct);
|
|
throw new InvalidOperationException("unreachable");
|
|
});
|
|
var forwarder = CreateForwarder(handler, "http://a:5000", "http://b:5000");
|
|
|
|
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
|
() => forwarder.ForwardAsync(BuildRequestContext("{}"), callerCts.Token));
|
|
Assert.Equal(1, calls);
|
|
}
|
|
|
|
[Fact]
|
|
public void Forwarder_IsConfigured_TracksPeerList()
|
|
{
|
|
Assert.False(CreateForwarder(new FakeHandler(_ => throw new InvalidOperationException())).IsConfigured);
|
|
Assert.True(CreateForwarder(new FakeHandler(_ => throw new InvalidOperationException()), "http://b:5000").IsConfigured);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RelayedResult_WritesStatusContentTypeAndBody()
|
|
{
|
|
var ctx = new DefaultHttpContext();
|
|
ctx.Response.Body = new MemoryStream();
|
|
var relayed = new HttpStandbyRequestForwarder.RelayedResult(418, "text/plain", Encoding.UTF8.GetBytes("tea"));
|
|
|
|
await relayed.ExecuteAsync(ctx);
|
|
|
|
Assert.Equal(418, ctx.Response.StatusCode);
|
|
Assert.Equal("text/plain", ctx.Response.ContentType);
|
|
Assert.Equal(3, ctx.Response.ContentLength);
|
|
ctx.Response.Body.Position = 0;
|
|
Assert.Equal("tea", new StreamReader(ctx.Response.Body).ReadToEnd());
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// helpers
|
|
// ------------------------------------------------------------------
|
|
|
|
private static InboundApiEndpointFilter CreateFilter(InboundApiOptions? options = null) =>
|
|
new(NullLogger<InboundApiEndpointFilter>.Instance,
|
|
new StaticOptionsMonitor<InboundApiOptions>(options ?? new InboundApiOptions()));
|
|
|
|
private static (DefaultHttpContext ctx, DefaultEndpointFilterInvocationContext invocation)
|
|
BuildInvocation(bool activeNode, IStandbyRequestForwarder forwarder, long contentLength = 2)
|
|
{
|
|
var services = new ServiceCollection();
|
|
services.AddSingleton<IActiveNodeGate>(new StubActiveNodeGate(activeNode));
|
|
services.AddSingleton(forwarder);
|
|
var ctx = new DefaultHttpContext { RequestServices = services.BuildServiceProvider() };
|
|
ctx.Request.Method = "POST";
|
|
ctx.Request.ContentLength = contentLength;
|
|
return (ctx, new DefaultEndpointFilterInvocationContext(ctx));
|
|
}
|
|
|
|
private static EndpointFilterDelegate NextSentinel(out Func<bool> wasCalled)
|
|
{
|
|
var called = false;
|
|
wasCalled = () => called;
|
|
return _ =>
|
|
{
|
|
called = true;
|
|
return ValueTask.FromResult<object?>(Results.Ok("handler-ran"));
|
|
};
|
|
}
|
|
|
|
private static void AssertStatus(object? result, int expected)
|
|
{
|
|
var status = Assert.IsAssignableFrom<IStatusCodeHttpResult>(result);
|
|
Assert.Equal(expected, status.StatusCode);
|
|
}
|
|
|
|
private static DefaultHttpContext BuildRequestContext(string body)
|
|
{
|
|
var ctx = new DefaultHttpContext();
|
|
ctx.Request.Method = "POST";
|
|
ctx.Request.Path = "/api/GetRecipe";
|
|
ctx.Request.QueryString = new QueryString("?v=2");
|
|
ctx.Request.ContentType = "application/json";
|
|
ctx.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(body));
|
|
return ctx;
|
|
}
|
|
|
|
private static HttpStandbyRequestForwarder CreateForwarder(FakeHandler handler, params string[] peers) =>
|
|
CreateForwarder(handler, TimeSpan.FromSeconds(5), peers);
|
|
|
|
private static HttpStandbyRequestForwarder CreateForwarder(FakeHandler handler, TimeSpan timeout, params string[] peers) =>
|
|
new(new FakeHttpClientFactory(handler),
|
|
new StaticOptionsMonitor<InboundApiOptions>(new InboundApiOptions
|
|
{
|
|
StandbyForwardPeerEndpoints = peers.ToList(),
|
|
StandbyForwardTimeout = timeout,
|
|
}),
|
|
NullLogger<HttpStandbyRequestForwarder>.Instance);
|
|
|
|
private sealed class StubForwarder : IStandbyRequestForwarder
|
|
{
|
|
public StubForwarder(bool configured, IResult result) { IsConfigured = configured; Result = result; }
|
|
public bool IsConfigured { get; }
|
|
public IResult Result { get; }
|
|
public int Calls { get; private set; }
|
|
public Task<IResult> ForwardAsync(HttpContext context, CancellationToken cancellationToken)
|
|
{
|
|
Calls++;
|
|
return Task.FromResult(Result);
|
|
}
|
|
}
|
|
|
|
private sealed class StubActiveNodeGate : IActiveNodeGate
|
|
{
|
|
public StubActiveNodeGate(bool isActive) => IsActiveNode = isActive;
|
|
public bool IsActiveNode { get; }
|
|
}
|
|
|
|
private sealed class FakeHandler : HttpMessageHandler
|
|
{
|
|
private readonly Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> _send;
|
|
public FakeHandler(Func<HttpRequestMessage, Task<HttpResponseMessage>> send) => _send = (r, _) => send(r);
|
|
public FakeHandler(Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>> send) => _send = send;
|
|
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken ct) =>
|
|
_send(request, ct);
|
|
}
|
|
|
|
private sealed class FakeHttpClientFactory : IHttpClientFactory
|
|
{
|
|
private readonly HttpMessageHandler _handler;
|
|
public FakeHttpClientFactory(HttpMessageHandler handler) => _handler = handler;
|
|
public HttpClient CreateClient(string name) =>
|
|
new(_handler, disposeHandler: false) { Timeout = Timeout.InfiniteTimeSpan };
|
|
}
|
|
|
|
private sealed class StaticOptionsMonitor<T> : IOptionsMonitor<T>
|
|
{
|
|
public StaticOptionsMonitor(T value) => CurrentValue = value;
|
|
public T CurrentValue { get; }
|
|
public T Get(string? name) => CurrentValue;
|
|
public IDisposable? OnChange(Action<T, string?> listener) => null;
|
|
}
|
|
}
|