diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs index 96b69c07..ad0ec231 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs @@ -757,21 +757,10 @@ try // metrics listener never collides with the Akka remoting port on site nodes. var metricsPort = configuration.GetValue("ScadaBridge:Node:MetricsPort", 8084); - // Configure Kestrel for HTTP/2 only on the gRPC port + // Configure Kestrel for HTTP/2 only on the gRPC port (plus the h2 keepalive that + // makes a half-open inbound stream detectable) — see SiteKestrelSetup. builder.WebHost.ConfigureKestrel(options => - { - options.ListenAnyIP(grpcPort, listenOptions => - { - listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http2; - }); - - // Dedicated HTTP/1.1 (and HTTP/2) listener for the Prometheus /metrics - // scrape endpoint, reachable by an HTTP/1.1 scraper. - options.ListenAnyIP(metricsPort, listenOptions => - { - listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.Http1AndHttp2; - }); - }); + SiteKestrelSetup.Configure(options, grpcPort, metricsPort)); // gRPC server registration. Two interceptors, two disjoint service prefixes, two // separate keys — neither one's absence weakens the other: diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/SiteKestrelSetup.cs b/src/ZB.MOM.WW.ScadaBridge.Host/SiteKestrelSetup.cs new file mode 100644 index 00000000..eb74d00c --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.Host/SiteKestrelSetup.cs @@ -0,0 +1,66 @@ +using Microsoft.AspNetCore.Server.Kestrel.Core; + +namespace ZB.MOM.WW.ScadaBridge.Host; + +/// +/// Kestrel configuration for a Site node: the h2c gRPC listener, the HTTP/1.1 metrics listener, and +/// the HTTP/2 keepalive that governs both. +/// +/// +/// Extracted from Program.cs's Site branch so the keepalive settings are assertable — the +/// existing Host wiring tests re-declare their own ConfigureKestrel lambda and therefore +/// prove nothing about the one that ships. +/// +internal static class SiteKestrelSetup +{ + /// + /// How long an HTTP/2 connection may sit idle before Kestrel sends a keepalive PING. + /// + /// + /// + /// Kestrel's default is — no server-initiated pings at all — and + /// every long-lived stream this port carries is legitimately silent for long stretches: the + /// LocalDb replication sync stream on a converged pair, and a SiteStreamService + /// subscription for an idle site. A hard crash or network partition of the peer therefore leaves + /// this node holding a half-open stream that will never produce another byte and never fail + /// either. For LocalDb sync that is the expensive one: the handler sits inside its single-stream + /// guard, so the rebooted peer's reconnects are refused while this node still reports itself + /// replication-connected (ScadaBridge arch-review 2026-08-19, P4-3). + /// + /// + /// 60s costs one frame per idle minute per connection and bounds detection at roughly + /// + . Pings are + /// ordinary HTTP/2 frames, so this is unrelated to the listener being h2c rather than TLS. + /// + /// + internal static readonly TimeSpan Http2KeepAlivePingDelay = TimeSpan.FromSeconds(60); + + /// How long Kestrel waits for a keepalive PING ack before closing the connection. + internal static readonly TimeSpan Http2KeepAlivePingTimeout = TimeSpan.FromSeconds(20); + + /// + /// Binds the site node's two listeners and applies the server-wide HTTP/2 keepalive. + /// + /// + /// The keepalive lives on Limits, which is per-server rather than per-listener, so it + /// governs the gRPC port and the metrics port alike. That is the intent: the metrics listener + /// also speaks HTTP/2, and a scrape connection has nothing to lose from a ping. + /// + public static void Configure(KestrelServerOptions options, int grpcPort, int metricsPort) + { + options.Limits.Http2.KeepAlivePingDelay = Http2KeepAlivePingDelay; + options.Limits.Http2.KeepAlivePingTimeout = Http2KeepAlivePingTimeout; + + options.ListenAnyIP(grpcPort, listenOptions => + { + listenOptions.Protocols = HttpProtocols.Http2; + }); + + // Dedicated HTTP/1.1 (and HTTP/2) listener for the Prometheus /metrics + // scrape endpoint, reachable by an HTTP/1.1 scraper. + options.ListenAnyIP(metricsPort, listenOptions => + { + listenOptions.Protocols = HttpProtocols.Http1AndHttp2; + }); + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteKestrelSetupTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteKestrelSetupTests.cs new file mode 100644 index 00000000..138c69b6 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteKestrelSetupTests.cs @@ -0,0 +1,56 @@ +using Microsoft.AspNetCore.Server.Kestrel.Core; +using ZB.MOM.WW.ScadaBridge.Host; + +namespace ZB.MOM.WW.ScadaBridge.Host.Tests; + +/// +/// The site node's Kestrel configuration, asserted against the code that actually ships (the other +/// Host startup tests re-declare their own ConfigureKestrel lambda, which can never catch a +/// regression in Program.cs's). +/// +public class SiteKestrelSetupTests +{ + [Fact] + public void SiteListener_EnablesHttp2Keepalive() + { + // Arch-review 2026-08-19 P4-3. Kestrel's default KeepAlivePingDelay is TimeSpan.MaxValue — + // no server-initiated pings — so a half-open HTTP/2 connection is never detected on a + // stream that is legitimately silent. The LocalDb sync stream is exactly that: with the + // dialling node crashed or partitioned, this node's handler stays parked on a dead stream + // holding the single-stream guard, refusing the reconnect that would heal the pair. + var options = new KestrelServerOptions(); + + SiteKestrelSetup.Configure(options, grpcPort: 8083, metricsPort: 8084); + + Assert.NotEqual(TimeSpan.MaxValue, options.Limits.Http2.KeepAlivePingDelay); + Assert.Equal(SiteKestrelSetup.Http2KeepAlivePingDelay, options.Limits.Http2.KeepAlivePingDelay); + Assert.Equal(SiteKestrelSetup.Http2KeepAlivePingTimeout, options.Limits.Http2.KeepAlivePingTimeout); + // Detection has to be quick enough to matter, and a ping must not be its own timeout. + Assert.True(SiteKestrelSetup.Http2KeepAlivePingTimeout < SiteKestrelSetup.Http2KeepAlivePingDelay); + Assert.True( + SiteKestrelSetup.Http2KeepAlivePingDelay + SiteKestrelSetup.Http2KeepAlivePingTimeout + <= TimeSpan.FromMinutes(2)); + } + + [Fact] + public void SiteListener_KeepsGrpcHttp2Only_AndMetricsHttp1Capable() + { + // The keepalive change must not disturb the protocol split: h2c-only on the gRPC port so + // Grpc.AspNetCore can serve it without TLS ALPN, HTTP/1.1-capable on the metrics port so a + // standard Prometheus scraper can reach /metrics. + var options = new KestrelServerOptions(); + + SiteKestrelSetup.Configure(options, grpcPort: 8083, metricsPort: 8084); + + var endpoints = options.GetType() + .GetProperty("CodeBackedListenOptions", System.Reflection.BindingFlags.Instance + | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public)! + .GetValue(options) as System.Collections.IEnumerable; + Assert.NotNull(endpoints); + + var protocols = endpoints!.Cast().Select(o => o.Protocols).ToList(); + Assert.Equal(2, protocols.Count); + Assert.Equal(HttpProtocols.Http2, protocols[0]); + Assert.Equal(HttpProtocols.Http1AndHttp2, protocols[1]); + } +}