mbproxy: add keepalive / connection monitoring

The DL205/DL260 ECOM emits no TCP keepalives, so an idle backend socket can be silently dropped by a middlebox (switch, firewall, NAT) after 2-5 minutes. Enable OS SO_KEEPALIVE on backend and accepted upstream sockets, and drive a periodic synthetic FC03 heartbeat on each idle backend socket so a dead path is detected before a real client request hits it. Controlled by Connection.Keepalive (ON by default). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-15 09:40:54 -04:00
parent 7466a46aa7
commit 0868613890
25 changed files with 1135 additions and 25 deletions
@@ -438,6 +438,69 @@ public sealed class MultiplexerE2ETests
        }
    }

+    // ── E2E 6: Backend keepalive heartbeat keeps an idle connection warm ─────────────
+
+    /// <summary>
+    /// With keepalive enabled, an idle backend connection receives periodic FC03 heartbeat
+    /// probes. This test idles a simulator-backed connection past
+    /// <c>BackendHeartbeatIdleMs</c>, verifies <c>backendHeartbeatsSent</c> climbs on the
+    /// status page, and confirms a later real read still round-trips on the same
+    /// (un-cascaded) connection.
+    /// </summary>
+    [Fact(Timeout = 8_000)]
+    public async Task E2E_Keepalive_IdleBackend_ReceivesHeartbeats_AndStaysUsable()
+    {
+        if (_sim.SkipReason is not null) Assert.Skip(_sim.SkipReason);
+
+        int proxyPort = PickFreePort();
+        int adminPort = PickFreePort();
+
+        var config = MakeBaseConfig(proxyPort);
+        config["Mbproxy:AdminPort"] = adminPort.ToString();
+        // Short idle window so the heartbeat fires several times within the test budget.
+        config["Mbproxy:Connection:Keepalive:Enabled"] = "true";
+        config["Mbproxy:Connection:Keepalive:BackendHeartbeatIdleMs"] = "700";
+
+        var host = BuildBcdHost(config);
+        using var startCts = new CancellationTokenSource(TimeSpan.FromSeconds(3));
+        await host.StartAsync(startCts.Token);
+        await using var hd = new AsyncHostDispose(host);
+        await Task.Delay(200, TestContext.Current.CancellationToken);
+
+        using (var client = new TcpClient())
+        {
+            await client.ConnectAsync("127.0.0.1", proxyPort, TestContext.Current.CancellationToken);
+            var master = new ModbusFactory().CreateMaster(client);
+
+            // One read brings the backend up and starts the heartbeat loop.
+            _ = master.ReadHoldingRegisters(1, 0, 1);
+
+            // Idle the connection so the heartbeat loop fires repeatedly.
+            await Task.Delay(2500, TestContext.Current.CancellationToken);
+
+            // A later read still succeeds — the connection was never cascaded.
+            ushort[] regs = master.ReadHoldingRegisters(1, 0, 1);
+            regs.Length.ShouldBe(1, "the idle-then-active connection must still serve reads");
+        }
+
+        using var httpClient = new HttpClient();
+        var resp = await httpClient.GetStringAsync(
+            $"http://127.0.0.1:{adminPort}/status.json",
+            TestContext.Current.CancellationToken);
+
+        using var doc = JsonDocument.Parse(resp);
+        var backend = doc.RootElement.GetProperty("plcs")[0].GetProperty("backend");
+
+        backend.TryGetProperty("backendHeartbeatsSent", out _)
+            .ShouldBeTrue("status.json must expose backend.backendHeartbeatsSent");
+        backend.GetProperty("backendHeartbeatsSent").GetInt64()
+            .ShouldBeGreaterThanOrEqualTo(1, "an idle backend must have received at least one heartbeat");
+        backend.GetProperty("backendHeartbeatsFailed").GetInt64()
+            .ShouldBe(0, "every heartbeat against the live simulator must be answered");
+        backend.GetProperty("backendIdleDisconnects").GetInt64()
+            .ShouldBe(0, "an answered heartbeat must never tear the backend down");
+    }
+
    // ── Helpers ──────────────────────────────────────────────────────────────────────

    private Dictionary<string, string?> MakeBaseConfig(int proxyPort) => new()