test(sql): blackhole/timeout live-gate on a dedicated mssql container

Adds the frozen-peer / BadTimeout gate for the Sql driver — the single highest-value integration test. Mirrors the S7 R2-01 blackhole gate: docker pause a DEDICATED mssql mid-poll, assert the next read surfaces BadTimeout within the client-side operationTimeout (≈3s) and well below the server-side CommandTimeout backstop (30s), the driver degrades (Degraded + host Stopped), and docker unpause recovers to Healthy/Running. - Docker/docker-compose.yml: disposable `otopcua-sql-blackhole` mssql on :14333 (never the shared :14330 ConfigDb server) + one-shot seed. - Docker/seed.sql: the two sample tables. - SqlBlackholeTimeoutTests.cs: env-gated (SQL_BLACKHOLE_ENDPOINT); pauses ONLY the hard-coded container name; skips loudly if the endpoint is the shared port 14330; bounds its own wait so a wedged impl fails, not hangs. Offline: clean skip (no socket, no docker shell-out). Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
2026-07-24 15:20:24 -04:00
parent a55b7e51ca
commit 10a6a37ca5
3 changed files with 520 additions and 0 deletions
@@ -0,0 +1,68 @@
+# Dedicated, DISPOSABLE SQL Server for the Sql driver's blackhole / frozen-peer live gate.
+#
+# ── SAFETY ─────────────────────────────────────────────────────────────────────
+# This stack owns its OWN mssql container, `otopcua-sql-blackhole`, on host port
+# 14333. The blackhole gate `docker pause`s THIS container by that hard-coded name.
+# It is NOT — and must never be — the rig's shared always-on SQL Server, which is
+# `10.100.0.35,14330` and hosts ConfigDb for the whole dev rig. Two independent
+# guardrails keep them apart:
+#   1. a distinct container_name (`otopcua-sql-blackhole`) — the pause target is a
+#      compile-time constant in the test, and the shared server is not named this; and
+#   2. a distinct host port (14333, not 14330) — the test SKIPS LOUDLY if its
+#      endpoint resolves to port 14330.
+# `restart: "no"` so a paused/killed container stays down rather than respawning.
+#
+# ── DEPLOY (from this VM; docker runs on the shared Linux host) ──────────────────
+#   lmxopcua-fix sync sql-blackhole      # rsync this Docker/ dir → /opt/otopcua-sql-blackhole/
+#   lmxopcua-fix up   sql-blackhole      # single-service stack (+ one-shot seed), no profile arg
+# `lmxopcua-fix` applies the host-side `project=lmxopcua` label; the explicit label
+# below keeps the stack discoverable even when brought up with plain `docker compose`.
+#
+# The one-shot `mssql-seed` service applies seed.sql once the server is healthy, so
+# the stack is self-seeding. The blackhole test ALSO seeds defensively on connect
+# (create-if-missing, same schema), so a stack that never ran the seed still works.
+
+name: otopcua-sql-blackhole
+
+services:
+  mssql:
+    image: mcr.microsoft.com/mssql/server:2022-latest
+    container_name: otopcua-sql-blackhole
+    labels:
+      project: lmxopcua
+    restart: "no"
+    environment:
+      ACCEPT_EULA: "Y"
+      # Dev-only sandbox password for a throwaway container. Override via the shell env
+      # (the same value the test reads from SQL_BLACKHOLE_PASSWORD).
+      MSSQL_SA_PASSWORD: "${SQL_BLACKHOLE_SA_PASSWORD:-Blackhole_dev_pw1}"
+      MSSQL_PID: "Developer"
+    ports:
+      - "14333:1433"
+    healthcheck:
+      # -C trusts the self-signed dev cert; the login succeeding is the readiness signal.
+      test:
+        - "CMD-SHELL"
+        - "/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P \"$${MSSQL_SA_PASSWORD}\" -Q 'SELECT 1' || exit 1"
+      interval: 10s
+      timeout: 5s
+      retries: 12
+      start_period: 30s
+
+  # One-shot seed: waits for the server to pass its healthcheck, applies seed.sql, exits.
+  mssql-seed:
+    image: mcr.microsoft.com/mssql/server:2022-latest
+    labels:
+      project: lmxopcua
+    restart: "no"
+    depends_on:
+      mssql:
+        condition: service_healthy
+    environment:
+      MSSQL_SA_PASSWORD: "${SQL_BLACKHOLE_SA_PASSWORD:-Blackhole_dev_pw1}"
+    volumes:
+      - ./seed.sql:/seed.sql:ro
+    entrypoint:
+      - "/bin/bash"
+      - "-c"
+      - "/opt/mssql-tools18/bin/sqlcmd -C -S mssql -U sa -P \"$${MSSQL_SA_PASSWORD}\" -i /seed.sql"
@@ -0,0 +1,43 @@
+-- Seed for the Sql driver's blackhole / frozen-peer live gate (dedicated container only).
+--
+-- Applied once by the compose `mssql-seed` one-shot against the OWNED
+-- `otopcua-sql-blackhole` container. The blackhole test (SqlBlackholeTimeoutTests)
+-- also creates this exact shape defensively on connect, so the two are kept in
+-- parity: a Good baseline poll before the pause depends on `sqlpoll.TagValues`
+-- holding the one seeded row.
+--
+-- Two sample tables mirror the offline SqlitePollFixture / live SqlPollServerFixture
+-- shapes so the gate polls something representative, not a bespoke schema.
+
+IF DB_ID(N'SqlBlackholeFixture') IS NULL CREATE DATABASE [SqlBlackholeFixture];
+GO
+
+USE [SqlBlackholeFixture];
+GO
+
+IF SCHEMA_ID(N'sqlpoll') IS NULL EXEC(N'CREATE SCHEMA [sqlpoll]');
+GO
+
+-- Key-value (EAV) source: tag_name → num_value, with a source timestamp.
+DROP TABLE IF EXISTS [sqlpoll].[TagValues];
+CREATE TABLE [sqlpoll].[TagValues] (
+    [tag_name]  nvarchar(128) NOT NULL,
+    [num_value] float         NULL,
+    [sample_ts] datetime2(3)  NOT NULL
+);
+INSERT INTO [sqlpoll].[TagValues] ([tag_name], [num_value], [sample_ts])
+VALUES (N'Line1.Speed',    42.0, '2026-07-24T10:00:00'),
+       (N'Line1.Pressure', 3.5,  '2026-07-24T10:00:01');
+GO
+
+-- Wide-row source: one row per station, several value columns.
+DROP TABLE IF EXISTS [sqlpoll].[LatestStatus];
+CREATE TABLE [sqlpoll].[LatestStatus] (
+    [station_id] int          NOT NULL,
+    [oven_temp]  float        NULL,
+    [pressure]   float        NULL,
+    [sample_ts]  datetime2(3) NOT NULL
+);
+INSERT INTO [sqlpoll].[LatestStatus] ([station_id], [oven_temp], [pressure], [sample_ts])
+VALUES (7, 180.5, 1.2, '2026-07-24T10:00:00');
+GO
@@ -0,0 +1,409 @@
+using System.Diagnostics;
+using System.Globalization;
+using System.Text.Json.Nodes;
+using Microsoft.Data.SqlClient;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests;
+
+/// <summary>
+///     The single highest-value integration test for the <c>Sql</c> driver: the <b>frozen-peer /
+///     blackhole gate</b>. When the database stops answering mid-poll, a reused pooled connection hangs
+///     inside <c>ExecuteReaderAsync</c>; the driver must surface <see cref="SqlStatusCodes.BadTimeout"/>
+///     within its client-side <c>operationTimeout</c> (a real linked-CTS cancellation) and NOT wedge the
+///     poll loop. This is the exact bug class the repo shipped and fixed once in its S7 driver (async
+///     reads ignored the socket read timeout → a frozen peer wedged the poll loop; see
+///     <c>S7_1500ConnectTimeoutOutageTests</c>, which this mirrors).
+///     <para><b>The wall-clock bound is the whole point.</b> <c>operationTimeout</c> is set well BELOW
+///     <c>commandTimeout</c> here, deliberately inverted from the production rule, so the two backstops
+///     are distinguishable: a working client-side cancellation returns at ≈ <c>operationTimeout</c>; if it
+///     were broken and only the ADO.NET <c>CommandTimeout</c> server-side backstop fired, the read would
+///     return at ≈ <c>commandTimeout</c> instead — which the upper-bound assertion fails on. A test that
+///     merely asserted "eventually BadTimeout" would pass even with the client-side cancellation broken.</para>
+///     <para><b>SAFETY — this gate `docker pause`s a DEDICATED container it owns, never shared infra.</b>
+///     The pause target is the compile-time constant <see cref="ContainerName"/>
+///     (<c>otopcua-sql-blackhole</c>), never anything derived from an env var, so a mis-set endpoint can
+///     never aim a pause at the rig's shared always-on SQL Server (<c>10.100.0.35,14330</c>, which hosts
+///     ConfigDb for the whole rig). As a second guardrail the gate <b>skips loudly</b> if its endpoint
+///     resolves to the shared server's port <see cref="SharedCentralPort"/>. Bring the dedicated stack up
+///     with <c>Docker/docker-compose.yml</c> (container <c>otopcua-sql-blackhole</c>, host port 14333).</para>
+///     <para><b>Env-gated; offline skip is the normal outcome.</b> With <see cref="EndpointEnvVar"/> unset
+///     nothing here opens a socket, shells out to docker, or waits — the suite is instant and green on the
+///     macOS dev box, exactly like every other <c>*.IntegrationTests</c> live gate in this tree.</para>
+///     <para>
+///         <b>Operator run recipe</b> (docker runs on the shared Linux host; drive it from this VM):
+///         <code>
+///         # 1. Deploy + start the DEDICATED disposable mssql (NOT the shared 14330 server):
+///         lmxopcua-fix sync sql-blackhole
+///         lmxopcua-fix up   sql-blackhole            # container otopcua-sql-blackhole on :14333, self-seeds
+///
+///         # 2. Point the gate at the dedicated container + tell it how to reach docker:
+///         export SQL_BLACKHOLE_ENDPOINT=10.100.0.35,14333      # the dedicated container; MUST NOT be ,14330
+///         export SQL_BLACKHOLE_PASSWORD=Blackhole_dev_pw1      # matches the compose SA password
+///         export SQL_BLACKHOLE_DOCKER_SSH=dohertj2@10.100.0.35 # empty ⇒ run docker locally
+///
+///         # 3. Run just this gate:
+///         dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests \
+///           --filter "FullyQualifiedName~SqlBlackholeTimeoutTests"
+///
+///         # 4. Tear the disposable stack down:
+///         lmxopcua-fix down sql-blackhole
+///         </code>
+///     </para>
+/// </summary>
+[Trait("Category", "Integration")]
+[Trait("Category", "Blackhole")]
+public sealed class SqlBlackholeTimeoutTests
+{
+    /// <summary>Endpoint (<c>host,port</c>) of the DEDICATED disposable container. Absent ⇒ offline skip.</summary>
+    private const string EndpointEnvVar = "SQL_BLACKHOLE_ENDPOINT";
+
+    /// <summary>SA password for the dedicated container. Required by the run.</summary>
+    private const string PasswordEnvVar = "SQL_BLACKHOLE_PASSWORD";
+
+    /// <summary>SQL login. Defaults to <c>sa</c>.</summary>
+    private const string UserEnvVar = "SQL_BLACKHOLE_USER";
+
+    /// <summary>
+    ///     Optional SSH target the <c>docker pause</c>/<c>unpause</c> runs through (e.g.
+    ///     <c>dohertj2@10.100.0.35</c>), since <c>docker -H ssh://</c> does not work from this VM. Empty ⇒
+    ///     run docker locally.
+    /// </summary>
+    private const string DockerSshEnvVar = "SQL_BLACKHOLE_DOCKER_SSH";
+
+    /// <summary>
+    ///     The ONLY container this gate ever pauses — a compile-time constant, never derived from the
+    ///     environment. The shared central SQL Server is not named this, so no env misconfiguration can
+    ///     cause this gate to freeze shared infra. It matches <c>Docker/docker-compose.yml</c>'s
+    ///     <c>container_name</c>.
+    /// </summary>
+    private const string ContainerName = "otopcua-sql-blackhole";
+
+    /// <summary>
+    ///     The rig's shared always-on SQL Server port (hosts ConfigDb for the whole rig). An endpoint on
+    ///     this port is refused with a loud skip — the dedicated container must be on a different port.
+    /// </summary>
+    private const string SharedCentralPort = "14330";
+
+    /// <summary>The database the seed lives in; created if missing (never dropped).</summary>
+    private const string Database = "SqlBlackholeFixture";
+
+    /// <summary>The seeded key-value table + its columns and the one baseline key.</summary>
+    private const string Schema = "sqlpoll";
+    private const string KeyValueTable = "TagValues";
+    private const string KeyColumn = "tag_name";
+    private const string ValueColumn = "num_value";
+    private const string TimestampColumn = "sample_ts";
+    private const string BaselineKey = "Line1.Speed";
+    private const double BaselineValue = 42.0;
+
+    /// <summary>The RawPath the driver serves the baseline key under.</summary>
+    private const string RawPath = "Sql/Line1/Speed";
+
+    // ── deadlines chosen so client-side and server-side backstops are DISTINGUISHABLE ──
+
+    /// <summary>Client-side wall-clock ceiling on one group. A frozen peer must surface BadTimeout at ≈ this.</summary>
+    private static readonly TimeSpan OperationTimeout = TimeSpan.FromSeconds(3);
+
+    /// <summary>
+    ///     ADO.NET server-side backstop — set an order of magnitude ABOVE <see cref="OperationTimeout"/> so
+    ///     that if the client-side cancellation were broken and only this fired, the read would return at
+    ///     ≈ 30 s and the upper-bound assertion would catch it.
+    /// </summary>
+    private static readonly TimeSpan CommandTimeout = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    ///     Upper bound on the post-pause read. Comfortably above <see cref="OperationTimeout"/> (docker
+    ///     pause propagation + pooled-connection validation + thread-pool scheduling) yet comfortably below
+    ///     <see cref="CommandTimeout"/> — so a return here proves the CLIENT-side bound fired, not the
+    ///     server-side backstop.
+    /// </summary>
+    private static readonly TimeSpan UpperBound = TimeSpan.FromSeconds(15);
+
+    /// <summary>Hard cap on the test's own wait, so a genuinely wedged implementation FAILS rather than hangs CI.</summary>
+    private static readonly TimeSpan HardCap = TimeSpan.FromSeconds(60);
+
+    /// <summary>
+    ///     Start reading against the dedicated mssql, <c>docker pause</c> it mid-poll, and assert the next
+    ///     read surfaces <see cref="SqlStatusCodes.BadTimeout"/> within <see cref="OperationTimeout"/> (a
+    ///     client-side linked-CTS cancellation, not the poll thread wedging and not the server-side
+    ///     backstop), that the driver degrades (health <see cref="DriverState.Degraded"/> + host
+    ///     <see cref="HostState.Stopped"/>), and that after <c>docker unpause</c> a subsequent poll recovers
+    ///     to <see cref="DriverState.Healthy"/> / <see cref="HostState.Running"/>.
+    /// </summary>
+    [Fact]
+    public async Task PausedServer_surfacesBadTimeout_withinDeadline_notWedged()
+    {
+        var ep = Environment.GetEnvironmentVariable(EndpointEnvVar);
+        Assert.SkipWhen(
+            string.IsNullOrWhiteSpace(ep),
+            $"{EndpointEnvVar} not set — offline skip. Bring up Docker/docker-compose.yml (dedicated " +
+            $"container {ContainerName} on :14333) and export {EndpointEnvVar}/{PasswordEnvVar} to run this gate.");
+
+        var password = Environment.GetEnvironmentVariable(PasswordEnvVar);
+        Assert.SkipWhen(
+            string.IsNullOrWhiteSpace(password),
+            $"{PasswordEnvVar} not set — the blackhole gate needs the dedicated container's SA password.");
+
+        var (host, port) = ParseEndpoint(ep!);
+
+        // ── SAFETY GUARD: refuse to run against the rig's shared central SQL Server. ──
+        // The pause target (ContainerName) is a constant and could never be the shared container regardless,
+        // but refusing the shared PORT stops us even connecting-and-polling against shared infra.
+        Assert.SkipWhen(
+            string.Equals(port, SharedCentralPort, StringComparison.Ordinal),
+            $"{EndpointEnvVar} resolves to port {SharedCentralPort} — that is the rig's SHARED always-on SQL " +
+            "Server (ConfigDb for the whole rig), which this destructive gate must NEVER pause. Point it at " +
+            $"the dedicated {ContainerName} container (host port 14333), not the shared server.");
+
+        var user = Environment.GetEnvironmentVariable(UserEnvVar) is { Length: > 0 } named ? named : "sa";
+        var ssh = Environment.GetEnvironmentVariable(DockerSshEnvVar);
+        var ct = TestContext.Current.CancellationToken;
+
+        // Seed defensively (create-if-missing) so the baseline poll is Good even if compose never ran seed.sql.
+        await SeedAsync(host, port, user, password!, ct);
+
+        await using var driver = await StartDriverAsync(host, port, user, password!, ct);
+
+        // Good baseline — prove the endpoint answers before we freeze it.
+        var baseline = (await ReadOnceAsync(driver, ct)).ShouldHaveSingleItem();
+        baseline.StatusCode.ShouldBe(SqlStatusCodes.Good);
+        Convert.ToDouble(baseline.Value).ShouldBe(BaselineValue);
+        driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+
+        var paused = false;
+        try
+        {
+            // Freeze the container mid-poll: its SQL process is SIGSTOPped, so the pooled connection's next
+            // ExecuteReaderAsync hangs against a live-but-silent socket — the frozen-peer wedge exactly.
+            await RunShellCommandAsync(DockerCommand("pause", ssh), ct);
+            paused = true;
+
+            var clock = Stopwatch.StartNew();
+            // Bounded by HardCap so a wedged implementation FAILS here instead of hanging CI.
+            var snapshots = await ReadOnceAsync(driver, ct).WaitAsync(HardCap, ct);
+            clock.Stop();
+
+            var frozen = snapshots.ShouldHaveSingleItem();
+            frozen.StatusCode.ShouldBe(
+                SqlStatusCodes.BadTimeout,
+                "a frozen database must surface BadTimeout, not a stale Good or another Bad class");
+
+            // The wall-clock contract: returned at ≈ operationTimeout (client-side cancellation), NOT at
+            // commandTimeout (server-side backstop) and NOT never. UpperBound < CommandTimeout is what
+            // distinguishes a working client-side abort from a broken one that only the backstop rescued.
+            clock.Elapsed.ShouldBeLessThan(
+                UpperBound,
+                $"BadTimeout must arrive on the client-side {OperationTimeout.TotalSeconds:0}s bound, not the " +
+                $"server-side {CommandTimeout.TotalSeconds:0}s CommandTimeout backstop");
+            clock.Elapsed.ShouldBeLessThan(CommandTimeout, "the server-side backstop must not be what fired");
+            clock.Elapsed.ShouldBeGreaterThanOrEqualTo(
+                OperationTimeout - TimeSpan.FromSeconds(2),
+                "returning far faster than operationTimeout would mean a fast-fail misclassified as BadTimeout");
+
+            // Driver-level classification, not just the reader: an all-BadTimeout poll degrades health and
+            // stops the host (SqlDriver.ObservePollOutcome), even though the reader returned successfully.
+            driver.GetHealth().State.ShouldBe(DriverState.Degraded);
+            driver.GetHostStatuses().ShouldHaveSingleItem().State.ShouldBe(HostState.Stopped);
+        }
+        finally
+        {
+            if (paused)
+            {
+                // ALWAYS restore the container once paused — never leave a frozen container behind.
+                await RunShellCommandAsync(DockerCommand("unpause", ssh), CancellationToken.None);
+            }
+        }
+
+        // Recovery: after unpause a subsequent poll must return Good and the whole stack must recover.
+        await PollUntilGoodAsync(driver, ct);
+        driver.GetHealth().State.ShouldBe(DriverState.Healthy);
+        driver.GetHostStatuses().ShouldHaveSingleItem().State.ShouldBe(HostState.Running);
+    }
+
+    // ── helpers ──
+
+    /// <summary>Runs one poll of the single baseline reference.</summary>
+    private static Task<IReadOnlyList<DataValueSnapshot>> ReadOnceAsync(SqlDriver driver, CancellationToken ct)
+        => driver.ReadAsync([RawPath], ct);
+
+    /// <summary>Polls until a Good snapshot returns, or the recovery deadline elapses.</summary>
+    private static async Task PollUntilGoodAsync(SqlDriver driver, CancellationToken ct)
+    {
+        var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(45);
+        while (DateTime.UtcNow < deadline)
+        {
+            var snapshot = (await ReadOnceAsync(driver, ct)).ShouldHaveSingleItem();
+            if (snapshot.StatusCode == SqlStatusCodes.Good) return;
+            await Task.Delay(TimeSpan.FromSeconds(1), ct);
+        }
+
+        throw new Xunit.Sdk.XunitException(
+            "the driver never recovered to a Good poll within 45 s of docker unpause — the poll loop did not survive the outage");
+    }
+
+    /// <summary>Builds a driver over the dedicated container, warmed to hold a pooled connection.</summary>
+    private static async Task<SqlDriver> StartDriverAsync(
+        string host, string port, string user, string password, CancellationToken ct)
+    {
+        var options = new SqlDriverOptions
+        {
+            RawTags = [KeyValueTag()],
+            OperationTimeout = OperationTimeout,
+            CommandTimeout = CommandTimeout,
+        };
+
+        var driver = new SqlDriver(
+            options,
+            driverInstanceId: "sql-blackhole",
+            dialect: new SqlServerDialect(),
+            connectionString: DriverConnectionString(host, port, user, password));
+
+        try
+        {
+            await driver.InitializeAsync("{}", ct);
+        }
+        catch
+        {
+            await driver.DisposeAsync();
+            throw;
+        }
+
+        return driver;
+    }
+
+    /// <summary>The KeyValue tag over the seeded <c>sqlpoll.TagValues</c> baseline row.</summary>
+    private static RawTagEntry KeyValueTag()
+    {
+        var config = new JsonObject
+        {
+            ["driver"] = "Sql",
+            ["model"] = "KeyValue",
+            ["table"] = $"{Schema}.{KeyValueTable}",
+            ["keyColumn"] = KeyColumn,
+            ["keyValue"] = BaselineKey,
+            ["valueColumn"] = ValueColumn,
+            ["timestampColumn"] = TimestampColumn,
+        };
+        return new RawTagEntry(RawPath, config.ToJsonString(), WriteIdempotent: false);
+    }
+
+    /// <summary>
+    ///     The driver's connection string. <c>Min Pool Size=1</c> keeps a warm pooled connection so the
+    ///     post-pause read reuses it and hangs inside <c>ExecuteReaderAsync</c> — exercising the
+    ///     query-execution wedge (the S7 bug class) rather than a fresh connect.
+    /// </summary>
+    private static string DriverConnectionString(string host, string port, string user, string password)
+        => new SqlConnectionStringBuilder
+        {
+            DataSource = $"{host},{port}",
+            InitialCatalog = Database,
+            UserID = user,
+            Password = password,
+            TrustServerCertificate = true,
+            Encrypt = false,
+            ConnectTimeout = 5,
+            MinPoolSize = 1,
+        }.ConnectionString;
+
+    /// <summary>Creates the database (if missing) and (re)seeds the baseline schema + row.</summary>
+    private static async Task SeedAsync(
+        string host, string port, string user, string password, CancellationToken ct)
+    {
+        var master = new SqlConnectionStringBuilder
+        {
+            DataSource = $"{host},{port}",
+            InitialCatalog = "master",
+            UserID = user,
+            Password = password,
+            TrustServerCertificate = true,
+            Encrypt = false,
+            ConnectTimeout = 10,
+        }.ConnectionString;
+
+        await using (var connection = new SqlConnection(master))
+        {
+            await connection.OpenAsync(ct);
+            await ExecuteAsync(connection, $"IF DB_ID(N'{Database}') IS NULL CREATE DATABASE [{Database}];", ct);
+        }
+
+        var db = new SqlConnectionStringBuilder(master) { InitialCatalog = Database }.ConnectionString;
+        await using (var connection = new SqlConnection(db))
+        {
+            await connection.OpenAsync(ct);
+            foreach (var statement in SeedStatements())
+                await ExecuteAsync(connection, statement, ct);
+        }
+    }
+
+    /// <summary>The seed, one batch per statement (T-SQL requires CREATE SCHEMA to be first in its batch).</summary>
+    private static IEnumerable<string> SeedStatements()
+    {
+        yield return $"IF SCHEMA_ID(N'{Schema}') IS NULL EXEC(N'CREATE SCHEMA [{Schema}]');";
+        yield return $"DROP TABLE IF EXISTS [{Schema}].[{KeyValueTable}];";
+        yield return $"""
+            CREATE TABLE [{Schema}].[{KeyValueTable}] (
+                [{KeyColumn}]       nvarchar(128) NOT NULL,
+                [{ValueColumn}]     float         NULL,
+                [{TimestampColumn}] datetime2(3)  NOT NULL
+            );
+            """;
+        yield return string.Create(CultureInfo.InvariantCulture, $"""
+            INSERT INTO [{Schema}].[{KeyValueTable}] ([{KeyColumn}], [{ValueColumn}], [{TimestampColumn}])
+            VALUES (N'{BaselineKey}', {BaselineValue}, '2026-07-24T10:00:00');
+            """);
+    }
+
+    /// <summary>Runs one non-query batch.</summary>
+    private static async Task ExecuteAsync(SqlConnection connection, string sql, CancellationToken ct)
+    {
+        await using var command = connection.CreateCommand();
+        command.CommandText = sql;
+        await command.ExecuteNonQueryAsync(ct);
+    }
+
+    /// <summary>Splits a <c>host,port</c> endpoint. A missing port is rejected — the guard needs the port.</summary>
+    private static (string Host, string Port) ParseEndpoint(string endpoint)
+    {
+        var parts = endpoint.Split(',', 2, StringSplitOptions.TrimEntries);
+        if (parts.Length != 2 || parts[1].Length == 0)
+        {
+            throw new Xunit.Sdk.XunitException(
+                $"{EndpointEnvVar} must be 'host,port' (e.g. 10.100.0.35,14333) so the shared-server port " +
+                $"guard can run — got '{endpoint}'.");
+        }
+
+        return (parts[0], parts[1]);
+    }
+
+    /// <summary>
+    ///     Builds the docker command for the OWNED container only. The verb and the constant container name
+    ///     are the sole inputs — the target is never taken from the environment.
+    /// </summary>
+    private static string DockerCommand(string verb, string? ssh)
+    {
+        var docker = $"docker {verb} {ContainerName}";
+        return string.IsNullOrWhiteSpace(ssh) ? docker : $"ssh {ssh} \"{docker}\"";
+    }
+
+    /// <summary>Runs a shell one-liner through the login shell, exactly as the S7 blackhole gate does.</summary>
+    private static async Task RunShellCommandAsync(string command, CancellationToken ct)
+    {
+        using var proc = new Process
+        {
+            StartInfo = new ProcessStartInfo("/bin/sh", $"-c \"{command.Replace("\"", "\\\"")}\"")
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false,
+            },
+        };
+        proc.Start();
+        await proc.WaitForExitAsync(ct);
+        proc.ExitCode.ShouldBe(0, $"docker command failed: {await proc.StandardError.ReadToEndAsync(ct)}");
+        // Give the pause/unpause a moment to take effect before the next poll tick.
+        await Task.Delay(TimeSpan.FromSeconds(1), ct);
+    }
+}