feat(cluster): simultaneous-cold-start split-brain guard (#33)
Port OtOpcUa's bootstrap guard (lmxopcua d1dac87f) to ScadaBridge's BuildHocon bootstrap. Both pair nodes are self-first seeds, so a true simultaneous cold start (shared site power event) races FirstSeedNodeProcess on both and forms two 1-node clusters that never merge. Opt-in dark switch ScadaBridge:Cluster:BootstrapGuard:Enabled (default off, guard-off behavior byte-identical). When on: BuildHocon emits an empty seed list so Akka does not auto-join, and ClusterBootstrapCoordinator (IHostedService, registered in both the Central and Site composition roots) picks the join order from ClusterBootstrapGuard's pure decision core — the lower canonical host:port is the founder (self-first, forms immediately); the higher node TCP-probes the founder up to PartnerProbeSeconds and joins peer-first if reachable, else self-first (cold-start-alone preserved). Decides before a single JoinSeedNodes, never re-forms mid-handshake. Review notes carried over: case-insensitive founder tie-break; fail-fast validation of probe timings when enabled; higher-node-cold-start-alone covered by a real-ActorSystem test. 21 unit + 5 real-cluster tests (incl. the headline both-cold-start-together-form-one-cluster).
This commit is contained in:
Joseph Doherty
+147
@@ -0,0 +1,147 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.ClusterInfrastructure.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for <see cref="ClusterBootstrapGuard"/> — the pure decision core of the
|
||||
/// simultaneous-cold-start split-brain guard (Gitea #33). The runtime probe + JoinSeedNodes wiring
|
||||
/// lives in <c>ClusterBootstrapCoordinator</c> and is covered by the real-ActorSystem
|
||||
/// <c>ClusterBootstrapCoordinatorTests</c> in the integration suite.
|
||||
/// </summary>
|
||||
public class ClusterBootstrapGuardTests
|
||||
{
|
||||
private const string A = "akka.tcp://scadabridge@scadabridge-site-a-a:8082";
|
||||
private const string B = "akka.tcp://scadabridge@scadabridge-site-a-b:8082";
|
||||
|
||||
[Fact]
|
||||
public void Lower_address_node_is_the_founder_and_needs_no_probe()
|
||||
{
|
||||
// scadabridge-site-a-a < scadabridge-site-a-b, so on node-a the guard makes it the founder.
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-site-a-a", 8082);
|
||||
|
||||
Assert.True(role.Applies);
|
||||
Assert.True(role.IsFounder);
|
||||
Assert.Equal(new[] { A, B }, role.SelfFirstOrder);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Higher_address_node_is_not_the_founder_and_targets_the_partner_for_probing()
|
||||
{
|
||||
// On node-b the partner is node-a (the lower/founder).
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
|
||||
|
||||
Assert.True(role.Applies);
|
||||
Assert.False(role.IsFounder);
|
||||
Assert.Equal("scadabridge-site-a-a", role.PartnerHost);
|
||||
Assert.Equal(8082, role.PartnerPort);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tie_break_is_symmetric_regardless_of_seed_order()
|
||||
{
|
||||
// Both nodes must agree on who founds no matter how each lists its seeds.
|
||||
var onLower = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-site-a-a", 8082);
|
||||
var onHigher = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
|
||||
|
||||
Assert.True(onLower.IsFounder);
|
||||
Assert.False(onHigher.IsFounder);
|
||||
// Exactly one founder.
|
||||
Assert.True(onLower.IsFounder ^ onHigher.IsFounder);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Tie_break_is_case_insensitive_so_a_casing_mismatch_cannot_make_both_founders()
|
||||
{
|
||||
// The two sides' seed config differ only in host casing (review note 1): if the tie-break
|
||||
// were case-sensitive, both could conclude they are the founder and re-open the split.
|
||||
var lowerSelf = "akka.tcp://scadabridge@SITE-A-A:8082";
|
||||
var higherSelf = "akka.tcp://scadabridge@site-a-b:8082";
|
||||
|
||||
var onLower = ClusterBootstrapGuard.Analyze(new[] { lowerSelf, higherSelf }, "SITE-A-A", 8082);
|
||||
var onHigher = ClusterBootstrapGuard.Analyze(new[] { higherSelf, lowerSelf }, "site-a-b", 8082);
|
||||
|
||||
Assert.True(onLower.IsFounder ^ onHigher.IsFounder);
|
||||
Assert.True(onLower.IsFounder); // "site-a-a" < "site-a-b" ordinal-ignore-case
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Higher_node_joins_the_founder_when_partner_is_reachable()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
|
||||
|
||||
// Reachable ⇒ peer-first ⇒ JoinSeedNodeProcess ⇒ joins the founder, never self-forms.
|
||||
Assert.Equal(new[] { A, B }, ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: true)); // partner (node-a) first
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Higher_node_forms_alone_when_partner_is_unreachable()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { B, A }, "scadabridge-site-a-b", 8082);
|
||||
|
||||
// Unreachable after the window ⇒ partner is dead ⇒ self-first ⇒ cold-start-alone preserved.
|
||||
Assert.Equal(new[] { B, A }, ClusterBootstrapGuard.HigherNodeOrder(role, partnerReachable: false)); // self (node-b) first
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Guard_does_not_apply_to_a_single_seed_node()
|
||||
{
|
||||
// A single-node install lists exactly one seed (itself) — not a pair, guard is inert.
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { "akka.tcp://scadabridge@lone:8081" }, "lone", 8081);
|
||||
|
||||
Assert.False(role.Applies);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Guard_does_not_apply_when_self_is_absent_from_the_two_seeds()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { A, B }, "scadabridge-central-a", 8081);
|
||||
|
||||
Assert.False(role.Applies);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Guard_does_not_apply_to_three_or_more_seeds()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(
|
||||
new[] { A, B, "akka.tcp://scadabridge@scadabridge-site-a-c:8082" }, "scadabridge-site-a-a", 8082);
|
||||
|
||||
Assert.False(role.Applies);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Guard_does_not_apply_when_a_seed_is_malformed()
|
||||
{
|
||||
var role = ClusterBootstrapGuard.Analyze(new[] { A, "not-a-seed" }, "scadabridge-site-a-a", 8082);
|
||||
|
||||
Assert.False(role.Applies);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("akka.tcp://scadabridge@scadabridge-site-a-a:8082", "scadabridge-site-a-a", 8082)]
|
||||
[InlineData("akka.tcp://scadabridge@10.0.0.5:8082/", "10.0.0.5", 8082)]
|
||||
[InlineData(" akka.tcp://scadabridge@host.lan:1234 ", "host.lan", 1234)]
|
||||
public void TryParse_extracts_host_and_port(string seed, string expectedHost, int expectedPort)
|
||||
{
|
||||
Assert.True(ClusterBootstrapGuard.TryParse(seed, out var host, out var port));
|
||||
Assert.Equal(expectedHost, host);
|
||||
Assert.Equal(expectedPort, port);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData("garbage")]
|
||||
[InlineData("akka.tcp://scadabridge@host")] // no port
|
||||
public void TryParse_rejects_malformed_seeds(string seed)
|
||||
{
|
||||
Assert.False(ClusterBootstrapGuard.TryParse(seed, out _, out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Port_participates_in_the_tie_break_when_hosts_are_equal()
|
||||
{
|
||||
// Shared-host loopback pair (test rig): same host, different ports — port breaks the tie.
|
||||
const string low = "akka.tcp://scadabridge@127.0.0.1:8081";
|
||||
const string high = "akka.tcp://scadabridge@127.0.0.1:8082";
|
||||
|
||||
Assert.True(ClusterBootstrapGuard.Analyze(new[] { low, high }, "127.0.0.1", 8081).IsFounder);
|
||||
Assert.False(ClusterBootstrapGuard.Analyze(new[] { high, low }, "127.0.0.1", 8082).IsFounder);
|
||||
}
|
||||
}
|
||||
+64
@@ -215,4 +215,68 @@ public class ClusterOptionsValidatorTests
|
||||
Assert.Contains("StableAfter", result.FailureMessage);
|
||||
Assert.Contains("HeartbeatInterval", result.FailureMessage);
|
||||
}
|
||||
|
||||
// ---- BootstrapGuard timing validation (Gitea #33) ----
|
||||
|
||||
[Fact]
|
||||
public void BootstrapGuard_disabled_with_zero_timings_passes_validation()
|
||||
{
|
||||
// The knobs are inert when the guard is off, so a disabled guard must never block a boot —
|
||||
// even if the timing values are nonsense (0 here).
|
||||
var options = ValidOptions();
|
||||
options.BootstrapGuard = new ClusterBootstrapGuardOptions
|
||||
{
|
||||
Enabled = false,
|
||||
PartnerProbeSeconds = 0,
|
||||
PartnerProbeIntervalMs = 0,
|
||||
ProbeConnectTimeoutMs = 0,
|
||||
};
|
||||
|
||||
var result = new ClusterOptionsValidator().Validate(null, options);
|
||||
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BootstrapGuard_enabled_with_default_timings_passes_validation()
|
||||
{
|
||||
var options = ValidOptions();
|
||||
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true };
|
||||
|
||||
var result = new ClusterOptionsValidator().Validate(null, options);
|
||||
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BootstrapGuard_enabled_with_nonpositive_probe_seconds_fails_validation()
|
||||
{
|
||||
// A zero PartnerProbeSeconds silently degrades the guard to "never wait, always conclude the
|
||||
// partner is dead" — the higher node forms alone immediately and re-opens the split. Reject it.
|
||||
var options = ValidOptions();
|
||||
options.BootstrapGuard = new ClusterBootstrapGuardOptions { Enabled = true, PartnerProbeSeconds = 0 };
|
||||
|
||||
var result = new ClusterOptionsValidator().Validate(null, options);
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("PartnerProbeSeconds", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BootstrapGuard_enabled_with_nonpositive_interval_or_timeout_fails_validation()
|
||||
{
|
||||
var options = ValidOptions();
|
||||
options.BootstrapGuard = new ClusterBootstrapGuardOptions
|
||||
{
|
||||
Enabled = true,
|
||||
PartnerProbeIntervalMs = 0,
|
||||
ProbeConnectTimeoutMs = -1,
|
||||
};
|
||||
|
||||
var result = new ClusterOptionsValidator().Validate(null, options);
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("PartnerProbeIntervalMs", result.FailureMessage);
|
||||
Assert.Contains("ProbeConnectTimeoutMs", result.FailureMessage);
|
||||
}
|
||||
}
|
||||
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.Host;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Cluster;
|
||||
|
||||
/// <summary>
|
||||
/// Real-ActorSystem tests for the simultaneous-cold-start split-brain guard (Gitea #33):
|
||||
/// <see cref="ClusterBootstrapCoordinator"/> + <see cref="ClusterBootstrapGuard"/>, driven exactly as
|
||||
/// production drives them — the node's ActorSystem is built from the production <c>BuildHocon</c> output
|
||||
/// with the guard enabled (which emits an EMPTY seed list, so Akka does not auto-join), then the real
|
||||
/// coordinator issues the reachability-gated <c>JoinSeedNodes</c>. This subsystem has a history of bugs
|
||||
/// invisible to pure unit tests, so the load-bearing correctness properties are asserted against running
|
||||
/// nodes, mirroring <see cref="SelfFirstSeedBootstrapTests"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Ephemeral loopback ports share host <c>127.0.0.1</c>, so the port breaks the guard's ordinal
|
||||
/// <c>host:port</c> tie — <c>Min(port)</c> is the founder, <c>Max(port)</c> is the higher (probing) node.
|
||||
/// </remarks>
|
||||
public sealed class ClusterBootstrapCoordinatorTests : IAsyncLifetime
|
||||
{
|
||||
private readonly List<ActorSystem> _systems = new();
|
||||
private readonly List<ClusterBootstrapCoordinator> _coordinators = new();
|
||||
|
||||
/// <summary>Starts a node with the bootstrap guard ENABLED, wired exactly as the composition roots
|
||||
/// do: the ActorSystem is built with an empty HOCON seed list (guard on) and the real coordinator
|
||||
/// issues the reachability-gated join. Seeds are listed self-first (as every shipped config is); the
|
||||
/// guard, not the order, decides founder vs. joiner.</summary>
|
||||
private async Task<ActorSystem> StartGuardedNodeAsync(int selfPort, int peerPort, int probeSeconds = 4)
|
||||
{
|
||||
var self = $"akka.tcp://scadabridge@127.0.0.1:{selfPort}";
|
||||
var peer = $"akka.tcp://scadabridge@127.0.0.1:{peerPort}";
|
||||
var nodeOptions = new NodeOptions { Role = "Central", NodeHostname = "127.0.0.1", RemotingPort = selfPort };
|
||||
var clusterOptions = new ClusterOptions
|
||||
{
|
||||
SeedNodes = new List<string> { self, peer },
|
||||
BootstrapGuard = new ClusterBootstrapGuardOptions
|
||||
{
|
||||
Enabled = true,
|
||||
PartnerProbeSeconds = probeSeconds,
|
||||
PartnerProbeIntervalMs = 250,
|
||||
ProbeConnectTimeoutMs = 500,
|
||||
},
|
||||
StableAfter = TimeSpan.FromSeconds(3),
|
||||
HeartbeatInterval = TimeSpan.FromMilliseconds(500),
|
||||
FailureDetectionThreshold = TimeSpan.FromSeconds(2),
|
||||
MinNrOfMembers = 1,
|
||||
};
|
||||
|
||||
var hocon = AkkaHostedService.BuildHocon(
|
||||
nodeOptions, clusterOptions, new[] { "Central" },
|
||||
TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(3));
|
||||
var system = ActorSystem.Create("scadabridge", ConfigurationFactory.ParseString(hocon));
|
||||
_systems.Add(system);
|
||||
|
||||
var coordinator = new ClusterBootstrapCoordinator(
|
||||
() => system, Options.Create(clusterOptions), Options.Create(nodeOptions),
|
||||
NullLogger<ClusterBootstrapCoordinator>.Instance);
|
||||
_coordinators.Add(coordinator);
|
||||
await coordinator.StartAsync(CancellationToken.None);
|
||||
return system;
|
||||
}
|
||||
|
||||
private static async Task<bool> WaitForUpAsync(ActorSystem system, int expected, TimeSpan timeout)
|
||||
{
|
||||
var cluster = Akka.Cluster.Cluster.Get(system);
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expected) return true;
|
||||
await Task.Delay(200);
|
||||
}
|
||||
return cluster.State.Members.Count(m => m.Status == MemberStatus.Up) >= expected;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The FOUNDER (lower address) with the guard on forms a cluster alone when its partner is dead —
|
||||
/// exactly as un-guarded self-first does. The guard must not regress the preferred founder.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Founder_forms_alone_when_partner_is_dead()
|
||||
{
|
||||
var low = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var high = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var founderPort = Math.Min(low, high);
|
||||
var deadPeerPort = Math.Max(low, high); // nothing listening
|
||||
|
||||
var system = await StartGuardedNodeAsync(founderPort, deadPeerPort);
|
||||
|
||||
Assert.True(await WaitForUpAsync(system, 1, TimeSpan.FromSeconds(30)),
|
||||
"the founder (lower address) must self-form immediately when its partner is down");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// THE LOAD-BEARING CASE. The HIGHER node with the guard on must still cold-start ALONE when its
|
||||
/// partner is genuinely dead: it probes the (dead) founder, times out, and falls back to self-first.
|
||||
/// If the guard's peer-first logic were wrong, this node would hang in JoinSeedNodeProcess forever —
|
||||
/// the very regression a naive "always let the lower node found" rule would introduce.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Higher_node_forms_alone_when_partner_is_dead_after_probing()
|
||||
{
|
||||
var low = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var high = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var higherPort = Math.Max(low, high);
|
||||
var deadFounderPort = Math.Min(low, high); // the founder is dead
|
||||
|
||||
var system = await StartGuardedNodeAsync(higherPort, deadFounderPort, probeSeconds: 3);
|
||||
|
||||
// Must come Up AFTER the probe window (~3s) expires and it falls back to self-first.
|
||||
Assert.True(await WaitForUpAsync(system, 1, TimeSpan.FromSeconds(30)),
|
||||
"the higher node must fall back to self-first and form alone once the dead partner never answers the probe");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// FALSIFIABILITY CONTROL for the test above: the higher node does NOT form alone during the probe
|
||||
/// window — it is genuinely waiting/probing, not self-forming immediately (which would mean the
|
||||
/// guard is inert). If this ever starts seeing a member before the window, the probe is not gating.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Higher_node_does_not_form_before_the_probe_window_expires()
|
||||
{
|
||||
var low = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var high = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var higherPort = Math.Max(low, high);
|
||||
var deadFounderPort = Math.Min(low, high);
|
||||
|
||||
var system = await StartGuardedNodeAsync(higherPort, deadFounderPort, probeSeconds: 10);
|
||||
|
||||
// Well within the 10s probe window: still no cluster, because it is probing the dead founder.
|
||||
Assert.False(await WaitForUpAsync(system, 1, TimeSpan.FromSeconds(3)),
|
||||
"the higher node must probe for the founder, not self-form immediately");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The higher node JOINS a live founder rather than forming a second cluster: start the founder,
|
||||
/// then the higher node — its probe finds the founder reachable, it commits peer-first, and the
|
||||
/// pair converges to ONE cluster of two.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Higher_node_joins_a_live_founder()
|
||||
{
|
||||
var low = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var high = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var founderPort = Math.Min(low, high);
|
||||
var higherPort = Math.Max(low, high);
|
||||
|
||||
var founder = await StartGuardedNodeAsync(founderPort, higherPort);
|
||||
Assert.True(await WaitForUpAsync(founder, 1, TimeSpan.FromSeconds(30)),
|
||||
"the founder must be Up before the higher node probes it");
|
||||
|
||||
var higher = await StartGuardedNodeAsync(higherPort, founderPort);
|
||||
|
||||
Assert.True(await WaitForUpAsync(higher, 2, TimeSpan.FromSeconds(60)),
|
||||
"the higher node must join the founder, forming one cluster of two");
|
||||
Assert.True(await WaitForUpAsync(founder, 2, TimeSpan.FromSeconds(60)),
|
||||
"the founder must see the higher node as a member of ITS cluster");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// THE HEADLINE #33 CASE. Both nodes cold-start truly simultaneously with the guard on. Without the
|
||||
/// guard each would run FirstSeedNodeProcess, race, and form its own 1-node cluster (split brain).
|
||||
/// With it, the founder forms and the higher node probes-then-joins, so the pair converges to
|
||||
/// EXACTLY ONE cluster of two — no two oldest, no dual singletons.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Both_nodes_cold_starting_together_form_exactly_one_cluster()
|
||||
{
|
||||
var low = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var high = TwoNodeClusterFixture.GetFreeTcpPort();
|
||||
var founderPort = Math.Min(low, high);
|
||||
var higherPort = Math.Max(low, high);
|
||||
|
||||
// Start both without serialization — as a shared power event does.
|
||||
var founderTask = StartGuardedNodeAsync(founderPort, higherPort);
|
||||
var higherTask = StartGuardedNodeAsync(higherPort, founderPort);
|
||||
var founder = await founderTask;
|
||||
var higher = await higherTask;
|
||||
|
||||
Assert.True(await WaitForUpAsync(founder, 2, TimeSpan.FromSeconds(60)),
|
||||
"the pair must converge to one 2-member cluster, not two 1-member clusters");
|
||||
Assert.True(await WaitForUpAsync(higher, 2, TimeSpan.FromSeconds(60)),
|
||||
"the higher node must be in the SAME cluster as the founder");
|
||||
}
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public async Task DisposeAsync()
|
||||
{
|
||||
foreach (var c in _coordinators)
|
||||
{
|
||||
try { await c.StopAsync(CancellationToken.None); } catch { /* teardown */ }
|
||||
}
|
||||
foreach (var s in _systems)
|
||||
{
|
||||
try { await s.Terminate().WaitAsync(TimeSpan.FromSeconds(10)); } catch { /* teardown */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user