From bec07378cc17536fa808e895f635d8bd9aab9b91 Mon Sep 17 00:00:00 2001
From: Joseph Doherty <dohejw01@gmail.com>
Date: Mon, 9 Nov 2026 13:44:02 -0500
Subject: [PATCH] test(deployment): de-flake unreachable-site reconcile breaker
 test; assert structurally, not by 2 s wall clock (ScadaBridge#92)

Under a loaded full-project run the serial prepare of 6 instances with a 300 ms
reconcile deadline took 2.6 s and tripped the fixed 2 s cap (5/10 full runs
failed under CPU load). Deadline raised to 2 s so it cannot expire before the
query is sent; the elapsed bound is now derived from the deadline (must beat
(n-1) deadlines) while the one-query assertion proves the breaker.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
---
 .../DeploySiteAsyncTests.cs                       | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tests/ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests/DeploySiteAsyncTests.cs b/tests/ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests/DeploySiteAsyncTests.cs
index d4fb73b7..463c98a1 100644
--- a/tests/ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests/DeploySiteAsyncTests.cs
+++ b/tests/ZB.MOM.WW.ScadaBridge.DeploymentManager.Tests/DeploySiteAsyncTests.cs
@@ -434,7 +434,14 @@ public class DeploySiteAsyncTests : TestKit
         var commActor = Sys.ActorOf(Props.Create(() =>
             new SilentQuerySiteActor(() => Interlocked.Increment(ref queryCount))));
 
-        var reconcileTimeout = TimeSpan.FromMilliseconds(300);
+        // The deadline must dwarf scheduler/JIT jitter (#92): the reconcile deadline
+        // covers the repo reads and the Ask itself, so a deadline of a few hundred ms
+        // can expire before the query is even sent when the full suite runs in
+        // parallel, and the old fixed 2 s wall-clock cap was blown by ~2.3 s of
+        // starvation on top of 6 x 300 ms. The breaker is proven STRUCTURALLY by the
+        // query count below; the elapsed check is only a coarse "not six deadlines"
+        // bound whose threshold is derived from the deadline, not a magic number.
+        var reconcileTimeout = TimeSpan.FromSeconds(2);
         var service = CreateService(commActor, maxParallelism: 3, reconcileTimeout: reconcileTimeout);
 
         var sw = Stopwatch.StartNew();
@@ -446,8 +453,10 @@ public class DeploySiteAsyncTests : TestKit
         // The circuit-breaker opened after the first unanswered reconcile.
         Assert.Equal(1, Volatile.Read(ref queryCount));
 
-        // Prepare therefore cost ONE reconcile deadline, not six.
-        Assert.True(sw.Elapsed < TimeSpan.FromSeconds(2),
+        // Prepare therefore cost ONE reconcile deadline, not six: without the breaker
+        // the serial prepare alone would take >= instanceCount x deadline.
+        var withoutBreakerFloor = instanceCount * reconcileTimeout;
+        Assert.True(sw.Elapsed < withoutBreakerFloor - reconcileTimeout,
             $"bulk prepare took {sw.ElapsedMilliseconds} ms — the per-batch reconcile bound is not holding");
 
         // A skipped reconcile is not a failed deploy: every instance still went out.