diff --git a/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/SiteAlarmLiveCacheServiceTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/SiteAlarmLiveCacheServiceTests.cs index 0aead6e8..e0f164bf 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/SiteAlarmLiveCacheServiceTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/SiteAlarmLiveCacheServiceTests.cs @@ -136,6 +136,30 @@ public class SiteAlarmLiveCacheServiceTests : TestKit return service; } + /// + /// Row-40 house pattern for an ABSENCE assertion (mirrors the Medium-wave + /// SiteAlarmAggregatorActorTests.AssertHoldsFor treatment in this same test + /// project): proves every check in keeps holding throughout + /// by polling, rather than a single blind + /// Thread.Sleep(window) followed by one end-of-window sample. A bare + /// sleep-then-assert cannot distinguish "the mechanism correctly stayed quiet" from "the + /// mechanism just hadn't run yet when the one sample was taken", and it can't catch a + /// value that flips wrong and then flips back before that single sample. This fails + /// immediately the first tick any check goes false, and only returns once the full + /// window has elapsed with every check still true. + /// + private static void AssertHoldsFor(TimeSpan window, params (Func Holds, string Because)[] checks) + { + var deadline = DateTime.UtcNow + window; + while (true) + { + foreach (var (holds, because) in checks) + Assert.True(holds(), because); + if (DateTime.UtcNow >= deadline) return; + Thread.Sleep(20); + } + } + [Fact] public void Seed_FanOut_Publishes_The_Instances_That_Failed_To_Answer() { @@ -450,8 +474,8 @@ public class SiteAlarmLiveCacheServiceTests : TestKit sub.Dispose(); AwaitAssert(() => Assert.False(service.IsLive(SiteId)), TimeSpan.FromSeconds(10)); var startsAfterStop = factory.GetOrCreateCount; - Thread.Sleep(300); - Assert.Equal(startsAfterStop, factory.GetOrCreateCount); // no self-heal restart fired + AssertHoldsFor(TimeSpan.FromMilliseconds(300), + (() => factory.GetOrCreateCount == startsAfterStop, "no self-heal restart fired")); } // ── WP2.3: IsLive tracks the STREAM, not merely "a snapshot was published once" ── @@ -468,8 +492,8 @@ public class SiteAlarmLiveCacheServiceTests : TestKit // the cache is not live — the page must keep polling. Pre-fix IsLive flipped true // here and the page grafted a never-updating snapshot over fresh poll data. AwaitCondition(() => service.GetCurrentAlarms(SiteId) is not null, TimeSpan.FromSeconds(5)); - Thread.Sleep(400); - Assert.False(service.IsLive(SiteId)); + AssertHoldsFor(TimeSpan.FromMilliseconds(400), + (() => !service.IsLive(SiteId), "must stay not-live until the site accepts the stream")); // The site accepts it → live. AwaitCondition(() => factory.Client.Connects.Count >= 1, TimeSpan.FromSeconds(5)); diff --git a/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/DataConnectionActorBatchTests.cs b/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/DataConnectionActorBatchTests.cs index 7d309df8..30fd13ae 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/DataConnectionActorBatchTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/DataConnectionActorBatchTests.cs @@ -177,11 +177,73 @@ public class DataConnectionActorBatchTests : TestKit ExpectMsg(u => u.Quality == QualityCode.Bad, TimeSpan.FromSeconds(5)); ExpectMsg(TimeSpan.FromSeconds(5)); + // This is a genuine real-time rate assertion (how many retry rounds a backoff + // schedule produces inside a fixed wall-clock window), not a synchronization + // primitive standing in for "wait until X happens" — proving the floor (>= 2 + // rounds) is unavoidably tied to actually waiting out the window. What the old + // blind Thread.Sleep(2000) + single end-of-window sample could NOT catch is a + // regression to full-rate (no-backoff) retrying: it would still land somewhere + // in [2, 8] by luck if the window elapsed while under load, or it would blow + // past 8 well before the 2s mark and nobody would know until the final sample. + // AssertRoundsWithinWindow polls throughout and fails immediately the first time + // the ceiling is breached, so a no-backoff regression is caught fast instead of + // being masked by (or racing) the fixed window. var initialCalls = adapter.SubscribeBatches.Count; - Thread.Sleep(2000); - var rounds = adapter.SubscribeBatches.Count - initialCalls; + AssertRoundsWithinWindow(TimeSpan.FromSeconds(2), + () => adapter.SubscribeBatches.Count - initialCalls, + minInclusive: 2, maxInclusive: 8, + context: "dead-device probe backoff (floor 150ms, ceiling 600ms)"); + } - Assert.InRange(rounds, 2, 8); + /// + /// Row-40 house pattern adapted for a RATE assertion: proves a monotonically-increasing + /// count stays within [minInclusive, maxInclusive] across the whole + /// , rather than a single blind sleep-then-sample at the end. + /// The upper bound is enforced continuously (fails fast the instant it is exceeded, + /// well before the window elapses); the lower bound can only be confirmed once the + /// full window has elapsed, since proving "at least N happened by wall-clock time T" + /// requires actually waiting until T. + /// + private static void AssertRoundsWithinWindow( + TimeSpan window, Func currentRounds, int minInclusive, int maxInclusive, string context) + { + var deadline = DateTime.UtcNow + window; + while (true) + { + var rounds = currentRounds(); + Assert.True(rounds <= maxInclusive, + $"{context}: {rounds} rounds already fired, exceeding the ceiling of {maxInclusive} " + + "before the window even elapsed — backoff is not slowing retries down."); + if (DateTime.UtcNow >= deadline) + { + Assert.True(rounds >= minInclusive, + $"{context}: only {rounds} rounds fired in the window (need >= {minInclusive})."); + return; + } + Thread.Sleep(20); + } + } + + /// + /// Row-40 house pattern for an ABSENCE assertion (mirrors the Medium-wave + /// SiteAlarmAggregatorActorTests.AssertHoldsFor treatment): proves + /// keeps being true for the whole by + /// polling, rather than a single blind Thread.Sleep(window) followed by one + /// end-of-window sample. A bare sleep-then-assert cannot distinguish "the mechanism + /// correctly stayed quiet" from "the mechanism just hadn't run yet when the one sample + /// was taken", and it can't catch a value that flips wrong and then flips back before + /// that single sample. This fails immediately the first time + /// goes false, and only returns once the full window has elapsed with it still true. + /// + private static void AssertHoldsFor(TimeSpan window, Func holds, string because) + { + var deadline = DateTime.UtcNow + window; + while (true) + { + Assert.True(holds(), because); + if (DateTime.UtcNow >= deadline) return; + Thread.Sleep(20); + } } [Fact] @@ -203,16 +265,18 @@ public class DataConnectionActorBatchTests : TestKit for (var i = 0; i < 20; i++) adapter.ValueCallback!("tag1", new TagValue(i, QualityCode.Good, DateTimeOffset.UtcNow)); - Thread.Sleep(400); - Assert.DoesNotContain(_health.ReceivedCalls(), c => c.GetMethodInfo().Name == "UpdateTagQuality"); + AssertHoldsFor(TimeSpan.FromMilliseconds(400), + () => !_health.ReceivedCalls().Any(c => c.GetMethodInfo().Name == "UpdateTagQuality"), + "20 unchanged-quality repeats must never push UpdateTagQuality"); // One genuine transition → exactly one coalesced push. adapter.ValueCallback!("tag1", new TagValue(1, QualityCode.Bad, DateTimeOffset.UtcNow)); AwaitCondition( () => _health.ReceivedCalls().Count(c => c.GetMethodInfo().Name == "UpdateTagQuality") == 1, TimeSpan.FromSeconds(3)); - Thread.Sleep(300); - Assert.Single(_health.ReceivedCalls(), c => c.GetMethodInfo().Name == "UpdateTagQuality"); + AssertHoldsFor(TimeSpan.FromMilliseconds(300), + () => _health.ReceivedCalls().Count(c => c.GetMethodInfo().Name == "UpdateTagQuality") == 1, + "the coalesced push must not repeat"); } [Fact] diff --git a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/ScadaBridgeTelemetryTests.cs b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/ScadaBridgeTelemetryTests.cs index c17b290b..a18c4875 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/ScadaBridgeTelemetryTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/ScadaBridgeTelemetryTests.cs @@ -15,9 +15,37 @@ public class ScadaBridgeTelemetryTests Assert.Equal("ZB.MOM.WW.ScadaBridge", ScadaBridgeTelemetry.MeterName); } + /// + /// Register row 32 class: SetQueueDepthProvider writes the SAME process-global + /// slot QueueDepthGaugeCollection (StoreAndForward.Tests) exists to serialize + /// access to — this test was mutating it with no clear and no [Collection], + /// leaving a stray () => 5 provider registered in the process for the rest of + /// this assembly's run. Since the meter/instrument state is a single static field on + /// ScadaBridgeTelemetry (Commons), the leak is scoped to whatever process this + /// assembly's test host runs in — it cannot reach StoreAndForward.Tests (a separate + /// assembly with its own test-host process), but it IS visible within Host.Tests, whose + /// MetricsEndpointTests boots a real in-process host (WebApplicationFactory + /// shares this AppDomain, not a child process) and scrapes /metrics, which + /// enumerates every registered gauge including this one. + /// + /// Honesty note on what was and wasn't proven: a search of this assembly (and the whole + /// test tree) found no test today that asserts on the VALUE of + /// scadabridge.store_and_forward.queue.depth from Host.Tests or any other + /// same-process reader — MetricsEndpointTests only asserts the endpoint returns + /// Prometheus-shaped text, not this series' content, and the only test that DOES assert + /// on the value (QueueDepthGaugeTests) lives in StoreAndForward.Tests, a distinct + /// test-host process the leak cannot reach. So there is no currently-reproducible flake + /// to demonstrate here (no fork of this test with the leak left in has an assertion to + /// break) — the defect is a landmine for a FUTURE same-process reader, not a live one. + /// The fix (compare-and-clear in finally) closes the landmine regardless; a + /// [Collection] is intentionally not added since no second same-process reader of + /// this slot exists to serialize against as of this writing — re-run this search before + /// removing the guard below if that changes. + /// [Fact] public void EmitHelpers_DoNotThrow() { + Func queueDepthProvider = () => 5; var ex = Record.Exception(() => { ScadaBridgeTelemetry.RecordDeploymentApplied(); @@ -27,9 +55,20 @@ public class ScadaBridgeTelemetryTests ScadaBridgeTelemetry.LiveAlarmAggregatorStarted(); ScadaBridgeTelemetry.LiveAlarmAggregatorStopped(); ScadaBridgeTelemetry.RecordLiveAlarmStreamReconnect(); - ScadaBridgeTelemetry.SetQueueDepthProvider(() => 5); + ScadaBridgeTelemetry.SetQueueDepthProvider(queueDepthProvider); }); - Assert.Null(ex); + try + { + Assert.Null(ex); + } + finally + { + // Compare-and-clear: only removes OUR delegate from the process-global slot, so + // a legitimate concurrent registrant (a real StoreAndForwardService started + // elsewhere in this process) is never stomped — mirrors + // ScadaBridgeTelemetry.ClearQueueDepthProvider's own doc'd contract. + ScadaBridgeTelemetry.ClearQueueDepthProvider(queueDepthProvider); + } } } diff --git a/tests/ZB.MOM.WW.ScadaBridge.KpiHistory.Tests/KpiHistoryRecorderActorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.KpiHistory.Tests/KpiHistoryRecorderActorTests.cs index a9f31382..65f2261c 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.KpiHistory.Tests/KpiHistoryRecorderActorTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.KpiHistory.Tests/KpiHistoryRecorderActorTests.cs @@ -497,6 +497,28 @@ public class KpiHistoryRecorderActorTests : TestKit NullLogger.Instance))); } + /// + /// Row-40 house pattern for an ABSENCE assertion (mirrors the Medium-wave + /// SiteAlarmAggregatorActorTests.AssertHoldsFor treatment): proves + /// keeps being true for the whole by + /// polling, rather than a single blind Thread.Sleep(window) followed by one + /// end-of-window sample. A bare sleep-then-assert cannot distinguish "the mechanism + /// correctly stayed quiet" from "the mechanism just hadn't run yet when the one sample + /// was taken", and it can't catch a value that flips wrong and then flips back before + /// that single sample. This fails immediately the first time + /// goes false, and only returns once the full window has elapsed with it still true. + /// + private static void AssertHoldsFor(TimeSpan window, Func holds, string because) + { + var deadline = DateTime.UtcNow + window; + while (true) + { + Assert.True(holds(), because); + if (DateTime.UtcNow >= deadline) return; + Thread.Sleep(20); + } + } + [Fact] public void SampleTick_WritesHealthySourceSamples_AndThrowingSourceDoesNotAbortTick() { @@ -1040,13 +1062,13 @@ public class KpiHistoryRecorderActorTests : TestKit interval: TimeSpan.FromMilliseconds(50)); // …and then skip: no slices are ever folded. - Thread.Sleep(300); - Assert.Equal(0, repository.FoldCount); + AssertHoldsFor(TimeSpan.FromMilliseconds(300), () => repository.FoldCount == 0, + "watermark inside the lookback tail must skip the backfill entirely"); // A second BackfillTick is a no-op (the done-latch is set). actor.Tell(KpiHistoryRecorderActor.BackfillTick.Instance); - Thread.Sleep(300); - Assert.Equal(0, repository.FoldCount); + AssertHoldsFor(TimeSpan.FromMilliseconds(300), () => repository.FoldCount == 0, + "the done-latch must make a second BackfillTick a no-op"); } [Fact] diff --git a/tests/ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests/StoreAndForwardServiceTests.cs b/tests/ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests/StoreAndForwardServiceTests.cs index c18a8a74..3ebe9b03 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests/StoreAndForwardServiceTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.StoreAndForward.Tests/StoreAndForwardServiceTests.cs @@ -786,6 +786,29 @@ public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable while (Interlocked.CompareExchange(ref target, value, current) != current); } + /// + /// Register row-40 house pattern for an ABSENCE assertion (mirrors the Medium-wave + /// SiteAlarmAggregatorActorTests.AssertHoldsFor treatment, ported here async since + /// this class's facts run on the async test thread): proves keeps + /// being true for the whole by polling, rather than a single + /// blind await Task.Delay(window) followed by one end-of-window sample. A bare + /// delay-then-assert cannot distinguish "the mechanism correctly stayed quiet" from "the + /// mechanism just hadn't run yet when the one sample was taken", and it can't catch a + /// value that flips wrong and then flips back before that single sample. This fails + /// immediately the first time goes false, and only returns once + /// the full window has elapsed with it still true throughout. + /// + private static async Task AssertHoldsForAsync(TimeSpan window, Func holds, string because) + { + var deadline = DateTime.UtcNow + window; + while (true) + { + Assert.True(holds(), because); + if (DateTime.UtcNow >= deadline) return; + await Task.Delay(20); + } + } + // ── Task 12: deferToSweep enqueue mode ── /// @@ -888,8 +911,13 @@ public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable service.TriggerSweep(); // the clobbering kick var stop = service.StopAsync(); - await Task.Delay(300); - Assert.False(stop.IsCompleted); // pre-fix: StopAsync already returned (awaited the no-op) + // The real sweep is blocked on `release` behind `entered`, so `stop` is genuinely + // unable to complete during this window (not merely "hasn't yet") — proving it stays + // incomplete throughout, not just at one end-of-delay sample, is what actually + // distinguishes "waited for the real sweep" from "pre-fix: awaited the no-op and + // returned instantly, then this single sample raced whether that had happened yet". + await AssertHoldsForAsync(TimeSpan.FromMilliseconds(300), () => !stop.IsCompleted, + "pre-fix: StopAsync already returned (awaited the no-op)"); release.TrySetResult(); await stop.WaitAsync(TimeSpan.FromSeconds(5)); // drains the real sweep promptly once released