fix(worker): resilient failover switch; FIPS-safe synthetic GUID; dup-reference guard + tests (Worker-026..028, Worker.Tests-031..033)

src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/FailoverAlarmConsumerTests.cs

@@ -27,6 +27,15 @@ public sealed class FailoverAlarmConsumerTests
         public event EventHandler<MxAlarmTransitionEvent>? AlarmTransitionEmitted;
 
         public bool ThrowOnPoll = true;
+
+        /// <summary>
+        ///     When set, <see cref="PollOnce"/> throws
+        ///     <see cref="OutOfMemoryException"/> instead of a
+        ///     <see cref="System.Runtime.InteropServices.COMException"/>, to
+        ///     exercise the OOM-safe exception filter (Worker.Tests-032).
+        /// </summary>
+        public bool ThrowOutOfMemoryOnPoll;
+
         public int Polls;
 
         /// <summary>
@@ -48,6 +57,11 @@ public sealed class FailoverAlarmConsumerTests
         public void PollOnce()
         {
             Polls++;
+            if (ThrowOutOfMemoryOnPoll)
+            {
+                throw new OutOfMemoryException("simulated allocation failure");
+            }
+
             if (ThrowOnPoll)
             {
                 throw new System.Runtime.InteropServices.COMException("boom", unchecked((int)0x80004005));
@@ -75,6 +89,15 @@ public sealed class FailoverAlarmConsumerTests
 
         public bool Subscribed;
 
+        /// <summary>
+        ///     When set, <see cref="SnapshotActiveAlarms"/> throws — modeling a
+        ///     priming-snapshot failure during failover (Worker-026).
+        /// </summary>
+        public bool ThrowOnSnapshot;
+
+        /// <summary>Number of <see cref="SnapshotActiveAlarms"/> calls.</summary>
+        public int SnapshotCalls;
+
         public void Subscribe(string s) => Subscribed = true;
 
         public void PollOnce() { }
@@ -83,7 +106,16 @@ public sealed class FailoverAlarmConsumerTests
 
         public int AcknowledgeByName(string n, string p, string gr, string c, string a, string b, string d, string e) => 22;
 
-        public IReadOnlyList<MxAlarmSnapshotRecord> SnapshotActiveAlarms() => Array.Empty<MxAlarmSnapshotRecord>();
+        public IReadOnlyList<MxAlarmSnapshotRecord> SnapshotActiveAlarms()
+        {
+            SnapshotCalls++;
+            if (ThrowOnSnapshot)
+            {
+                throw new InvalidOperationException("priming snapshot failed");
+            }
+
+            return Array.Empty<MxAlarmSnapshotRecord>();
+        }
 
         public void Dispose() { }
 
@@ -291,4 +323,151 @@ public sealed class FailoverAlarmConsumerTests
         sut.ProbeOnce();  // clean 3 → failback
         Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
     }
+
+    /// <summary>
+    ///     Worker-026 regression: when the standby's priming
+    ///     <c>SnapshotActiveAlarms</c> throws during failover, the switch must
+    ///     still (a) fire <c>ProviderModeChanged</c> so the gateway learns the
+    ///     feed went degraded, (b) leave <see cref="FailoverAlarmConsumer.Mode"/>
+    ///     in Subtag, and (c) not rethrow out of <c>PollOnce</c> (which on the
+    ///     real STA would land in the poll loop's trailing catch and permanently
+    ///     stop alarm delivery).
+    /// </summary>
+    [Fact]
+    public void Failover_WhenStandbyPrimingSnapshotThrows_StillRaisesModeChangeAndDoesNotRethrow()
+    {
+        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
+        StubStandby standby = new StubStandby { ThrowOnSnapshot = true };
+        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
+        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
+
+        List<AlarmProviderModeChange> changes = new List<AlarmProviderModeChange>();
+        sut.ProviderModeChanged += (_, e) => changes.Add(e);
+
+        // threshold=1 → the Subscribe failure triggers the switch, which primes
+        // the standby snapshot (throwing). The exception must be contained.
+        Exception? escaped = Record.Exception(() => sut.Subscribe(@"\\HOST\Galaxy!Area"));
+
+        Assert.Null(escaped);
+        Assert.Single(changes);
+        Assert.Equal(AlarmProviderMode.Subtag, changes[0].Mode);
+        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
+        Assert.True(standby.SnapshotCalls >= 1); // priming was attempted
+
+        // A subsequent degraded PollOnce (standby.PollOnce + ProbeOnce) must also
+        // not rethrow the snapshot failure.
+        Exception? pollEscaped = Record.Exception(() => sut.PollOnce());
+        Assert.Null(pollEscaped);
+    }
+
+    /// <summary>
+    ///     Worker-026 regression: when a <c>ProviderModeChanged</c> subscriber's
+    ///     handler throws (modeling the AlarmCommandHandler's event-queue enqueue
+    ///     overflowing at capacity), the switch must still take effect and the
+    ///     exception must not escape the switch path into the poll loop.
+    /// </summary>
+    [Fact]
+    public void Failover_WhenModeChangedHandlerThrows_SwitchStillTakesEffectAndDoesNotRethrow()
+    {
+        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
+        StubStandby standby = new StubStandby();
+        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
+        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
+
+        int handlerInvocations = 0;
+        sut.ProviderModeChanged += (_, _) =>
+        {
+            handlerInvocations++;
+            throw new InvalidOperationException("subscriber handler blew up");
+        };
+
+        Exception? escaped = Record.Exception(() => sut.Subscribe(@"\\HOST\Galaxy!Area"));
+
+        Assert.Null(escaped);
+        Assert.Equal(1, handlerInvocations);          // the event still fired
+        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode); // the switch still took effect
+    }
+
+    /// <summary>
+    ///     Worker.Tests-031 regression: with a non-zero
+    ///     <see cref="FailoverSettings.ProbeIntervalSeconds"/>, two back-to-back
+    ///     <c>ProbeOnce</c> calls must throttle — the second falls inside the
+    ///     interval and must NOT re-poll the primary. Two consecutive calls
+    ///     reliably fall inside any interval of one second or more, so this needs
+    ///     no injected clock.
+    /// </summary>
+    [Fact]
+    public void ProbeOnce_WithNonZeroInterval_ThrottlesSecondProbeWithinInterval()
+    {
+        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = true };
+        StubStandby standby = new StubStandby();
+        // stableProbes high enough that a single clean probe cannot fail back,
+        // so Mode stays Subtag and ProbeOnce remains the throttled path.
+        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 3600, stableProbes: 5);
+        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
+
+        sut.Subscribe(@"\\HOST\Galaxy!Area"); // threshold=1 → switch to Subtag
+        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
+
+        primary.ThrowOnPoll = false; // primary healthy so a probe would poll cleanly
+
+        sut.ProbeOnce(); // first probe runs: re-polls the primary
+        int pollsAfterFirstProbe = primary.Polls;
+        Assert.Equal(1, pollsAfterFirstProbe);
+
+        sut.ProbeOnce(); // within the 3600s interval → throttled, must NOT re-poll
+
+        Assert.Equal(pollsAfterFirstProbe, primary.Polls);
+        Assert.Equal(AlarmProviderMode.Subtag, sut.Mode);
+    }
+
+    /// <summary>
+    ///     Worker.Tests-032 regression: <c>RunPrimary</c>'s
+    ///     <c>when (ex is not OutOfMemoryException)</c> filter must let an
+    ///     <see cref="OutOfMemoryException"/> propagate rather than swallowing it
+    ///     and counting it toward the failover threshold. No mode change must
+    ///     fire — a fatal allocation failure is not a clean degraded handoff.
+    /// </summary>
+    [Fact]
+    public void RunPrimary_WhenPrimaryThrowsOutOfMemory_PropagatesAndDoesNotFailOver()
+    {
+        FlakyPrimary primary = new FlakyPrimary { ThrowOnPoll = false, ThrowOutOfMemoryOnPoll = true };
+        StubStandby standby = new StubStandby();
+        FailoverSettings settings = new FailoverSettings(threshold: 1, probeIntervalSeconds: 0, stableProbes: 1);
+        using FailoverAlarmConsumer sut = new FailoverAlarmConsumer(primary, standby, settings);
+
+        bool modeChanged = false;
+        sut.ProviderModeChanged += (_, _) => modeChanged = true;
+
+        sut.Subscribe(@"\\HOST\Galaxy!Area"); // Subscribe path does not poll; no throw here
+
+        Assert.Throws<OutOfMemoryException>(() => sut.PollOnce());
+        Assert.False(modeChanged);
+        Assert.Equal(AlarmProviderMode.Alarmmgr, sut.Mode);
+    }
+
+    /// <summary>
+    ///     Worker.Tests-032 regression: <see cref="FailoverSettings"/> clamps
+    ///     sub-1 <c>threshold</c> and <c>stableProbes</c> (and sub-0
+    ///     <c>probeIntervalSeconds</c>) to their safe minimums so a misconfigured
+    ///     bind cannot change failover semantics.
+    /// </summary>
+    [Theory]
+    [InlineData(0, 0, 0, 1, 0, 1)]
+    [InlineData(-5, -5, -5, 1, 0, 1)]
+    [InlineData(3, 7, 2, 3, 7, 2)]
+    public void FailoverSettings_ClampsSubMinimumValues(
+        int threshold,
+        int probeInterval,
+        int stableProbes,
+        int expectedThreshold,
+        int expectedProbeInterval,
+        int expectedStableProbes)
+    {
+        FailoverSettings settings = new FailoverSettings(threshold, probeInterval, stableProbes);
+
+        Assert.Equal(expectedThreshold, settings.Threshold);
+        Assert.Equal(expectedProbeInterval, settings.ProbeIntervalSeconds);
+        Assert.Equal(expectedStableProbes, settings.StableProbes);
+    }
 }