From 15018e4158482f794bf53328791eb3f5dd0a7c06 Mon Sep 17 00:00:00 2001
From: Joseph Doherty <dohejw01@gmail.com>
Date: Mon, 13 Jul 2026 09:50:40 -0400
Subject: [PATCH] fix(siteruntime): faulted expression-eval task clears
 _evalInFlight instead of killing the AlarmActor trigger (plan R2-03 T3)

---
 .../Actors/AlarmActor.cs                      | 32 ++++++++++++++++++-
 .../Actors/AlarmActorTests.cs                 | 27 ++++++++++++++++
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmActor.cs b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmActor.cs
index 019e39f3..477c1fb1 100644
--- a/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmActor.cs
+++ b/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Actors/AlarmActor.cs
@@ -186,6 +186,10 @@ public class AlarmActor : ReceiveActor
 
         // Handle the off-dispatcher trigger-expression evaluation result (P1).
         Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
+
+        // Handle a faulted off-dispatcher evaluation task (N2) — clears in-flight
+        // instead of stranding the alarm trigger with an unhandled Status.Failure.
+        Receive<ExpressionEvalFailed>(HandleExpressionEvalFailed);
     }
 
     /// <inheritdoc />
@@ -551,7 +555,8 @@ public class AlarmActor : ReceiveActor
             }
         }, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
             ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
-            success: r => new ExpressionEvalResult(r));
+            success: r => new ExpressionEvalResult(r),
+            failure: ex => new ExpressionEvalFailed(ex));
     }
 
     /// <summary>
@@ -576,6 +581,22 @@ public class AlarmActor : ReceiveActor
         if (_evalPending) { _evalPending = false; StartExpressionEvaluation(); }
     }
 
+    /// <summary>
+    /// Applies a faulted off-dispatcher evaluation task (N2): logs the fault, then
+    /// delegates to the false result (per the class contract — a throwing expression
+    /// is treated as false so the state machine still runs, an Active alarm clears)
+    /// which clears in-flight and drains any pending evaluation. Without this the
+    /// alarm trigger would be permanently parked with _evalInFlight stuck true.
+    /// </summary>
+    private void HandleExpressionEvalFailed(ExpressionEvalFailed msg)
+    {
+        _healthCollector?.IncrementAlarmError();
+        _logger.LogError(msg.Cause,
+            "Alarm {Alarm} trigger-expression evaluation task faulted on {Instance}; treated as false",
+            _alarmName, _instanceName);
+        HandleExpressionEvalResult(new ExpressionEvalResult(false));
+    }
+
     /// <summary>
     /// HiLo level evaluator: returns the most-severe matching band for the
     /// given value. Severity order checked from highest to lowest so that a
@@ -795,6 +816,15 @@ public class AlarmActor : ReceiveActor
     /// actor thread.
     /// </summary>
     private sealed record ExpressionEvalResult(bool Result);
+
+    /// <summary>
+    /// Piped back to self when the off-dispatcher evaluation TASK itself faults
+    /// (e.g. ObjectDisposedException from a disposed ScriptExecutionScheduler
+    /// during shutdown) — the inner body's catch never sees that. Without this
+    /// mapping the actor receives an unhandled Status.Failure and _evalInFlight
+    /// stays true forever, permanently parking the expression trigger (N2).
+    /// </summary>
+    internal sealed record ExpressionEvalFailed(Exception Cause);
 }
 
 internal enum RateOfChangeDirection { Either, Rising, Falling }
diff --git a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/AlarmActorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/AlarmActorTests.cs
index 2707607d..3d5d2050 100644
--- a/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/AlarmActorTests.cs
+++ b/tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/AlarmActorTests.cs
@@ -1054,4 +1054,31 @@ public class AlarmActorTests : TestKit, IDisposable
         var change = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
         Assert.Equal(AlarmState.Active, change.State);
     }
+
+    [Fact]
+    public void ExpressionEvalTaskFault_DoesNotKillTheAlarmTrigger()
+    {
+        var expr = CompileRawTriggerExpression("true");
+        var alarmConfig = new ResolvedAlarm
+        {
+            CanonicalName = "ExprFaultAlarm",
+            TriggerType = "Expression",
+            TriggerConfiguration = "{\"expression\":\"true\"}",
+            PriorityLevel = 1
+        };
+        var instanceProbe = CreateTestProbe();
+        var alarm = ActorOf(Props.Create(() => new AlarmActor(
+            "ExprFaultAlarm", "Pump1", instanceProbe.Ref, alarmConfig,
+            null, _sharedLibrary, _options,
+            NullLogger<AlarmActor>.Instance, expr)));
+
+        // Simulate the faulted scheduler task: must be handled (logged, in-flight
+        // cleared, false applied) — NOT an unhandled Status.Failure.
+        alarm.Tell(new AlarmActor.ExpressionEvalFailed(new ObjectDisposedException("scheduler")));
+
+        // The trigger must still be alive: the next change evaluates and activates.
+        alarm.Tell(new AttributeValueChanged("Pump1", "A", "A", 1, "Good", DateTimeOffset.UtcNow));
+        var change = instanceProbe.ExpectMsg<AlarmStateChanged>(TimeSpan.FromSeconds(10));
+        Assert.Equal(AlarmState.Active, change.State);
+    }
 }