fix(worker): review 2026-10-21 stream E — cache eviction, phased ReadBulk, poll/STA fault handling

- RemoveItem/Unregister evict value + write-completion caches (leak under churn)
- ReadBulk snapshot path: add/advise all, one shared deadline, then tear down
- alarm poll faults only after 3 consecutive failures; transient ones logged
- AlarmCommandHandler disposes primary when standby build throws
- StaRuntime pumps between queued work items
- WRK-02: post-start STA loop death surfaces as StaHung WorkerFault
- WRK-09: AppDomain unhandled-exception hook, exit code 7
- RETRIGGER documented as reserved, never synthesized
- TST-05: live facts for Ping/GetSessionState/GetWorkerInfo/DrainEvents/ShutdownWorker

src/ZB.MOM.WW.MxGateway.Worker/Sta/StaRuntime.cs

@@ -8,7 +8,7 @@ namespace ZB.MOM.WW.MxGateway.Worker.Sta;
 public sealed class StaRuntime : IDisposable
 {
     private readonly IStaComApartmentInitializer comApartmentInitializer;
-    private readonly StaMessagePump messagePump;
+    private readonly IStaMessagePump messagePump;
     private readonly ConcurrentQueue<IStaWorkItem> commandQueue = new();
     private readonly AutoResetEvent commandWakeEvent = new(false);
     private readonly ManualResetEventSlim startedEvent = new(false);
@@ -20,6 +20,8 @@ public sealed class StaRuntime : IDisposable
     private bool startRequested;
     private bool shutdownRequested;
     private Exception? startupException;
+    private Exception? loopFault;
+    private Action<Exception>? loopFaultHandler;
     private long lastActivityUtcTicks;
     private bool comInitialized;
 
@@ -41,6 +43,21 @@ public sealed class StaRuntime : IDisposable
         IStaComApartmentInitializer comApartmentInitializer,
         StaMessagePump messagePump,
         TimeSpan idlePumpInterval)
+        : this(comApartmentInitializer, (IStaMessagePump)messagePump, idlePumpInterval)
+    {
+    }
+
+    /// <summary>
+    /// Initializes a new instance of <see cref="StaRuntime"/> over any
+    /// <see cref="IStaMessagePump"/> (test seam).
+    /// </summary>
+    /// <param name="comApartmentInitializer">COM apartment initializer.</param>
+    /// <param name="messagePump">Message pump for the STA thread.</param>
+    /// <param name="idlePumpInterval">Interval for idle message pump waits.</param>
+    internal StaRuntime(
+        IStaComApartmentInitializer comApartmentInitializer,
+        IStaMessagePump messagePump,
+        TimeSpan idlePumpInterval)
     {
         this.comApartmentInitializer = comApartmentInitializer
             ?? throw new ArgumentNullException(nameof(comApartmentInitializer));
@@ -79,6 +96,59 @@ public sealed class StaRuntime : IDisposable
     /// </summary>
     public bool IsRunning => startedEvent.IsSet && !stoppedEvent.IsSet;
 
+    /// <summary>
+    /// Gets the exception that terminated the STA loop after a successful
+    /// <see cref="Start"/>, or <see langword="null"/> while the loop is healthy
+    /// (or stopped by a requested shutdown). Startup failures are not reported
+    /// here — <see cref="Start"/> throws them.
+    /// </summary>
+    public Exception? LoopFault
+    {
+        get
+        {
+            lock (gate)
+            {
+                return loopFault;
+            }
+        }
+    }
+
+    /// <summary>
+    /// Registers the callback invoked once, on the dying STA thread, when the
+    /// STA loop terminates with an exception after a successful
+    /// <see cref="Start"/>. When the loop has already died the callback runs
+    /// immediately on the calling thread, so a fault that lands between
+    /// <see cref="Start"/> and registration is never lost. Only one handler is
+    /// kept; a later registration replaces an earlier one.
+    /// </summary>
+    /// <remarks>
+    /// Without this, a post-start loop death (a failed
+    /// <c>MsgWaitForMultipleObjectsEx</c>, or an exception escaping a dispatched
+    /// window message) left the worker process alive with no STA: every later
+    /// command queued forever and the gateway only learned of it through the
+    /// heartbeat watchdog, if at all.
+    /// </remarks>
+    /// <param name="handler">Callback receiving the terminating exception.</param>
+    public void RegisterLoopFaultHandler(Action<Exception> handler)
+    {
+        if (handler is null)
+        {
+            throw new ArgumentNullException(nameof(handler));
+        }
+
+        Exception? alreadyFaulted;
+        lock (gate)
+        {
+            loopFaultHandler = handler;
+            alreadyFaulted = loopFault;
+        }
+
+        if (alreadyFaulted is not null)
+        {
+            handler(alreadyFaulted);
+        }
+    }
+
     /// <summary>
     ///     Pumps any pending Windows messages on the calling thread and refreshes
     ///     the STA activity timestamp. Intended for commands that synchronously
@@ -176,6 +246,14 @@ public sealed class StaRuntime : IDisposable
 
         lock (gate)
         {
+            if (loopFault is not null)
+            {
+                // The loop is gone; nothing would ever dequeue this item.
+                return Task.FromException<T>(new InvalidOperationException(
+                    "The worker STA loop terminated unexpectedly.",
+                    loopFault));
+            }
+
             if (shutdownRequested)
             {
                 return Task.FromException<T>(new StaRuntimeShutdownException());
@@ -244,12 +322,14 @@ public sealed class StaRuntime : IDisposable
 
     private void ThreadMain()
     {
+        bool started = false;
         try
         {
             StaThreadId = Thread.CurrentThread.ManagedThreadId;
             comApartmentInitializer.Initialize();
             comInitialized = true;
             MarkActivity();
+            started = true;
             startedEvent.Set();
 
             while (!IsShutdownRequested())
@@ -264,8 +344,40 @@ public sealed class StaRuntime : IDisposable
         }
         catch (Exception exception)
         {
-            startupException = exception;
-            startedEvent.Set();
+            if (!started)
+            {
+                // Start() is blocked on startedEvent and rethrows this.
+                startupException = exception;
+                startedEvent.Set();
+            }
+            else
+            {
+                // Post-start loop death. Start() returned long ago, so nothing
+                // would ever read startupException: record the fault where
+                // InvokeAsync and the registered handler see it. Set under the
+                // gate so every item enqueued before it is still in the queue
+                // for the finally block's CancelQueuedCommands, and every
+                // InvokeAsync after it fails fast instead of queueing forever.
+                Action<Exception>? handler;
+                lock (gate)
+                {
+                    loopFault = exception;
+                    handler = loopFaultHandler;
+                }
+
+                if (handler is not null)
+                {
+                    try
+                    {
+                        handler(exception);
+                    }
+                    catch
+                    {
+                        // The handler reports the fault; a failure inside it
+                        // must not skip the apartment cleanup below.
+                    }
+                }
+            }
         }
         finally
         {
@@ -292,6 +404,17 @@ public sealed class StaRuntime : IDisposable
             MarkActivity();
             workItem.Execute();
             MarkActivity();
+
+            // Pump between work items, not only once the whole queue is
+            // drained: a burst of queued commands (bulk callers, the alarm
+            // poll, a reconnecting client replaying subscriptions) would
+            // otherwise hold every inbound COM callback — OnDataChange,
+            // OnWriteComplete — behind the entire backlog. PeekMessage with an
+            // empty queue is cheap, so the per-item cost is negligible.
+            if (!commandQueue.IsEmpty)
+            {
+                messagePump.PumpPendingMessages();
+            }
         }
     }