fix(worker): review follow-up — ghost cache entries, ReadBulk dedupe/pumping, fault hygiene

- AddItem/AddItem2/AddBufferedItem drop stale cache entries for a reused handle id
- ReadBulk: dedupe handles within a batch (advise/teardown once, fan out result);
  pump every 16 items in add/advise and teardown phases
- alarm poll stops without recording a fault once StaRuntime.LoopFault is set
- StaRuntime skips the loop-fault handler during a requested shutdown
- StaHung loop fault diagnostic carries type + HRESULT only
- gateway.md documents ReadBulk's shared deadline

src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessSessionTests.cs

@@ -305,6 +305,129 @@ public sealed class MxAccessSessionTests
         }
     }
 
+    /// <summary>
+    ///     Ghost entry: an OnDataChange/OnWriteComplete already queued when
+    ///     RemoveItem ran is dispatched after the eviction and recreates the
+    ///     entries. When MXAccess later reuses that item id, AddItem must drop
+    ///     them so ReadBulk's cached path cannot serve the stale value.
+    /// </summary>
+    [Fact]
+    public void AddItem_ReusingAnEvictedHandleId_DropsGhostCacheEntries()
+    {
+        RecordingMxAccessServer server = new();
+        CompletionCacheSink sink = new();
+        MxAccessValueCache valueCache = new();
+        using MxAccessSession session = MxAccessSession.CreateForTesting(server, sink, valueCache: valueCache);
+        int serverHandle = session.Register("client");
+
+        int first = session.AddItem(serverHandle, "Tag.Old");
+        session.Advise(serverHandle, first);
+        session.RemoveItem(serverHandle, first);
+
+        // Late dispatch of callbacks queued before the RemoveItem.
+        valueCache.Set(serverHandle, first, BuildDataChange(serverHandle, first, 999));
+        RecordCompletion(sink.WriteCompletionCache, serverHandle, first);
+
+        server.NextItemHandleOverride = first;
+        int reused = session.AddItem(serverHandle, "Tag.New");
+        Assert.Equal(first, reused);
+
+        Assert.False(valueCache.TryGet(serverHandle, reused, out _));
+        Assert.Equal(0UL, sink.WriteCompletionCache.CurrentVersion(serverHandle, reused));
+
+        // Once advised, ReadBulk on the new tag does not report the ghost as cached.
+        session.Advise(serverHandle, reused);
+        BulkReadResult result = Assert.Single(session.ReadBulk(serverHandle, new[] { "Tag.New" }, TimeSpan.FromMilliseconds(30), () => { }));
+        Assert.False(result.WasCached);
+        Assert.False(result.WasSuccessful);
+    }
+
+    /// <summary>
+    ///     A handle MXAccess returns while the registry still tracks it is a
+    ///     live item, so its cached value must survive the AddItem.
+    /// </summary>
+    [Fact]
+    public void AddItem_ReturningAStillRegisteredHandle_KeepsItsCachedValue()
+    {
+        RecordingMxAccessServer server = new() { ReuseHandleForSameTag = true };
+        MxAccessValueCache valueCache = new();
+        using MxAccessSession session = MxAccessSession.CreateForTesting(server, new NoopEventSink(), valueCache: valueCache);
+        int serverHandle = session.Register("client");
+        int handle = session.AddItem(serverHandle, "Tag.Live");
+        valueCache.Set(serverHandle, handle, BuildDataChange(serverHandle, handle, 5));
+
+        Assert.Equal(handle, session.AddItem(serverHandle, "Tag.Live"));
+
+        Assert.True(valueCache.TryGet(serverHandle, handle, out _));
+    }
+
+    /// <summary>
+    ///     Duplicate addresses that MXAccess resolves to one handle: the handle
+    ///     is advised, unadvised, and removed exactly once, and every duplicate
+    ///     receives the shared outcome.
+    /// </summary>
+    [Fact]
+    public void ReadBulk_DuplicateTagsResolvingToOneHandle_AdviseAndTearDownOnceAndFanOutResult()
+    {
+        RecordingMxAccessServer server = new() { ReuseHandleForSameTag = true };
+        MxAccessValueCache valueCache = new();
+        using MxAccessSession session = MxAccessSession.CreateForTesting(server, new NoopEventSink(), valueCache: valueCache);
+        int serverHandle = session.Register("client");
+
+        Action pumpStep = () =>
+        {
+            if (server.TryGetAdvisedHandle("Tag.Dup", out int handle) && !valueCache.TryGet(serverHandle, handle, out _))
+            {
+                valueCache.Set(serverHandle, handle, BuildDataChange(serverHandle, handle, 77));
+            }
+        };
+
+        IReadOnlyList<BulkReadResult> results = session.ReadBulk(
+            serverHandle,
+            new[] { "Tag.Dup", "Tag.Other", "Tag.Dup" },
+            TimeSpan.FromMilliseconds(100),
+            pumpStep);
+
+        int dupHandle = server.HandleForTag("Tag.Dup");
+        string advise = "Advise:" + serverHandle + ":" + dupHandle;
+        string unAdvise = "UnAdvise:" + serverHandle + ":" + dupHandle;
+        string remove = "RemoveItem:" + serverHandle + ":" + dupHandle;
+        Assert.True(results[0].WasSuccessful);
+        Assert.True(results[2].WasSuccessful);
+        Assert.Equal(dupHandle, results[0].ItemHandle);
+        Assert.Equal(dupHandle, results[2].ItemHandle);
+        Assert.Equal(77, results[2].Value.Int32Value);
+        Assert.Equal("Tag.Dup", results[2].TagAddress);
+        Assert.False(results[1].WasSuccessful);
+        Assert.Equal("Tag.Other", results[1].TagAddress);
+
+        Assert.Equal(1, server.Operations.Count(op => op == advise));
+        Assert.Equal(1, server.Operations.Count(op => op == unAdvise));
+        Assert.Equal(1, server.Operations.Count(op => op == remove));
+        Assert.Empty(session.HandleRegistry.ItemHandles);
+    }
+
+    /// <summary>
+    ///     Large batches pump during the add/advise and teardown phases too,
+    ///     not only inside the wait, so the STA heartbeat stays fresh.
+    /// </summary>
+    [Fact]
+    public void ReadBulk_LargeBatch_PumpsDuringAddAdviseAndTeardownPhases()
+    {
+        RecordingMxAccessServer server = new();
+        using MxAccessSession session = MxAccessSession.CreateForTesting(server, new NoopEventSink());
+        int serverHandle = session.Register("client");
+        string[] tags = Enumerable.Range(0, 3 * MxAccessSession.ReadBulkPumpInterval).Select(i => "Tag." + i).ToArray();
+        List<int> pumpAtOperation = new();
+
+        session.ReadBulk(serverHandle, tags, TimeSpan.FromMilliseconds(20), () => pumpAtOperation.Add(server.Operations.Count));
+
+        int lastAdvise = server.Operations.FindLastIndex(op => op.StartsWith("Advise:", StringComparison.Ordinal));
+        int firstUnAdvise = server.Operations.FindIndex(op => op.StartsWith("UnAdvise:", StringComparison.Ordinal));
+        Assert.Contains(pumpAtOperation, at => at < lastAdvise);
+        Assert.Contains(pumpAtOperation, at => at > firstUnAdvise);
+    }
+
     private static void RecordCompletion(MxAccessWriteCompletionCache cache, int serverHandle, int itemHandle)
     {
         RepeatedField<MxStatusProxy> statuses = new() { new MxStatusProxy { Category = MxStatusCategory.Ok } };
@@ -368,6 +491,12 @@ public sealed class MxAccessSessionTests
 
         public HashSet<string> FailAdviseFor { get; } = new(StringComparer.Ordinal);
 
+        /// <summary>When set, the next AddItem returns this handle (simulates MXAccess reusing an id).</summary>
+        public int? NextItemHandleOverride { get; set; }
+
+        /// <summary>When true, AddItem of an already-added address returns the existing handle.</summary>
+        public bool ReuseHandleForSameTag { get; set; }
+
         public int HandleForTag(string tag) => handleByTag[tag];
 
         public bool TryGetAdvisedHandle(string tag, out int handle)
@@ -391,7 +520,21 @@ public sealed class MxAccessSessionTests
                 throw new InvalidOperationException($"AddItem failed for {itemDefinition}");
             }
 
-            int handle = nextItemHandle++;
+            int handle;
+            if (NextItemHandleOverride is int forced)
+            {
+                NextItemHandleOverride = null;
+                handle = forced;
+            }
+            else if (ReuseHandleForSameTag && handleByTag.TryGetValue(itemDefinition, out int existing))
+            {
+                handle = existing;
+            }
+            else
+            {
+                handle = nextItemHandle++;
+            }
+
             handleByTag[itemDefinition] = handle;
             tagByHandle[handle] = itemDefinition;
             return handle;

src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs

@@ -687,7 +687,7 @@ public sealed class MxAccessStaSessionTests
         await session.StartAsync("session-1", workerProcessId: 1);
         Assert.False(eventQueue.IsFaulted);
 
-        pump.ThrowOnNextPump(new InvalidOperationException("The worker STA message pump failed."));
+        pump.ThrowOnNextPump(new InvalidOperationException("secret-tag-name pump failed."));
 
         DateTime deadline = DateTime.UtcNow.AddSeconds(5);
         while (!eventQueue.IsFaulted && DateTime.UtcNow < deadline)
@@ -701,9 +701,54 @@ public sealed class MxAccessStaSessionTests
         Assert.Equal(WorkerFaultCategory.StaHung, fault!.Category);
         Assert.Equal(typeof(InvalidOperationException).FullName, fault.ExceptionType);
         Assert.Contains("STA loop terminated", fault.DiagnosticMessage, StringComparison.Ordinal);
+        Assert.Contains("exceptionType=System.InvalidOperationException", fault.DiagnosticMessage, StringComparison.Ordinal);
+        Assert.Contains("hresult=0x", fault.DiagnosticMessage, StringComparison.Ordinal);
+        Assert.DoesNotContain("secret-tag-name", fault.DiagnosticMessage, StringComparison.Ordinal);
+        Assert.DoesNotContain("secret-tag-name", fault.ProtocolStatus.Message, StringComparison.Ordinal);
         Assert.Equal(ProtocolStatusCode.WorkerUnavailable, fault.ProtocolStatus.Code);
     }
 
+    /// <summary>
+    /// After the STA loop dies, the alarm poll's fast-failed InvokeAsync must
+    /// not record a second, mislabelled alarm-poll fault: once the StaHung
+    /// fault is drained, nothing else is recorded.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task RunAlarmPollLoop_AfterStaLoopFault_DoesNotRecordAlarmPollFault()
+    {
+        ControllableMessagePump pump = new();
+        FakeAlarmCommandHandler handler = new();
+        using StaRuntime runtime = new(new NoopComApartmentInitializer(), pump, TimeSpan.FromMilliseconds(25));
+        MxAccessEventQueue eventQueue = new();
+        using MxAccessStaSession session = new(
+            runtime,
+            new FakeMxAccessComObjectFactory(),
+            new FakeMxAccessEventSink(),
+            eventQueue,
+            (_eq, _affinity, _comFactory) => handler);
+        session.AlarmPollInterval = TimeSpan.FromMilliseconds(10);
+        session.AlarmPollFailureThreshold = 1;
+        session.AlarmPollWarningSink = _ => { };
+
+        await session.StartAsync("session-1", workerProcessId: 1);
+        pump.ThrowOnNextPump(new InvalidOperationException("pump failed"));
+
+        DateTime deadline = DateTime.UtcNow.AddSeconds(5);
+        while (!eventQueue.IsFaulted && DateTime.UtcNow < deadline)
+        {
+            await Task.Delay(10);
+        }
+
+        WorkerFault? first = session.DrainFault();
+        Assert.NotNull(first);
+        Assert.Equal(WorkerFaultCategory.StaHung, first!.Category);
+
+        // Give the poll loop several ticks to (wrongly) record its own fault.
+        await Task.Delay(300);
+        Assert.Null(session.DrainFault());
+    }
+
     /// <summary>
     /// The STA-affinity guard throws when an
     /// IMxAccessAlarmConsumer call is attempted off the thread that created

src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaRuntimeTests.cs

@@ -274,6 +274,27 @@ public sealed class StaRuntimeTests
         Assert.False(invoked);
     }
 
+    /// <summary>
+    ///     A loop that throws while a requested shutdown is in progress is
+    ///     being torn down on purpose: the loop-fault handler must not run.
+    /// </summary>
+    [Fact]
+    public void LoopException_DuringRequestedShutdown_DoesNotInvokeHandler()
+    {
+        ControllableMessagePump pump = new();
+        using StaRuntime runtime = new(new RecordingComApartmentInitializer(), pump, TimeSpan.FromSeconds(30));
+        runtime.Start();
+        bool invoked = false;
+        runtime.RegisterLoopFaultHandler(_ => invoked = true);
+
+        // The loop is parked in the 30 s idle wait; Shutdown wakes it and the
+        // very next pump throws with shutdownRequested already set.
+        pump.ThrowOnNextPump(new InvalidOperationException("pump failed during shutdown"));
+        Assert.True(runtime.Shutdown(TimeSpan.FromSeconds(5)));
+
+        Assert.False(invoked);
+    }
+
     private static StaRuntime CreateRuntime(RecordingComApartmentInitializer initializer)
     {
         return new StaRuntime(