wip: item handle reuse probe (#166)

Add MxAccessWriteCompletionCache.Remove and RemoveServer and call them
from AlarmAckWriter when it drops an item (dirty re-add, LRU eviction,
failed AddItem/Advise/Write) and when it tears down its registration, so
the cache no longer keeps one entry per dropped handle for the life of
the worker.

Fixes #165

Claude-Session: https://claude.ai/code/session_016J6isfcgqy4SB2RK3ZvFvT

docs/Grpc.md

@@ -115,7 +115,7 @@ The worker calls `AuthenticateUser` as the operator, then writes `operator_comme
 | `AuthenticateUser` failed with a server-gone HRESULT (`RPC_E_DISCONNECTED`, `RPC_S_SERVER_UNAVAILABLE`, `RPC_E_SERVERFAULT`); the writer tears down its MXAccess registration and re-registers on the next ack | `MXACCESS_FAILURE` | that HRESULT |
 | `AddItem`, `Advise` or `Write` on the `AckMsg` item raised an MXAccess error (`COMException`, or `ArgumentException` for `E_INVALIDARG`); the item is dropped and re-added on the next ack, and a server-gone HRESULT also tears down the registration | `MXACCESS_FAILURE` | the exception's HRESULT |
 | Write rejected (`OnWriteComplete` non-OK) | `MXACCESS_FAILURE` | the `detail` of the `OnWriteComplete` `MXSTATUS_PROXY`, or `-1` when `detail` is `0` |
-| Any other exception escaping the ack (e.g. the ack writer's MXAccess `Register` failing, or the monitor's worker session having no active alarm subscription — `InvalidOperationException`) | `MXACCESS_FAILURE` | the exception's `HResult`; the diagnostic names only the exception type and HRESULT |
+| Any other exception escaping the ack (e.g. the ack writer's MXAccess `Register` failing, the monitor's worker session having no active alarm subscription — `InvalidOperationException` — or an argument error that is not the interop's `E_INVALIDARG`: `ArgumentNullException`, `ArgumentOutOfRangeException`, or an `ArgumentException` with another HRESULT, which signal a bug rather than an MXAccess failure) | `MXACCESS_FAILURE` | the exception's `HResult`; the diagnostic names only the exception type and HRESULT |
 
 `reply.Statuses` carries the `OnWriteComplete` `MxStatusProxy` rows when they were observed. Because an unconfirmed write also returns `OK`, a client that must be certain should watch for the resulting transition on `StreamAlarms`. `AcknowledgeAlarm` stays under the `invoke:write` scope; see [Authorization](./Authorization.md).
 

docs/MxAccessWorkerInstanceDesign.md

@@ -351,6 +351,17 @@ native client correlating by handle has the identical ambiguity). The exposure
 is bounded: `statuses` is an enrichment field, and `Hresult`/`ProtocolStatus`
 stay acceptance-only regardless of which completion (if any) was attributed.
 
+**The alarm ack writer narrows the window for acknowledgements.**
+`AlarmAckWriter` registers its own MXAccess server with its own
+`MxAccessWriteCompletionCache` and reuses one advised `AckMsg` item per alarm
+(least-recently-used, capped at 256). After an ack ends unconfirmed it marks the
+item dirty, and the next ack to that alarm removes and re-adds the item so the
+write waits on a fresh item handle the stale completion cannot match. Every
+drop of an item (dirty re-add, eviction, or a failed `AddItem`/`Advise`/`Write`)
+also removes that handle's completion entry, and tearing down the registration
+removes every entry for its server handle, so the cache does not grow by one
+entry per dropped handle for the life of the worker.
+
 ## Live alarm watch-list replacement
 
 An auto-mode `SubscribeAlarms` with `fallback_enabled = true` always builds a

src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/AlarmAckWriterTests.cs

@@ -187,6 +187,70 @@ public sealed class AlarmAckWriterTests
         Assert.DoesNotContain("Unregister", server.Operations);
     }
 
+    [Fact]
+    public void Acknowledge_WhenAuthenticateUserThrowsArgumentExceptionWithEInvalidArg_ReturnsAuthenticationFailed()
+    {
+        // The HRESULT is what the narrowed filter keys on; set it explicitly so the
+        // test does not depend on ArgumentException's default HResult.
+        RecordingAckServer server = new RecordingAckServer
+        {
+            AuthenticateResult = 5,
+            AuthenticateException = new HResultArgumentException("bad credentials", unchecked((int)0x80070057)),
+        };
+        using AlarmAckWriter writer = new AlarmAckWriter(server, new MxAccessWriteCompletionCache(), () => { }, TimeSpan.FromMilliseconds(10), "MxGateway");
+
+        AlarmAckResult result = writer.Acknowledge("T.A.AckMsg", "one", Alice());
+
+        Assert.Equal(AlarmAckOutcome.AuthenticationFailed, result.Outcome);
+        Assert.Equal(unchecked((int)0x80070057), result.NativeStatus);
+    }
+
+    [Fact]
+    public void Acknowledge_WhenAuthenticateUserThrowsArgumentNullException_Propagates()
+    {
+        // A programming error is a bug, not an operator authentication failure.
+        RecordingAckServer server = new RecordingAckServer
+        {
+            AuthenticateResult = 5,
+            AuthenticateException = new ArgumentNullException("verifyUser"),
+        };
+        using AlarmAckWriter writer = new AlarmAckWriter(server, new MxAccessWriteCompletionCache(), () => { }, TimeSpan.FromMilliseconds(10), "MxGateway");
+
+        Assert.Throws<ArgumentNullException>(() => writer.Acknowledge("T.A.AckMsg", "one", Alice()));
+    }
+
+    [Fact]
+    public void Acknowledge_WhenAuthenticateUserThrowsArgumentExceptionWithOtherHResult_Propagates()
+    {
+        RecordingAckServer server = new RecordingAckServer
+        {
+            AuthenticateResult = 5,
+            AuthenticateException = new HResultArgumentException("not interop", unchecked((int)0x80131500)),
+        };
+        using AlarmAckWriter writer = new AlarmAckWriter(server, new MxAccessWriteCompletionCache(), () => { }, TimeSpan.FromMilliseconds(10), "MxGateway");
+
+        Assert.Throws<HResultArgumentException>(() => writer.Acknowledge("T.A.AckMsg", "one", Alice()));
+    }
+
+    [Fact]
+    public void Acknowledge_WhenWriteThrowsArgumentOutOfRangeException_Propagates()
+    {
+        RecordingAckServer server = new RecordingAckServer { AuthenticateResult = 5, WriteException = new ArgumentOutOfRangeException("value") };
+        using AlarmAckWriter writer = new AlarmAckWriter(server, new MxAccessWriteCompletionCache(), () => { }, TimeSpan.FromMilliseconds(10), "MxGateway");
+
+        Assert.Throws<ArgumentOutOfRangeException>(() => writer.Acknowledge("T.A.AckMsg", "one", Alice()));
+    }
+
+    [Fact]
+    public void Acknowledge_WhenOperatorUserNameIsNull_ThrowsBeforeAuthenticating()
+    {
+        RecordingAckServer server = new RecordingAckServer { AuthenticateResult = 5 };
+        using AlarmAckWriter writer = new AlarmAckWriter(server, new MxAccessWriteCompletionCache(), () => { }, TimeSpan.FromMilliseconds(10), "MxGateway");
+
+        Assert.Throws<ArgumentException>(() => writer.Acknowledge("T.A.AckMsg", "one", new AlarmAckOperator { UserName = null! }));
+        Assert.DoesNotContain(server.Operations, op => op.StartsWith("AuthenticateUser", StringComparison.Ordinal));
+    }
+
     [Fact]
     public void Acknowledge_WhenWriteThrowsArgumentException_RejectsAndDropsItem()
     {
@@ -269,6 +333,76 @@ public sealed class AlarmAckWriterTests
         Assert.Empty(server.RemovedItemHandles);
     }
 
+    [Fact]
+    public void Acknowledge_AfterDirtyReAdd_ForgetsDroppedHandleCompletion()
+    {
+        RecordingAckServer server = new RecordingAckServer { AuthenticateResult = 5 };
+        MxAccessWriteCompletionCache cache = new MxAccessWriteCompletionCache();
+        using AlarmAckWriter writer = new AlarmAckWriter(server, cache, () => { }, TimeSpan.FromMilliseconds(20), "MxGateway");
+
+        writer.Acknowledge("T.A.AckMsg", "one", Alice()); // handle 100, Unconfirmed
+        cache.Record(7, 100, Rows(MxStatusCategory.Ok, 0)); // its late completion
+        Assert.Equal(1, cache.Count);
+        server.OnWrite = (handle, item) => cache.Record(handle, item, Rows(MxStatusCategory.Ok, 0));
+
+        AlarmAckResult second = writer.Acknowledge("T.A.AckMsg", "two", Alice()); // drops 100, adds 101
+
+        Assert.Equal(AlarmAckOutcome.Applied, second.Outcome);
+        Assert.Equal(1, cache.Count);
+        Assert.Equal(0UL, cache.CurrentVersion(7, 100));
+        Assert.Equal(1UL, cache.CurrentVersion(7, 101));
+    }
+
+    [Fact]
+    public void Acknowledge_PastCacheCap_ForgetsEvictedHandleCompletion()
+    {
+        RecordingAckServer server = new RecordingAckServer { AuthenticateResult = 5 };
+        MxAccessWriteCompletionCache cache = new MxAccessWriteCompletionCache();
+        server.OnWrite = (handle, item) => cache.Record(handle, item, Rows(MxStatusCategory.Ok, 0));
+        using AlarmAckWriter writer = new AlarmAckWriter(server, cache, () => { }, TimeSpan.FromSeconds(1), "MxGateway", maxCachedItems: 2);
+
+        writer.Acknowledge("A.X.AckMsg", "1", Alice()); // handle 100
+        writer.Acknowledge("B.X.AckMsg", "2", Alice()); // handle 101
+        Assert.Equal(2, cache.Count);
+        writer.Acknowledge("C.X.AckMsg", "3", Alice()); // handle 102, evicts A (100)
+
+        Assert.Equal(2, cache.Count);
+        Assert.Equal(0UL, cache.CurrentVersion(7, 100));
+    }
+
+    [Fact]
+    public void Dispose_ForgetsServerCompletions()
+    {
+        RecordingAckServer server = new RecordingAckServer { AuthenticateResult = 5 };
+        MxAccessWriteCompletionCache cache = new MxAccessWriteCompletionCache();
+        server.OnWrite = (handle, item) => cache.Record(handle, item, Rows(MxStatusCategory.Ok, 0));
+        AlarmAckWriter writer = new AlarmAckWriter(server, cache, () => { }, TimeSpan.FromSeconds(1), "MxGateway");
+        writer.Acknowledge("A.X.AckMsg", "1", Alice());
+        writer.Acknowledge("B.X.AckMsg", "2", Alice());
+        cache.Record(8, 100, Rows(MxStatusCategory.Ok, 0)); // another server's entry survives
+
+        writer.Dispose();
+
+        Assert.Equal(1, cache.Count);
+        Assert.Equal(1UL, cache.CurrentVersion(8, 100));
+    }
+
+    [Fact]
+    public void Acknowledge_WhenWriteDisconnected_ForgetsServerCompletions()
+    {
+        RecordingAckServer server = new RecordingAckServer { AuthenticateResult = 5 };
+        MxAccessWriteCompletionCache cache = new MxAccessWriteCompletionCache();
+        server.OnWrite = (handle, item) => cache.Record(handle, item, Rows(MxStatusCategory.Ok, 0));
+        using AlarmAckWriter writer = new AlarmAckWriter(server, cache, () => { }, TimeSpan.FromSeconds(1), "MxGateway");
+        writer.Acknowledge("A.X.AckMsg", "1", Alice()); // handle 100 recorded
+        server.WriteException = new COMException("x", unchecked((int)0x80010108));
+
+        writer.Acknowledge("A.X.AckMsg", "2", Alice());
+
+        Assert.Contains("Unregister", server.Operations);
+        Assert.Equal(0, cache.Count);
+    }
+
     [Fact]
     public void Acknowledge_WhenNonOkCompletionHasNoDetail_ReturnsMinusOne()
     {
@@ -290,6 +424,16 @@ public sealed class AlarmAckWriterTests
     }
 }
 
+/// <summary>ArgumentException with an explicit HRESULT (the setter is protected on net48).</summary>
+internal sealed class HResultArgumentException : ArgumentException
+{
+    public HResultArgumentException(string message, int hresult)
+        : base(message)
+    {
+        HResult = hresult;
+    }
+}
+
 internal sealed class RecordingAckServer : IMxAccessServer
 {
     private int nextItemHandle = 100;

src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs

@@ -183,6 +183,39 @@ public sealed class MxAccessWriteCompletionCacheTests
         Assert.Equal(77, Assert.Single(statuses).Detail);
     }
 
+    /// <summary>Verifies that Remove forgets only the given handle pair.</summary>
+    [Fact]
+    public void Remove_ForgetsOnlyThatHandlePair()
+    {
+        MxAccessWriteCompletionCache cache = new();
+        cache.Record(7, 21, BuildStatuses(detail: 1));
+        cache.Record(7, 22, BuildStatuses(detail: 2));
+
+        cache.Remove(7, 21);
+        cache.Remove(7, 99); // unknown pair is a no-op
+
+        Assert.Equal(1, cache.Count);
+        Assert.Equal(0UL, cache.CurrentVersion(7, 21));
+        Assert.Equal(1UL, cache.CurrentVersion(7, 22));
+    }
+
+    /// <summary>Verifies that RemoveServer forgets every pair of that server and no other.</summary>
+    [Fact]
+    public void RemoveServer_ForgetsOnlyThatServersPairs()
+    {
+        MxAccessWriteCompletionCache cache = new();
+        cache.Record(7, 21, BuildStatuses(detail: 1));
+        cache.Record(7, -5, BuildStatuses(detail: 2));
+        cache.Record(8, 21, BuildStatuses(detail: 3));
+        cache.Record(-7, 21, BuildStatuses(detail: 4));
+
+        cache.RemoveServer(7);
+
+        Assert.Equal(2, cache.Count);
+        Assert.Equal(1UL, cache.CurrentVersion(8, 21));
+        Assert.Equal(1UL, cache.CurrentVersion(-7, 21));
+    }
+
     private static RepeatedField<MxStatusProxy> BuildStatuses(int detail)
     {
         return new RepeatedField<MxStatusProxy>

src/ZB.MOM.WW.MxGateway.Worker.Tests/Probes/ItemHandleReuseProbeTests.cs

@@ -0,0 +1,160 @@
+// ItemHandleReuseProbeTests.cs
+//
+// Probe for issue #166: does MXAccess hand out a new item handle number when the
+// same item is removed and re-added on one LMXProxyServer registration? The
+// AlarmAckWriter's Unconfirmed mitigation (drop the item, re-add it, wait on the
+// fresh handle) relies on the number changing, because OnWriteComplete carries
+// only the (server, item) handle pair.
+//
+// The probe only adds, advises and removes items. It never writes, so it does
+// not acknowledge the alarm or create Historian rows.
+//
+// HOW TO RUN (dev rig with MXAccess installed and the DEV Galaxy running):
+//   1. Remove (or set to null) the Skip parameter on the [Fact] below.
+//   2. dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj
+//        -p:Platform=x86 --filter FullyQualifiedName~ItemHandleReuseProbeTests
+//        --logger "console;verbosity=detailed"
+//   3. Read the logged handles. Results are recorded in docs/AlarmProbeFindings.md.
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+using System.Threading;
+using ArchestrA.MxAccess;
+using ZB.MOM.WW.MxGateway.Worker.MxAccess;
+using Xunit.Abstractions;
+
+namespace ZB.MOM.WW.MxGateway.Worker.Tests.Probes;
+
+/// <summary>
+///     Live probe that logs the item handles MXAccess returns when the same
+///     <c>AckMsg</c> item is removed and re-added, to decide whether a late
+///     <c>OnWriteComplete</c> for a dropped handle could match a re-added one.
+/// </summary>
+public sealed class ItemHandleReuseProbeTests
+{
+    private const string AckItem = "TestMachine_001.TestAlarm001.AckMsg";
+    private const string OtherItem = "TestMachine_001.TestAlarm001.Acked";
+    private const int Rounds = 5;
+
+    private readonly ITestOutputHelper output;
+    private readonly List<string> log = new List<string>();
+
+    /// <summary>Initializes a new instance of the <see cref="ItemHandleReuseProbeTests"/> class.</summary>
+    /// <param name="output">Test output helper for logging.</param>
+    public ItemHandleReuseProbeTests(ITestOutputHelper output)
+    {
+        this.output = output;
+    }
+
+    /// <summary>Adds, removes and re-adds the AckMsg item on a fresh registration and logs every handle.</summary>
+    [Fact(Skip = "Live dev-rig probe (issue #166) — flip Skip=null on a host with MXAccess and the DEV Galaxy running.")]
+    public void RemoveThenReAdd_LogsItemHandles()
+    {
+        Exception? threadException = null;
+        Thread thread = new Thread(() =>
+        {
+            try { RunProbe(); }
+            catch (Exception ex) { threadException = ex; }
+        });
+        thread.SetApartmentState(ApartmentState.STA);
+        thread.Start();
+        thread.Join();
+
+        foreach (string line in log)
+        {
+            output.WriteLine(line);
+        }
+
+        if (threadException is not null)
+        {
+            throw new InvalidOperationException("Probe failed: " + threadException.Message, threadException);
+        }
+    }
+
+    private void RunProbe()
+    {
+        LMXProxyServerClass com = new LMXProxyServerClass();
+        try
+        {
+            IMxAccessServer mx = new MxAccessComServer(com);
+            int server = mx.Register("MxGatewayProbe166");
+            Log($"Register -> server handle {server}");
+
+            // A: the AlarmAckWriter drop path (Advise, then UnAdvise + RemoveItem).
+            for (int i = 0; i < Rounds; i++)
+            {
+                int h = mx.AddItem(server, AckItem);
+                mx.Advise(server, h);
+                Log($"A{i}: AddItem+Advise {AckItem} -> {h}");
+                mx.UnAdvise(server, h);
+                mx.RemoveItem(server, h);
+                Log($"A{i}: UnAdvise+RemoveItem {h}");
+            }
+
+            // B: RemoveItem without Advise.
+            for (int i = 0; i < Rounds; i++)
+            {
+                int h = mx.AddItem(server, AckItem);
+                Log($"B{i}: AddItem {AckItem} -> {h}");
+                mx.RemoveItem(server, h);
+                Log($"B{i}: RemoveItem {h}");
+            }
+
+            // C: interleave another item to see whether a freed number is handed to any item.
+            int c1 = mx.AddItem(server, AckItem);
+            Log($"C: AddItem {AckItem} -> {c1}");
+            mx.RemoveItem(server, c1);
+            Log($"C: RemoveItem {c1}");
+            int c2 = mx.AddItem(server, OtherItem);
+            Log($"C: AddItem {OtherItem} -> {c2}");
+            int c3 = mx.AddItem(server, AckItem);
+            Log($"C: AddItem {AckItem} -> {c3}");
+            mx.RemoveItem(server, c2);
+            mx.RemoveItem(server, c3);
+            Log($"C: RemoveItem {c2}, {c3}");
+
+            // D: the same item added twice while the first is still held.
+            int d1 = mx.AddItem(server, AckItem);
+            int d2 = mx.AddItem(server, AckItem);
+            Log($"D: AddItem {AckItem} twice while held -> {d1}, {d2}");
+            mx.RemoveItem(server, d1);
+            mx.RemoveItem(server, d2);
+
+            mx.Unregister(server);
+            Log($"Unregister {server}");
+
+            // E: the TearDownServer path: re-register on the same COM object and add again.
+            int server2 = mx.Register("MxGatewayProbe166");
+            int e1 = mx.AddItem(server2, AckItem);
+            Log($"E: re-Register -> server handle {server2}; AddItem {AckItem} -> {e1}");
+            mx.RemoveItem(server2, e1);
+            mx.Unregister(server2);
+        }
+        finally
+        {
+            Marshal.FinalReleaseComObject(com);
+        }
+
+        // F: a fresh COM object, as AlarmAckWriter creates after a teardown.
+        LMXProxyServerClass com2 = new LMXProxyServerClass();
+        try
+        {
+            IMxAccessServer mx2 = new MxAccessComServer(com2);
+            int server3 = mx2.Register("MxGatewayProbe166");
+            int f1 = mx2.AddItem(server3, AckItem);
+            Log($"F: new COM object Register -> server handle {server3}; AddItem {AckItem} -> {f1}");
+            mx2.RemoveItem(server3, f1);
+            mx2.Unregister(server3);
+        }
+        finally
+        {
+            Marshal.FinalReleaseComObject(com2);
+        }
+    }
+
+    private void Log(string line)
+    {
+        log.Add(DateTime.Now.ToString("HH:mm:ss.fff") + " " + line);
+    }
+}

src/ZB.MOM.WW.MxGateway.Worker/MxAccess/AlarmAckWriter.cs

@@ -34,6 +34,9 @@ public sealed class AlarmAckWriter : IAlarmAckWriter
     /// </summary>
     public const int DefaultMaxCachedItems = 256;
 
+    /// <summary>E_INVALIDARG, which the interop raises as ArgumentException.</summary>
+    private const int EInvalidArg = unchecked((int)0x80070057);
+
     private readonly IMxAccessComObjectFactory? factory;
     private readonly MxAccessWriteCompletionCache completionCache;
     private readonly Action pumpStep;
@@ -104,12 +107,20 @@ public sealed class AlarmAckWriter : IAlarmAckWriter
         if (string.IsNullOrWhiteSpace(itemAddress)) throw new ArgumentException("Ack item address is required.", nameof(itemAddress));
         if (ackOperator is null) throw new ArgumentNullException(nameof(ackOperator));
 
+        // Resolve every argument before the interop try blocks: their catch
+        // filters accept E_INVALIDARG ArgumentExceptions as MXAccess failures, so
+        // an argument error raised by this code must fail before reaching them.
+        if (ackOperator.UserName is null) throw new ArgumentException("Operator user name is required.", nameof(ackOperator));
+        string verifyUser = ackOperator.VerifyUser;
+        string password = ackOperator.Password ?? string.Empty;
+        string commentValue = comment ?? string.Empty;
+
         IMxAccessServer mx = EnsureRegistered();
 
         int userId;
         try
         {
-            userId = mx.AuthenticateUser(serverHandle, ackOperator.VerifyUser, ackOperator.Password ?? string.Empty);
+            userId = mx.AuthenticateUser(serverHandle, verifyUser, password);
         }
         catch (Exception ex) when (IsInteropFailure(ex))
         {
@@ -130,7 +141,7 @@ public sealed class AlarmAckWriter : IAlarmAckWriter
             return AlarmAckResult.Create(
                 AlarmAckOutcome.AuthenticationFailed,
                 ex.HResult,
-                $"Operator authentication failed for '{ackOperator.VerifyUser}' (HRESULT 0x{ex.HResult:X8}).");
+                $"Operator authentication failed for '{verifyUser}' (HRESULT 0x{ex.HResult:X8}).");
         }
 
         if (userId == 0)
@@ -138,7 +149,7 @@ public sealed class AlarmAckWriter : IAlarmAckWriter
             return AlarmAckResult.Create(
                 AlarmAckOutcome.AuthenticationFailed,
                 -1,
-                $"Operator authentication failed for '{ackOperator.VerifyUser}' (no user id returned).");
+                $"Operator authentication failed for '{verifyUser}' (no user id returned).");
         }
 
         ulong baselineVersion = 0;
@@ -177,7 +188,7 @@ public sealed class AlarmAckWriter : IAlarmAckWriter
             pumpStep();
             step = "Write";
             baselineVersion = completionCache.CurrentVersion(serverHandle, itemHandle);
-            mx.Write(serverHandle, itemHandle, comment ?? string.Empty, userId);
+            mx.Write(serverHandle, itemHandle, commentValue, userId);
         }
         catch (Exception ex) when (IsInteropFailure(ex))
         {
@@ -249,10 +260,23 @@ public sealed class AlarmAckWriter : IAlarmAckWriter
     ///     ArgumentException, E_ACCESSDENIED to UnauthorizedAccessException) and
     ///     the rest to COMException. Each carries the HRESULT in Exception.HResult.
     /// </summary>
+    /// <remarks>
+    ///     The ArgumentException arm exists because a wrong password or unknown
+    ///     user makes <c>AuthenticateUser</c> throw ArgumentException with
+    ///     E_INVALIDARG (0x80070057), observed live on windev on 2026-10-19. It
+    ///     is narrowed to that HRESULT, and excludes ArgumentNullException and
+    ///     ArgumentOutOfRangeException, so a programming error inside the try
+    ///     blocks propagates as a bug instead of being reported as an MXAccess
+    ///     authentication or write failure. The interop never raises those two
+    ///     subtypes for a COM HRESULT.
+    /// </remarks>
     private static bool IsInteropFailure(Exception ex) =>
         ex is ExternalException
-        || ex is ArgumentException
-        || ex is UnauthorizedAccessException;
+        || ex is UnauthorizedAccessException
+        || (ex is ArgumentException
+            && ex is not ArgumentNullException
+            && ex is not ArgumentOutOfRangeException
+            && ex.HResult == EInvalidArg);
 
     private static bool IsServerGone(int hresult) =>
         hresult == unchecked((int)0x80010108) // RPC_E_DISCONNECTED
@@ -279,6 +303,9 @@ public sealed class AlarmAckWriter : IAlarmAckWriter
             try { mx.Unregister(serverHandle); } catch { /* best effort */ }
         }
 
+        // The registration is gone, so no completion can arrive for its handles.
+        completionCache.RemoveServer(serverHandle);
+
         itemHandles.Clear();
         itemLru.Clear();
         server = null;
@@ -311,6 +338,10 @@ public sealed class AlarmAckWriter : IAlarmAckWriter
         itemLru.Remove(node);
         try { mx.UnAdvise(serverHandle, node.Value.Handle); } catch { /* best effort */ }
         try { mx.RemoveItem(serverHandle, node.Value.Handle); } catch { /* best effort */ }
+
+        // Without this the completion cache keeps one entry per dropped handle
+        // (each Unconfirmed re-add and each eviction) for the worker's life.
+        completionCache.Remove(serverHandle, node.Value.Handle);
     }
 
     private IMxAccessServer EnsureRegistered()

src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs

@@ -184,6 +184,57 @@ public sealed class MxAccessWriteCompletionCache
         }
     }
 
+    /// <summary>
+    ///     Forgets the completion entry of a handle pair whose MXAccess item was
+    ///     removed, so a long-lived writer that drops and re-adds items does not
+    ///     keep one entry per dropped handle for the life of the worker.
+    /// </summary>
+    /// <param name="serverHandle">MXAccess server handle.</param>
+    /// <param name="itemHandle">MXAccess item handle that was removed.</param>
+    public void Remove(
+        int serverHandle,
+        int itemHandle)
+    {
+        lock (syncRoot)
+        {
+            entries.Remove(CreateItemKey(serverHandle, itemHandle));
+        }
+    }
+
+    /// <summary>Forgets every completion entry recorded under a server handle that was unregistered.</summary>
+    /// <param name="serverHandle">MXAccess server handle that was unregistered.</param>
+    public void RemoveServer(int serverHandle)
+    {
+        lock (syncRoot)
+        {
+            List<long> stale = new();
+            foreach (long key in entries.Keys)
+            {
+                if ((int)(key >> 32) == serverHandle)
+                {
+                    stale.Add(key);
+                }
+            }
+
+            foreach (long key in stale)
+            {
+                entries.Remove(key);
+            }
+        }
+    }
+
+    /// <summary>Gets the number of handle pairs with a recorded completion (test seam).</summary>
+    internal int Count
+    {
+        get
+        {
+            lock (syncRoot)
+            {
+                return entries.Count;
+            }
+        }
+    }
+
     private static long CreateItemKey(
         int serverHandle,
         int itemHandle)