Implement graceful worker shutdown

src/MxGateway.Worker/MxAccess/MxAccessSession.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Runtime.InteropServices;
 using Google.Protobuf.WellKnownTypes;
 using MxGateway.Contracts.Proto;
@@ -188,6 +189,23 @@ public sealed class MxAccessSession : IDisposable
             MxAccessAdviceKind.Supervisory);
     }
 
+    public MxAccessShutdownResult ShutdownGracefully()
+    {
+        if (disposed)
+        {
+            return new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>());
+        }
+
+        List<MxAccessShutdownFailure> failures = new();
+
+        CleanupAdviceHandles(failures);
+        CleanupItemHandles(failures);
+        CleanupServerHandles(failures);
+        DisposeCore(failures);
+
+        return new MxAccessShutdownResult(failures);
+    }
+
     public void Dispose()
     {
         if (disposed)
@@ -195,11 +213,112 @@ public sealed class MxAccessSession : IDisposable
             return;
         }
 
-        eventSink.Detach();
+        DisposeCore(failures: null);
+    }
 
-        if (Marshal.IsComObject(mxAccessComObject))
+    private void CleanupAdviceHandles(ICollection<MxAccessShutdownFailure> failures)
+    {
+        HashSet<long> cleanedPairs = new();
+        foreach (RegisteredAdviceHandle adviceHandle in handleRegistry.AdviceHandles)
         {
-            Marshal.FinalReleaseComObject(mxAccessComObject);
+            long key = CreateItemKey(adviceHandle.ServerHandle, adviceHandle.ItemHandle);
+            if (!cleanedPairs.Add(key))
+            {
+                continue;
+            }
+
+            try
+            {
+                mxAccessServer.UnAdvise(adviceHandle.ServerHandle, adviceHandle.ItemHandle);
+                handleRegistry.RemoveAdviceHandles(adviceHandle.ServerHandle, adviceHandle.ItemHandle);
+            }
+            catch (Exception exception)
+            {
+                failures.Add(new MxAccessShutdownFailure(
+                    nameof(UnAdvise),
+                    adviceHandle.ServerHandle,
+                    adviceHandle.ItemHandle,
+                    exception));
+            }
+        }
+    }
+
+    private void CleanupItemHandles(ICollection<MxAccessShutdownFailure> failures)
+    {
+        foreach (RegisteredItemHandle itemHandle in handleRegistry.ItemHandles)
+        {
+            try
+            {
+                mxAccessServer.RemoveItem(itemHandle.ServerHandle, itemHandle.ItemHandle);
+                handleRegistry.RemoveItemHandle(itemHandle.ServerHandle, itemHandle.ItemHandle);
+            }
+            catch (Exception exception)
+            {
+                failures.Add(new MxAccessShutdownFailure(
+                    nameof(RemoveItem),
+                    itemHandle.ServerHandle,
+                    itemHandle.ItemHandle,
+                    exception));
+            }
+        }
+    }
+
+    private void CleanupServerHandles(ICollection<MxAccessShutdownFailure> failures)
+    {
+        foreach (RegisteredServerHandle serverHandle in handleRegistry.ServerHandles)
+        {
+            try
+            {
+                mxAccessServer.Unregister(serverHandle.ServerHandle);
+                handleRegistry.UnregisterServerHandle(serverHandle.ServerHandle);
+            }
+            catch (Exception exception)
+            {
+                failures.Add(new MxAccessShutdownFailure(
+                    nameof(Unregister),
+                    serverHandle.ServerHandle,
+                    itemHandle: null,
+                    exception));
+            }
+        }
+    }
+
+    private static long CreateItemKey(
+        int serverHandle,
+        int itemHandle)
+    {
+        return ((long)serverHandle << 32) | (uint)itemHandle;
+    }
+
+    private void DisposeCore(ICollection<MxAccessShutdownFailure>? failures)
+    {
+        try
+        {
+            eventSink.Detach();
+        }
+        catch (Exception exception) when (failures is not null)
+        {
+            failures.Add(new MxAccessShutdownFailure(
+                "DetachEvents",
+                serverHandle: null,
+                itemHandle: null,
+                exception));
+        }
+
+        try
+        {
+            if (Marshal.IsComObject(mxAccessComObject))
+            {
+                Marshal.FinalReleaseComObject(mxAccessComObject);
+            }
+        }
+        catch (Exception exception) when (failures is not null)
+        {
+            failures.Add(new MxAccessShutdownFailure(
+                "ReleaseComObject",
+                serverHandle: null,
+                itemHandle: null,
+                exception));
         }
 
         disposed = true;

src/MxGateway.Worker/MxAccess/MxAccessShutdownFailure.cs

@@ -0,0 +1,34 @@
+using System;
+
+namespace MxGateway.Worker.MxAccess;
+
+public sealed class MxAccessShutdownFailure
+{
+    public MxAccessShutdownFailure(
+        string operation,
+        int? serverHandle,
+        int? itemHandle,
+        Exception exception)
+    {
+        if (string.IsNullOrWhiteSpace(operation))
+        {
+            throw new ArgumentException("Shutdown failure operation is required.", nameof(operation));
+        }
+
+        Operation = operation;
+        ServerHandle = serverHandle;
+        ItemHandle = itemHandle;
+        ExceptionType = exception?.GetType().FullName ?? string.Empty;
+        HResult = exception?.HResult;
+    }
+
+    public string Operation { get; }
+
+    public int? ServerHandle { get; }
+
+    public int? ItemHandle { get; }
+
+    public string ExceptionType { get; }
+
+    public int? HResult { get; }
+}

src/MxGateway.Worker/MxAccess/MxAccessShutdownResult.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+
+namespace MxGateway.Worker.MxAccess;
+
+public sealed class MxAccessShutdownResult
+{
+    public MxAccessShutdownResult(IReadOnlyList<MxAccessShutdownFailure> failures)
+    {
+        Failures = failures ?? throw new ArgumentNullException(nameof(failures));
+    }
+
+    public IReadOnlyList<MxAccessShutdownFailure> Failures { get; }
+
+    public bool Succeeded => Failures.Count == 0;
+}

src/MxGateway.Worker/MxAccess/MxAccessStaSession.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 using MxGateway.Contracts.Proto;
@@ -141,6 +142,61 @@ public sealed class MxAccessStaSession : IDisposable
             cancellationToken);
     }
 
+    public async Task<MxAccessShutdownResult> ShutdownGracefullyAsync(
+        TimeSpan timeout,
+        CancellationToken cancellationToken = default)
+    {
+        if (timeout <= TimeSpan.Zero)
+        {
+            throw new ArgumentOutOfRangeException(
+                nameof(timeout),
+                "MXAccess graceful shutdown timeout must be greater than zero.");
+        }
+
+        if (disposed)
+        {
+            return new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>());
+        }
+
+        commandDispatcher?.RequestShutdown();
+
+        Stopwatch stopwatch = Stopwatch.StartNew();
+        MxAccessShutdownResult result;
+        if (session is null)
+        {
+            result = new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>());
+        }
+        else
+        {
+            using CancellationTokenSource shutdownCancellation =
+                CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            shutdownCancellation.CancelAfter(timeout);
+
+            Task<MxAccessShutdownResult> cleanupTask = staRuntime.InvokeAsync(
+                () => session.ShutdownGracefully(),
+                shutdownCancellation.Token);
+            Task delayTask = Task.Delay(timeout, cancellationToken);
+            Task completedTask = await Task.WhenAny(cleanupTask, delayTask).ConfigureAwait(false);
+            if (completedTask != cleanupTask)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+                throw new TimeoutException($"MXAccess graceful shutdown exceeded {timeout}.");
+            }
+
+            result = await cleanupTask.ConfigureAwait(false);
+        }
+
+        TimeSpan remaining = timeout - stopwatch.Elapsed;
+        if (remaining <= TimeSpan.Zero || !staRuntime.Shutdown(remaining))
+        {
+            throw new TimeoutException($"MXAccess graceful shutdown exceeded {timeout}.");
+        }
+
+        staRuntime.Dispose();
+        disposed = true;
+        return result;
+    }
+
     public void Dispose()
     {
         if (disposed)