Merge remote-tracking branch 'origin/main' into agent-2/issue-33-implement-graceful-shutdown

# Conflicts: # src/MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs # src/MxGateway.Worker/Ipc/WorkerPipeClient.cs # src/MxGateway.Worker/Ipc/WorkerPipeSession.cs
2026-04-26 19:41:04 -04:00
parent d890eff862 9dcd4baff2
commit f7929cc12f
56 changed files with 9855 additions and 167 deletions
@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using System.IO.Pipes;
 using System.Threading;
 using System.Threading.Tasks;
@@ -11,6 +12,7 @@ public sealed class WorkerPipeClient : IWorkerPipeClient
    public const int DefaultConnectTimeoutMilliseconds = 30000;

    private readonly int _connectTimeoutMilliseconds;
+    private readonly Func<Stream, WorkerFrameProtocolOptions, IWorkerLogger?, WorkerPipeSession> _sessionFactory;
    private readonly IWorkerLogger? _logger;

    public WorkerPipeClient()
@@ -28,9 +30,30 @@ public sealed class WorkerPipeClient : IWorkerPipeClient
    {
    }

+    public WorkerPipeClient(
+        int connectTimeoutMilliseconds,
+        Func<Stream, WorkerFrameProtocolOptions, WorkerPipeSession> sessionFactory)
+        : this(
+            null,
+            connectTimeoutMilliseconds,
+            (stream, frameOptions, _) => sessionFactory(stream, frameOptions))
+    {
+    }
+
    public WorkerPipeClient(
        IWorkerLogger? logger,
        int connectTimeoutMilliseconds)
+        : this(
+            logger,
+            connectTimeoutMilliseconds,
+            (stream, frameOptions, workerLogger) => new WorkerPipeSession(stream, frameOptions, workerLogger))
+    {
+    }
+
+    public WorkerPipeClient(
+        IWorkerLogger? logger,
+        int connectTimeoutMilliseconds,
+        Func<Stream, WorkerFrameProtocolOptions, IWorkerLogger?, WorkerPipeSession> sessionFactory)
    {
        if (connectTimeoutMilliseconds <= 0)
        {
@@ -39,8 +62,9 @@ public sealed class WorkerPipeClient : IWorkerPipeClient
                "Worker pipe connect timeout must be greater than zero.");
        }

-        _connectTimeoutMilliseconds = connectTimeoutMilliseconds;
        _logger = logger;
+        _sessionFactory = sessionFactory ?? throw new ArgumentNullException(nameof(sessionFactory));
+        _connectTimeoutMilliseconds = connectTimeoutMilliseconds;
    }

    public async Task RunAsync(
@@ -62,8 +86,7 @@ public sealed class WorkerPipeClient : IWorkerPipeClient

        await ConnectAsync(pipe, cancellationToken).ConfigureAwait(false);

-        WorkerPipeSession session = new(pipe, frameOptions, _logger);
-        await session.CompleteStartupHandshakeAsync(cancellationToken).ConfigureAwait(false);
+        WorkerPipeSession session = _sessionFactory(pipe, frameOptions, _logger);
        await session.RunAsync(cancellationToken).ConfigureAwait(false);
    }

@@ -16,12 +16,15 @@ public sealed class WorkerPipeSession
 {
    private readonly WorkerFrameProtocolOptions _options;
    private readonly Func<int> _processIdProvider;
+    private readonly Func<IWorkerRuntimeSession> _runtimeSessionFactory;
+    private readonly WorkerPipeSessionOptions _sessionOptions;
    private readonly IWorkerLogger? _logger;
    private readonly WorkerFrameReader _reader;
    private readonly WorkerFrameWriter _writer;
-    private MxAccessStaSession? _mxAccessStaSession;
+    private IWorkerRuntimeSession? _runtimeSession;
    private long _nextSequence;
-    private bool _shutdownCompleted;
+    private WorkerState _state = WorkerState.Starting;
+    private bool _watchdogFaultSent;
    private bool _shutdownTimedOut;

    public WorkerPipeSession(
@@ -33,22 +36,67 @@ public sealed class WorkerPipeSession
            new WorkerFrameWriter(stream, options),
            options,
            () => Process.GetCurrentProcess().Id,
+            new WorkerPipeSessionOptions(),
+            () => new MxAccessStaSession(),
            logger)
    {
    }

+    public WorkerPipeSession(
+        WorkerFrameReader reader,
+        WorkerFrameWriter writer,
+        WorkerFrameProtocolOptions options,
+        Func<int> processIdProvider)
+        : this(
+            reader,
+            writer,
+            options,
+            processIdProvider,
+            new WorkerPipeSessionOptions(),
+            () => new MxAccessStaSession(),
+            logger: null)
+    {
+    }
+
    public WorkerPipeSession(
        WorkerFrameReader reader,
        WorkerFrameWriter writer,
        WorkerFrameProtocolOptions options,
        Func<int> processIdProvider,
+        WorkerPipeSessionOptions sessionOptions,
+        Func<IWorkerRuntimeSession> runtimeSessionFactory,
        IWorkerLogger? logger = null)
    {
        _reader = reader ?? throw new ArgumentNullException(nameof(reader));
        _writer = writer ?? throw new ArgumentNullException(nameof(writer));
        _options = options ?? throw new ArgumentNullException(nameof(options));
        _processIdProvider = processIdProvider ?? throw new ArgumentNullException(nameof(processIdProvider));
+        _sessionOptions = sessionOptions ?? throw new ArgumentNullException(nameof(sessionOptions));
+        _runtimeSessionFactory = runtimeSessionFactory ?? throw new ArgumentNullException(nameof(runtimeSessionFactory));
        _logger = logger;
+        _sessionOptions.Validate();
+    }
+
+    public async Task RunAsync(CancellationToken cancellationToken = default)
+    {
+        _runtimeSession = _runtimeSessionFactory();
+        try
+        {
+            await CompleteStartupHandshakeAsync(
+                token => _runtimeSession.StartAsync(_options.SessionId, _processIdProvider(), token),
+                cancellationToken).ConfigureAwait(false);
+            await RunMessageLoopAsync(cancellationToken).ConfigureAwait(false);
+        }
+        finally
+        {
+            if (!_shutdownTimedOut)
+            {
+                _runtimeSession?.Dispose();
+            }
+
+            _runtimeSession = null;
+            _state = WorkerState.Stopped;
+        }
    }

    public Task CompleteStartupHandshakeAsync(CancellationToken cancellationToken = default)
@@ -86,11 +134,14 @@ public sealed class WorkerPipeSession
        try
        {
            WorkerEnvelope envelope = await _reader.ReadAsync(cancellationToken).ConfigureAwait(false);
+            _state = WorkerState.Handshaking;
            ValidateGatewayHello(envelope);

            await WriteWorkerHelloAsync(cancellationToken).ConfigureAwait(false);
+            _state = WorkerState.InitializingSta;
            WorkerReady ready = await initializeMxAccessAsync(cancellationToken).ConfigureAwait(false);
            await WriteWorkerReadyAsync(ready, cancellationToken).ConfigureAwait(false);
+            _state = WorkerState.Ready;
        }
        catch (WorkerFrameProtocolException exception)
        {
@@ -105,44 +156,6 @@ public sealed class WorkerPipeSession
        }
    }

-    public async Task RunAsync(CancellationToken cancellationToken = default)
-    {
-        try
-        {
-            while (true)
-            {
-                WorkerEnvelope envelope = await _reader.ReadAsync(cancellationToken).ConfigureAwait(false);
-                switch (envelope.BodyCase)
-                {
-                    case WorkerEnvelope.BodyOneofCase.WorkerCommand:
-                        await HandleCommandAsync(envelope, cancellationToken).ConfigureAwait(false);
-                        break;
-                    case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
-                        await HandleShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
-                        return;
-                    case WorkerEnvelope.BodyOneofCase.WorkerCancel:
-                        break;
-                    default:
-                        throw new WorkerFrameProtocolException(
-                            WorkerFrameProtocolErrorCode.UnexpectedEnvelopeBody,
-                            $"Worker received unexpected gateway envelope body {envelope.BodyCase} after startup.");
-                }
-            }
-        }
-        catch (WorkerFrameProtocolException exception)
-        {
-            await TryWriteFaultAsync(exception, cancellationToken).ConfigureAwait(false);
-            throw;
-        }
-        finally
-        {
-            if (!_shutdownCompleted && !_shutdownTimedOut)
-            {
-                _mxAccessStaSession?.Dispose();
-            }
-        }
-    }
-
    private void ValidateGatewayHello(WorkerEnvelope envelope)
    {
        if (envelope.BodyCase != WorkerEnvelope.BodyOneofCase.GatewayHello)
@@ -188,6 +201,189 @@ public sealed class WorkerPipeSession
        return _writer.WriteAsync(CreateEnvelope(ready), cancellationToken);
    }

+    private async Task RunMessageLoopAsync(CancellationToken cancellationToken)
+    {
+        using CancellationTokenSource heartbeatCancellation = CancellationTokenSource
+            .CreateLinkedTokenSource(cancellationToken);
+        Task heartbeatTask = RunHeartbeatLoopAsync(heartbeatCancellation.Token);
+
+        try
+        {
+            while (!cancellationToken.IsCancellationRequested)
+            {
+                Task<WorkerEnvelope> readTask = _reader.ReadAsync(cancellationToken);
+                Task completedTask = await Task.WhenAny(readTask, heartbeatTask).ConfigureAwait(false);
+                if (completedTask == heartbeatTask)
+                {
+                    await heartbeatTask.ConfigureAwait(false);
+                }
+
+                WorkerEnvelope envelope = await readTask.ConfigureAwait(false);
+                bool keepReading = await DispatchGatewayEnvelopeAsync(envelope, cancellationToken).ConfigureAwait(false);
+                if (!keepReading)
+                {
+                    return;
+                }
+            }
+        }
+        finally
+        {
+            heartbeatCancellation.Cancel();
+            try
+            {
+                await heartbeatTask.ConfigureAwait(false);
+            }
+            catch (OperationCanceledException)
+            {
+            }
+        }
+    }
+
+    private async Task<bool> DispatchGatewayEnvelopeAsync(
+        WorkerEnvelope envelope,
+        CancellationToken cancellationToken)
+    {
+        switch (envelope.BodyCase)
+        {
+            case WorkerEnvelope.BodyOneofCase.WorkerCommand:
+                _ = ProcessCommandAsync(envelope, cancellationToken);
+                return true;
+            case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
+                await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
+                return false;
+            case WorkerEnvelope.BodyOneofCase.WorkerCancel:
+                return true;
+            default:
+                throw new WorkerFrameProtocolException(
+                    WorkerFrameProtocolErrorCode.UnexpectedEnvelopeBody,
+                    $"Worker received unexpected gateway envelope body {envelope.BodyCase}.");
+        }
+    }
+
+    private async Task ProcessCommandAsync(
+        WorkerEnvelope envelope,
+        CancellationToken cancellationToken)
+    {
+        IWorkerRuntimeSession runtimeSession = _runtimeSession
+            ?? throw new InvalidOperationException("Worker runtime session has not been initialized.");
+        WorkerCommand workerCommand = envelope.WorkerCommand;
+        MxCommand command = workerCommand.Command;
+        StaCommand staCommand = new(
+            _options.SessionId,
+            envelope.CorrelationId,
+            command,
+            workerCommand.EnqueueTimestamp,
+            cancellationToken);
+
+        try
+        {
+            MxCommandReply reply = await runtimeSession.DispatchAsync(staCommand).ConfigureAwait(false);
+            await _writer
+                .WriteAsync(
+                    CreateEnvelope(new WorkerCommandReply
+                    {
+                        Reply = reply,
+                        CompletedTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
+                    }),
+                    cancellationToken)
+                .ConfigureAwait(false);
+        }
+        catch (Exception exception) when (exception is not OperationCanceledException)
+        {
+            _state = WorkerState.Faulted;
+            await TryWriteFaultAsync(
+                CreateFault(
+                    WorkerFaultCategory.MxaccessCommandFailed,
+                    staCommand.MethodName,
+                    exception),
+                cancellationToken).ConfigureAwait(false);
+        }
+    }
+
+    private async Task ShutdownAsync(
+        WorkerShutdown shutdown,
+        CancellationToken cancellationToken)
+    {
+        _state = WorkerState.ShuttingDown;
+        IWorkerRuntimeSession? runtimeSession = _runtimeSession;
+        if (runtimeSession is null)
+        {
+            await WriteShutdownAckAsync(
+                CreateShutdownAck(new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>()), shutdown),
+                cancellationToken).ConfigureAwait(false);
+            return;
+        }
+
+        TimeSpan gracePeriod = ResolveGracePeriod(shutdown);
+        try
+        {
+            MxAccessShutdownResult result = await runtimeSession
+                .ShutdownGracefullyAsync(gracePeriod, cancellationToken)
+                .ConfigureAwait(false);
+            LogShutdownFailures(result.Failures);
+            await WriteShutdownAckAsync(CreateShutdownAck(result, shutdown), cancellationToken).ConfigureAwait(false);
+        }
+        catch (TimeoutException exception)
+        {
+            _shutdownTimedOut = true;
+            _state = WorkerState.Faulted;
+            await TryWriteFaultAsync(CreateShutdownTimeoutFault(exception), cancellationToken).ConfigureAwait(false);
+            throw;
+        }
+    }
+
+    private Task WriteShutdownAckAsync(
+        WorkerShutdownAck shutdownAck,
+        CancellationToken cancellationToken)
+    {
+        return _writer.WriteAsync(CreateEnvelope(shutdownAck), cancellationToken);
+    }
+
+    private async Task RunHeartbeatLoopAsync(CancellationToken cancellationToken)
+    {
+        while (!cancellationToken.IsCancellationRequested)
+        {
+            await Task.Delay(_sessionOptions.HeartbeatInterval, cancellationToken).ConfigureAwait(false);
+            IWorkerRuntimeSession? runtimeSession = _runtimeSession;
+            if (runtimeSession is null)
+            {
+                continue;
+            }
+
+            WorkerRuntimeHeartbeatSnapshot snapshot = runtimeSession.CaptureHeartbeat();
+            await _writer
+                .WriteAsync(CreateEnvelope(CreateHeartbeat(snapshot)), cancellationToken)
+                .ConfigureAwait(false);
+
+            await ReportWatchdogFaultIfNeededAsync(snapshot, cancellationToken).ConfigureAwait(false);
+        }
+    }
+
+    private async Task ReportWatchdogFaultIfNeededAsync(
+        WorkerRuntimeHeartbeatSnapshot snapshot,
+        CancellationToken cancellationToken)
+    {
+        TimeSpan staleFor = DateTimeOffset.UtcNow - snapshot.LastStaActivityUtc;
+        if (staleFor <= _sessionOptions.HeartbeatGrace)
+        {
+            _watchdogFaultSent = false;
+            return;
+        }
+
+        if (_watchdogFaultSent)
+        {
+            return;
+        }
+
+        _watchdogFaultSent = true;
+        await TryWriteFaultAsync(
+            CreateFault(
+                WorkerFaultCategory.StaHung,
+                snapshot.CurrentCommandCorrelationId,
+                $"STA activity is stale by {staleFor}."),
+            cancellationToken).ConfigureAwait(false);
+    }
+
    private async Task TryWriteFaultAsync(
        WorkerFrameProtocolException exception,
        CancellationToken cancellationToken)
@@ -241,7 +437,7 @@ public sealed class WorkerPipeSession
            || faultWriteException is ObjectDisposedException
            || faultWriteException is WorkerFrameProtocolException)
        {
-            // The shutdown timeout is the actionable error.
+            // The runtime fault remains observable through worker exit or pipe closure.
        }
    }

@@ -265,16 +461,16 @@ public sealed class WorkerPipeSession
        return CreateBaseEnvelope(reply);
    }

-    private WorkerEnvelope CreateEnvelope(WorkerEvent workerEvent)
-    {
-        return CreateBaseEnvelope(workerEvent);
-    }
-
    private WorkerEnvelope CreateEnvelope(WorkerShutdownAck shutdownAck)
    {
        return CreateBaseEnvelope(shutdownAck);
    }

+    private WorkerEnvelope CreateEnvelope(WorkerHeartbeat heartbeat)
+    {
+        return CreateBaseEnvelope(heartbeat);
+    }
+
    private WorkerEnvelope CreateBaseEnvelope(WorkerHello body)
    {
        WorkerEnvelope envelope = CreateBaseEnvelope();
@@ -304,13 +500,6 @@ public sealed class WorkerPipeSession
        return envelope;
    }

-    private WorkerEnvelope CreateBaseEnvelope(WorkerEvent body)
-    {
-        WorkerEnvelope envelope = CreateBaseEnvelope();
-        envelope.WorkerEvent = body.Clone();
-        return envelope;
-    }
-
    private WorkerEnvelope CreateBaseEnvelope(WorkerShutdownAck body)
    {
        WorkerEnvelope envelope = CreateBaseEnvelope();
@@ -318,6 +507,13 @@ public sealed class WorkerPipeSession
        return envelope;
    }

+    private WorkerEnvelope CreateBaseEnvelope(WorkerHeartbeat body)
+    {
+        WorkerEnvelope envelope = CreateBaseEnvelope();
+        envelope.WorkerHeartbeat = body;
+        return envelope;
+    }
+
    private WorkerEnvelope CreateBaseEnvelope()
    {
        return new WorkerEnvelope
@@ -335,88 +531,37 @@ public sealed class WorkerPipeSession

    private async Task<WorkerReady> InitializeMxAccessAsync(CancellationToken cancellationToken)
    {
-        _mxAccessStaSession = new MxAccessStaSession();
+        _runtimeSession = new MxAccessStaSession();
        try
        {
-            return await _mxAccessStaSession
+            return await _runtimeSession
                .StartAsync(_options.SessionId, _processIdProvider(), cancellationToken)
                .ConfigureAwait(false);
        }
        catch
        {
-            _mxAccessStaSession.Dispose();
-            _mxAccessStaSession = null;
+            _runtimeSession.Dispose();
+            _runtimeSession = null;
            throw;
        }
    }

-    private async Task HandleCommandAsync(
-        WorkerEnvelope envelope,
-        CancellationToken cancellationToken)
+    private WorkerHeartbeat CreateHeartbeat(WorkerRuntimeHeartbeatSnapshot snapshot)
    {
-        if (_mxAccessStaSession is null)
-        {
-            throw new InvalidOperationException("MXAccess STA session is not initialized.");
-        }
+        WorkerState state = string.IsNullOrWhiteSpace(snapshot.CurrentCommandCorrelationId)
+            ? _state
+            : WorkerState.ExecutingCommand;

-        StaCommand command = new(
-            _options.SessionId,
-            envelope.CorrelationId,
-            envelope.WorkerCommand.Command,
-            envelope.WorkerCommand.EnqueueTimestamp,
-            cancellationToken);
-
-        MxCommandReply mxReply = await _mxAccessStaSession
-            .DispatchAsync(command)
-            .ConfigureAwait(false);
-        WorkerCommandReply reply = new()
+        return new WorkerHeartbeat
        {
-            Reply = mxReply,
-            CompletedTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
+            WorkerProcessId = _processIdProvider(),
+            State = state,
+            LastStaActivityTimestamp = Timestamp.FromDateTimeOffset(snapshot.LastStaActivityUtc),
+            PendingCommandCount = snapshot.PendingCommandCount,
+            OutboundEventQueueDepth = snapshot.OutboundEventQueueDepth,
+            LastEventSequence = snapshot.LastEventSequence,
+            CurrentCommandCorrelationId = snapshot.CurrentCommandCorrelationId,
        };
-
-        await _writer.WriteAsync(CreateEnvelope(reply), cancellationToken).ConfigureAwait(false);
-        await DrainEventsAsync(cancellationToken).ConfigureAwait(false);
-    }
-
-    private async Task HandleShutdownAsync(
-        WorkerShutdown shutdown,
-        CancellationToken cancellationToken)
-    {
-        TimeSpan gracePeriod = ResolveGracePeriod(shutdown);
-        try
-        {
-            MxAccessShutdownResult result = _mxAccessStaSession is null
-                ? new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>())
-                : await _mxAccessStaSession
-                    .ShutdownGracefullyAsync(gracePeriod, cancellationToken)
-                    .ConfigureAwait(false);
-
-            LogShutdownFailures(result.Failures);
-            await _writer
-                .WriteAsync(CreateEnvelope(CreateShutdownAck(result)), cancellationToken)
-                .ConfigureAwait(false);
-            _shutdownCompleted = true;
-        }
-        catch (TimeoutException exception)
-        {
-            _shutdownTimedOut = true;
-            await TryWriteFaultAsync(CreateShutdownTimeoutFault(exception), cancellationToken).ConfigureAwait(false);
-            throw;
-        }
-    }
-
-    private async Task DrainEventsAsync(CancellationToken cancellationToken)
-    {
-        if (_mxAccessStaSession is null)
-        {
-            return;
-        }
-
-        foreach (WorkerEvent workerEvent in _mxAccessStaSession.DrainEvents(maxEvents: 0))
-        {
-            await _writer.WriteAsync(CreateEnvelope(workerEvent), cancellationToken).ConfigureAwait(false);
-        }
    }

    private WorkerReady CreateWorkerReady()
@@ -443,16 +588,24 @@ public sealed class WorkerPipeSession
            : gracePeriod;
    }

-    private static WorkerShutdownAck CreateShutdownAck(MxAccessShutdownResult result)
+    private static WorkerShutdownAck CreateShutdownAck(
+        MxAccessShutdownResult result,
+        WorkerShutdown shutdown)
    {
+        string message = result.Succeeded
+            ? "Graceful shutdown completed."
+            : $"Graceful shutdown completed with {result.Failures.Count} cleanup failure(s).";
+        if (!string.IsNullOrWhiteSpace(shutdown.Reason))
+        {
+            message = $"{message} Reason: {shutdown.Reason}";
+        }
+
        return new WorkerShutdownAck
        {
            Status = new ProtocolStatus
            {
                Code = ProtocolStatusCode.Ok,
-                Message = result.Succeeded
-                    ? "Graceful shutdown completed."
-                    : $"Graceful shutdown completed with {result.Failures.Count} cleanup failure(s).",
+                Message = message,
            },
        };
    }
@@ -511,22 +664,50 @@ public sealed class WorkerPipeSession
        return fault;
    }

-    private static WorkerFault CreateShutdownTimeoutFault(TimeoutException exception)
+    private static WorkerFault CreateFault(
+        WorkerFaultCategory category,
+        string commandMethod,
+        Exception exception)
+    {
+        WorkerFault fault = CreateFault(
+            category,
+            commandMethod,
+            exception.Message);
+        fault.ExceptionType = exception.GetType().FullName ?? string.Empty;
+        fault.ProtocolStatus = new ProtocolStatus
+        {
+            Code = ProtocolStatusCode.WorkerUnavailable,
+            Message = exception.Message,
+        };
+        return fault;
+    }
+
+    private static WorkerFault CreateFault(
+        WorkerFaultCategory category,
+        string commandMethod,
+        string diagnosticMessage)
    {
-        string message = exception.Message;
        return new WorkerFault
        {
-            Category = WorkerFaultCategory.ShutdownTimeout,
-            ExceptionType = exception.GetType().FullName ?? string.Empty,
-            DiagnosticMessage = message,
+            Category = category,
+            CommandMethod = commandMethod ?? string.Empty,
+            DiagnosticMessage = diagnosticMessage,
            ProtocolStatus = new ProtocolStatus
            {
                Code = ProtocolStatusCode.WorkerUnavailable,
-                Message = message,
+                Message = diagnosticMessage,
            },
        };
    }

+    private static WorkerFault CreateShutdownTimeoutFault(TimeoutException exception)
+    {
+        return CreateFault(
+            WorkerFaultCategory.ShutdownTimeout,
+            commandMethod: string.Empty,
+            exception);
+    }
+
    private static WorkerFaultCategory MapFaultCategory(WorkerFrameProtocolErrorCode errorCode)
    {
        return errorCode switch
@@ -0,0 +1,36 @@
+using System;
+
+namespace MxGateway.Worker.Ipc;
+
+public sealed class WorkerPipeSessionOptions
+{
+    public static readonly TimeSpan DefaultHeartbeatInterval = TimeSpan.FromSeconds(5);
+    public static readonly TimeSpan DefaultHeartbeatGrace = TimeSpan.FromSeconds(15);
+
+    public WorkerPipeSessionOptions()
+    {
+        HeartbeatInterval = DefaultHeartbeatInterval;
+        HeartbeatGrace = DefaultHeartbeatGrace;
+    }
+
+    public TimeSpan HeartbeatInterval { get; set; }
+
+    public TimeSpan HeartbeatGrace { get; set; }
+
+    public void Validate()
+    {
+        if (HeartbeatInterval <= TimeSpan.Zero)
+        {
+            throw new ArgumentOutOfRangeException(
+                nameof(HeartbeatInterval),
+                "Worker heartbeat interval must be greater than zero.");
+        }
+
+        if (HeartbeatGrace <= TimeSpan.Zero)
+        {
+            throw new ArgumentOutOfRangeException(
+                nameof(HeartbeatGrace),
+                "Worker heartbeat grace must be greater than zero.");
+        }
+    }
+}
@@ -0,0 +1,25 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using MxGateway.Contracts.Proto;
+using MxGateway.Worker.Sta;
+
+namespace MxGateway.Worker.MxAccess;
+
+public interface IWorkerRuntimeSession : IDisposable
+{
+    Task<WorkerReady> StartAsync(
+        string sessionId,
+        int workerProcessId,
+        CancellationToken cancellationToken = default);
+
+    Task<MxCommandReply> DispatchAsync(StaCommand command);
+
+    WorkerRuntimeHeartbeatSnapshot CaptureHeartbeat();
+
+    void RequestShutdown();
+
+    Task<MxAccessShutdownResult> ShutdownGracefullyAsync(
+        TimeSpan timeout,
+        CancellationToken cancellationToken = default);
+}
@@ -8,7 +8,7 @@ using MxGateway.Worker.Sta;

 namespace MxGateway.Worker.MxAccess;

-public sealed class MxAccessStaSession : IDisposable
+public sealed class MxAccessStaSession : IWorkerRuntimeSession
 {
    private readonly IMxAccessComObjectFactory factory;
    private readonly IMxAccessEventSink eventSink;
@@ -98,6 +98,30 @@ public sealed class MxAccessStaSession : IDisposable
        return commandDispatcher.DispatchAsync(command);
    }

+    public WorkerRuntimeHeartbeatSnapshot CaptureHeartbeat()
+    {
+        uint pendingCommandCount = 0;
+        string currentCommandCorrelationId = string.Empty;
+
+        if (commandDispatcher is not null)
+        {
+            pendingCommandCount = (uint)commandDispatcher.PendingCommandCount;
+            currentCommandCorrelationId = commandDispatcher.CurrentCommandCorrelationId;
+        }
+
+        return new WorkerRuntimeHeartbeatSnapshot(
+            staRuntime.LastActivityUtc,
+            pendingCommandCount,
+            (uint)eventQueue.Count,
+            eventQueue.LastEventSequence,
+            currentCommandCorrelationId);
+    }
+
+    public void RequestShutdown()
+    {
+        commandDispatcher?.RequestShutdown();
+    }
+
    public IReadOnlyList<WorkerEvent> DrainEvents(uint maxEvents)
    {
        return eventQueue.Drain(maxEvents);
@@ -204,7 +228,7 @@ public sealed class MxAccessStaSession : IDisposable
            return;
        }

-        commandDispatcher?.RequestShutdown();
+        RequestShutdown();

        if (session is not null)
        {
@@ -0,0 +1,30 @@
+using System;
+
+namespace MxGateway.Worker.MxAccess;
+
+public sealed class WorkerRuntimeHeartbeatSnapshot
+{
+    public WorkerRuntimeHeartbeatSnapshot(
+        DateTimeOffset lastStaActivityUtc,
+        uint pendingCommandCount,
+        uint outboundEventQueueDepth,
+        ulong lastEventSequence,
+        string currentCommandCorrelationId)
+    {
+        LastStaActivityUtc = lastStaActivityUtc;
+        PendingCommandCount = pendingCommandCount;
+        OutboundEventQueueDepth = outboundEventQueueDepth;
+        LastEventSequence = lastEventSequence;
+        CurrentCommandCorrelationId = currentCommandCorrelationId ?? string.Empty;
+    }
+
+    public DateTimeOffset LastStaActivityUtc { get; }
+
+    public uint PendingCommandCount { get; }
+
+    public uint OutboundEventQueueDepth { get; }
+
+    public ulong LastEventSequence { get; }
+
+    public string CurrentCommandCorrelationId { get; }
+}
@@ -83,7 +83,7 @@ public static class WorkerApplication

            pipeClient.RunAsync(options).GetAwaiter().GetResult();

-            logger.Information("WorkerPipeHandshakeSucceeded", new Dictionary<string, object?>
+            logger.Information("WorkerPipeSessionCompleted", new Dictionary<string, object?>
            {
                ["session_id"] = options.SessionId,
                ["pipe_name"] = options.PipeName,