Merge remote-tracking branch 'origin/main' into agent-2/issue-33-implement-graceful-shutdown

# Conflicts:
#	src/MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs
#	src/MxGateway.Worker/Ipc/WorkerPipeClient.cs
#	src/MxGateway.Worker/Ipc/WorkerPipeSession.cs
This commit is contained in:
Joseph Doherty
2026-04-26 19:41:04 -04:00
56 changed files with 9855 additions and 167 deletions
@@ -0,0 +1,379 @@
using System.Text.Json;
using Google.Protobuf;
using MxGateway.Contracts;
using MxGateway.Contracts.Proto;
namespace MxGateway.Tests.Contracts;
public sealed class ClientBehaviorFixtureTests
{
private static readonly JsonParser ProtobufJsonParser = new(JsonParser.Settings.Default);
[Fact]
public void BehaviorManifest_DeclaresCurrentProtocolVersionsAndExistingFixtures()
{
using JsonDocument manifest = LoadBehaviorManifest();
JsonElement root = manifest.RootElement;
Assert.Equal(1, root.GetProperty("schemaVersion").GetInt32());
Assert.Equal("mxaccess-gateway-client-behavior", root.GetProperty("fixtureSet").GetString());
Assert.Equal(GatewayContractInfo.GatewayProtocolVersion, root.GetProperty("gatewayProtocolVersion").GetUInt32());
Assert.Equal(GatewayContractInfo.WorkerProtocolVersion, root.GetProperty("workerProtocolVersion").GetUInt32());
HashSet<string> fixtureIds = new(StringComparer.Ordinal);
foreach (JsonElement fixture in root.GetProperty("fixtures").EnumerateArray())
{
string id = fixture.GetProperty("id").GetString()!;
string path = fixture.GetProperty("path").GetString()!;
string category = fixture.GetProperty("category").GetString()!;
string messageType = fixture.GetProperty("messageType").GetString()!;
Assert.True(fixtureIds.Add(id), $"Duplicate behavior fixture id '{id}'.");
Assert.Contains(category, KnownCategories);
Assert.Contains(messageType, KnownMessageTypes);
Assert.True(
File.Exists(Path.Combine(GetBehaviorFixtureRoot().FullName, path)),
$"Expected behavior fixture '{path}' to exist.");
Assert.False(Path.IsPathRooted(path), $"Fixture path '{path}' must be relative.");
Assert.NotEmpty(fixture.GetProperty("expectation").GetString()!);
}
}
[Fact]
public void ProtoInputManifest_ReferencesBehaviorFixtureRoot()
{
DirectoryInfo repositoryRoot = FindRepositoryRoot();
string manifestPath = Path.Combine(repositoryRoot.FullName, "clients", "proto", "proto-inputs.json");
using JsonDocument manifest = JsonDocument.Parse(File.ReadAllText(manifestPath));
string fixtureRoot = manifest.RootElement.GetProperty("behaviorFixtureRoot").GetString()!;
Assert.Equal("clients/proto/fixtures/behavior", fixtureRoot);
Assert.True(Directory.Exists(Path.Combine(repositoryRoot.FullName, fixtureRoot)));
}
[Fact]
public void CommandReplyFixtures_ParseWithCurrentContractAndPreserveMxAccessDetails()
{
IReadOnlyList<JsonElement> fixtures = LoadManifestFixtures("command_replies");
Assert.NotEmpty(fixtures);
foreach (JsonElement fixture in fixtures)
{
MxCommandReply reply = ParseFixture<MxCommandReply>(
fixture,
MxCommandReply.Parser);
Assert.NotEqual(MxCommandKind.Unspecified, reply.Kind);
Assert.NotEqual(ProtocolStatusCode.Unspecified, reply.ProtocolStatus.Code);
Assert.True(reply.HasHresult, $"Fixture '{GetFixtureId(fixture)}' must carry an HRESULT.");
Assert.NotEmpty(reply.Statuses);
Assert.NotEqual(MxDataType.Unspecified, reply.ReturnValue.DataType);
Assert.True(
reply.ReturnValue.KindCase != MxValue.KindOneofCase.None || reply.ReturnValue.IsNull,
$"Fixture '{GetFixtureId(fixture)}' must carry a typed value, raw value, or explicit null.");
}
MxCommandReply failedWrite = ParseFixture<MxCommandReply>(
Assert.Single(fixtures, fixture => GetFixtureId(fixture) == "command-reply.write.mxaccess-failure"),
MxCommandReply.Parser);
Assert.Equal(ProtocolStatusCode.MxaccessFailure, failedWrite.ProtocolStatus.Code);
Assert.Equal(-2147220992, failedWrite.Hresult);
Assert.True(failedWrite.Statuses.Count > 1);
Assert.All(failedWrite.Statuses, status => Assert.Equal(0, status.Success));
}
[Fact]
public void EventStreamFixtures_ParseWithMonotonicSequencesAndExpectedFamilies()
{
IReadOnlyList<JsonElement> fixtures = LoadManifestFixtures("event_streams");
Assert.NotEmpty(fixtures);
foreach (JsonElement fixture in fixtures)
{
using JsonDocument document = JsonDocument.Parse(File.ReadAllText(GetFixturePath(fixture)));
ulong previousSequence = 0;
List<MxEventFamily> families = [];
foreach (JsonElement eventElement in document.RootElement.GetProperty("events").EnumerateArray())
{
MxEvent gatewayEvent = ProtobufJsonParser.Parse<MxEvent>(eventElement.GetRawText());
Assert.True(gatewayEvent.WorkerSequence > previousSequence);
Assert.Equal(document.RootElement.GetProperty("sessionId").GetString(), gatewayEvent.SessionId);
Assert.NotEmpty(gatewayEvent.Statuses);
AssertEventBodyMatchesFamily(gatewayEvent);
previousSequence = gatewayEvent.WorkerSequence;
families.Add(gatewayEvent.Family);
}
Assert.Contains(MxEventFamily.OnDataChange, families);
Assert.Contains(MxEventFamily.OnWriteComplete, families);
Assert.Contains(MxEventFamily.OperationComplete, families);
Assert.Contains(MxEventFamily.OnBufferedDataChange, families);
}
}
[Fact]
public void ValueConversionFixtures_ParseTypedValuesAndRawFallbacks()
{
JsonElement cases = LoadCaseSet("value_conversion", "cases");
bool sawRawFallback = false;
bool sawRawArrayFallback = false;
bool sawTypedArray = false;
foreach (JsonElement valueCase in cases.EnumerateArray())
{
MxValue value = ProtobufJsonParser.Parse<MxValue>(
valueCase.GetProperty("value").GetRawText());
string expectedKind = valueCase.GetProperty("expectedKind").GetString()!;
Assert.NotEqual(MxDataType.Unspecified, value.DataType);
AssertJsonKindMatchesValueKind(expectedKind, value);
sawRawFallback |= value.KindCase == MxValue.KindOneofCase.RawValue
&& !string.IsNullOrWhiteSpace(value.RawDiagnostic)
&& value.RawDataType != 0;
sawRawArrayFallback |= value.KindCase == MxValue.KindOneofCase.ArrayValue
&& value.ArrayValue.ValuesCase == MxArray.ValuesOneofCase.RawValues
&& !string.IsNullOrWhiteSpace(value.ArrayValue.RawDiagnostic)
&& value.ArrayValue.RawElementDataType != 0;
sawTypedArray |= value.KindCase == MxValue.KindOneofCase.ArrayValue
&& value.ArrayValue.ValuesCase != MxArray.ValuesOneofCase.RawValues;
}
Assert.True(sawRawFallback, "Expected at least one raw scalar fallback case.");
Assert.True(sawRawArrayFallback, "Expected at least one raw array fallback case.");
Assert.True(sawTypedArray, "Expected at least one typed array case.");
}
[Fact]
public void StatusConversionFixtures_ParseStatusArraysAndRawFields()
{
JsonElement cases = LoadCaseSet("status_conversion", "cases");
bool sawRawUnknown = false;
foreach (JsonElement statusCase in cases.EnumerateArray())
{
MxStatusProxy status = ProtobufJsonParser.Parse<MxStatusProxy>(
statusCase.GetProperty("status").GetRawText());
Assert.NotEqual(MxStatusCategory.Unspecified, status.Category);
Assert.NotEqual(MxStatusSource.Unspecified, status.DetectedBy);
Assert.NotEmpty(status.DiagnosticText);
sawRawUnknown |= status.Category == MxStatusCategory.Unknown
&& status.RawCategory != 0
&& status.RawDetectedBy != 0;
}
Assert.True(sawRawUnknown, "Expected a status case with unknown raw native fields.");
}
[Fact]
public void AuthErrorFixtures_MapAuthenticationAuthorizationAndRedactCredentials()
{
JsonElement cases = LoadCaseSet("auth_errors", "cases");
HashSet<string> statusCodes = new(StringComparer.Ordinal);
foreach (JsonElement authCase in cases.EnumerateArray())
{
string grpcStatusCode = authCase.GetProperty("grpcStatusCode").GetString()!;
string category = authCase.GetProperty("clientErrorCategory").GetString()!;
string redactedOutput = authCase.GetProperty("expectedRedactedOutput").GetString()!;
string serialized = authCase.GetRawText();
Assert.Contains(grpcStatusCode, AuthGrpcStatusCodes);
Assert.Contains(category, AuthClientErrorCategories);
string authorization = authCase.GetProperty("inputMetadata").GetProperty("authorization").GetString()!;
if (!string.IsNullOrEmpty(authorization))
{
Assert.Contains("<redacted>", serialized);
}
Assert.DoesNotContain("mxgw_", serialized, StringComparison.Ordinal);
Assert.DoesNotContain("secret", redactedOutput, StringComparison.OrdinalIgnoreCase);
statusCodes.Add(grpcStatusCode);
}
Assert.Contains("UNAUTHENTICATED", statusCodes);
Assert.Contains("PERMISSION_DENIED", statusCodes);
}
[Fact]
public void TimeoutCancelFixtures_DocumentClientWaitAndWorkerCommandBehavior()
{
JsonElement cases = LoadCaseSet("timeout_cancel", "cases");
HashSet<string> statusCodes = new(StringComparer.Ordinal);
foreach (JsonElement timeoutCase in cases.EnumerateArray())
{
string grpcStatusCode = timeoutCase.GetProperty("grpcStatusCode").GetString()!;
Assert.Contains(grpcStatusCode, TimeoutGrpcStatusCodes);
Assert.NotEmpty(timeoutCase.GetProperty("clientDeadline").GetString()!);
Assert.NotEmpty(timeoutCase.GetProperty("gatewayWaitBehavior").GetString()!);
Assert.NotEmpty(timeoutCase.GetProperty("workerCommandBehavior").GetString()!);
Assert.NotEmpty(timeoutCase.GetProperty("expectedClientAction").GetString()!);
statusCodes.Add(grpcStatusCode);
}
Assert.Contains("DEADLINE_EXCEEDED", statusCodes);
Assert.Contains("CANCELLED", statusCodes);
}
private static readonly string[] KnownCategories =
[
"command_replies",
"event_streams",
"value_conversion",
"status_conversion",
"auth_errors",
"timeout_cancel",
];
private static readonly string[] KnownMessageTypes =
[
"mxaccess_gateway.v1.MxCommandReply",
"mxaccess_gateway.v1.MxEvent",
"mxaccess_gateway.v1.MxValue",
"mxaccess_gateway.v1.MxStatusProxy",
"client_behavior.v1.AuthErrorCase",
"client_behavior.v1.TimeoutCancelCase",
];
private static readonly string[] AuthGrpcStatusCodes =
[
"UNAUTHENTICATED",
"PERMISSION_DENIED",
];
private static readonly string[] AuthClientErrorCategories =
[
"AuthenticationError",
"AuthorizationError",
];
private static readonly string[] TimeoutGrpcStatusCodes =
[
"DEADLINE_EXCEEDED",
"CANCELLED",
];
private static T ParseFixture<T>(
JsonElement fixture,
MessageParser<T> parser)
where T : IMessage<T>
{
return parser.ParseJson(File.ReadAllText(GetFixturePath(fixture)));
}
private static JsonElement LoadCaseSet(
string category,
string propertyName)
{
JsonElement fixture = Assert.Single(LoadManifestFixtures(category));
using JsonDocument document = JsonDocument.Parse(File.ReadAllText(GetFixturePath(fixture)));
return document.RootElement.GetProperty(propertyName).Clone();
}
private static IReadOnlyList<JsonElement> LoadManifestFixtures(string category)
{
using JsonDocument manifest = LoadBehaviorManifest();
return manifest.RootElement
.GetProperty("fixtures")
.EnumerateArray()
.Where(fixture => fixture.GetProperty("category").GetString() == category)
.Select(fixture => fixture.Clone())
.ToArray();
}
private static JsonDocument LoadBehaviorManifest()
{
return JsonDocument.Parse(File.ReadAllText(Path.Combine(GetBehaviorFixtureRoot().FullName, "manifest.json")));
}
private static string GetFixturePath(JsonElement fixture)
{
return Path.Combine(GetBehaviorFixtureRoot().FullName, fixture.GetProperty("path").GetString()!);
}
private static string GetFixtureId(JsonElement fixture)
{
return fixture.GetProperty("id").GetString()!;
}
private static DirectoryInfo GetBehaviorFixtureRoot()
{
DirectoryInfo repositoryRoot = FindRepositoryRoot();
return new DirectoryInfo(Path.Combine(repositoryRoot.FullName, "clients", "proto", "fixtures", "behavior"));
}
private static DirectoryInfo FindRepositoryRoot()
{
DirectoryInfo? current = new(AppContext.BaseDirectory);
while (current is not null)
{
if (File.Exists(Path.Combine(current.FullName, "AGENTS.md"))
&& Directory.Exists(Path.Combine(current.FullName, "src"))
&& Directory.Exists(Path.Combine(current.FullName, "clients")))
{
return current;
}
current = current.Parent;
}
throw new DirectoryNotFoundException("Could not locate the repository root from the test output directory.");
}
private static void AssertEventBodyMatchesFamily(MxEvent gatewayEvent)
{
switch (gatewayEvent.Family)
{
case MxEventFamily.OnDataChange:
Assert.Equal(MxEvent.BodyOneofCase.OnDataChange, gatewayEvent.BodyCase);
break;
case MxEventFamily.OnWriteComplete:
Assert.Equal(MxEvent.BodyOneofCase.OnWriteComplete, gatewayEvent.BodyCase);
break;
case MxEventFamily.OperationComplete:
Assert.Equal(MxEvent.BodyOneofCase.OperationComplete, gatewayEvent.BodyCase);
break;
case MxEventFamily.OnBufferedDataChange:
Assert.Equal(MxEvent.BodyOneofCase.OnBufferedDataChange, gatewayEvent.BodyCase);
break;
default:
throw new InvalidOperationException($"Unexpected event family '{gatewayEvent.Family}'.");
}
}
private static void AssertJsonKindMatchesValueKind(
string expectedKind,
MxValue value)
{
MxValue.KindOneofCase expected = expectedKind switch
{
"boolValue" => MxValue.KindOneofCase.BoolValue,
"int32Value" => MxValue.KindOneofCase.Int32Value,
"int64Value" => MxValue.KindOneofCase.Int64Value,
"floatValue" => MxValue.KindOneofCase.FloatValue,
"doubleValue" => MxValue.KindOneofCase.DoubleValue,
"stringValue" => MxValue.KindOneofCase.StringValue,
"timestampValue" => MxValue.KindOneofCase.TimestampValue,
"arrayValue" => MxValue.KindOneofCase.ArrayValue,
"rawValue" => MxValue.KindOneofCase.RawValue,
_ => throw new InvalidOperationException($"Unexpected expected value kind '{expectedKind}'."),
};
Assert.Equal(expected, value.KindCase);
}
}
@@ -105,6 +105,25 @@ public sealed class FakeWorkerHarnessTests
Assert.Equal(WorkerClientState.Faulted, client.State);
}
[Fact]
public async Task SendHeartbeatAsync_UpdatesClientHeartbeatState()
{
await using FakeWorkerHarness fakeWorker = await FakeWorkerHarness.CreateConnectedPairAsync();
await using WorkerClient client = fakeWorker.CreateClient();
await StartClientAsync(fakeWorker, client);
DateTimeOffset previousHeartbeat = client.LastHeartbeatAt;
await Task.Delay(TimeSpan.FromMilliseconds(20));
await fakeWorker.SendHeartbeatAsync(
configureHeartbeat: heartbeat => heartbeat.WorkerProcessId = 2468);
await WaitUntilAsync(
() => client.ProcessId == 2468 && client.LastHeartbeatAt > previousHeartbeat,
TestTimeout);
Assert.Equal(WorkerClientState.Ready, client.State);
}
[Fact]
public async Task InvokeAsync_WithHungWorker_TimesOutPendingCommand()
{
@@ -284,6 +284,26 @@ public sealed class FakeWorkerHarness : IAsyncDisposable
cancellationToken).ConfigureAwait(false);
}
public async Task SendHeartbeatAsync(
WorkerState state = WorkerState.Ready,
CancellationToken cancellationToken = default,
Action<WorkerHeartbeat>? configureHeartbeat = null)
{
WorkerHeartbeat heartbeat = new()
{
WorkerProcessId = DefaultWorkerProcessId,
State = state,
LastStaActivityTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
};
configureHeartbeat?.Invoke(heartbeat);
await _writer.WriteAsync(
CreateEnvelope(
correlationId: string.Empty,
envelope => envelope.WorkerHeartbeat = heartbeat),
cancellationToken).ConfigureAwait(false);
}
public async Task SendShutdownAckAsync(
ProtocolStatusCode statusCode = ProtocolStatusCode.Ok,
CancellationToken cancellationToken = default)
@@ -1,4 +1,5 @@
using System.IO.Pipes;
using Google.Protobuf.WellKnownTypes;
using MxGateway.Contracts;
using MxGateway.Contracts.Proto;
using MxGateway.Server.Workers;
@@ -151,6 +152,24 @@ public sealed class WorkerClientTests
Assert.Equal(WorkerClientState.Faulted, client.State);
}
[Fact]
public async Task ReadLoop_WhenHeartbeatArrives_UpdatesLastHeartbeatAndWorkerProcess()
{
await using PipePair pipePair = await PipePair.CreateAsync();
await using WorkerClient client = CreateClient(pipePair);
await CompleteHandshakeAsync(client, pipePair);
DateTimeOffset previousHeartbeat = client.LastHeartbeatAt;
await Task.Delay(TimeSpan.FromMilliseconds(20));
await pipePair.WorkerWriter.WriteAsync(CreateHeartbeatEnvelope(workerProcessId: 9876));
await WaitUntilAsync(
() => client.ProcessId == 9876 && client.LastHeartbeatAt > previousHeartbeat,
TestTimeout);
Assert.Equal(WorkerClientState.Ready, client.State);
}
[Fact]
public async Task HeartbeatMonitor_WhenHeartbeatExpires_FaultsClient()
{
@@ -276,6 +295,21 @@ public sealed class WorkerClientTests
});
}
private static WorkerEnvelope CreateHeartbeatEnvelope(int workerProcessId)
{
return CreateWorkerEnvelope(
correlationId: string.Empty,
sequence: 20,
envelope => envelope.WorkerHeartbeat = new WorkerHeartbeat
{
WorkerProcessId = workerProcessId,
State = WorkerState.Ready,
LastStaActivityTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
PendingCommandCount = 0,
OutboundEventQueueDepth = 0,
});
}
private static WorkerEnvelope CreateWorkerEnvelope(
string correlationId,
ulong sequence,
@@ -30,7 +30,7 @@ public sealed class WorkerApplicationTests
Assert.Equal("mxaccess-gateway-123-session-1", entry.Fields["pipe_name"]);
Assert.Equal(GatewayContractInfo.WorkerProtocolVersion, entry.Fields["protocol_version"]);
Assert.Equal("[redacted]", entry.Fields["nonce"]);
Assert.Equal("WorkerPipeHandshakeSucceeded", logger.Entries[1].EventName);
Assert.Equal("WorkerPipeSessionCompleted", logger.Entries[1].EventName);
}
[Fact]
@@ -1,10 +1,15 @@
using System;
using System.IO;
using System.IO.Pipes;
using System.Threading;
using System.Threading.Tasks;
using Google.Protobuf.WellKnownTypes;
using MxGateway.Contracts;
using MxGateway.Contracts.Proto;
using MxGateway.Worker.Bootstrap;
using MxGateway.Worker.Ipc;
using MxGateway.Worker.MxAccess;
using MxGateway.Worker.Sta;
namespace MxGateway.Worker.Tests.Ipc;
@@ -28,7 +33,9 @@ public sealed class WorkerPipeClientTests
PipeTransmissionMode.Byte,
PipeOptions.Asynchronous);
WorkerPipeClient client = new(connectTimeoutMilliseconds: 5000);
WorkerPipeClient client = new(
connectTimeoutMilliseconds: 5000,
(stream, options) => CreateSession(stream, options));
Task clientTask = client.RunAsync(workerOptions);
await Task.Factory.FromAsync(server.BeginWaitForConnection, server.EndWaitForConnection, null);
@@ -56,6 +63,94 @@ public sealed class WorkerPipeClientTests
WorkerEnvelope ready = await reader.ReadAsync();
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerReady, ready.BodyCase);
await writer.WriteAsync(new WorkerEnvelope
{
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
SessionId = "session-1",
Sequence = 2,
WorkerShutdown = new WorkerShutdown
{
GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(1)),
Reason = "test-complete",
},
});
WorkerEnvelope shutdownAck = await reader.ReadAsync();
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, shutdownAck.BodyCase);
await clientTask;
}
private static WorkerPipeSession CreateSession(
Stream stream,
WorkerFrameProtocolOptions options)
{
return new WorkerPipeSession(
new WorkerFrameReader(stream, options),
new WorkerFrameWriter(stream, options),
options,
() => 1234,
new WorkerPipeSessionOptions
{
HeartbeatInterval = TimeSpan.FromSeconds(30),
HeartbeatGrace = TimeSpan.FromSeconds(30),
},
() => new FakeRuntimeSession());
}
private sealed class FakeRuntimeSession : IWorkerRuntimeSession
{
public Task<WorkerReady> StartAsync(
string sessionId,
int workerProcessId,
CancellationToken cancellationToken = default)
{
return Task.FromResult(new WorkerReady
{
WorkerProcessId = workerProcessId,
MxaccessProgid = MxGateway.Worker.MxAccess.MxAccessInteropInfo.ProgId,
MxaccessClsid = MxGateway.Worker.MxAccess.MxAccessInteropInfo.Clsid,
ReadyTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
});
}
public Task<MxCommandReply> DispatchAsync(StaCommand command)
{
return Task.FromResult(new MxCommandReply
{
SessionId = command.SessionId,
CorrelationId = command.CorrelationId,
Kind = command.Kind,
ProtocolStatus = new ProtocolStatus
{
Code = ProtocolStatusCode.Ok,
Message = "OK",
},
});
}
public WorkerRuntimeHeartbeatSnapshot CaptureHeartbeat()
{
return new WorkerRuntimeHeartbeatSnapshot(
DateTimeOffset.UtcNow,
pendingCommandCount: 0,
outboundEventQueueDepth: 0,
lastEventSequence: 0,
currentCommandCorrelationId: string.Empty);
}
public void RequestShutdown()
{
}
public Task<MxAccessShutdownResult> ShutdownGracefullyAsync(
TimeSpan timeout,
CancellationToken cancellationToken = default)
{
return Task.FromResult(new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>()));
}
public void Dispose()
{
}
}
}
@@ -1,12 +1,16 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Pipes;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Google.Protobuf.WellKnownTypes;
using MxGateway.Contracts;
using MxGateway.Contracts.Proto;
using MxGateway.Worker.Ipc;
using MxGateway.Worker.MxAccess;
using MxGateway.Worker.Sta;
namespace MxGateway.Worker.Tests.Ipc;
@@ -149,26 +153,124 @@ public sealed class WorkerPipeSessionTests
}
[Fact]
public async Task RunAsync_WithWorkerShutdown_WritesShutdownAckAndReturns()
public async Task RunAsync_SendsHeartbeatPayloadFromRuntimeSnapshot()
{
WorkerFrameProtocolOptions options = CreateOptions();
MemoryStream inbound = new();
WorkerFrameWriter inboundWriter = new(inbound, options);
await inboundWriter.WriteAsync(CreateGatewayHelloEnvelope());
await inboundWriter.WriteAsync(CreateWorkerShutdownEnvelope());
inbound.Position = 0;
MemoryStream outbound = new();
WorkerPipeSession session = CreateSession(inbound, outbound, options);
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5));
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
FakeRuntimeSession runtime = new();
runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot(
DateTimeOffset.UtcNow,
pendingCommandCount: 2,
outboundEventQueueDepth: 3,
lastEventSequence: 42,
currentCommandCorrelationId: "current-command"));
WorkerPipeSession session = CreatePipeSession(
pipePair.WorkerStream,
runtime,
new WorkerPipeSessionOptions
{
HeartbeatInterval = TimeSpan.FromMilliseconds(20),
HeartbeatGrace = TimeSpan.FromSeconds(5),
});
Task runTask = session.RunAsync(cancellation.Token);
await session.CompleteStartupHandshakeAsync(_ => Task.CompletedTask);
await session.RunAsync();
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
await ThrowIfCompletedAsync(runTask);
WorkerEnvelope[] written = ReadWrittenFrames(outbound, options);
Assert.Equal(3, written.Length);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerHello, written[0].BodyCase);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerReady, written[1].BodyCase);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerShutdownAck, written[2].BodyCase);
Assert.Equal(ProtocolStatusCode.Ok, written[2].WorkerShutdownAck.Status.Code);
WorkerEnvelope heartbeat = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerHeartbeat,
cancellation.Token);
Assert.Equal(WorkerState.ExecutingCommand, heartbeat.WorkerHeartbeat.State);
Assert.Equal(1234, heartbeat.WorkerHeartbeat.WorkerProcessId);
Assert.Equal(2u, heartbeat.WorkerHeartbeat.PendingCommandCount);
Assert.Equal(3u, heartbeat.WorkerHeartbeat.OutboundEventQueueDepth);
Assert.Equal(42UL, heartbeat.WorkerHeartbeat.LastEventSequence);
Assert.Equal("current-command", heartbeat.WorkerHeartbeat.CurrentCommandCorrelationId);
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
[Fact]
public async Task RunAsync_WhenCommandIsExecuting_HeartbeatReportsCurrentCorrelation()
{
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5));
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
FakeRuntimeSession runtime = new()
{
BlockDispatch = true,
};
WorkerPipeSession session = CreatePipeSession(
pipePair.WorkerStream,
runtime,
new WorkerPipeSessionOptions
{
HeartbeatInterval = TimeSpan.FromMilliseconds(20),
HeartbeatGrace = TimeSpan.FromSeconds(5),
});
Task runTask = session.RunAsync(cancellation.Token);
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
await pipePair.GatewayWriter.WriteAsync(
CreateCommandEnvelope("command-1"),
cancellation.Token);
Assert.True(runtime.DispatchStarted.Wait(TimeSpan.FromSeconds(2)));
WorkerEnvelope heartbeat = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerHeartbeat,
envelope => envelope.WorkerHeartbeat.CurrentCommandCorrelationId == "command-1",
cancellation.Token);
Assert.Equal("command-1", heartbeat.WorkerHeartbeat.CurrentCommandCorrelationId);
Assert.Equal(WorkerState.ExecutingCommand, heartbeat.WorkerHeartbeat.State);
runtime.ReleaseDispatch();
WorkerEnvelope reply = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
cancellation.Token);
Assert.Equal("command-1", reply.CorrelationId);
Assert.Equal(ProtocolStatusCode.Ok, reply.WorkerCommandReply.Reply.ProtocolStatus.Code);
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
[Fact]
public async Task RunAsync_WhenStaActivityIsStale_WritesWatchdogFault()
{
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5));
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
FakeRuntimeSession runtime = new();
runtime.SetSnapshot(new WorkerRuntimeHeartbeatSnapshot(
DateTimeOffset.UtcNow - TimeSpan.FromSeconds(5),
pendingCommandCount: 0,
outboundEventQueueDepth: 0,
lastEventSequence: 0,
currentCommandCorrelationId: "stuck-command"));
WorkerPipeSession session = CreatePipeSession(
pipePair.WorkerStream,
runtime,
new WorkerPipeSessionOptions
{
HeartbeatInterval = TimeSpan.FromMilliseconds(20),
HeartbeatGrace = TimeSpan.FromMilliseconds(50),
});
Task runTask = session.RunAsync(cancellation.Token);
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
WorkerEnvelope fault = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerFault,
cancellation.Token);
Assert.Equal(WorkerFaultCategory.StaHung, fault.WorkerFault.Category);
Assert.Equal("stuck-command", fault.WorkerFault.CommandMethod);
Assert.Contains("STA activity is stale", fault.WorkerFault.DiagnosticMessage);
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
}
private static WorkerPipeSession CreateSession(
@@ -183,6 +285,21 @@ public sealed class WorkerPipeSessionTests
() => 1234);
}
private static WorkerPipeSession CreatePipeSession(
Stream stream,
FakeRuntimeSession runtime,
WorkerPipeSessionOptions sessionOptions)
{
WorkerFrameProtocolOptions options = CreateOptions();
return new WorkerPipeSession(
new WorkerFrameReader(stream, options),
new WorkerFrameWriter(stream, options),
options,
() => 1234,
sessionOptions,
() => runtime);
}
private static WorkerFrameProtocolOptions CreateOptions()
{
return new WorkerFrameProtocolOptions(
@@ -209,21 +326,119 @@ public sealed class WorkerPipeSessionTests
};
}
private static WorkerEnvelope CreateWorkerShutdownEnvelope()
private static WorkerEnvelope CreateCommandEnvelope(string correlationId)
{
return new WorkerEnvelope
{
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
SessionId = SessionId,
Sequence = 2,
WorkerShutdown = new WorkerShutdown
CorrelationId = correlationId,
WorkerCommand = new WorkerCommand
{
GracePeriod = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(TimeSpan.FromSeconds(1)),
Reason = "test-shutdown",
Command = new MxCommand
{
Kind = MxCommandKind.Ping,
Ping = new PingCommand
{
Message = "ping",
},
},
EnqueueTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
},
};
}
private static WorkerEnvelope CreateShutdownEnvelope()
{
return new WorkerEnvelope
{
ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
SessionId = SessionId,
Sequence = 3,
WorkerShutdown = new WorkerShutdown
{
GracePeriod = Duration.FromTimeSpan(TimeSpan.FromSeconds(1)),
Reason = "test-complete",
},
};
}
private static async Task CompleteGatewayHandshakeAsync(
PipePair pipePair,
CancellationToken cancellationToken)
{
await pipePair.GatewayWriter
.WriteAsync(CreateGatewayHelloEnvelope(), cancellationToken)
.ConfigureAwait(false);
WorkerEnvelope hello = await pipePair.GatewayReader.ReadAsync(cancellationToken).ConfigureAwait(false);
WorkerEnvelope ready = await pipePair.GatewayReader.ReadAsync(cancellationToken).ConfigureAwait(false);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerHello, hello.BodyCase);
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerReady, ready.BodyCase);
}
private static async Task SendShutdownAndWaitAsync(
PipePair pipePair,
Task runTask,
CancellationToken cancellationToken)
{
await pipePair.GatewayWriter
.WriteAsync(CreateShutdownEnvelope(), cancellationToken)
.ConfigureAwait(false);
WorkerEnvelope shutdownAck = await ReadUntilAsync(
pipePair.GatewayReader,
WorkerEnvelope.BodyOneofCase.WorkerShutdownAck,
cancellationToken);
Assert.Equal(ProtocolStatusCode.Ok, shutdownAck.WorkerShutdownAck.Status.Code);
Task completedTask = await Task
.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(2), cancellationToken))
.ConfigureAwait(false);
Assert.Same(runTask, completedTask);
await runTask.ConfigureAwait(false);
}
private static async Task ThrowIfCompletedAsync(Task task)
{
await Task.Delay(TimeSpan.FromMilliseconds(100));
if (task.IsCompleted)
{
await task;
}
}
private static Task<WorkerEnvelope> ReadUntilAsync(
WorkerFrameReader reader,
WorkerEnvelope.BodyOneofCase expectedBody,
CancellationToken cancellationToken)
{
return ReadUntilAsync(
reader,
expectedBody,
_ => true,
cancellationToken);
}
private static async Task<WorkerEnvelope> ReadUntilAsync(
WorkerFrameReader reader,
WorkerEnvelope.BodyOneofCase expectedBody,
Func<WorkerEnvelope, bool> predicate,
CancellationToken cancellationToken)
{
while (true)
{
WorkerEnvelope envelope = await reader.ReadAsync(cancellationToken).ConfigureAwait(false);
if (envelope.BodyCase == expectedBody && predicate(envelope))
{
return envelope;
}
}
}
private static WorkerEnvelope[] ReadWrittenFrames(
MemoryStream stream,
WorkerFrameProtocolOptions options)
@@ -258,4 +473,166 @@ public sealed class WorkerPipeSessionTests
buffer[2] = (byte)(value >> 16);
buffer[3] = (byte)(value >> 24);
}
private sealed class FakeRuntimeSession : IWorkerRuntimeSession
{
private readonly ManualResetEventSlim releaseDispatch = new(false);
private readonly object gate = new();
private WorkerRuntimeHeartbeatSnapshot snapshot = new(
DateTimeOffset.UtcNow,
pendingCommandCount: 0,
outboundEventQueueDepth: 0,
lastEventSequence: 0,
currentCommandCorrelationId: string.Empty);
public ManualResetEventSlim DispatchStarted { get; } = new(false);
public bool BlockDispatch { get; set; }
public Task<WorkerReady> StartAsync(
string sessionId,
int workerProcessId,
CancellationToken cancellationToken = default)
{
return Task.FromResult(new WorkerReady
{
WorkerProcessId = workerProcessId,
MxaccessProgid = MxGateway.Worker.MxAccess.MxAccessInteropInfo.ProgId,
MxaccessClsid = MxGateway.Worker.MxAccess.MxAccessInteropInfo.Clsid,
ReadyTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
});
}
public Task<MxCommandReply> DispatchAsync(StaCommand command)
{
return Task.Run(
() =>
{
SetSnapshot(new WorkerRuntimeHeartbeatSnapshot(
DateTimeOffset.UtcNow,
pendingCommandCount: 0,
outboundEventQueueDepth: 0,
lastEventSequence: 0,
command.CorrelationId));
DispatchStarted.Set();
if (BlockDispatch)
{
releaseDispatch.Wait(TimeSpan.FromSeconds(5));
}
SetSnapshot(new WorkerRuntimeHeartbeatSnapshot(
DateTimeOffset.UtcNow,
pendingCommandCount: 0,
outboundEventQueueDepth: 0,
lastEventSequence: 0,
currentCommandCorrelationId: string.Empty));
return new MxCommandReply
{
SessionId = command.SessionId,
CorrelationId = command.CorrelationId,
Kind = command.Kind,
ProtocolStatus = new ProtocolStatus
{
Code = ProtocolStatusCode.Ok,
Message = "OK",
},
};
});
}
public WorkerRuntimeHeartbeatSnapshot CaptureHeartbeat()
{
lock (gate)
{
return snapshot;
}
}
public void RequestShutdown()
{
releaseDispatch.Set();
}
public Task<MxAccessShutdownResult> ShutdownGracefullyAsync(
TimeSpan timeout,
CancellationToken cancellationToken = default)
{
releaseDispatch.Set();
return Task.FromResult(new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>()));
}
public void ReleaseDispatch()
{
releaseDispatch.Set();
}
public void SetSnapshot(WorkerRuntimeHeartbeatSnapshot value)
{
lock (gate)
{
snapshot = value;
}
}
public void Dispose()
{
releaseDispatch.Set();
releaseDispatch.Dispose();
DispatchStarted.Dispose();
}
}
private sealed class PipePair : IDisposable
{
private readonly NamedPipeServerStream gatewayStream;
private PipePair(
NamedPipeServerStream gatewayStream,
NamedPipeClientStream workerStream)
{
this.gatewayStream = gatewayStream;
WorkerStream = workerStream;
WorkerFrameProtocolOptions options = CreateOptions();
GatewayReader = new WorkerFrameReader(gatewayStream, options);
GatewayWriter = new WorkerFrameWriter(gatewayStream, options);
}
public Stream WorkerStream { get; }
public WorkerFrameReader GatewayReader { get; }
public WorkerFrameWriter GatewayWriter { get; }
public static async Task<PipePair> CreateAsync(CancellationToken cancellationToken)
{
string pipeName = $"mxaccessgw-worker-session-tests-{Guid.NewGuid():N}";
NamedPipeServerStream gatewayStream = new(
pipeName,
PipeDirection.InOut,
maxNumberOfServerInstances: 1,
PipeTransmissionMode.Byte,
PipeOptions.Asynchronous);
NamedPipeClientStream workerStream = new(
".",
pipeName,
PipeDirection.InOut,
PipeOptions.Asynchronous);
Task waitForConnectionTask = gatewayStream.WaitForConnectionAsync();
await Task
.Run(() => workerStream.Connect(5000), cancellationToken)
.ConfigureAwait(false);
await waitForConnectionTask.ConfigureAwait(false);
return new PipePair(gatewayStream, workerStream);
}
public void Dispose()
{
WorkerStream.Dispose();
gatewayStream.Dispose();
}
}
}
+26 -3
View File
@@ -1,4 +1,5 @@
using System;
using System.IO;
using System.IO.Pipes;
using System.Threading;
using System.Threading.Tasks;
@@ -11,6 +12,7 @@ public sealed class WorkerPipeClient : IWorkerPipeClient
public const int DefaultConnectTimeoutMilliseconds = 30000;
private readonly int _connectTimeoutMilliseconds;
private readonly Func<Stream, WorkerFrameProtocolOptions, IWorkerLogger?, WorkerPipeSession> _sessionFactory;
private readonly IWorkerLogger? _logger;
public WorkerPipeClient()
@@ -28,9 +30,30 @@ public sealed class WorkerPipeClient : IWorkerPipeClient
{
}
public WorkerPipeClient(
int connectTimeoutMilliseconds,
Func<Stream, WorkerFrameProtocolOptions, WorkerPipeSession> sessionFactory)
: this(
null,
connectTimeoutMilliseconds,
(stream, frameOptions, _) => sessionFactory(stream, frameOptions))
{
}
public WorkerPipeClient(
IWorkerLogger? logger,
int connectTimeoutMilliseconds)
: this(
logger,
connectTimeoutMilliseconds,
(stream, frameOptions, workerLogger) => new WorkerPipeSession(stream, frameOptions, workerLogger))
{
}
public WorkerPipeClient(
IWorkerLogger? logger,
int connectTimeoutMilliseconds,
Func<Stream, WorkerFrameProtocolOptions, IWorkerLogger?, WorkerPipeSession> sessionFactory)
{
if (connectTimeoutMilliseconds <= 0)
{
@@ -39,8 +62,9 @@ public sealed class WorkerPipeClient : IWorkerPipeClient
"Worker pipe connect timeout must be greater than zero.");
}
_connectTimeoutMilliseconds = connectTimeoutMilliseconds;
_logger = logger;
_sessionFactory = sessionFactory ?? throw new ArgumentNullException(nameof(sessionFactory));
_connectTimeoutMilliseconds = connectTimeoutMilliseconds;
}
public async Task RunAsync(
@@ -62,8 +86,7 @@ public sealed class WorkerPipeClient : IWorkerPipeClient
await ConnectAsync(pipe, cancellationToken).ConfigureAwait(false);
WorkerPipeSession session = new(pipe, frameOptions, _logger);
await session.CompleteStartupHandshakeAsync(cancellationToken).ConfigureAwait(false);
WorkerPipeSession session = _sessionFactory(pipe, frameOptions, _logger);
await session.RunAsync(cancellationToken).ConfigureAwait(false);
}
+311 -130
View File
@@ -16,12 +16,15 @@ public sealed class WorkerPipeSession
{
private readonly WorkerFrameProtocolOptions _options;
private readonly Func<int> _processIdProvider;
private readonly Func<IWorkerRuntimeSession> _runtimeSessionFactory;
private readonly WorkerPipeSessionOptions _sessionOptions;
private readonly IWorkerLogger? _logger;
private readonly WorkerFrameReader _reader;
private readonly WorkerFrameWriter _writer;
private MxAccessStaSession? _mxAccessStaSession;
private IWorkerRuntimeSession? _runtimeSession;
private long _nextSequence;
private bool _shutdownCompleted;
private WorkerState _state = WorkerState.Starting;
private bool _watchdogFaultSent;
private bool _shutdownTimedOut;
public WorkerPipeSession(
@@ -33,22 +36,67 @@ public sealed class WorkerPipeSession
new WorkerFrameWriter(stream, options),
options,
() => Process.GetCurrentProcess().Id,
new WorkerPipeSessionOptions(),
() => new MxAccessStaSession(),
logger)
{
}
public WorkerPipeSession(
WorkerFrameReader reader,
WorkerFrameWriter writer,
WorkerFrameProtocolOptions options,
Func<int> processIdProvider)
: this(
reader,
writer,
options,
processIdProvider,
new WorkerPipeSessionOptions(),
() => new MxAccessStaSession(),
logger: null)
{
}
public WorkerPipeSession(
WorkerFrameReader reader,
WorkerFrameWriter writer,
WorkerFrameProtocolOptions options,
Func<int> processIdProvider,
WorkerPipeSessionOptions sessionOptions,
Func<IWorkerRuntimeSession> runtimeSessionFactory,
IWorkerLogger? logger = null)
{
_reader = reader ?? throw new ArgumentNullException(nameof(reader));
_writer = writer ?? throw new ArgumentNullException(nameof(writer));
_options = options ?? throw new ArgumentNullException(nameof(options));
_processIdProvider = processIdProvider ?? throw new ArgumentNullException(nameof(processIdProvider));
_sessionOptions = sessionOptions ?? throw new ArgumentNullException(nameof(sessionOptions));
_runtimeSessionFactory = runtimeSessionFactory ?? throw new ArgumentNullException(nameof(runtimeSessionFactory));
_logger = logger;
_sessionOptions.Validate();
}
public async Task RunAsync(CancellationToken cancellationToken = default)
{
_runtimeSession = _runtimeSessionFactory();
try
{
await CompleteStartupHandshakeAsync(
token => _runtimeSession.StartAsync(_options.SessionId, _processIdProvider(), token),
cancellationToken).ConfigureAwait(false);
await RunMessageLoopAsync(cancellationToken).ConfigureAwait(false);
}
finally
{
if (!_shutdownTimedOut)
{
_runtimeSession?.Dispose();
}
_runtimeSession = null;
_state = WorkerState.Stopped;
}
}
public Task CompleteStartupHandshakeAsync(CancellationToken cancellationToken = default)
@@ -86,11 +134,14 @@ public sealed class WorkerPipeSession
try
{
WorkerEnvelope envelope = await _reader.ReadAsync(cancellationToken).ConfigureAwait(false);
_state = WorkerState.Handshaking;
ValidateGatewayHello(envelope);
await WriteWorkerHelloAsync(cancellationToken).ConfigureAwait(false);
_state = WorkerState.InitializingSta;
WorkerReady ready = await initializeMxAccessAsync(cancellationToken).ConfigureAwait(false);
await WriteWorkerReadyAsync(ready, cancellationToken).ConfigureAwait(false);
_state = WorkerState.Ready;
}
catch (WorkerFrameProtocolException exception)
{
@@ -105,44 +156,6 @@ public sealed class WorkerPipeSession
}
}
public async Task RunAsync(CancellationToken cancellationToken = default)
{
try
{
while (true)
{
WorkerEnvelope envelope = await _reader.ReadAsync(cancellationToken).ConfigureAwait(false);
switch (envelope.BodyCase)
{
case WorkerEnvelope.BodyOneofCase.WorkerCommand:
await HandleCommandAsync(envelope, cancellationToken).ConfigureAwait(false);
break;
case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
await HandleShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
return;
case WorkerEnvelope.BodyOneofCase.WorkerCancel:
break;
default:
throw new WorkerFrameProtocolException(
WorkerFrameProtocolErrorCode.UnexpectedEnvelopeBody,
$"Worker received unexpected gateway envelope body {envelope.BodyCase} after startup.");
}
}
}
catch (WorkerFrameProtocolException exception)
{
await TryWriteFaultAsync(exception, cancellationToken).ConfigureAwait(false);
throw;
}
finally
{
if (!_shutdownCompleted && !_shutdownTimedOut)
{
_mxAccessStaSession?.Dispose();
}
}
}
private void ValidateGatewayHello(WorkerEnvelope envelope)
{
if (envelope.BodyCase != WorkerEnvelope.BodyOneofCase.GatewayHello)
@@ -188,6 +201,189 @@ public sealed class WorkerPipeSession
return _writer.WriteAsync(CreateEnvelope(ready), cancellationToken);
}
private async Task RunMessageLoopAsync(CancellationToken cancellationToken)
{
using CancellationTokenSource heartbeatCancellation = CancellationTokenSource
.CreateLinkedTokenSource(cancellationToken);
Task heartbeatTask = RunHeartbeatLoopAsync(heartbeatCancellation.Token);
try
{
while (!cancellationToken.IsCancellationRequested)
{
Task<WorkerEnvelope> readTask = _reader.ReadAsync(cancellationToken);
Task completedTask = await Task.WhenAny(readTask, heartbeatTask).ConfigureAwait(false);
if (completedTask == heartbeatTask)
{
await heartbeatTask.ConfigureAwait(false);
}
WorkerEnvelope envelope = await readTask.ConfigureAwait(false);
bool keepReading = await DispatchGatewayEnvelopeAsync(envelope, cancellationToken).ConfigureAwait(false);
if (!keepReading)
{
return;
}
}
}
finally
{
heartbeatCancellation.Cancel();
try
{
await heartbeatTask.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
}
}
private async Task<bool> DispatchGatewayEnvelopeAsync(
WorkerEnvelope envelope,
CancellationToken cancellationToken)
{
switch (envelope.BodyCase)
{
case WorkerEnvelope.BodyOneofCase.WorkerCommand:
_ = ProcessCommandAsync(envelope, cancellationToken);
return true;
case WorkerEnvelope.BodyOneofCase.WorkerShutdown:
await ShutdownAsync(envelope.WorkerShutdown, cancellationToken).ConfigureAwait(false);
return false;
case WorkerEnvelope.BodyOneofCase.WorkerCancel:
return true;
default:
throw new WorkerFrameProtocolException(
WorkerFrameProtocolErrorCode.UnexpectedEnvelopeBody,
$"Worker received unexpected gateway envelope body {envelope.BodyCase}.");
}
}
private async Task ProcessCommandAsync(
WorkerEnvelope envelope,
CancellationToken cancellationToken)
{
IWorkerRuntimeSession runtimeSession = _runtimeSession
?? throw new InvalidOperationException("Worker runtime session has not been initialized.");
WorkerCommand workerCommand = envelope.WorkerCommand;
MxCommand command = workerCommand.Command;
StaCommand staCommand = new(
_options.SessionId,
envelope.CorrelationId,
command,
workerCommand.EnqueueTimestamp,
cancellationToken);
try
{
MxCommandReply reply = await runtimeSession.DispatchAsync(staCommand).ConfigureAwait(false);
await _writer
.WriteAsync(
CreateEnvelope(new WorkerCommandReply
{
Reply = reply,
CompletedTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
}),
cancellationToken)
.ConfigureAwait(false);
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
_state = WorkerState.Faulted;
await TryWriteFaultAsync(
CreateFault(
WorkerFaultCategory.MxaccessCommandFailed,
staCommand.MethodName,
exception),
cancellationToken).ConfigureAwait(false);
}
}
private async Task ShutdownAsync(
WorkerShutdown shutdown,
CancellationToken cancellationToken)
{
_state = WorkerState.ShuttingDown;
IWorkerRuntimeSession? runtimeSession = _runtimeSession;
if (runtimeSession is null)
{
await WriteShutdownAckAsync(
CreateShutdownAck(new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>()), shutdown),
cancellationToken).ConfigureAwait(false);
return;
}
TimeSpan gracePeriod = ResolveGracePeriod(shutdown);
try
{
MxAccessShutdownResult result = await runtimeSession
.ShutdownGracefullyAsync(gracePeriod, cancellationToken)
.ConfigureAwait(false);
LogShutdownFailures(result.Failures);
await WriteShutdownAckAsync(CreateShutdownAck(result, shutdown), cancellationToken).ConfigureAwait(false);
}
catch (TimeoutException exception)
{
_shutdownTimedOut = true;
_state = WorkerState.Faulted;
await TryWriteFaultAsync(CreateShutdownTimeoutFault(exception), cancellationToken).ConfigureAwait(false);
throw;
}
}
private Task WriteShutdownAckAsync(
WorkerShutdownAck shutdownAck,
CancellationToken cancellationToken)
{
return _writer.WriteAsync(CreateEnvelope(shutdownAck), cancellationToken);
}
private async Task RunHeartbeatLoopAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
await Task.Delay(_sessionOptions.HeartbeatInterval, cancellationToken).ConfigureAwait(false);
IWorkerRuntimeSession? runtimeSession = _runtimeSession;
if (runtimeSession is null)
{
continue;
}
WorkerRuntimeHeartbeatSnapshot snapshot = runtimeSession.CaptureHeartbeat();
await _writer
.WriteAsync(CreateEnvelope(CreateHeartbeat(snapshot)), cancellationToken)
.ConfigureAwait(false);
await ReportWatchdogFaultIfNeededAsync(snapshot, cancellationToken).ConfigureAwait(false);
}
}
private async Task ReportWatchdogFaultIfNeededAsync(
WorkerRuntimeHeartbeatSnapshot snapshot,
CancellationToken cancellationToken)
{
TimeSpan staleFor = DateTimeOffset.UtcNow - snapshot.LastStaActivityUtc;
if (staleFor <= _sessionOptions.HeartbeatGrace)
{
_watchdogFaultSent = false;
return;
}
if (_watchdogFaultSent)
{
return;
}
_watchdogFaultSent = true;
await TryWriteFaultAsync(
CreateFault(
WorkerFaultCategory.StaHung,
snapshot.CurrentCommandCorrelationId,
$"STA activity is stale by {staleFor}."),
cancellationToken).ConfigureAwait(false);
}
private async Task TryWriteFaultAsync(
WorkerFrameProtocolException exception,
CancellationToken cancellationToken)
@@ -241,7 +437,7 @@ public sealed class WorkerPipeSession
|| faultWriteException is ObjectDisposedException
|| faultWriteException is WorkerFrameProtocolException)
{
// The shutdown timeout is the actionable error.
// The runtime fault remains observable through worker exit or pipe closure.
}
}
@@ -265,16 +461,16 @@ public sealed class WorkerPipeSession
return CreateBaseEnvelope(reply);
}
private WorkerEnvelope CreateEnvelope(WorkerEvent workerEvent)
{
return CreateBaseEnvelope(workerEvent);
}
private WorkerEnvelope CreateEnvelope(WorkerShutdownAck shutdownAck)
{
return CreateBaseEnvelope(shutdownAck);
}
private WorkerEnvelope CreateEnvelope(WorkerHeartbeat heartbeat)
{
return CreateBaseEnvelope(heartbeat);
}
private WorkerEnvelope CreateBaseEnvelope(WorkerHello body)
{
WorkerEnvelope envelope = CreateBaseEnvelope();
@@ -304,13 +500,6 @@ public sealed class WorkerPipeSession
return envelope;
}
private WorkerEnvelope CreateBaseEnvelope(WorkerEvent body)
{
WorkerEnvelope envelope = CreateBaseEnvelope();
envelope.WorkerEvent = body.Clone();
return envelope;
}
private WorkerEnvelope CreateBaseEnvelope(WorkerShutdownAck body)
{
WorkerEnvelope envelope = CreateBaseEnvelope();
@@ -318,6 +507,13 @@ public sealed class WorkerPipeSession
return envelope;
}
private WorkerEnvelope CreateBaseEnvelope(WorkerHeartbeat body)
{
WorkerEnvelope envelope = CreateBaseEnvelope();
envelope.WorkerHeartbeat = body;
return envelope;
}
private WorkerEnvelope CreateBaseEnvelope()
{
return new WorkerEnvelope
@@ -335,88 +531,37 @@ public sealed class WorkerPipeSession
private async Task<WorkerReady> InitializeMxAccessAsync(CancellationToken cancellationToken)
{
_mxAccessStaSession = new MxAccessStaSession();
_runtimeSession = new MxAccessStaSession();
try
{
return await _mxAccessStaSession
return await _runtimeSession
.StartAsync(_options.SessionId, _processIdProvider(), cancellationToken)
.ConfigureAwait(false);
}
catch
{
_mxAccessStaSession.Dispose();
_mxAccessStaSession = null;
_runtimeSession.Dispose();
_runtimeSession = null;
throw;
}
}
private async Task HandleCommandAsync(
WorkerEnvelope envelope,
CancellationToken cancellationToken)
private WorkerHeartbeat CreateHeartbeat(WorkerRuntimeHeartbeatSnapshot snapshot)
{
if (_mxAccessStaSession is null)
{
throw new InvalidOperationException("MXAccess STA session is not initialized.");
}
WorkerState state = string.IsNullOrWhiteSpace(snapshot.CurrentCommandCorrelationId)
? _state
: WorkerState.ExecutingCommand;
StaCommand command = new(
_options.SessionId,
envelope.CorrelationId,
envelope.WorkerCommand.Command,
envelope.WorkerCommand.EnqueueTimestamp,
cancellationToken);
MxCommandReply mxReply = await _mxAccessStaSession
.DispatchAsync(command)
.ConfigureAwait(false);
WorkerCommandReply reply = new()
return new WorkerHeartbeat
{
Reply = mxReply,
CompletedTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
WorkerProcessId = _processIdProvider(),
State = state,
LastStaActivityTimestamp = Timestamp.FromDateTimeOffset(snapshot.LastStaActivityUtc),
PendingCommandCount = snapshot.PendingCommandCount,
OutboundEventQueueDepth = snapshot.OutboundEventQueueDepth,
LastEventSequence = snapshot.LastEventSequence,
CurrentCommandCorrelationId = snapshot.CurrentCommandCorrelationId,
};
await _writer.WriteAsync(CreateEnvelope(reply), cancellationToken).ConfigureAwait(false);
await DrainEventsAsync(cancellationToken).ConfigureAwait(false);
}
private async Task HandleShutdownAsync(
WorkerShutdown shutdown,
CancellationToken cancellationToken)
{
TimeSpan gracePeriod = ResolveGracePeriod(shutdown);
try
{
MxAccessShutdownResult result = _mxAccessStaSession is null
? new MxAccessShutdownResult(Array.Empty<MxAccessShutdownFailure>())
: await _mxAccessStaSession
.ShutdownGracefullyAsync(gracePeriod, cancellationToken)
.ConfigureAwait(false);
LogShutdownFailures(result.Failures);
await _writer
.WriteAsync(CreateEnvelope(CreateShutdownAck(result)), cancellationToken)
.ConfigureAwait(false);
_shutdownCompleted = true;
}
catch (TimeoutException exception)
{
_shutdownTimedOut = true;
await TryWriteFaultAsync(CreateShutdownTimeoutFault(exception), cancellationToken).ConfigureAwait(false);
throw;
}
}
private async Task DrainEventsAsync(CancellationToken cancellationToken)
{
if (_mxAccessStaSession is null)
{
return;
}
foreach (WorkerEvent workerEvent in _mxAccessStaSession.DrainEvents(maxEvents: 0))
{
await _writer.WriteAsync(CreateEnvelope(workerEvent), cancellationToken).ConfigureAwait(false);
}
}
private WorkerReady CreateWorkerReady()
@@ -443,16 +588,24 @@ public sealed class WorkerPipeSession
: gracePeriod;
}
private static WorkerShutdownAck CreateShutdownAck(MxAccessShutdownResult result)
private static WorkerShutdownAck CreateShutdownAck(
MxAccessShutdownResult result,
WorkerShutdown shutdown)
{
string message = result.Succeeded
? "Graceful shutdown completed."
: $"Graceful shutdown completed with {result.Failures.Count} cleanup failure(s).";
if (!string.IsNullOrWhiteSpace(shutdown.Reason))
{
message = $"{message} Reason: {shutdown.Reason}";
}
return new WorkerShutdownAck
{
Status = new ProtocolStatus
{
Code = ProtocolStatusCode.Ok,
Message = result.Succeeded
? "Graceful shutdown completed."
: $"Graceful shutdown completed with {result.Failures.Count} cleanup failure(s).",
Message = message,
},
};
}
@@ -511,22 +664,50 @@ public sealed class WorkerPipeSession
return fault;
}
private static WorkerFault CreateShutdownTimeoutFault(TimeoutException exception)
private static WorkerFault CreateFault(
WorkerFaultCategory category,
string commandMethod,
Exception exception)
{
WorkerFault fault = CreateFault(
category,
commandMethod,
exception.Message);
fault.ExceptionType = exception.GetType().FullName ?? string.Empty;
fault.ProtocolStatus = new ProtocolStatus
{
Code = ProtocolStatusCode.WorkerUnavailable,
Message = exception.Message,
};
return fault;
}
private static WorkerFault CreateFault(
WorkerFaultCategory category,
string commandMethod,
string diagnosticMessage)
{
string message = exception.Message;
return new WorkerFault
{
Category = WorkerFaultCategory.ShutdownTimeout,
ExceptionType = exception.GetType().FullName ?? string.Empty,
DiagnosticMessage = message,
Category = category,
CommandMethod = commandMethod ?? string.Empty,
DiagnosticMessage = diagnosticMessage,
ProtocolStatus = new ProtocolStatus
{
Code = ProtocolStatusCode.WorkerUnavailable,
Message = message,
Message = diagnosticMessage,
},
};
}
private static WorkerFault CreateShutdownTimeoutFault(TimeoutException exception)
{
return CreateFault(
WorkerFaultCategory.ShutdownTimeout,
commandMethod: string.Empty,
exception);
}
private static WorkerFaultCategory MapFaultCategory(WorkerFrameProtocolErrorCode errorCode)
{
return errorCode switch
@@ -0,0 +1,36 @@
using System;
namespace MxGateway.Worker.Ipc;
public sealed class WorkerPipeSessionOptions
{
public static readonly TimeSpan DefaultHeartbeatInterval = TimeSpan.FromSeconds(5);
public static readonly TimeSpan DefaultHeartbeatGrace = TimeSpan.FromSeconds(15);
public WorkerPipeSessionOptions()
{
HeartbeatInterval = DefaultHeartbeatInterval;
HeartbeatGrace = DefaultHeartbeatGrace;
}
public TimeSpan HeartbeatInterval { get; set; }
public TimeSpan HeartbeatGrace { get; set; }
public void Validate()
{
if (HeartbeatInterval <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(
nameof(HeartbeatInterval),
"Worker heartbeat interval must be greater than zero.");
}
if (HeartbeatGrace <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(
nameof(HeartbeatGrace),
"Worker heartbeat grace must be greater than zero.");
}
}
}
@@ -0,0 +1,25 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using MxGateway.Contracts.Proto;
using MxGateway.Worker.Sta;
namespace MxGateway.Worker.MxAccess;
public interface IWorkerRuntimeSession : IDisposable
{
Task<WorkerReady> StartAsync(
string sessionId,
int workerProcessId,
CancellationToken cancellationToken = default);
Task<MxCommandReply> DispatchAsync(StaCommand command);
WorkerRuntimeHeartbeatSnapshot CaptureHeartbeat();
void RequestShutdown();
Task<MxAccessShutdownResult> ShutdownGracefullyAsync(
TimeSpan timeout,
CancellationToken cancellationToken = default);
}
@@ -8,7 +8,7 @@ using MxGateway.Worker.Sta;
namespace MxGateway.Worker.MxAccess;
public sealed class MxAccessStaSession : IDisposable
public sealed class MxAccessStaSession : IWorkerRuntimeSession
{
private readonly IMxAccessComObjectFactory factory;
private readonly IMxAccessEventSink eventSink;
@@ -98,6 +98,30 @@ public sealed class MxAccessStaSession : IDisposable
return commandDispatcher.DispatchAsync(command);
}
public WorkerRuntimeHeartbeatSnapshot CaptureHeartbeat()
{
uint pendingCommandCount = 0;
string currentCommandCorrelationId = string.Empty;
if (commandDispatcher is not null)
{
pendingCommandCount = (uint)commandDispatcher.PendingCommandCount;
currentCommandCorrelationId = commandDispatcher.CurrentCommandCorrelationId;
}
return new WorkerRuntimeHeartbeatSnapshot(
staRuntime.LastActivityUtc,
pendingCommandCount,
(uint)eventQueue.Count,
eventQueue.LastEventSequence,
currentCommandCorrelationId);
}
public void RequestShutdown()
{
commandDispatcher?.RequestShutdown();
}
public IReadOnlyList<WorkerEvent> DrainEvents(uint maxEvents)
{
return eventQueue.Drain(maxEvents);
@@ -204,7 +228,7 @@ public sealed class MxAccessStaSession : IDisposable
return;
}
commandDispatcher?.RequestShutdown();
RequestShutdown();
if (session is not null)
{
@@ -0,0 +1,30 @@
using System;
namespace MxGateway.Worker.MxAccess;
public sealed class WorkerRuntimeHeartbeatSnapshot
{
public WorkerRuntimeHeartbeatSnapshot(
DateTimeOffset lastStaActivityUtc,
uint pendingCommandCount,
uint outboundEventQueueDepth,
ulong lastEventSequence,
string currentCommandCorrelationId)
{
LastStaActivityUtc = lastStaActivityUtc;
PendingCommandCount = pendingCommandCount;
OutboundEventQueueDepth = outboundEventQueueDepth;
LastEventSequence = lastEventSequence;
CurrentCommandCorrelationId = currentCommandCorrelationId ?? string.Empty;
}
public DateTimeOffset LastStaActivityUtc { get; }
public uint PendingCommandCount { get; }
public uint OutboundEventQueueDepth { get; }
public ulong LastEventSequence { get; }
public string CurrentCommandCorrelationId { get; }
}
+1 -1
View File
@@ -83,7 +83,7 @@ public static class WorkerApplication
pipeClient.RunAsync(options).GetAwaiter().GetResult();
logger.Information("WorkerPipeHandshakeSucceeded", new Dictionary<string, object?>
logger.Information("WorkerPipeSessionCompleted", new Dictionary<string, object?>
{
["session_id"] = options.SessionId,
["pipe_name"] = options.PipeName,