rename: prefix gateway projects/namespaces with ZB.MOM.WW + sln→slnx
Apply the ZB.MOM.WW. prefix to all gateway-side projects, folders,
.csproj/.sln contents, C# namespaces, using directives, generated proto
C# (csharp_namespace + checked-in generated files), InternalsVisibleTo
attributes, project-name string literals (LoadProject, .sln lookups,
worker exe paths, staticwebassets manifest), and the install/script/doc
references that point at any of the above. Migrate the solution from
.sln to .slnx via `dotnet sln migrate` and delete the old file.
External-runtime identifiers are intentionally NOT prefixed so external
configuration keeps working:
- GatewayMetrics.cs MeterName ("MxGateway.Server")
- DashboardAuthenticationDefaults Scheme/Policy ("MxGateway.Dashboard")
- GatewayRequestLoggingMiddleware logger category ("MxGateway.Request")
- StaRuntime thread name ("MxGateway.Worker.STA")
- appsettings.json root section "MxGateway" + env-var prefix
MxGateway__... and secret-name MxGateway:ApiKeyPepper
- C:\ProgramData\MxGateway\ data dir paths
Also fixes two tests that were not rename-related but became visible
while validating the rename:
- WorkerLiveMxAccessSmokeTests.ShutDownAsync: cancellation that the
gateway service correctly maps to RpcException(Cancelled) per gRPC
convention was being misclassified as a stream fault. Added a sibling
catch on RpcException with StatusCode.Cancelled.
- IntegrationTestEnvironment.ResolveRepositoryRoot: extracted IsRepositoryRoot
and made it accept either a .git marker OR a .sln/.slnx next to src/
so the worker-exe walker works in non-git working copies.
clients/proto/proto-inputs.json's protoRoot updated to point at
src/ZB.MOM.WW.MxGateway.Contracts/Protos.
Verified by `dotnet build` and a full `dotnet test` of the .slnx with
MXGATEWAY_RUN_LIVE_{MXACCESS,LDAP,GALAXY}_TESTS=1:
Tests: 472/472 pass
Worker.Tests: 280/280 pass (4 dev-rig [Fact(Skip=...)] skipped)
IntegrationTests: 18/18 pass
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,336 @@
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using Polly;
|
||||
using Polly.Retry;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Launches worker processes with startup probing and error handling.
|
||||
/// </summary>
|
||||
public sealed class WorkerProcessLauncher : IWorkerProcessLauncher
|
||||
{
|
||||
/// <summary>Environment variable for worker nonce.</summary>
|
||||
public const string WorkerNonceEnvironmentVariableName = "MXGATEWAY_WORKER_NONCE";
|
||||
|
||||
/// <summary>Environment variable for worker pipe connect attempt timeout.</summary>
|
||||
public const string WorkerPipeConnectAttemptTimeoutEnvironmentVariableName =
|
||||
"MXGATEWAY_WORKER_PIPE_CONNECT_ATTEMPT_TIMEOUT_MS";
|
||||
|
||||
private readonly IWorkerProcessFactory _processFactory;
|
||||
private readonly IWorkerStartupProbe _startupProbe;
|
||||
private readonly GatewayMetrics _metrics;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly WorkerOptions _workerOptions;
|
||||
private readonly ILogger<WorkerProcessLauncher> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes the worker process launcher with gateway options and dependencies.
|
||||
/// </summary>
|
||||
/// <param name="gatewayOptions">Gateway configuration options.</param>
|
||||
/// <param name="processFactory">Factory for creating worker processes.</param>
|
||||
/// <param name="startupProbe">Probe for checking worker startup completion.</param>
|
||||
/// <param name="metrics">Gateway metrics collector.</param>
|
||||
/// <param name="logger">Optional logger for diagnostic output.</param>
|
||||
/// <param name="timeProvider">Optional time provider for timestamps.</param>
|
||||
public WorkerProcessLauncher(
|
||||
IOptions<GatewayOptions> gatewayOptions,
|
||||
IWorkerProcessFactory processFactory,
|
||||
IWorkerStartupProbe startupProbe,
|
||||
GatewayMetrics metrics,
|
||||
ILogger<WorkerProcessLauncher>? logger = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(gatewayOptions);
|
||||
ArgumentNullException.ThrowIfNull(processFactory);
|
||||
ArgumentNullException.ThrowIfNull(startupProbe);
|
||||
ArgumentNullException.ThrowIfNull(metrics);
|
||||
|
||||
_workerOptions = gatewayOptions.Value.Worker;
|
||||
_processFactory = processFactory;
|
||||
_startupProbe = startupProbe;
|
||||
_metrics = metrics;
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? NullLogger<WorkerProcessLauncher>.Instance;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Launches a worker process and waits for startup.
|
||||
/// </summary>
|
||||
/// <param name="request">Request payload.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>Handle to the launched worker process.</returns>
|
||||
public async Task<WorkerProcessHandle> LaunchAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await LaunchCoreAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
request.PipeReservation?.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<WorkerProcessHandle> LaunchCoreAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ValidateRequest(request);
|
||||
|
||||
DateTimeOffset startedAt = _timeProvider.GetUtcNow();
|
||||
ProcessStartInfo startInfo = CreateStartInfo(request, out WorkerProcessCommandLine commandLine);
|
||||
|
||||
IWorkerProcess process;
|
||||
try
|
||||
{
|
||||
process = _processFactory.Start(startInfo);
|
||||
}
|
||||
catch (Exception exception) when (exception is not WorkerProcessLaunchException)
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.StartFailed,
|
||||
"Worker process failed to start.",
|
||||
exception);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using CancellationTokenSource startupTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
startupTimeout.CancelAfter(TimeSpan.FromSeconds(_workerOptions.StartupTimeoutSeconds));
|
||||
|
||||
await CreateStartupProbePipeline(process)
|
||||
.ExecuteAsync(
|
||||
async token =>
|
||||
{
|
||||
await _startupProbe
|
||||
.WaitUntilReadyAsync(process, request, token)
|
||||
.ConfigureAwait(false);
|
||||
},
|
||||
startupTimeout.Token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return new WorkerProcessHandle(process, commandLine, startedAt);
|
||||
}
|
||||
catch (OperationCanceledException exception) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
KillAndDispose(process, "StartupTimeout");
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.StartupTimeout,
|
||||
"Worker process did not complete startup before the configured timeout.",
|
||||
exception);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
KillAndDispose(process, "LaunchCanceled");
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception) when (exception is not WorkerProcessLaunchException)
|
||||
{
|
||||
KillAndDispose(process, "StartupFailed");
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.StartupFailed,
|
||||
"Worker process failed during startup.",
|
||||
exception);
|
||||
}
|
||||
catch (WorkerProcessLaunchException)
|
||||
{
|
||||
KillAndDispose(process, "StartupFailed");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private ProcessStartInfo CreateStartInfo(
|
||||
WorkerProcessLaunchRequest request,
|
||||
out WorkerProcessCommandLine commandLine)
|
||||
{
|
||||
string executablePath = ResolveExecutablePath();
|
||||
string workingDirectory = ResolveWorkingDirectory(executablePath);
|
||||
string[] arguments =
|
||||
[
|
||||
"--session-id",
|
||||
request.SessionId,
|
||||
"--pipe-name",
|
||||
request.PipeName,
|
||||
"--protocol-version",
|
||||
request.ProtocolVersion.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
];
|
||||
|
||||
ProcessStartInfo startInfo = new()
|
||||
{
|
||||
FileName = executablePath,
|
||||
WorkingDirectory = workingDirectory,
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
ErrorDialog = false,
|
||||
};
|
||||
|
||||
foreach (string argument in arguments)
|
||||
{
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
startInfo.Environment[WorkerNonceEnvironmentVariableName] = request.Nonce;
|
||||
startInfo.Environment[WorkerPipeConnectAttemptTimeoutEnvironmentVariableName] =
|
||||
_workerOptions.PipeConnectAttemptTimeoutMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
|
||||
commandLine = new WorkerProcessCommandLine(executablePath, arguments);
|
||||
|
||||
return startInfo;
|
||||
}
|
||||
|
||||
private string ResolveExecutablePath()
|
||||
{
|
||||
string executablePath;
|
||||
try
|
||||
{
|
||||
executablePath = Path.GetFullPath(_workerOptions.ExecutablePath);
|
||||
}
|
||||
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.InvalidExecutable,
|
||||
"Worker executable path is not a valid filesystem path.",
|
||||
exception);
|
||||
}
|
||||
|
||||
if (!string.Equals(Path.GetExtension(executablePath), ".exe", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.InvalidExecutable,
|
||||
"Worker executable path must point to a .exe file.");
|
||||
}
|
||||
|
||||
if (!File.Exists(executablePath))
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.ExecutableNotFound,
|
||||
"Worker executable does not exist.");
|
||||
}
|
||||
|
||||
WorkerExecutableValidator.Validate(executablePath, _workerOptions.RequiredArchitecture);
|
||||
|
||||
return executablePath;
|
||||
}
|
||||
|
||||
private string ResolveWorkingDirectory(string executablePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(_workerOptions.WorkingDirectory))
|
||||
{
|
||||
return Path.GetDirectoryName(executablePath) ?? Environment.CurrentDirectory;
|
||||
}
|
||||
|
||||
string workingDirectory;
|
||||
try
|
||||
{
|
||||
workingDirectory = Path.GetFullPath(_workerOptions.WorkingDirectory);
|
||||
}
|
||||
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.InvalidWorkingDirectory,
|
||||
"Worker working directory is not a valid filesystem path.",
|
||||
exception);
|
||||
}
|
||||
|
||||
if (!Directory.Exists(workingDirectory))
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.InvalidWorkingDirectory,
|
||||
"Worker working directory does not exist.");
|
||||
}
|
||||
|
||||
return workingDirectory;
|
||||
}
|
||||
|
||||
private void KillAndDispose(IWorkerProcess process, string reason)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
_metrics.WorkerKilled(reason);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
process.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private ResiliencePipeline CreateStartupProbePipeline(IWorkerProcess process)
|
||||
{
|
||||
RetryStrategyOptions retryOptions = new()
|
||||
{
|
||||
MaxRetryAttempts = Math.Max(0, _workerOptions.StartupProbeRetryAttempts - 1),
|
||||
BackoffType = DelayBackoffType.Exponential,
|
||||
UseJitter = true,
|
||||
Delay = TimeSpan.FromMilliseconds(_workerOptions.StartupProbeRetryDelayMilliseconds),
|
||||
MaxDelay = TimeSpan.FromSeconds(2),
|
||||
ShouldHandle = new PredicateBuilder().Handle<Exception>(exception =>
|
||||
ShouldRetryStartupProbe(exception, process)),
|
||||
OnRetry = args =>
|
||||
{
|
||||
_metrics.RetryAttempted("worker_startup");
|
||||
_logger.LogDebug(
|
||||
args.Outcome.Exception,
|
||||
"Retrying worker startup probe after transient failure. Attempt {Attempt}.",
|
||||
args.AttemptNumber + 1);
|
||||
return default;
|
||||
},
|
||||
};
|
||||
|
||||
return new ResiliencePipelineBuilder()
|
||||
.AddRetry(retryOptions)
|
||||
.Build();
|
||||
}
|
||||
|
||||
private static bool ShouldRetryStartupProbe(Exception exception, IWorkerProcess process)
|
||||
{
|
||||
if (exception is OperationCanceledException or WorkerProcessLaunchException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return !process.HasExited;
|
||||
}
|
||||
|
||||
private static void ValidateRequest(WorkerProcessLaunchRequest request)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(request.SessionId))
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.InvalidRequest,
|
||||
"Worker launch requires a session id.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.PipeName))
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.InvalidRequest,
|
||||
"Worker launch requires a pipe name.");
|
||||
}
|
||||
|
||||
if (request.ProtocolVersion == 0)
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.InvalidRequest,
|
||||
"Worker launch requires a non-zero protocol version.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Nonce))
|
||||
{
|
||||
throw new WorkerProcessLaunchException(
|
||||
WorkerProcessLaunchErrorCode.InvalidRequest,
|
||||
"Worker launch requires a nonce.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user