feat(worker): log exception messages and per-command start/end (mxaccessgw#184)

R21-O3: WorkerApplication.Run's failure lines logged only the exception
type. Each now also carries exception_message, scrubbed by the new
WorkerLogRedactor.RedactMessage: the value of any name=value / name: value
pair whose name looks sensitive and any mxgw_ API key token are replaced,
line breaks are flattened and the text is capped at 512 characters. The
AppDomain unhandled-exception hook stays type-only, since an exception
from an arbitrary thread can carry command values.

R21-O7: StaCommandDispatcher.ExecuteQueuedCommandAsync logs
WorkerCommandStarted (correlation id, method, queued_ms) and
WorkerCommandCompleted (status, HRESULT, elapsed_ms) when a CommandLogger
is set. IWorkerLogger has no Debug level, so the lines are opt-in through
MXGATEWAY_WORKER_LOG_COMMANDS=1|true (WorkerCommandLogging), which the
worker inherits from the gateway process environment. No payloads or
diagnostic messages are logged. Docs updated. (mxaccessgw#184)

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 12:56:29 -05:00
parent f49b7a0cc1
commit d9c60a78eb
11 changed files with 365 additions and 12 deletions
@@ -0,0 +1,36 @@
using System;
namespace ZB.MOM.WW.MxGateway.Worker.Bootstrap;
/// <summary>
/// Gate for the per-command <c>WorkerCommandStarted</c> / <c>WorkerCommandCompleted</c> log
/// lines. <see cref="IWorkerLogger"/> has no Debug level, and a pair of lines for every STA
/// command is too much for the default log, so the lines are opt-in through an environment
/// variable. The gateway's launcher does not set it; the worker inherits it from the gateway
/// process environment, so setting it on the gateway service turns the lines on for every worker
/// that service launches.
/// </summary>
public static class WorkerCommandLogging
{
/// <summary>
/// Environment variable that enables per-command logging when set to <c>1</c> or
/// <c>true</c> (case-insensitive). Any other value, or none, leaves it off.
/// </summary>
public const string EnvironmentVariable = "MXGATEWAY_WORKER_LOG_COMMANDS";
/// <summary>Gets whether per-command logging is enabled for this process.</summary>
/// <returns><c>true</c> when <see cref="EnvironmentVariable"/> enables it.</returns>
public static bool IsEnabled()
{
return IsEnabledValue(Environment.GetEnvironmentVariable(EnvironmentVariable));
}
/// <summary>Interprets a raw <see cref="EnvironmentVariable"/> value.</summary>
/// <param name="value">The variable's value, or <c>null</c> when unset.</param>
/// <returns><c>true</c> for <c>1</c> or <c>true</c>; otherwise <c>false</c>.</returns>
public static bool IsEnabledValue(string? value)
{
string trimmed = value?.Trim() ?? string.Empty;
return trimmed == "1" || string.Equals(trimmed, "true", StringComparison.OrdinalIgnoreCase);
}
}
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
namespace ZB.MOM.WW.MxGateway.Worker.Bootstrap;
@@ -24,6 +25,50 @@ public static class WorkerLogRedactor
"api_key",
];
/// <summary>
/// Longest exception message <see cref="RedactMessage"/> passes through; the rest is cut and
/// marked, so one runaway message cannot flood the worker's stderr log.
/// </summary>
public const int MaxMessageLength = 512;
// A "name=value" or "name: value" pair whose name contains a sensitive part, e.g.
// "password=hunter2" or "ApiKey: mxgw_k_s". The name is kept; the value is replaced.
private static readonly Regex SensitiveAssignmentPattern = new(
@"(?<name>[\w.-]*(?:nonce|secret|password|token|credential|apikey|api_key)[\w.-]*)(?<sep>\s*[=:]\s*)(?<value>""[^""]*""|'[^']*'|[^\s,;]+)",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled);
// A gateway API key token (mxgw_<key-id>_<secret>) anywhere in the text.
private static readonly Regex ApiKeyTokenPattern = new(
@"mxgw_[A-Za-z0-9_-]+",
RegexOptions.CultureInvariant | RegexOptions.Compiled);
/// <summary>
/// Makes an exception message safe to log. Field-name redaction cannot see inside a free-text
/// message, so this scrubs the content: the value of any <c>name=value</c> or
/// <c>name: value</c> pair whose name looks sensitive, and any gateway API key token. Line
/// breaks are flattened so the message stays on one log line, and the result is capped at
/// <see cref="MaxMessageLength"/> characters.
/// </summary>
/// <param name="message">Exception message to scrub; <c>null</c> yields an empty string.</param>
/// <returns>The scrubbed message.</returns>
public static string RedactMessage(string? message)
{
if (string.IsNullOrEmpty(message))
{
return string.Empty;
}
string redacted = SensitiveAssignmentPattern.Replace(
message,
match => match.Groups["name"].Value + match.Groups["sep"].Value + RedactedValue);
redacted = ApiKeyTokenPattern.Replace(redacted, RedactedValue);
redacted = redacted.Replace("\r\n", " ").Replace('\r', ' ').Replace('\n', ' ');
return redacted.Length <= MaxMessageLength
? redacted
: redacted.Substring(0, MaxMessageLength) + "...[truncated]";
}
/// <summary>
/// Redacts sensitive field values from a log field dictionary.
/// </summary>
@@ -76,7 +76,7 @@ public sealed class WorkerPipeSession
options,
() => Process.GetCurrentProcess().Id,
new WorkerPipeSessionOptions(),
() => new MxAccessStaSession((eq, affinity, comFactory, pump) => new AlarmCommandHandler(eq, () => new WnWrapAlarmConsumer(), affinity, comFactory, standbyFactory: null, ackPumpStep: pump)),
() => CreateDefaultRuntimeSession(WorkerCommandLogging.IsEnabled() ? logger : null),
logger,
stream)
{
@@ -98,7 +98,7 @@ public sealed class WorkerPipeSession
options,
processIdProvider,
new WorkerPipeSessionOptions(),
() => new MxAccessStaSession((eq, affinity, comFactory, pump) => new AlarmCommandHandler(eq, () => new WnWrapAlarmConsumer(), affinity, comFactory, standbyFactory: null, ackPumpStep: pump)),
() => CreateDefaultRuntimeSession(commandLogger: null),
logger: null)
{
}
@@ -138,6 +138,19 @@ public sealed class WorkerPipeSession
_sessionOptions.Validate();
}
/// <summary>
/// Creates the production runtime session: MXAccess on its own STA with the wnwrap alarm
/// consumer. <paramref name="commandLogger"/> receives the per-command start/end lines when
/// non-null; the caller decides that through <see cref="WorkerCommandLogging"/>.
/// </summary>
private static MxAccessStaSession CreateDefaultRuntimeSession(IWorkerLogger? commandLogger)
{
return new MxAccessStaSession((eq, affinity, comFactory, pump) => new AlarmCommandHandler(eq, () => new WnWrapAlarmConsumer(), affinity, comFactory, standbyFactory: null, ackPumpStep: pump))
{
CommandLogger = commandLogger,
};
}
/// <summary>Runs the worker session, completing the handshake and processing messages until cancellation.</summary>
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
@@ -251,6 +251,14 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
/// </summary>
internal Action<string>? AlarmPollWarningSink { get; set; }
/// <summary>
/// Logger handed to the STA command dispatcher for the per-command start/end lines, or
/// <c>null</c> to leave them off (see <see cref="Bootstrap.WorkerCommandLogging"/>). Set
/// before <see cref="StartAsync(string, int, CancellationToken)"/>, which creates the
/// dispatcher.
/// </summary>
internal Bootstrap.IWorkerLogger? CommandLogger { get; set; }
/// <summary>
/// Resolves the write-completion wait from the launcher-provided
/// environment variable; a missing or invalid value falls back to
@@ -363,7 +371,10 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
// closes over staRuntime so it always pumps the pump
// tied to the apartment that owns this session.
pumpStep: () => staRuntime.PumpPendingMessages(),
writeCompletionTimeout: WriteCompletionTimeout));
writeCompletionTimeout: WriteCompletionTimeout))
{
CommandLogger = this.CommandLogger,
};
return session.CreateWorkerReady(workerProcessId);
},
@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
using ZB.MOM.WW.MxGateway.Worker.Conversion;
namespace ZB.MOM.WW.MxGateway.Worker.Sta;
@@ -72,6 +74,14 @@ public sealed class StaCommandDispatcher
this.maxPendingCommands = maxPendingCommands;
}
/// <summary>
/// Gets or sets the logger for the per-command <c>WorkerCommandStarted</c> /
/// <c>WorkerCommandCompleted</c> lines. <c>null</c> (the default) turns them off: they are one
/// pair per STA command, so they are opt-in rather than always-on (see
/// <see cref="WorkerCommandLogging"/>). Set it before commands are dispatched.
/// </summary>
public IWorkerLogger? CommandLogger { get; set; }
/// <summary>
/// Gets the count of pending commands in the queue.
/// </summary>
@@ -263,22 +273,90 @@ public sealed class StaCommandDispatcher
}
SetCurrentCommand(command.CorrelationId);
IWorkerLogger? logger = CommandLogger;
Stopwatch? stopwatch = null;
if (logger is not null)
{
LogCommandStarted(logger, command);
stopwatch = Stopwatch.StartNew();
}
MxCommandReply completedReply;
try
{
MxCommandReply reply = await staRuntime
.InvokeAsync(() => commandExecutor.Execute(command))
.ConfigureAwait(false);
queuedCommand.Complete(NormalizeReply(command, reply));
completedReply = NormalizeReply(command, reply);
}
catch (Exception exception)
{
queuedCommand.Complete(CreateExceptionReply(command, exception));
completedReply = CreateExceptionReply(command, exception);
}
finally
{
SetCurrentCommand(string.Empty);
}
if (logger is not null && stopwatch is not null)
{
LogCommandCompleted(logger, command, completedReply, stopwatch.Elapsed);
}
queuedCommand.Complete(completedReply);
}
// Identity and timing only: never the command payload, whose values (and WriteSecured
// credentials) must not reach the log.
private static void LogCommandStarted(IWorkerLogger logger, StaCommand command)
{
Dictionary<string, object?> fields = new()
{
["session_id"] = command.SessionId,
["correlation_id"] = command.CorrelationId,
["command_method"] = command.MethodName,
};
if (command.EnqueueTimestamp is not null)
{
double queuedMilliseconds = (DateTime.UtcNow - command.EnqueueTimestamp.ToDateTime()).TotalMilliseconds;
fields["queued_ms"] = Math.Max(0, Math.Round(queuedMilliseconds, 3));
}
TryLog(() => logger.Information("WorkerCommandStarted", fields));
}
// Status summary only: the protocol status code and HRESULT, not the diagnostic message,
// which can echo provider detail.
private static void LogCommandCompleted(
IWorkerLogger logger,
StaCommand command,
MxCommandReply reply,
TimeSpan elapsed)
{
Dictionary<string, object?> fields = new()
{
["session_id"] = command.SessionId,
["correlation_id"] = command.CorrelationId,
["command_method"] = command.MethodName,
["protocol_status"] = (reply.ProtocolStatus?.Code ?? ProtocolStatusCode.Unspecified).ToString(),
["hresult"] = reply.HasHresult ? $"0x{reply.Hresult:X8}" : string.Empty,
["elapsed_ms"] = Math.Round(elapsed.TotalMilliseconds, 3),
};
TryLog(() => logger.Information("WorkerCommandCompleted", fields));
}
// A diagnostics line must never fail the command it describes.
private static void TryLog(Action log)
{
try
{
log();
}
catch
{
}
}
private void SetCurrentCommand(string correlationId)
@@ -40,10 +40,14 @@ public static class WorkerApplication
/// CLR's crash code.
/// </summary>
/// <remarks>
/// Logs the exception type only — never its message or stack, which
/// can carry tag values or credentials passed through a command — the
/// same rule every other worker failure line follows. A logging
/// failure is swallowed so the exit always happens.
/// Logs the exception type only — never its message or stack. An
/// exception reaching this hook can come from any thread, including a
/// COM callback or a command path whose message can carry tag values
/// or credentials, so it gets none of the content that
/// <see cref="Run(string[], IWorkerEnvironment, IWorkerLogger, IWorkerPipeClient)"/>'s
/// own catch blocks log through <see cref="WorkerLogRedactor.RedactMessage"/>
/// (those see only pipe-session failures). A logging failure is
/// swallowed so the exit always happens.
/// </remarks>
/// <param name="logger">Worker logger.</param>
/// <param name="exceptionObject">The unhandled exception object (not necessarily an <see cref="Exception"/>).</param>
@@ -171,6 +175,7 @@ public static class WorkerApplication
["exit_code"] = WorkerExitCode.PipeConnectionFailed,
["error_code"] = exception.ErrorCode,
["exception_type"] = exception.GetType().FullName,
["exception_message"] = WorkerLogRedactor.RedactMessage(exception.Message),
});
return (int)WorkerExitCode.PipeConnectionFailed;
@@ -182,6 +187,7 @@ public static class WorkerApplication
["exit_code"] = WorkerExitCode.ProtocolViolation,
["error_code"] = exception.ErrorCode,
["exception_type"] = exception.GetType().FullName,
["exception_message"] = WorkerLogRedactor.RedactMessage(exception.Message),
});
return (int)WorkerExitCode.ProtocolViolation;
@@ -193,6 +199,7 @@ public static class WorkerApplication
{
["exit_code"] = WorkerExitCode.EventQueueOverflow,
["exception_type"] = exception.GetType().FullName,
["exception_message"] = WorkerLogRedactor.RedactMessage(exception.Message),
});
return (int)WorkerExitCode.EventQueueOverflow;
@@ -203,6 +210,7 @@ public static class WorkerApplication
{
["exit_code"] = WorkerExitCode.PipeConnectionFailed,
["exception_type"] = exception.GetType().FullName,
["exception_message"] = WorkerLogRedactor.RedactMessage(exception.Message),
});
return (int)WorkerExitCode.PipeConnectionFailed;
@@ -213,6 +221,7 @@ public static class WorkerApplication
{
["exit_code"] = WorkerExitCode.UnexpectedFailure,
["exception_type"] = exception.GetType().FullName,
["exception_message"] = WorkerLogRedactor.RedactMessage(exception.Message),
});
return (int)WorkerExitCode.UnexpectedFailure;