Merge stream C (gRPC surface: audit aggregation, streaming auth, alarm overflow/snapshot, write cancellation, correlation echo, disconnect reasons, verifier single-flight, opt-in /metrics auth)

This commit is contained in:
Joseph Doherty
2026-10-22 06:47:49 -04:00
39 changed files with 2535 additions and 112 deletions
@@ -1704,6 +1704,20 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
/// </summary>
public WriteOptions? WriteOptions { get; set; }
/// <summary>
/// Records the message unless the token is already cancelled.
/// </summary>
/// <param name="message">The message to write.</param>
/// <param name="cancellationToken">Token that cancels the write.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public Task WriteAsync(T message, CancellationToken cancellationToken)
{
// The interface default throws NotSupportedException for a cancelable token; the
// gateway passes the call's token on every streamed write.
cancellationToken.ThrowIfCancellationRequested();
return WriteAsync(message);
}
/// <summary>
/// Records the message and signals any pending waiter.
/// </summary>
@@ -2112,26 +2126,26 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
private sealed class AllowAllConstraintEnforcer : IConstraintEnforcer
{
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadTagAsync(
public ValueTask<ConstraintFailure?> CheckReadTagAsync(
ApiKeyIdentity? identity,
string tagAddress,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadHandleAsync(
public ValueTask<ConstraintFailure?> CheckReadHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
int itemHandle,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task<ConstraintFailure?> CheckWriteHandleAsync(
public ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
int itemHandle,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task RecordDenialAsync(
@@ -2141,6 +2155,14 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
ConstraintFailure failure,
string? correlationId,
CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task RecordBulkDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
ConstraintDenialSummary summary,
string? correlationId,
CancellationToken cancellationToken) => Task.CompletedTask;
}
}
@@ -0,0 +1,39 @@
namespace ZB.MOM.WW.MxGateway.Server.Alarms;
/// <summary>
/// Completes an alarm-feed subscriber that fell behind its bounded queue and was dropped.
/// </summary>
/// <remarks>
/// A typed exception rather than a bare <see cref="InvalidOperationException"/> so the gRPC
/// layer can map it to <c>ResourceExhausted</c> — the status a <c>StreamEvents</c> overflow
/// carries — without treating a slow client as a gateway failure (an unrecognized exception is
/// logged as a warning with its stack and surfaced as <c>Unavailable</c>). The remedy is the
/// same as for an event-stream overflow: reconnect, which re-snapshots the alarm set.
/// </remarks>
public sealed class AlarmFeedSubscriberOverflowException : Exception
{
/// <summary>The message every dropped subscriber is completed with.</summary>
public const string DefaultMessage =
"Alarm feed subscriber fell behind and was dropped; reconnect to re-snapshot.";
/// <summary>Initializes a new instance of the <see cref="AlarmFeedSubscriberOverflowException"/> class.</summary>
public AlarmFeedSubscriberOverflowException()
: base(DefaultMessage)
{
}
/// <summary>Initializes a new instance of the <see cref="AlarmFeedSubscriberOverflowException"/> class.</summary>
/// <param name="message">The error message.</param>
public AlarmFeedSubscriberOverflowException(string message)
: base(message)
{
}
/// <summary>Initializes a new instance of the <see cref="AlarmFeedSubscriberOverflowException"/> class.</summary>
/// <param name="message">The error message.</param>
/// <param name="innerException">The exception that caused this one.</param>
public AlarmFeedSubscriberOverflowException(string message, Exception innerException)
: base(message, innerException)
{
}
}
@@ -967,7 +967,9 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
}
// Caller holds _sync. Pushes a feed message to every matching subscriber;
// a subscriber that has fallen behind is completed with an error and dropped.
// a subscriber that has fallen behind is completed with an AlarmFeedSubscriberOverflowException
// and dropped. The typed fault is what lets StreamAlarms answer ResourceExhausted instead of
// treating a slow client as a gateway failure.
private void Broadcast(AlarmFeedMessage message, string reference)
{
for (int index = _subscribers.Count - 1; index >= 0; index--)
@@ -980,8 +982,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
if (!subscriber.Channel.Writer.TryWrite(message))
{
subscriber.Channel.Writer.TryComplete(new InvalidOperationException(
"Alarm feed subscriber fell behind and was dropped; reconnect to re-snapshot."));
subscriber.Channel.Writer.TryComplete(new AlarmFeedSubscriberOverflowException());
_subscribers.RemoveAt(index);
}
}
@@ -997,8 +998,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
Subscriber subscriber = _subscribers[index];
if (!subscriber.Channel.Writer.TryWrite(message))
{
subscriber.Channel.Writer.TryComplete(new InvalidOperationException(
"Alarm feed subscriber fell behind and was dropped; reconnect to re-snapshot."));
subscriber.Channel.Writer.TryComplete(new AlarmFeedSubscriberOverflowException());
_subscribers.RemoveAt(index);
}
}
@@ -1046,9 +1046,14 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
_subscribers.Add(subscriber);
providerStatus = BuildProviderStatus();
snapshotTruncated = _snapshotTruncated;
snapshot = _alarms.Values
.Where(alarm => prefix.Length == 0
|| alarm.AlarmFullReference.StartsWith(prefix, StringComparison.Ordinal))
// Take the memoized projection rather than cloning the whole set per attach: the
// clone runs at most once per alarm-set generation (shared with CurrentAlarms), so a
// burst of reconnecting subscribers no longer holds the broadcast lock for one full
// copy each. The array is never mutated once built — a write to _alarms replaces it
// with null — so capturing the reference here pins exactly the generation this
// subscriber registered against, which is what keeps snapshot-then-live ordering.
snapshot = _currentAlarmsProjection ??= _alarms.Values
.Select(alarm => alarm.Clone())
.ToArray();
}
@@ -1068,8 +1073,15 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
SnapshotStatus = new AlarmSnapshotStatus { Truncated = snapshotTruncated },
};
// The prefix filter runs here, outside _sync, over the immutable projection.
foreach (ActiveAlarmSnapshot alarm in snapshot)
{
if (prefix.Length != 0
&& !alarm.AlarmFullReference.StartsWith(prefix, StringComparison.Ordinal))
{
continue;
}
yield return new AlarmFeedMessage { ActiveAlarm = alarm };
}
@@ -52,4 +52,7 @@ public sealed class GatewayOptions
/// login / API-key rate limiting).
/// </summary>
public SecurityOptions Security { get; init; } = new();
/// <summary>Gets Prometheus <c>/metrics</c> endpoint options.</summary>
public MetricsOptions Metrics { get; init; } = new();
}
@@ -0,0 +1,15 @@
namespace ZB.MOM.WW.MxGateway.Server.Configuration;
/// <summary>Options for the Prometheus <c>/metrics</c> scrape endpoint (<c>MxGateway:Metrics</c>).</summary>
public sealed class MetricsOptions
{
/// <summary>
/// Gets a value indicating whether <c>/metrics</c> requires an authenticated caller. When
/// <see langword="true"/>, a scrape must present either an API key carrying the <c>admin</c>
/// scope (<c>Authorization: Bearer mxgw_…</c>) or a dashboard login cookie holding a real
/// dashboard role; anonymous localhost access does not qualify. Default <see langword="false"/>
/// keeps the endpoint anonymous for existing scrapers. The <c>/health</c> endpoints stay
/// anonymous either way.
/// </summary>
public bool RequireAuthentication { get; init; }
}
@@ -113,6 +113,7 @@ public static class GatewayApplication
builder.Services.AddZbSecrets(builder.Configuration, "Secrets");
builder.Services.AddSqliteAuthStore(builder.Configuration);
builder.Services.AddGatewayGrpcAuthorization();
builder.Services.AddGatewayMetricsAuthorization();
AddLoginRateLimiter(builder);
builder.Services.AddHealthChecks()
.AddTypeActivatedCheck<AuthStoreHealthCheck>(
@@ -350,8 +351,19 @@ public static class GatewayApplication
{
endpoints.MapStaticAssets(ResolveStaticAssetsManifestPath());
// /health stays anonymous unconditionally: an off-repo Overview dashboard polls
// /health/active without credentials.
endpoints.MapZbHealth();
endpoints.MapZbMetrics();
IEndpointConventionBuilder metricsEndpoint = endpoints.MapZbMetrics();
// Read from configuration rather than IOptions<GatewayOptions>: resolving the options here
// would run whole-options validation during Build instead of at StartAsync.
if (endpoints.ServiceProvider
.GetRequiredService<IConfiguration>()
.GetValue<bool>("MxGateway:Metrics:RequireAuthentication"))
{
// Opt-in (SEC-14): an admin-scoped API key or a dashboard login with a real role.
metricsEndpoint.RequireAuthorization(MetricsAuthorizationDefaults.Policy);
}
endpoints.MapGrpcService<MxAccessGatewayService>();
endpoints.MapZbGalaxyRepository();
@@ -0,0 +1,30 @@
namespace ZB.MOM.WW.MxGateway.Server.Grpc;
/// <summary>
/// The bounded set of <c>reason</c> tag values recorded on
/// <c>mxgateway.grpc.streams.disconnected</c> when a <c>StreamEvents</c> stream ends.
/// </summary>
/// <remarks>
/// Constants rather than exception messages or free text so the metric's tag cardinality stays
/// fixed: an exporter mints one time series per distinct tag value.
/// </remarks>
public static class EventStreamDisconnectReasons
{
/// <summary>The session's event stream completed cleanly (session closed or the worker stream ended).</summary>
public const string SessionClosed = "SessionClosed";
/// <summary>The subscriber fell behind its bounded queue and was disconnected.</summary>
public const string Overflow = "Overflow";
/// <summary>The worker event stream terminated with a fault.</summary>
public const string WorkerFault = "WorkerFault";
/// <summary>The client cancelled the call or disconnected.</summary>
public const string ClientCancelled = "ClientCancelled";
/// <summary>The stream ended on an unclassified error.</summary>
public const string Error = "Error";
/// <summary>The consumer stopped reading without cancelling the call.</summary>
public const string Detached = "Detached";
}
@@ -129,6 +129,10 @@ public sealed class EventStreamService(
IDisposable backlogRegistration = metrics.RegisterEventStreamBacklogSource(
() => subscriber.Reader.CanCount ? subscriber.Reader.Count : 0);
// Why this stream ended, recorded as a bounded constant on the disconnect metric. Set at
// the point the end is observed; a consumer that simply stops iterating keeps Detached.
string disconnectReason = EventStreamDisconnectReasons.Detached;
try
{
// Emit order for a resume: the ReplayGap sentinel FIRST (only when events were
@@ -173,13 +177,31 @@ public sealed class EventStreamService(
// here. Mirror the original ProduceEventsAsync behavior: fault the
// session and record the metric, then propagate the terminal fault to the
// gRPC client.
disconnectReason = EventStreamDisconnectReasons.WorkerFault;
session.MarkFaulted(workerException.Message);
metrics.Fault(WorkerClientErrorCode.WorkerFaulted.ToString());
throw;
}
catch (SessionManagerException sessionException)
when (sessionException.ErrorCode == SessionManagerErrorCode.EventQueueOverflow)
{
disconnectReason = EventStreamDisconnectReasons.Overflow;
throw;
}
catch (OperationCanceledException)
{
disconnectReason = EventStreamDisconnectReasons.ClientCancelled;
throw;
}
catch (Exception)
{
disconnectReason = EventStreamDisconnectReasons.Error;
throw;
}
if (!hasMore)
{
disconnectReason = EventStreamDisconnectReasons.SessionClosed;
break;
}
@@ -214,7 +236,15 @@ public sealed class EventStreamService(
backlogRegistration.Dispose();
subscriber.Dispose();
metrics.StreamDisconnected("Detached");
// A consumer that stopped iterating mid-stream lands here with the Detached default;
// when the call token has fired, that early stop was the client going away.
if (disconnectReason == EventStreamDisconnectReasons.Detached
&& cancellationToken.IsCancellationRequested)
{
disconnectReason = EventStreamDisconnectReasons.ClientCancelled;
}
metrics.StreamDisconnected(disconnectReason);
}
}
@@ -144,6 +144,25 @@ public sealed class MxAccessGatewayService(
publicReply = bulkConstraintPlan.MergeDeniedInto(publicReply);
}
// MxCommandReply.correlation_id always echoes the caller's client_correlation_id —
// the same value the constraint-denied replies carry — so a client can match every
// reply to its request. The worker reply carries the gateway's internal envelope id,
// which is a gateway↔worker routing detail and is overwritten here, empty included.
// Record the pairing once, before the envelope id is overwritten, so a client error
// report quoting its correlation id can be matched to gateway and worker log lines
// (which carry the envelope id). Ids and the command kind only — never values.
if (logger.IsEnabled(LogLevel.Debug))
{
logger.LogDebug(
"Invoke {CommandKind} for session {SessionId}: client correlation id {ClientCorrelationId} maps to worker envelope correlation id {WorkerCorrelationId}.",
command.Kind,
request.SessionId,
request.ClientCorrelationId,
publicReply.CorrelationId);
}
publicReply.CorrelationId = request.ClientCorrelationId;
session.TrackCommandReply(commandToInvoke, publicReply);
return publicReply;
}
@@ -172,14 +191,22 @@ public sealed class MxAccessGatewayService(
// volume gateway path. Stopwatch.GetTimestamp/GetElapsedTime measure the exact
// same wall-clock span the former Stopwatch.StartNew()/.Elapsed did.
long sendStartTimestamp = Stopwatch.GetTimestamp();
await responseStream.WriteAsync(publicEvent).ConfigureAwait(false);
// The call token releases a write parked on a slow or vanished client when the
// call is cancelled, instead of leaving it to the transport's own timeout.
await responseStream.WriteAsync(publicEvent, context.CancellationToken).ConfigureAwait(false);
metrics.RecordEventStreamSend(
publicEvent.Family.ToString(),
MxEventFamilyNames.Get(publicEvent.Family),
Stopwatch.GetElapsedTime(sendStartTimestamp));
}
}
catch (Exception exception) when (exception is not RpcException)
{
if (IsClientAbort(exception, context))
{
throw ClientCancelled();
}
throw MapException(exception);
}
}
@@ -225,11 +252,24 @@ public sealed class MxAccessGatewayService(
.WithCancellation(context.CancellationToken)
.ConfigureAwait(false))
{
await responseStream.WriteAsync(message).ConfigureAwait(false);
await responseStream.WriteAsync(message, context.CancellationToken).ConfigureAwait(false);
}
}
catch (AlarmFeedSubscriberOverflowException overflow)
{
// The monitor dropped this subscriber for falling behind its bounded queue. That is a
// client-side slowness condition with a client-side remedy (reconnect and
// re-snapshot), so it maps to ResourceExhausted like a StreamEvents overflow rather
// than reaching MapException's warning-with-stack + Unavailable fallback.
throw new RpcException(new Status(StatusCode.ResourceExhausted, overflow.Message));
}
catch (Exception exception) when (exception is not RpcException)
{
if (IsClientAbort(exception, context))
{
throw ClientCancelled();
}
throw MapException(exception);
}
}
@@ -264,6 +304,17 @@ public sealed class MxAccessGatewayService(
}
}
// A response write fails with an IOException (or an operation-cancelled exception) when the
// client disconnects mid-stream. With the call token already cancelled that is a routine
// client abort, not a gateway fault: it maps to Cancelled with no warning. The same exception
// on a live call token is a genuine transport failure and keeps the MapException path.
private static bool IsClientAbort(Exception exception, ServerCallContext context) =>
context.CancellationToken.IsCancellationRequested
&& exception is IOException or OperationCanceledException;
private static RpcException ClientCancelled() =>
new(new Status(StatusCode.Cancelled, "gRPC request was canceled."));
private string? ResolveClientIdentity()
{
return identityAccessor.Current?.DisplayName ?? identityAccessor.Current?.KeyId;
@@ -488,6 +539,7 @@ public sealed class MxAccessGatewayService(
Dictionary<int, SubscribeResult> denied = [];
List<string> allowed = [];
ConstraintDenialSummary? denialSummary = null;
for (int index = 0; index < tagAddresses.Count; index++)
{
string tagAddress = tagAddresses[index];
@@ -500,8 +552,7 @@ public sealed class MxAccessGatewayService(
continue;
}
await constraintEnforcer.RecordDenialAsync(identity, command.Kind.ToString(), tagAddress, failure, correlationId, cancellationToken)
.ConfigureAwait(false);
(denialSummary ??= new ConstraintDenialSummary(tagAddresses.Count)).Add(tagAddress, failure);
denied[index] = new SubscribeResult
{
ServerHandle = serverHandle,
@@ -511,11 +562,17 @@ public sealed class MxAccessGatewayService(
};
}
if (denied.Count == 0)
if (denialSummary is null)
{
return null;
}
// One aggregated audit event per bulk command, not one per denied item (see
// ConstraintDenialSummary for why).
await constraintEnforcer
.RecordBulkDenialAsync(identity, command.Kind.ToString(), denialSummary, correlationId, cancellationToken)
.ConfigureAwait(false);
// Build the filtered command directly instead of cloning the original and clearing it:
// the clone deep-copied every denied address only to drop it. The payload's other fields
// (server_handle) are copied across explicitly. Nothing aliases the request here — these
@@ -556,6 +613,7 @@ public sealed class MxAccessGatewayService(
Dictionary<int, BulkReadResult> denied = [];
List<string> allowed = [];
ConstraintDenialSummary? denialSummary = null;
for (int index = 0; index < tagAddresses.Count; index++)
{
string tagAddress = tagAddresses[index];
@@ -568,8 +626,7 @@ public sealed class MxAccessGatewayService(
continue;
}
await constraintEnforcer.RecordDenialAsync(identity, command.Kind.ToString(), tagAddress, failure, correlationId, cancellationToken)
.ConfigureAwait(false);
(denialSummary ??= new ConstraintDenialSummary(tagAddresses.Count)).Add(tagAddress, failure);
denied[index] = new BulkReadResult
{
ServerHandle = serverHandle,
@@ -580,11 +637,17 @@ public sealed class MxAccessGatewayService(
};
}
if (denied.Count == 0)
if (denialSummary is null)
{
return null;
}
// One aggregated audit event per bulk command, not one per denied item (see
// ConstraintDenialSummary for why).
await constraintEnforcer
.RecordBulkDenialAsync(identity, command.Kind.ToString(), denialSummary, correlationId, cancellationToken)
.ConfigureAwait(false);
MxCommand filtered = new() { Kind = command.Kind };
ReadBulkCommand payload = new()
{
@@ -621,6 +684,7 @@ public sealed class MxAccessGatewayService(
Dictionary<int, BulkWriteResult> denied = [];
List<TEntry> allowed = [];
ConstraintDenialSummary? denialSummary = null;
for (int index = 0; index < entries.Count; index++)
{
TEntry entry = entries[index];
@@ -634,14 +698,7 @@ public sealed class MxAccessGatewayService(
continue;
}
await constraintEnforcer.RecordDenialAsync(
identity,
command.Kind.ToString(),
itemHandle.ToString(System.Globalization.CultureInfo.InvariantCulture),
failure,
correlationId,
cancellationToken)
.ConfigureAwait(false);
(denialSummary ??= new ConstraintDenialSummary(entries.Count)).Add(itemHandle, failure);
denied[index] = new BulkWriteResult
{
ServerHandle = serverHandle,
@@ -651,11 +708,17 @@ public sealed class MxAccessGatewayService(
};
}
if (denied.Count == 0)
if (denialSummary is null)
{
return null;
}
// One aggregated audit event per bulk command, not one per denied item (see
// ConstraintDenialSummary for why).
await constraintEnforcer
.RecordBulkDenialAsync(identity, command.Kind.ToString(), denialSummary, correlationId, cancellationToken)
.ConfigureAwait(false);
return new WriteBulkConstraintPlan(
BuildFilteredWriteBulkCommand(command, allowed),
entries.Count,
@@ -742,6 +805,7 @@ public sealed class MxAccessGatewayService(
Dictionary<int, SubscribeResult> denied = [];
List<int> allowed = [];
ConstraintDenialSummary? denialSummary = null;
for (int index = 0; index < itemHandles.Count; index++)
{
int itemHandle = itemHandles[index];
@@ -754,8 +818,7 @@ public sealed class MxAccessGatewayService(
continue;
}
await constraintEnforcer.RecordDenialAsync(identity, command.Kind.ToString(), itemHandle.ToString(System.Globalization.CultureInfo.InvariantCulture), failure, correlationId, cancellationToken)
.ConfigureAwait(false);
(denialSummary ??= new ConstraintDenialSummary(itemHandles.Count)).Add(itemHandle, failure);
denied[index] = new SubscribeResult
{
ServerHandle = serverHandle,
@@ -765,11 +828,17 @@ public sealed class MxAccessGatewayService(
};
}
if (denied.Count == 0)
if (denialSummary is null)
{
return null;
}
// One aggregated audit event per bulk command, not one per denied item (see
// ConstraintDenialSummary for why).
await constraintEnforcer
.RecordBulkDenialAsync(identity, command.Kind.ToString(), denialSummary, correlationId, cancellationToken)
.ConfigureAwait(false);
MxCommand filtered = new() { Kind = command.Kind };
AdviseItemBulkCommand payload = new() { ServerHandle = command.AdviseItemBulk.ServerHandle };
payload.ItemHandles.Add(allowed);
@@ -0,0 +1,28 @@
using System.Collections.Frozen;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
namespace ZB.MOM.WW.MxGateway.Server.Metrics;
/// <summary>
/// Cached <see cref="MxEventFamily"/> names for the per-event metric tags.
/// </summary>
/// <remarks>
/// <see cref="Enum.ToString()"/> allocates a new string on every call, and the event-received
/// and stream-send metrics tag every event with its family — once per event at the worker
/// boundary and once per event per subscriber on the stream. The names are fixed by the
/// contract, so they are computed once here. A value the contract does not define falls back
/// to <see cref="Enum.ToString()"/>, which is the same string the uncached call produced.
/// </remarks>
public static class MxEventFamilyNames
{
private static readonly FrozenDictionary<MxEventFamily, string> Names =
Enum.GetValues<MxEventFamily>()
.Distinct()
.ToFrozenDictionary(family => family, family => family.ToString());
/// <summary>Gets the name of an event family.</summary>
/// <param name="family">The event family.</param>
/// <returns>The family's name, identical to <see cref="Enum.ToString()"/>.</returns>
public static string Get(MxEventFamily family) =>
Names.TryGetValue(family, out string? name) ? name : family.ToString();
}
@@ -58,6 +58,19 @@ public interface IApiKeyCacheInvalidator
/// window and it is intentionally kept short (<see cref="SecurityOptions.ApiKeyVerificationCacheSeconds"/>,
/// default 15 s).
/// </para>
/// <para>
/// Misses are single-flight: concurrent misses for the same token and the same key generation share
/// one inner verification, so a TTL expiry under load costs one SQLite read rather than one per
/// in-flight RPC. A miss that starts after an <see cref="Invalidate"/> never joins a flight that
/// started before it (the flight is keyed on the generation too), so the revoke guarantee holds for
/// joiners as well. The shared inner call is not bound to any one caller's cancellation token; each
/// caller stops waiting on its own token without cancelling the others.
/// </para>
/// <para>
/// The key-id index is pruned by a post-eviction callback whenever the cache drops an entry on its
/// own (expiry, compaction), not only by <see cref="Invalidate"/>, so it stays proportional to the
/// live cache rather than to every token ever verified.
/// </para>
/// </remarks>
public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalidator
{
@@ -69,9 +82,20 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
private readonly TimeSpan _ttl;
// keyId -> set of cache keys currently held for that id, so a revoke/rotate can evict every
// secret variant (e.g. an old secret still within TTL after a rotate).
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, byte>> _keyIdIndex =
new(StringComparer.Ordinal);
// secret variant (e.g. an old secret still within TTL after a rotate). Guarded by _indexSync,
// which also covers the cache Set that an index entry describes: the post-eviction callback
// runs asynchronously, so Set + index and check-cache + unindex must each be atomic with
// respect to the other, or a late callback could unindex a freshly re-cached entry and leave
// it out of reach of a later Invalidate. Every operation under the lock is O(1) and the lock is
// only taken on misses, evictions and invalidations — never on a cache hit.
private readonly object _indexSync = new();
private readonly Dictionary<string, HashSet<string>> _keyIdIndex = new(StringComparer.Ordinal);
// (cache key, key generation) -> the in-flight inner verification for that pair (single-flight).
private readonly ConcurrentDictionary<(string CacheKey, long Generation), Lazy<Task<ApiKeyVerification>>> _inflight =
new();
private readonly PostEvictionDelegate _onEvicted;
// keyId -> monotonic generation, bumped by Invalidate before it evicts. A VerifyAsync snapshots
// the generation before calling the inner verifier and only caches if it is unchanged after the
@@ -103,6 +127,37 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
_inner = inner;
_cache = cache;
_ttl = ttl;
_onEvicted = OnCacheEntryEvicted;
}
/// <summary>Gets the number of cache keys currently held in the key-id index (test seam).</summary>
internal int IndexedCacheKeyCount
{
get
{
lock (_indexSync)
{
int count = 0;
foreach (HashSet<string> set in _keyIdIndex.Values)
{
count += set.Count;
}
return count;
}
}
}
/// <summary>Gets the number of key ids currently present in the key-id index (test seam).</summary>
internal int IndexedKeyIdCount
{
get
{
lock (_indexSync)
{
return _keyIdIndex.Count;
}
}
}
/// <summary>
@@ -126,11 +181,50 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
}
// Snapshot the key's generation BEFORE the inner verify so a concurrent Invalidate that runs
// while we are in the library is detected below and its repopulation discarded.
// while we are in the library is detected and its repopulation discarded. The snapshot is
// also half of the single-flight key, so a miss that starts after an Invalidate cannot join
// a flight that started before it.
string? tokenKeyId = TryParseKeyId(authorizationHeader);
long generationAtStart = tokenKeyId is null ? 0 : ReadGeneration(tokenKeyId);
(string CacheKey, long Generation) flightKey = (cacheKey, generationAtStart);
ApiKeyVerification result = await _inner.VerifyAsync(authorizationHeader, ct).ConfigureAwait(false);
Lazy<Task<ApiKeyVerification>> created = new(
() => VerifyMissAsync(authorizationHeader, cacheKey, tokenKeyId, generationAtStart));
Lazy<Task<ApiKeyVerification>> flight = _inflight.GetOrAdd(flightKey, created);
Task<ApiKeyVerification> verification = flight.Value;
if (ReferenceEquals(flight, created))
{
// The flight's creator retires it once it completes (success, failure or fault), so the
// next miss for this token starts fresh. Removing by (key, value) pair never retires a
// newer flight that replaced this one.
_ = verification.ContinueWith(
static (_, state) =>
{
var (owner, key, completed) =
((CachingApiKeyVerifier, (string, long), Lazy<Task<ApiKeyVerification>>))state!;
owner._inflight.TryRemove(
new KeyValuePair<(string CacheKey, long Generation), Lazy<Task<ApiKeyVerification>>>(key, completed));
},
(this, flightKey, flight),
CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}
return await verification.WaitAsync(ct).ConfigureAwait(false);
}
// The shared body of one single-flight miss. Not bound to any caller's token: a caller that
// gives up must not cancel the verification other callers are waiting on.
private async Task<ApiKeyVerification> VerifyMissAsync(
string authorizationHeader,
string cacheKey,
string? tokenKeyId,
long generationAtStart)
{
ApiKeyVerification result = await _inner
.VerifyAsync(authorizationHeader, CancellationToken.None)
.ConfigureAwait(false);
if (result.Succeeded && result.Identity is not null)
{
@@ -158,11 +252,21 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
}
string keyId = result.Identity!.KeyId;
_cache.Set(cacheKey, result, new MemoryCacheEntryOptions
MemoryCacheEntryOptions entryOptions = new()
{
AbsoluteExpirationRelativeToNow = _ttl,
};
entryOptions.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration
{
EvictionCallback = _onEvicted,
State = keyId,
});
IndexCacheKey(keyId, cacheKey);
lock (_indexSync)
{
_cache.Set(cacheKey, result, entryOptions);
IndexCacheKey(keyId, cacheKey);
}
// Set-then-recheck: an Invalidate that landed between the pre-Set check and IndexCacheKey
// would have missed this entry (it was not yet indexed). If the generation has moved, evict
@@ -170,13 +274,38 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
if (tokenKeyId is not null && ReadGeneration(tokenKeyId) != generationAtStart)
{
_cache.Remove(cacheKey);
if (_keyIdIndex.TryGetValue(keyId, out ConcurrentDictionary<string, byte>? set))
lock (_indexSync)
{
set.TryRemove(cacheKey, out _);
UnindexCacheKey(keyId, cacheKey);
}
}
}
// Post-eviction callback: drops the evicted entry's key-id index slot so the index tracks the
// live cache. MemoryCache raises it asynchronously, so by the time it runs the same cache key
// may already hold a newer entry (re-verified after the expiry); that entry owns the slot, and
// the check below runs under the lock that also covers Set + index, so it cannot interleave
// with a re-cache.
private void OnCacheEntryEvicted(object key, object? value, EvictionReason reason, object? state)
{
if (reason == EvictionReason.Replaced || key is not string cacheKey || state is not string keyId)
{
// Replaced: a newer entry under the same key was Set in this entry's place and is
// indexed under the same slot already.
return;
}
lock (_indexSync)
{
if (_cache.TryGetValue(cacheKey, out _))
{
return;
}
UnindexCacheKey(keyId, cacheKey);
}
}
/// <inheritdoc />
public void Invalidate(string keyId)
{
@@ -189,12 +318,22 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
// generation refuses to cache (or self-evicts) its now-stale repopulation.
_generations.AddOrUpdate(keyId, 1, static (_, current) => current + 1);
if (_keyIdIndex.TryRemove(keyId, out ConcurrentDictionary<string, byte>? cacheKeys))
string[] cacheKeys;
lock (_indexSync)
{
foreach (string cacheKey in cacheKeys.Keys)
if (!_keyIdIndex.Remove(keyId, out HashSet<string>? indexed))
{
_cache.Remove(cacheKey);
return;
}
cacheKeys = [.. indexed];
}
// Evicted outside the lock: the eviction callbacks these raise take it themselves (and find
// the slot already gone).
foreach (string cacheKey in cacheKeys)
{
_cache.Remove(cacheKey);
}
}
@@ -257,12 +396,28 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
return new string(afterPrefix[..separator]);
}
// Caller holds _indexSync.
private void IndexCacheKey(string keyId, string cacheKey)
{
ConcurrentDictionary<string, byte> set = _keyIdIndex.GetOrAdd(
keyId,
static _ => new ConcurrentDictionary<string, byte>(StringComparer.Ordinal));
set.TryAdd(cacheKey, 0);
if (!_keyIdIndex.TryGetValue(keyId, out HashSet<string>? set))
{
set = new HashSet<string>(StringComparer.Ordinal);
_keyIdIndex[keyId] = set;
}
set.Add(cacheKey);
}
// Caller holds _indexSync. Drops the key id's slot entirely once its last cache key goes, so
// the index does not accumulate empty sets for keys that are no longer cached.
private void UnindexCacheKey(string keyId, string cacheKey)
{
if (_keyIdIndex.TryGetValue(keyId, out HashSet<string>? set)
&& set.Remove(cacheKey)
&& set.Count == 0)
{
_keyIdIndex.Remove(keyId);
}
}
private static bool TryComputeCacheKey(string? authorizationHeader, out string cacheKey)
@@ -0,0 +1,90 @@
using System.Globalization;
namespace ZB.MOM.WW.MxGateway.Server.Security.Authorization;
/// <summary>
/// Accumulates the constraint denials of one bulk command so they are audited as a single
/// aggregated event rather than one event per denied item.
/// </summary>
/// <remarks>
/// A bulk command can carry thousands of items, and the audit writer is a bounded, drop-on-full
/// channel. One audit event per denied item let a single low-privilege key flood that channel and
/// push every other audit record out of it. The summary keeps the forensic signal — how many items
/// were denied, which constraints denied them, and a bounded sample of the targets — at a fixed
/// cost of one audit event per command.
/// </remarks>
public sealed class ConstraintDenialSummary
{
/// <summary>The maximum number of denied targets retained as a sample.</summary>
public const int MaxSampleTargets = 20;
private readonly List<string> _constraintNames = [];
private readonly List<string> _sampleTargets = [];
/// <summary>Initializes a new instance of the <see cref="ConstraintDenialSummary"/> class.</summary>
/// <param name="totalCount">The number of items the bulk command carried.</param>
public ConstraintDenialSummary(int totalCount)
{
TotalCount = totalCount;
}
/// <summary>Gets the number of items the bulk command carried.</summary>
public int TotalCount { get; }
/// <summary>Gets the number of items denied.</summary>
public int DeniedCount { get; private set; }
/// <summary>Gets the distinct constraint names that denied at least one item, in first-seen order.</summary>
public IReadOnlyList<string> ConstraintNames => _constraintNames;
/// <summary>Gets up to <see cref="MaxSampleTargets"/> denied targets, in item order.</summary>
public IReadOnlyList<string> SampleTargets => _sampleTargets;
/// <summary>Gets a value indicating whether more items were denied than the sample retains.</summary>
public bool SampleTruncated => DeniedCount > _sampleTargets.Count;
/// <summary>Records one denied tag-address item.</summary>
/// <param name="target">The denied tag address.</param>
/// <param name="failure">The constraint failure that denied it.</param>
public void Add(string target, ConstraintFailure failure)
{
ArgumentNullException.ThrowIfNull(failure);
AddConstraint(failure.ConstraintName);
if (_sampleTargets.Count < MaxSampleTargets)
{
_sampleTargets.Add(target);
}
DeniedCount++;
}
/// <summary>Records one denied item-handle item.</summary>
/// <param name="itemHandle">The denied MXAccess item handle.</param>
/// <param name="failure">The constraint failure that denied it.</param>
/// <remarks>The handle is formatted only while the sample still has room.</remarks>
public void Add(int itemHandle, ConstraintFailure failure)
{
ArgumentNullException.ThrowIfNull(failure);
AddConstraint(failure.ConstraintName);
if (_sampleTargets.Count < MaxSampleTargets)
{
_sampleTargets.Add(itemHandle.ToString(CultureInfo.InvariantCulture));
}
DeniedCount++;
}
private void AddConstraint(string constraintName)
{
// The constraint vocabulary is a handful of fixed names, so a linear scan beats a set.
for (int index = 0; index < _constraintNames.Count; index++)
{
if (string.Equals(_constraintNames[index], constraintName, StringComparison.Ordinal))
{
return;
}
}
_constraintNames.Add(constraintName);
}
}
@@ -25,7 +25,7 @@ public sealed class ConstraintEnforcer(
identity?.EffectiveConstraints.HasWriteConstraints ?? false;
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadTagAsync(
public ValueTask<ConstraintFailure?> CheckReadTagAsync(
ApiKeyIdentity? identity,
string tagAddress,
CancellationToken cancellationToken)
@@ -33,14 +33,14 @@ public sealed class ConstraintEnforcer(
ApiKeyConstraints constraints = identity?.EffectiveConstraints ?? ApiKeyConstraints.Empty;
if (!constraints.HasReadConstraints)
{
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
return Task.FromResult(CheckReadTarget(constraints, tagAddress));
return ValueTask.FromResult(CheckReadTarget(constraints, tagAddress));
}
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadHandleAsync(
public ValueTask<ConstraintFailure?> CheckReadHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
@@ -50,19 +50,19 @@ public sealed class ConstraintEnforcer(
ApiKeyConstraints constraints = identity?.EffectiveConstraints ?? ApiKeyConstraints.Empty;
if (!constraints.HasReadConstraints)
{
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
if (!session.TryGetItemRegistration(serverHandle, itemHandle, out SessionItemRegistration registration))
{
return Task.FromResult<ConstraintFailure?>(new ConstraintFailure("item_handle", "Item handle is not registered in the constrained session."));
return ValueTask.FromResult<ConstraintFailure?>(new ConstraintFailure("item_handle", "Item handle is not registered in the constrained session."));
}
return Task.FromResult(CheckReadTarget(constraints, registration.TagAddress));
return ValueTask.FromResult(CheckReadTarget(constraints, registration.TagAddress));
}
/// <inheritdoc />
public Task<ConstraintFailure?> CheckWriteHandleAsync(
public ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
@@ -72,23 +72,23 @@ public sealed class ConstraintEnforcer(
ApiKeyConstraints constraints = identity?.EffectiveConstraints ?? ApiKeyConstraints.Empty;
if (!constraints.HasWriteConstraints)
{
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
if (!session.TryGetItemRegistration(serverHandle, itemHandle, out SessionItemRegistration registration))
{
return Task.FromResult<ConstraintFailure?>(new ConstraintFailure("item_handle", "Item handle is not registered in the constrained session."));
return ValueTask.FromResult<ConstraintFailure?>(new ConstraintFailure("item_handle", "Item handle is not registered in the constrained session."));
}
GalaxyTagLookup? target = ResolveTarget(registration.TagAddress);
if (target is null)
{
return Task.FromResult<ConstraintFailure?>(new ConstraintFailure("tag_metadata", "Tag metadata is not available in the Galaxy hierarchy cache."));
return ValueTask.FromResult<ConstraintFailure?>(new ConstraintFailure("tag_metadata", "Tag metadata is not available in the Galaxy hierarchy cache."));
}
if (!MatchesPathOrTag(target.ContainedPath, registration.TagAddress, constraints.WriteSubtrees, constraints.WriteTagGlobs))
{
return Task.FromResult<ConstraintFailure?>(new ConstraintFailure("write_scope", "Tag is outside the API key write scope."));
return ValueTask.FromResult<ConstraintFailure?>(new ConstraintFailure("write_scope", "Tag is outside the API key write scope."));
}
if (constraints.MaxWriteClassification is { } maxClassification)
@@ -96,18 +96,18 @@ public sealed class ConstraintEnforcer(
GalaxyAttribute? attribute = target.Attribute;
if (attribute is null)
{
return Task.FromResult<ConstraintFailure?>(new ConstraintFailure("max_write_classification", "Attribute security classification is not available."));
return ValueTask.FromResult<ConstraintFailure?>(new ConstraintFailure("max_write_classification", "Attribute security classification is not available."));
}
if (attribute.SecurityClassification > maxClassification)
{
return Task.FromResult<ConstraintFailure?>(new ConstraintFailure(
return ValueTask.FromResult<ConstraintFailure?>(new ConstraintFailure(
"max_write_classification",
$"Attribute security classification {attribute.SecurityClassification} exceeds allowed maximum {maxClassification}."));
}
}
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
/// <inheritdoc />
@@ -150,6 +150,46 @@ public sealed class ConstraintEnforcer(
await auditWriter.WriteAsync(auditEvent, cancellationToken).ConfigureAwait(false);
}
/// <inheritdoc />
public async Task RecordBulkDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
ConstraintDenialSummary summary,
string? correlationId,
CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(summary);
if (summary.DeniedCount == 0)
{
return;
}
// One event for the whole command. Target stays "<commandKind>:<target>"-shaped so audit
// queries that split on ':' keep working; the "bulk" target marks the aggregated form.
AuditEvent auditEvent = new()
{
EventId = Guid.NewGuid(),
OccurredAtUtc = DateTimeOffset.UtcNow,
Actor = identity?.KeyId ?? "anonymous",
Action = "constraint-denied",
Outcome = AuditOutcome.Denied,
Category = CanonicalForwardingApiKeyAuditStore.ApiKeyCategory,
Target = $"{commandKind}:bulk",
SourceNode = null,
CorrelationId = Guid.TryParse(correlationId, out var cid) ? cid : (Guid?)null,
DetailsJson = JsonSerializer.Serialize(new BulkDenialDetails(
summary.ConstraintNames,
commandKind,
summary.DeniedCount,
summary.TotalCount,
summary.SampleTargets,
summary.SampleTruncated,
correlationId ?? "")),
};
await auditWriter.WriteAsync(auditEvent, cancellationToken).ConfigureAwait(false);
}
private ConstraintFailure? CheckReadTarget(
ApiKeyConstraints constraints,
string tagAddress)
@@ -240,4 +280,15 @@ public sealed class ConstraintEnforcer(
return false;
}
// DetailsJson shape of an aggregated bulk denial. Serialized with camelCase names to match the
// single-item envelope's keys ("commandKind", "clientCorrelationId").
private sealed record BulkDenialDetails(
[property: System.Text.Json.Serialization.JsonPropertyName("constraints")] IReadOnlyList<string> Constraints,
[property: System.Text.Json.Serialization.JsonPropertyName("commandKind")] string CommandKind,
[property: System.Text.Json.Serialization.JsonPropertyName("deniedCount")] int DeniedCount,
[property: System.Text.Json.Serialization.JsonPropertyName("totalCount")] int TotalCount,
[property: System.Text.Json.Serialization.JsonPropertyName("sampleTargets")] IReadOnlyList<string> SampleTargets,
[property: System.Text.Json.Serialization.JsonPropertyName("sampleTruncated")] bool SampleTruncated,
[property: System.Text.Json.Serialization.JsonPropertyName("clientCorrelationId")] string ClientCorrelationId);
}
@@ -30,7 +30,10 @@ public sealed class GatewayGrpcAuthorizationInterceptor(
ServerCallContext context,
UnaryServerMethod<TRequest, TResponse> continuation)
{
ApiKeyIdentity? identity = await AuthenticateAndAuthorizeAsync(request, context).ConfigureAwait(false);
ApiKeyIdentity? identity = await AuthenticateAndAuthorizeAsync(
scopeResolver.ResolveRequiredScope(request),
context)
.ConfigureAwait(false);
IDisposable? identityScope = identity is null ? null : identityAccessor.Push(identity);
using (identityScope)
{
@@ -45,7 +48,10 @@ public sealed class GatewayGrpcAuthorizationInterceptor(
ServerCallContext context,
ServerStreamingServerMethod<TRequest, TResponse> continuation)
{
ApiKeyIdentity? identity = await AuthenticateAndAuthorizeAsync(request, context).ConfigureAwait(false);
ApiKeyIdentity? identity = await AuthenticateAndAuthorizeAsync(
scopeResolver.ResolveRequiredScope(request),
context)
.ConfigureAwait(false);
IDisposable? identityScope = identity is null ? null : identityAccessor.Push(identity);
using (identityScope)
{
@@ -53,15 +59,51 @@ public sealed class GatewayGrpcAuthorizationInterceptor(
}
}
// The base Interceptor passes client- and duplex-streaming calls straight to the handler, so
// without these overrides any such RPC (none exist today, but a .proto edit could add one, and
// the shared Galaxy Repository service is mapped through the same interceptor) would be
// reachable with no API key at all. Both run the identical authentication path. There is no
// single request message to derive a scope from before the handler reads the stream, so they
// fail closed to the scope the resolver assigns an unrecognized request: admin.
/// <inheritdoc />
public override async Task<TResponse> ClientStreamingServerHandler<TRequest, TResponse>(
IAsyncStreamReader<TRequest> requestStream,
ServerCallContext context,
ClientStreamingServerMethod<TRequest, TResponse> continuation)
{
ApiKeyIdentity? identity = await AuthenticateAndAuthorizeAsync(GatewayScopes.Admin, context)
.ConfigureAwait(false);
IDisposable? identityScope = identity is null ? null : identityAccessor.Push(identity);
using (identityScope)
{
return await continuation(requestStream, context).ConfigureAwait(false);
}
}
/// <inheritdoc />
public override async Task DuplexStreamingServerHandler<TRequest, TResponse>(
IAsyncStreamReader<TRequest> requestStream,
IServerStreamWriter<TResponse> responseStream,
ServerCallContext context,
DuplexStreamingServerMethod<TRequest, TResponse> continuation)
{
ApiKeyIdentity? identity = await AuthenticateAndAuthorizeAsync(GatewayScopes.Admin, context)
.ConfigureAwait(false);
IDisposable? identityScope = identity is null ? null : identityAccessor.Push(identity);
using (identityScope)
{
await continuation(requestStream, responseStream, context).ConfigureAwait(false);
}
}
/// <summary>Authenticates the API key and authorizes the RPC call by required scope.</summary>
/// <typeparam name="TRequest">Request message type.</typeparam>
/// <param name="request">Request payload.</param>
/// <param name="requiredScope">The scope the call requires, resolved by the caller.</param>
/// <param name="context">RPC server call context.</param>
/// <returns>Authenticated API key identity, or null if authentication is disabled.</returns>
private async Task<ApiKeyIdentity?> AuthenticateAndAuthorizeAsync<TRequest>(
TRequest request,
private async Task<ApiKeyIdentity?> AuthenticateAndAuthorizeAsync(
string requiredScope,
ServerCallContext context)
where TRequest : class
{
if (options.Value.Authentication.Mode == AuthenticationMode.Disabled)
{
@@ -116,7 +158,6 @@ public sealed class GatewayGrpcAuthorizationInterceptor(
ApiKeyIdentity identity = GatewayApiKeyIdentityMapper.ToGatewayIdentity(verification.Identity);
string requiredScope = scopeResolver.ResolveRequiredScope(request);
if (!identity.Scopes.Contains(requiredScope))
{
throw new RpcException(new Status(
@@ -3,6 +3,13 @@ using ZB.MOM.WW.MxGateway.Server.Sessions;
namespace ZB.MOM.WW.MxGateway.Server.Security.Authorization;
/// <summary>Enforces API-key read/write constraints and records constraint denials for audit.</summary>
/// <remarks>
/// The per-item checks return <see cref="ValueTask{TResult}"/> because every production check is
/// synchronous (an in-memory Galaxy hierarchy lookup) and runs once per item of a bulk command:
/// a <see cref="Task{TResult}"/> return allocated a task per item for a value that was already
/// known. An implementation that genuinely needs to await still can.
/// </remarks>
public interface IConstraintEnforcer
{
/// <summary>
@@ -34,7 +41,7 @@ public interface IConstraintEnforcer
/// <param name="tagAddress">Tag address to check.</param>
/// <param name="cancellationToken">Token to observe for cancellation.</param>
/// <returns>The constraint failure details if denied; otherwise null.</returns>
Task<ConstraintFailure?> CheckReadTagAsync(
ValueTask<ConstraintFailure?> CheckReadTagAsync(
ApiKeyIdentity? identity,
string tagAddress,
CancellationToken cancellationToken);
@@ -46,7 +53,7 @@ public interface IConstraintEnforcer
/// <param name="itemHandle">The MXAccess item handle.</param>
/// <param name="cancellationToken">Token to observe for cancellation.</param>
/// <returns>The constraint failure details if denied; otherwise null.</returns>
Task<ConstraintFailure?> CheckReadHandleAsync(
ValueTask<ConstraintFailure?> CheckReadHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
@@ -60,7 +67,7 @@ public interface IConstraintEnforcer
/// <param name="itemHandle">The MXAccess item handle.</param>
/// <param name="cancellationToken">Token to observe for cancellation.</param>
/// <returns>The constraint failure details if denied; otherwise null.</returns>
Task<ConstraintFailure?> CheckWriteHandleAsync(
ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
@@ -85,4 +92,25 @@ public interface IConstraintEnforcer
ConstraintFailure failure,
string? correlationId,
CancellationToken cancellationToken);
/// <summary>
/// Records every constraint denial of one bulk command as a single aggregated audit event.
/// </summary>
/// <param name="identity">The API key identity.</param>
/// <param name="commandKind">The kind of bulk command whose items were denied.</param>
/// <param name="summary">The accumulated denials: count, constraint names and a bounded target sample.</param>
/// <param name="correlationId">The per-request client correlation id, if any (same handling as <see cref="RecordDenialAsync"/>).</param>
/// <param name="cancellationToken">Token to observe for cancellation.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
/// <remarks>
/// Bulk callers use this instead of one <see cref="RecordDenialAsync"/> per item: the audit
/// channel is bounded and drops on full, so per-item events let one bulk command evict every
/// other pending audit record.
/// </remarks>
Task RecordBulkDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
ConstraintDenialSummary summary,
string? correlationId,
CancellationToken cancellationToken);
}
@@ -0,0 +1,222 @@
using System.Security.Claims;
using System.Text.Encodings.Web;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Server.Metrics;
namespace ZB.MOM.WW.MxGateway.Server.Security.Authorization;
/// <summary>Names shared by the optional <c>/metrics</c> authentication (<c>MxGateway:Metrics:RequireAuthentication</c>).</summary>
public static class MetricsAuthorizationDefaults
{
/// <summary>Authentication scheme that authenticates a metrics scrape.</summary>
public const string AuthenticationScheme = "MxGateway.Metrics";
/// <summary>Authorization policy attached to <c>/metrics</c> when authentication is required.</summary>
public const string Policy = "MxGateway.Metrics";
/// <summary>Claim type carrying one API-key scope on a principal the metrics scheme authenticated.</summary>
public const string ScopeClaimType = "mxgateway:scope";
}
/// <summary>
/// Authenticates a <c>/metrics</c> scrape from either credential the gateway already issues: an API
/// key presented as <c>Authorization: Bearer mxgw_…</c> (verified through the same cached
/// <see cref="IApiKeyVerifier"/> and per-peer failure limiter as gRPC calls), or, when no
/// authorization header is sent, the dashboard login cookie.
/// </summary>
/// <remarks>
/// One scheme fronting both credentials keeps the challenge a plain <c>401</c> with
/// <c>WWW-Authenticate: Bearer</c>, which is what a Prometheus scraper understands; listing the
/// cookie scheme on the policy directly would turn an anonymous scrape into a 302 redirect to the
/// login page. A throttled key partition answers <c>429</c>, mirroring the gRPC
/// <c>ResourceExhausted</c>. Failure reasons are never echoed to the caller and the presented
/// secret is never logged. The limiter's per-peer partition here is the remote IP, whereas the gRPC
/// interceptor's is the gRPC peer string (address and port, i.e. per connection), so the two paths
/// share only the per-key-id aggregate layer.
/// </remarks>
public sealed class MetricsAuthenticationHandler(
IOptionsMonitor<AuthenticationSchemeOptions> options,
ILoggerFactory logger,
UrlEncoder encoder,
IApiKeyVerifier apiKeyVerifier,
ApiKeyFailureLimiter failureLimiter,
IAuthenticationSchemeProvider schemeProvider,
IOptions<GatewayOptions> gatewayOptions,
GatewayMetrics metrics)
: AuthenticationHandler<AuthenticationSchemeOptions>(options, logger, encoder)
{
private bool _throttled;
/// <inheritdoc />
protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
{
string? authorizationHeader = Request.Headers.Authorization;
if (string.IsNullOrWhiteSpace(authorizationHeader))
{
return await AuthenticateDashboardCookieAsync().ConfigureAwait(false);
}
// Same throttle as the gRPC interceptor: checked before the store read so online guessing
// against /metrics cannot spend a verification per attempt or bypass the gRPC limiter.
ApiKeyThrottlePartition partition = new(
Context.Connection.RemoteIpAddress?.ToString() ?? "unknown",
GatewayGrpcAuthorizationInterceptor.TryResolveKeyId(authorizationHeader));
ApiKeyThrottleDecision decision = failureLimiter.Check(
partition,
out ApiKeyFailureLimiter.PartitionResolution resolution);
if (decision is ApiKeyThrottleDecision.ThrottledByPeer or ApiKeyThrottleDecision.ThrottledByAggregate)
{
_throttled = true;
metrics.RecordAuthThrottled(decision == ApiKeyThrottleDecision.ThrottledByPeer ? "peer" : "aggregate");
return AuthenticateResult.Fail("Too many authentication attempts.");
}
ApiKeyVerification verification = await apiKeyVerifier
.VerifyAsync(authorizationHeader, Context.RequestAborted)
.ConfigureAwait(false);
if (!verification.Succeeded || verification.Identity is null)
{
failureLimiter.RecordFailure(partition);
return AuthenticateResult.Fail("Missing or invalid API key.");
}
failureLimiter.Reset(partition, resolution);
List<Claim> claims = [new Claim(ClaimTypes.Name, verification.Identity.KeyId)];
foreach (string scope in verification.Identity.Scopes)
{
claims.Add(new Claim(MetricsAuthorizationDefaults.ScopeClaimType, scope));
}
ClaimsPrincipal principal = new(new ClaimsIdentity(claims, Scheme.Name));
return AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name));
}
/// <inheritdoc />
protected override Task HandleChallengeAsync(AuthenticationProperties properties)
{
if (_throttled)
{
Response.StatusCode = StatusCodes.Status429TooManyRequests;
return Task.CompletedTask;
}
Response.StatusCode = StatusCodes.Status401Unauthorized;
Response.Headers.WWWAuthenticate = "Bearer";
return Task.CompletedTask;
}
// No authorization header: fall back to the dashboard login cookie, so an operator already
// signed in to the dashboard can open /metrics in the same browser.
private async Task<AuthenticateResult> AuthenticateDashboardCookieAsync()
{
// Dashboard:DisableLogin registers an always-succeeding auto-login handler under the cookie
// scheme name. Falling back to it would let every anonymous request satisfy /metrics, so
// with DisableLogin on, RequireAuthentication=true accepts API keys only.
if (gatewayOptions.Value.Dashboard.DisableLogin)
{
return AuthenticateResult.NoResult();
}
if (await schemeProvider.GetSchemeAsync(DashboardAuthenticationDefaults.AuthenticationScheme).ConfigureAwait(false)
is null)
{
return AuthenticateResult.NoResult();
}
AuthenticateResult cookie = await Context
.AuthenticateAsync(DashboardAuthenticationDefaults.AuthenticationScheme)
.ConfigureAwait(false);
return cookie.Succeeded && cookie.Principal is not null
? AuthenticateResult.Success(new AuthenticationTicket(cookie.Principal, Scheme.Name))
: AuthenticateResult.NoResult();
}
}
/// <summary>Requirement for <c>/metrics</c> access; see <see cref="MetricsAccessHandler"/>.</summary>
public sealed class MetricsAccessRequirement : IAuthorizationRequirement
{
}
/// <summary>
/// Grants <c>/metrics</c> to an API key carrying the <c>admin</c> scope or to a dashboard login
/// holding a real dashboard role (Administrator or Viewer). It reads no options, so neither
/// anonymous localhost nor <c>Authentication:Mode=Disabled</c> can satisfy it.
/// </summary>
/// <remarks>
/// A Viewer login qualifies because every dashboard role already sees the same aggregate counters
/// on the dashboard itself; the metrics carry no per-session or per-tag data. An API key needs
/// <c>admin</c> because metrics are an operator surface, not one of the data-plane scopes a client
/// key is normally issued with.
/// </remarks>
public sealed class MetricsAccessHandler : AuthorizationHandler<MetricsAccessRequirement>
{
/// <inheritdoc />
protected override Task HandleRequirementAsync(
AuthorizationHandlerContext context,
MetricsAccessRequirement requirement)
{
foreach (ClaimsIdentity identity in context.User.Identities)
{
if (!identity.IsAuthenticated)
{
continue;
}
// Scope claims count only on an identity the metrics scheme minted from a verified key.
bool isApiKeyIdentity = string.Equals(
identity.AuthenticationType,
MetricsAuthorizationDefaults.AuthenticationScheme,
StringComparison.Ordinal);
if (isApiKeyIdentity
&& identity.HasClaim(MetricsAuthorizationDefaults.ScopeClaimType, GatewayScopes.Admin))
{
context.Succeed(requirement);
return Task.CompletedTask;
}
if (!isApiKeyIdentity
&& (identity.HasClaim(identity.RoleClaimType, DashboardRoles.Admin)
|| identity.HasClaim(identity.RoleClaimType, DashboardRoles.Viewer)))
{
context.Succeed(requirement);
return Task.CompletedTask;
}
}
return Task.CompletedTask;
}
}
/// <summary>Registration for the optional <c>/metrics</c> authentication.</summary>
public static class MetricsAuthorizationServiceCollectionExtensions
{
/// <summary>
/// Registers the metrics authentication scheme, the <see cref="MetricsAuthorizationDefaults.Policy"/>
/// policy and its handler. Registration is unconditional and inert: the policy only takes effect
/// when <c>MxGateway:Metrics:RequireAuthentication</c> attaches it to the endpoint.
/// </summary>
/// <param name="services">Service collection to register into.</param>
/// <returns>The same service collection, for chaining.</returns>
public static IServiceCollection AddGatewayMetricsAuthorization(this IServiceCollection services)
{
services.AddAuthentication()
.AddScheme<AuthenticationSchemeOptions, MetricsAuthenticationHandler>(
MetricsAuthorizationDefaults.AuthenticationScheme,
_ => { });
services.AddAuthorizationBuilder()
.AddPolicy(
MetricsAuthorizationDefaults.Policy,
policy => policy
.AddAuthenticationSchemes(MetricsAuthorizationDefaults.AuthenticationScheme)
.AddRequirements(new MetricsAccessRequirement()));
services.AddSingleton<IAuthorizationHandler, MetricsAccessHandler>();
return services;
}
}
@@ -705,7 +705,7 @@ public sealed class WorkerClient : IWorkerClient
{
if (workerEvent.Event is not null)
{
_metrics?.EventReceived(SessionId, workerEvent.Event.Family.ToString());
_metrics?.EventReceived(SessionId, MxEventFamilyNames.Get(workerEvent.Event.Family));
}
if (_eventStaging.Writer.TryWrite(workerEvent))
@@ -372,6 +372,105 @@ public sealed class AlarmTruncationSignalTests
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// A subscriber that stops reading is dropped once its bounded queue fills, and the fault it
/// observes is the typed <see cref="AlarmFeedSubscriberOverflowException"/> — the signal the
/// gRPC layer maps to ResourceExhausted — rather than a bare InvalidOperationException.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SubscriberThatStopsReading_IsDroppedWithTypedOverflowFault()
{
// Comfortably past the monitor's per-subscriber queue capacity (2048).
const int ProbeCount = 2300;
using GatewayMetrics metrics = new();
StubSessionManager sessions = new();
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForReconcileAsync(WaitTimeout);
// The slow subscriber: registered (first frame read) and then never read again.
using CancellationTokenSource slowCts = new(WaitTimeout);
await using IAsyncEnumerator<AlarmFeedMessage> slow =
monitor.StreamAsync(null, slowCts.Token).GetAsyncEnumerator(slowCts.Token);
Assert.True(await slow.MoveNextAsync());
// Raises on distinct references: each one is broadcast (no dedup) without forcing a
// reconcile, which keeps the burst cheap under a loaded parallel test run.
for (ulong sequence = 1; sequence <= ProbeCount; sequence++)
{
sessions.EmitEvent(UniqueRaise(sequence));
}
// The monitor caches each raise and broadcasts it under the same lock, so once the cache
// holds every raise, every broadcast has been offered to the slow subscriber. (A draining
// reader is no barrier here: under a loaded test run it can fall 2048 frames behind the
// burst and be dropped itself.)
await WaitUntilAsync(() => monitor.CurrentAlarms.Count >= ProbeCount, WaitTimeout);
// Drain what the slow subscriber had buffered; the terminal fault is the typed overflow.
await Assert.ThrowsAsync<AlarmFeedSubscriberOverflowException>(async () =>
{
while (await slow.MoveNextAsync())
{
}
});
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// A StreamAsync attach reuses the memoized alarm-set projection that CurrentAlarms serves
/// (no per-attach clone of the whole set under the broadcast lock) and still applies the
/// subscriber's prefix filter to the snapshot it emits.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SubscriberAttach_ReusesMemoizedProjectionAndAppliesPrefixFilter()
{
using GatewayMetrics metrics = new();
StubSessionManager sessions = new()
{
Snapshots =
[
NewSnapshot("Galaxy!Area.Tank01.Level.HiHi", fromTruncatedSnapshot: false),
NewSnapshot("Galaxy!Area.Pump02.Flow.Lo", fromTruncatedSnapshot: false),
],
};
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForReconcileAsync(WaitTimeout);
await WaitUntilAsync(() => monitor.CurrentAlarms.Count == 2, WaitTimeout);
IReadOnlyList<ActiveAlarmSnapshot> projection = monitor.CurrentAlarms;
List<ActiveAlarmSnapshot> streamed = [];
using CancellationTokenSource streamCts = new(WaitTimeout);
await foreach (AlarmFeedMessage message in monitor.StreamAsync("Galaxy!Area.Tank01", streamCts.Token))
{
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ActiveAlarm)
{
streamed.Add(message.ActiveAlarm);
}
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.SnapshotComplete)
{
break;
}
}
ActiveAlarmSnapshot only = Assert.Single(streamed);
Assert.Equal("Galaxy!Area.Tank01.Level.HiHi", only.AlarmFullReference);
Assert.Contains(projection, alarm => ReferenceEquals(alarm, only));
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
private static int CountOf(List<AlarmFeedMessage> received, AlarmFeedMessage.PayloadOneofCase payloadCase)
{
lock (received)
@@ -442,6 +541,23 @@ public sealed class AlarmTruncationSignalTests
},
};
// A Raise on a reference unique to the sequence, so the monitor never dedups it.
private static MxEvent UniqueRaise(ulong sequence) => new()
{
Family = MxEventFamily.OnAlarmTransition,
WorkerSequence = sequence,
OnAlarmTransition = new OnAlarmTransitionEvent
{
AlarmFullReference = $"Galaxy!Area.Tank{sequence}.Level.Hi",
SourceObjectReference = $"Tank{sequence}.Level",
AlarmTypeName = "AnalogLimitAlarm.Hi",
TransitionKind = AlarmTransitionKind.Raise,
Severity = 500,
SourceProvider = AlarmProviderMode.Alarmmgr,
TransitionTimestamp = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
},
};
private static ActiveAlarmSnapshot NewSnapshot(string reference, bool fromTruncatedSnapshot)
{
return new ActiveAlarmSnapshot
@@ -263,7 +263,7 @@ public sealed class EventStreamServiceTests
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.QueueOverflows);
Assert.Equal(1, snapshot.Faults);
// The finally block in StreamEventsAsync calls StreamDisconnected("Detached") on the
// The finally block in StreamEventsAsync records StreamDisconnected (reason Overflow) on the
// overflow+fault path too; pin it here so a regression removing that call is caught.
Assert.Equal(1, snapshot.StreamDisconnects);
}
@@ -572,6 +572,142 @@ public sealed class EventStreamServiceTests
}
}
/// <summary>
/// A stream that ends because the session's event stream completed records the
/// <c>SessionClosed</c> disconnect reason rather than a blanket constant.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenEventStreamCompletes_RecordsSessionClosedReason()
{
FakeWorkerClient workerClient = new();
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
GatewaySession session = CreateReadySession(workerClient);
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
workerClient.CompleteAfterConfiguredEvents = true;
await CollectEventsAsync(service, session.SessionId);
Assert.Equal([EventStreamDisconnectReasons.SessionClosed], reasons.Reasons);
}
/// <summary>A per-subscriber overflow records the <c>Overflow</c> disconnect reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenStreamQueueOverflows_RecordsOverflowReason()
{
FakeWorkerClient workerClient = new();
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
GatewaySession session = CreateReadySession(
workerClient,
queueCapacity: 1,
metrics: metrics,
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
EventStreamService service = CreateService(
new FakeSessionManager(session),
metrics,
queueCapacity: 1,
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
for (ulong sequence = 1; sequence <= 50; sequence++)
{
workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
}
workerClient.CompleteAfterConfiguredEvents = true;
await using (IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
.GetAsyncEnumerator())
{
await Assert.ThrowsAsync<SessionManagerException>(
async () =>
{
while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout))
{
}
});
}
Assert.Equal([EventStreamDisconnectReasons.Overflow], reasons.Reasons);
}
/// <summary>A terminal worker fault records the <c>WorkerFault</c> disconnect reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenWorkerEventStreamFaults_RecordsWorkerFaultReason()
{
FakeWorkerClient workerClient = new()
{
TerminalException = new WorkerClientException(
WorkerClientErrorCode.WorkerFaulted,
"worker terminal fault"),
};
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
await using (IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
.GetAsyncEnumerator())
{
await Assert.ThrowsAsync<WorkerClientException>(
async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
}
Assert.Equal([EventStreamDisconnectReasons.WorkerFault], reasons.Reasons);
}
/// <summary>A client cancellation records the <c>ClientCancelled</c> disconnect reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenCanceled_RecordsClientCancelledReason()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
using CancellationTokenSource cancellationTokenSource = new();
await using (IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, cancellationTokenSource.Token)
.GetAsyncEnumerator(cancellationTokenSource.Token))
{
Task<bool> moveTask = subscriber.MoveNextAsync().AsTask();
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1);
await cancellationTokenSource.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(
async () => await moveTask.WaitAsync(TestTimeout));
}
Assert.Equal([EventStreamDisconnectReasons.ClientCancelled], reasons.Reasons);
}
/// <summary>
/// A consumer that stops reading without cancelling (the gRPC writer gave up) keeps the
/// <c>Detached</c> reason.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenConsumerStopsEarly_RecordsDetachedReason()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
await using (IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
.GetAsyncEnumerator())
{
await ReadNextAsync(subscriber);
}
Assert.Equal([EventStreamDisconnectReasons.Detached], reasons.Reasons);
}
private static async Task<MxEvent> ReadNextAsync(IAsyncEnumerator<MxEvent> enumerator)
{
Assert.True(await enumerator.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
@@ -896,4 +1032,63 @@ public sealed class EventStreamServiceTests
return ValueTask.CompletedTask;
}
}
/// <summary>
/// Captures the <c>reason</c> tag of every <c>mxgateway.grpc.streams.disconnected</c>
/// measurement recorded on one <see cref="GatewayMetrics"/> instance (filtered by meter
/// reference, so parallel tests sharing the meter name cannot cross-talk).
/// </summary>
private sealed class DisconnectReasonRecorder : IDisposable
{
private readonly System.Diagnostics.Metrics.MeterListener _listener = new();
private readonly List<string> _reasons = [];
/// <summary>Initializes a new instance of the <see cref="DisconnectReasonRecorder"/> class.</summary>
/// <param name="metrics">The metrics instance whose meter to observe.</param>
public DisconnectReasonRecorder(GatewayMetrics metrics)
{
_listener.InstrumentPublished = (instrument, listener) =>
{
if (ReferenceEquals(instrument.Meter, metrics.Meter)
&& instrument.Name == "mxgateway.grpc.streams.disconnected")
{
listener.EnableMeasurementEvents(instrument);
}
};
_listener.SetMeasurementEventCallback<long>((instrument, _, tags, _) =>
{
if (!ReferenceEquals(instrument.Meter, metrics.Meter))
{
return;
}
foreach (KeyValuePair<string, object?> tag in tags)
{
if (tag.Key == "reason" && tag.Value is string reason)
{
lock (_reasons)
{
_reasons.Add(reason);
}
}
}
});
_listener.Start();
}
/// <summary>Gets the recorded reasons, in order.</summary>
public string[] Reasons
{
get
{
lock (_reasons)
{
return [.. _reasons];
}
}
}
/// <inheritdoc />
public void Dispose() => _listener.Dispose();
}
}
@@ -87,8 +87,56 @@ public sealed class MxAccessGatewayServiceConstraintTests
Assert.True(merged.Results[3].WasSuccessful);
Assert.Equal("Tank04.Public", merged.Results[3].TagAddress);
Assert.Equal(104, merged.Results[3].ItemHandle);
// Both denied tags recorded.
Assert.Equal(2, enforcer.RecordedDenials.Count);
// Both denied tags recorded in ONE aggregated bulk denial, not one audit call per item.
Assert.Empty(enforcer.RecordedDenials);
(string commandKind, ConstraintDenialSummary summary, _) = Assert.Single(enforcer.RecordedBulkDenials);
Assert.Equal(nameof(MxCommandKind.AddItemBulk), commandKind);
Assert.Equal(2, summary.DeniedCount);
Assert.Equal(4, summary.TotalCount);
Assert.Equal(["Tank01.Locked", "Tank03.Secret"], summary.SampleTargets);
}
/// <summary>
/// Audit-flood guard: a bulk write whose every entry is denied records exactly one
/// aggregated denial carrying the full denied count, the client correlation id and a sample
/// capped at <see cref="ConstraintDenialSummary.MaxSampleTargets"/> — so one low-privilege
/// key cannot push thousands of events into the bounded, drop-on-full audit channel.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Invoke_WriteBulk_WithManyDeniedEntries_RecordsOneAggregatedDenial()
{
const int EntryCount = 500;
PredicateConstraintEnforcer enforcer = new() { DenyWriteHandle = (_, _) => true };
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
WriteBulkCommand payload = new() { ServerHandle = 7 };
for (int index = 0; index < EntryCount; index++)
{
payload.Entries.Add(new WriteBulkEntry { ItemHandle = index + 1 });
}
MxCommandRequest request = new()
{
SessionId = SessionId,
ClientCorrelationId = "bulk-flood-1",
Command = new MxCommand { Kind = MxCommandKind.WriteBulk, WriteBulk = payload },
};
MxCommandReply reply = await service.Invoke(request, new TestServerCallContext());
Assert.Equal(0, sessionManager.InvokeCount);
Assert.Equal(EntryCount, reply.WriteBulk.Results.Count);
Assert.Empty(enforcer.RecordedDenials);
(string commandKind, ConstraintDenialSummary summary, string? correlationId) =
Assert.Single(enforcer.RecordedBulkDenials);
Assert.Equal(nameof(MxCommandKind.WriteBulk), commandKind);
Assert.Equal("bulk-flood-1", correlationId);
Assert.Equal(EntryCount, summary.DeniedCount);
Assert.Equal(EntryCount, summary.TotalCount);
Assert.Equal(ConstraintDenialSummary.MaxSampleTargets, summary.SampleTargets.Count);
Assert.True(summary.SampleTruncated);
Assert.Equal(["write-handle"], summary.ConstraintNames);
}
/// <summary>
@@ -2,9 +2,11 @@ using System.Diagnostics.Metrics;
using System.Runtime.CompilerServices;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
@@ -213,6 +215,8 @@ public sealed class MxAccessGatewayServiceTests
Assert.Equal(42, reply.AddItem.ItemHandle);
Assert.Equal("status detail", Assert.Single(reply.Statuses).DiagnosticText);
Assert.Equal("mxaccess diagnostic", reply.DiagnosticMessage);
// The reply echoes the caller's client_correlation_id, not the internal worker envelope id.
Assert.Equal("client-correlation", reply.CorrelationId);
}
/// <summary>Verifies that StreamEvents writes only events after the specified worker sequence.</summary>
@@ -463,11 +467,195 @@ public sealed class MxAccessGatewayServiceTests
Assert.Contains("server-stream-active-alarms", reply.Capabilities);
}
/// <summary>
/// Every reply echoes the request's <c>client_correlation_id</c>, including when the caller
/// sent none: the internal worker envelope id never reaches the client.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Invoke_WithoutClientCorrelationId_DoesNotLeakWorkerEnvelopeId()
{
FakeSessionManager sessionManager = new()
{
InvokeReply = new WorkerCommandReply
{
Reply = new MxCommandReply
{
SessionId = "session-1",
CorrelationId = "4f1d1b0e-internal-envelope-guid",
Kind = MxCommandKind.AddItem,
ProtocolStatus = MxAccessGrpcMapper.Ok(),
AddItem = new AddItemReply { ItemHandle = 42 },
},
},
};
MxAccessGatewayService service = CreateService(sessionManager);
MxCommandRequest request = new()
{
SessionId = "session-1",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand { ServerHandle = 12, ItemDefinition = "Galaxy.Tag.Value" },
},
};
MxCommandReply reply = await service.Invoke(request, new TestServerCallContext());
Assert.Equal(string.Empty, reply.CorrelationId);
}
/// <summary>
/// Invoke logs the client correlation id alongside the worker envelope id once, at Debug, so
/// a client error report can be matched to gateway and worker logs.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Invoke_LogsClientToWorkerCorrelationPairAtDebug()
{
FakeSessionManager sessionManager = new()
{
InvokeReply = new WorkerCommandReply
{
Reply = new MxCommandReply
{
SessionId = "session-1",
CorrelationId = "envelope-7f3a",
Kind = MxCommandKind.AddItem,
ProtocolStatus = MxAccessGrpcMapper.Ok(),
AddItem = new AddItemReply { ItemHandle = 42 },
},
},
};
RecordingLogger<MxAccessGatewayService> logger = new();
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
MxCommandRequest request = new()
{
SessionId = "session-1",
ClientCorrelationId = "client-trace-9",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand { ServerHandle = 12, ItemDefinition = "Galaxy.Tag.Value" },
},
};
await service.Invoke(request, new TestServerCallContext());
(LogLevel level, string message, _) = Assert.Single(
logger.Entries,
entry => entry.Message.Contains("envelope-7f3a", StringComparison.Ordinal));
Assert.Equal(LogLevel.Debug, level);
Assert.Contains("client-trace-9", message, StringComparison.Ordinal);
}
/// <summary>
/// StreamEvents hands the call's cancellation token to every response write, so a write
/// blocked on a slow or vanished client is released when the call is cancelled.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_PassesCallCancellationTokenToResponseWrites()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
MxAccessGatewayService service = CreateService(sessionManager);
RecordingServerStreamWriter<MxEvent> writer = new();
using CancellationTokenSource callCts = new();
await service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext(cancellationToken: callCts.Token));
Assert.Single(writer.Messages);
Assert.Equal(callCts.Token, writer.LastWriteCancellationToken);
}
/// <summary>
/// A response write that fails with an <see cref="IOException"/> because the client went
/// away (the call token is cancelled) maps to Cancelled and logs no warning: a client
/// disconnect is routine, not a gateway failure.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_WhenWriteFailsAfterClientCancel_ReturnsCancelledWithoutWarning()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
RecordingLogger<MxAccessGatewayService> logger = new();
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
using CancellationTokenSource callCts = new();
ThrowingStreamWriter<MxEvent> writer = new(() =>
{
callCts.Cancel();
return new IOException("The client reset the request stream.");
});
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext(cancellationToken: callCts.Token)));
Assert.Equal(StatusCode.Cancelled, exception.StatusCode);
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
}
/// <summary>
/// The same IOException with a live call token is a genuine transport failure and keeps the
/// existing Unavailable + warning mapping.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_WhenWriteFailsWithoutCancel_KeepsUnavailableMapping()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
RecordingLogger<MxAccessGatewayService> logger = new();
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
ThrowingStreamWriter<MxEvent> writer = new(() => new IOException("Transport failure."));
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext()));
Assert.Equal(StatusCode.Unavailable, exception.StatusCode);
Assert.Contains(logger.Entries, entry => entry.Level == LogLevel.Warning);
}
/// <summary>
/// A StreamAlarms subscriber the monitor drops for falling behind receives ResourceExhausted
/// (the same status a StreamEvents overflow carries), and the gateway logs no
/// warning-with-stack for what is a client-side slowness condition.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamAlarms_WhenSubscriberOverflows_ReturnsResourceExhaustedWithoutWarning()
{
RecordingLogger<MxAccessGatewayService> logger = new();
FakeGatewayAlarmService alarms = new() { StreamFault = new AlarmFeedSubscriberOverflowException() };
MxAccessGatewayService service = CreateService(new FakeSessionManager(), alarmService: alarms, logger: logger);
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamAlarms(
new StreamAlarmsRequest(),
new RecordingServerStreamWriter<AlarmFeedMessage>(),
new TestServerCallContext()));
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
}
private static MxAccessGatewayService CreateService(
FakeSessionManager sessionManager,
IGatewayRequestIdentityAccessor? identityAccessor = null,
GatewayMetrics? metrics = null,
IConstraintEnforcer? constraintEnforcer = null)
IConstraintEnforcer? constraintEnforcer = null,
IGatewayAlarmService? alarmService = null,
ILogger<MxAccessGatewayService>? logger = null)
{
return new MxAccessGatewayService(
sessionManager,
@@ -477,8 +665,8 @@ public sealed class MxAccessGatewayServiceTests
new MxAccessGrpcMapper(),
new FakeEventStreamService(sessionManager),
metrics ?? new GatewayMetrics(),
NullLogger<MxAccessGatewayService>.Instance,
new FakeGatewayAlarmService());
logger ?? NullLogger<MxAccessGatewayService>.Instance,
alarmService ?? new FakeGatewayAlarmService());
}
private static ApiKeyIdentity CreateIdentity()
@@ -786,4 +974,18 @@ public sealed class MxAccessGatewayServiceTests
}
}
/// <summary>A stream writer whose token-bearing write throws the supplied exception.</summary>
/// <typeparam name="T">The streamed message type.</typeparam>
private sealed class ThrowingStreamWriter<T>(Func<Exception> createException) : IServerStreamWriter<T>
{
/// <inheritdoc />
public WriteOptions? WriteOptions { get; set; }
/// <inheritdoc />
public Task WriteAsync(T message) => throw createException();
/// <inheritdoc />
public Task WriteAsync(T message, CancellationToken cancellationToken) => throw createException();
}
}
@@ -0,0 +1,182 @@
using System.Net;
using System.Net.Http;
using System.Security.Claims;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication.Cookies;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Auth.ApiKeys.Admin;
using ZB.MOM.WW.MxGateway.Server;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway;
/// <summary>
/// Host-level coverage of <c>MxGateway:Metrics:RequireAuthentication=true</c> (SEC-14): each case
/// builds the real gateway with <see cref="GatewayApplication.Build"/> and drives the full
/// middleware pipeline (authentication, authorization, the dashboard cookie scheme, the API-key
/// store) over HTTP on an ephemeral loopback port. Requests therefore arrive from loopback, which
/// is exactly the anonymous-localhost case the dashboard Viewer policy would admit and /metrics
/// must not.
/// </summary>
public sealed class MetricsEndpointAuthenticationTests
{
// A fixed test pepper: the configuration-backed pepper provider reads MxGateway:ApiKeyPepper,
// and the auth store itself is isolated per test process by TestHostEnvironmentInitializer.
private const string TestPepper = "metrics-endpoint-tests-pepper-0123456789abcdef";
/// <summary>
/// An anonymous loopback scrape gets a plain 401 Bearer challenge — not a 302 to /login, and not
/// the anonymous-localhost admission the dashboard Viewer policy grants.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task AnonymousLoopbackScrape_Returns401BearerNotLoginRedirect()
{
await using WebApplication app = await StartAsync();
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await client.GetAsync("/metrics");
Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
Assert.Null(response.Headers.Location);
Assert.Contains(response.Headers.WwwAuthenticate, value => value.Scheme == "Bearer");
}
/// <summary>/health/active stays anonymous: the off-repo Overview dashboard polls it without credentials.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task HealthActive_StaysAnonymous()
{
await using WebApplication app = await StartAsync();
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await client.GetAsync("/health/active");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
/// <summary>A dashboard login cookie holding the Viewer role is accepted.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DashboardViewerCookie_Returns200()
{
await using WebApplication app = await StartAsync();
using HttpClient client = CreateClient(app);
using HttpRequestMessage request = new(HttpMethod.Get, "/metrics");
request.Headers.Add("Cookie", MintDashboardCookie(app, DashboardRoles.Viewer));
using HttpResponseMessage response = await client.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
/// <summary>A real API key with the admin scope is accepted.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task AdminScopedApiKey_Returns200()
{
await using WebApplication app = await StartAsync();
string token = await CreateKeyAsync(app, [GatewayScopes.Admin]);
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await SendWithBearerAsync(client, token);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
/// <summary>A real API key without the admin scope authenticates but is forbidden.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task NonAdminApiKey_Returns403()
{
await using WebApplication app = await StartAsync();
string token = await CreateKeyAsync(app, [GatewayScopes.EventsRead, GatewayScopes.InvokeRead]);
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await SendWithBearerAsync(client, token);
Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode);
}
/// <summary>
/// Dev-only <c>Dashboard:DisableLogin</c> registers an always-succeeding auto-login handler under
/// the cookie scheme name; /metrics must not fall back to it, or the flag would silently open
/// an endpoint the operator explicitly asked to protect.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DisableLogin_DoesNotOpenMetrics()
{
await using WebApplication app = await StartAsync("--MxGateway:Dashboard:DisableLogin=true");
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await client.GetAsync("/metrics");
Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
}
private static async Task<WebApplication> StartAsync(params string[] extraArgs)
{
// Ephemeral port (:0): xUnit runs collections in parallel, so a fixed port would collide.
WebApplication app = GatewayApplication.Build(
[
"--urls=http://127.0.0.1:0",
"--MxGateway:Metrics:RequireAuthentication=true",
$"--MxGateway:ApiKeyPepper={TestPepper}",
.. extraArgs,
]);
await app.StartAsync();
return app;
}
private static HttpClient CreateClient(WebApplication app) =>
new(new HttpClientHandler { AllowAutoRedirect = false, UseCookies = false })
{
BaseAddress = new Uri(app.Urls.First()),
};
private static async Task<HttpResponseMessage> SendWithBearerAsync(HttpClient client, string token)
{
using HttpRequestMessage request = new(HttpMethod.Get, "/metrics");
request.Headers.TryAddWithoutValidation("Authorization", $"Bearer {token}");
return await client.SendAsync(request);
}
// Creates a key through the same admin command set the CLI and dashboard use. Key ids are unique
// per call because the isolated auth store is shared by every test in the process.
private static async Task<string> CreateKeyAsync(WebApplication app, string[] scopes)
{
ApiKeyAdminCommands commands = app.Services.GetRequiredService<ApiKeyAdminCommands>();
await commands.InitDbAsync(remoteAddress: null, CancellationToken.None);
CreateKeyResult created = await commands.CreateKeyAsync(
$"metrics{Guid.NewGuid():N}",
"Metrics test key",
new HashSet<string>(scopes, StringComparer.Ordinal),
null,
null,
remoteAddress: null,
CancellationToken.None);
return created.Token ?? throw new InvalidOperationException("Key creation returned no token.");
}
// Protects a ticket with the running host's own cookie options (data-protection keys, ticket
// format, cookie name) — the cookie a real dashboard login would issue, minus the LDAP bind.
private static string MintDashboardCookie(WebApplication app, string role)
{
CookieAuthenticationOptions cookieOptions = app.Services
.GetRequiredService<IOptionsMonitor<CookieAuthenticationOptions>>()
.Get(DashboardAuthenticationDefaults.AuthenticationScheme);
ClaimsIdentity identity = new(
[new Claim(ClaimTypes.Name, "viewer-user"), new Claim(ClaimTypes.Role, role)],
DashboardAuthenticationDefaults.AuthenticationScheme,
ClaimTypes.Name,
ClaimTypes.Role);
AuthenticationTicket ticket = new(
new ClaimsPrincipal(identity),
DashboardAuthenticationDefaults.AuthenticationScheme);
return $"{cookieOptions.Cookie.Name}={cookieOptions.TicketDataFormat.Protect(ticket)}";
}
}
@@ -0,0 +1,41 @@
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Metrics;
namespace ZB.MOM.WW.MxGateway.Tests.Metrics;
public sealed class MxEventFamilyNamesTests
{
/// <summary>
/// Every defined family maps to exactly the name <see cref="Enum.ToString()"/> produces, so
/// switching the per-event metric tags to the cached lookup changes no exported tag value.
/// </summary>
[Fact]
public void Get_ForEveryDefinedFamily_MatchesEnumToString()
{
foreach (MxEventFamily family in Enum.GetValues<MxEventFamily>())
{
Assert.Equal(family.ToString(), MxEventFamilyNames.Get(family));
}
}
/// <summary>A defined family returns the same cached string instance on every call (no per-event allocation).</summary>
[Fact]
public void Get_ForDefinedFamily_ReturnsCachedInstance()
{
Assert.Same(
MxEventFamilyNames.Get(MxEventFamily.OnDataChange),
MxEventFamilyNames.Get(MxEventFamily.OnDataChange));
}
/// <summary>
/// A numeric value the contract does not define (a newer worker, or a malformed frame) still
/// produces the <see cref="Enum.ToString()"/> form rather than throwing.
/// </summary>
[Fact]
public void Get_ForUndefinedValue_FallsBackToToString()
{
MxEventFamily undefined = (MxEventFamily)987654;
Assert.Equal(undefined.ToString(), MxEventFamilyNames.Get(undefined));
}
}
@@ -129,6 +129,120 @@ public sealed class CachingApiKeyVerifierTests
Assert.Equal(2, inner.CallCount);
}
/// <summary>
/// Single-flight: concurrent misses for the same token (a TTL expiry under load) share one
/// inner verification instead of each issuing its own SQLite read.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task VerifyAsync_ConcurrentMisses_ShareOneInnerVerification()
{
GatedVerifier inner = new(Success("operator01"));
using MemoryCache cache = NewCache();
CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30));
Task<ApiKeyVerification> first = verifier.VerifyAsync(Header, CancellationToken.None);
await inner.Entered;
Task<ApiKeyVerification>[] joiners = Enumerable
.Range(0, 16)
.Select(_ => verifier.VerifyAsync(Header, CancellationToken.None))
.ToArray();
inner.Release();
ApiKeyVerification[] results = await Task.WhenAll([first, .. joiners]);
Assert.All(results, result => Assert.True(result.Succeeded));
Assert.Equal(1, inner.CallCount);
}
/// <summary>
/// A miss that arrives after <see cref="IApiKeyCacheInvalidator.Invalidate"/> must not join a
/// verification that started before it: that flight may return the pre-revoke identity, so the
/// late caller starts its own flight against the inner verifier.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task VerifyAsync_MissAfterInvalidate_DoesNotJoinPreInvalidateFlight()
{
GatedVerifier inner = new(Success("operator01"));
using MemoryCache cache = NewCache();
CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30));
Task<ApiKeyVerification> preRevoke = verifier.VerifyAsync(Header, CancellationToken.None);
await inner.Entered;
((IApiKeyCacheInvalidator)verifier).Invalidate("operator01");
// The gated first call is still parked; a joined call would park with it.
ApiKeyVerification postRevoke = await verifier
.VerifyAsync(Header, CancellationToken.None)
.WaitAsync(TimeSpan.FromSeconds(5));
Assert.True(postRevoke.Succeeded);
Assert.Equal(2, inner.CallCount);
inner.Release();
await preRevoke;
}
/// <summary>
/// A caller whose own token is cancelled stops waiting, but the shared verification it joined
/// is not cancelled for the other waiters.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task VerifyAsync_CancelledWaiter_DoesNotCancelSharedVerification()
{
GatedVerifier inner = new(Success("operator01"));
using MemoryCache cache = NewCache();
CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30));
using CancellationTokenSource cancelled = new();
Task<ApiKeyVerification> survivor = verifier.VerifyAsync(Header, CancellationToken.None);
await inner.Entered;
Task<ApiKeyVerification> abandoned = verifier.VerifyAsync(Header, cancelled.Token);
await cancelled.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => abandoned);
inner.Release();
Assert.True((await survivor).Succeeded);
Assert.Equal(1, inner.CallCount);
}
/// <summary>
/// The key-id index is pruned when the cache evicts an entry on its own (expiry, compaction),
/// not only on <see cref="IApiKeyCacheInvalidator.Invalidate"/> — otherwise every token ever
/// verified leaves an index entry behind for the life of the process.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CacheEviction_PrunesKeyIdIndex()
{
FakeVerifier inner = new(Success("operator01"));
using MemoryCache cache = NewCache();
CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30));
await verifier.VerifyAsync(Header, CancellationToken.None);
Assert.Equal(1, verifier.IndexedCacheKeyCount);
cache.Compact(1.0);
DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
while (verifier.IndexedCacheKeyCount != 0 && DateTime.UtcNow < deadline)
{
await Task.Delay(10);
}
Assert.Equal(0, verifier.IndexedCacheKeyCount);
Assert.Equal(0, verifier.IndexedKeyIdCount);
// The evicted entry is gone, so the next request re-verifies, and Invalidate still reaches
// the repopulated entry through the index.
await verifier.VerifyAsync(Header, CancellationToken.None);
Assert.Equal(2, inner.CallCount);
((IApiKeyCacheInvalidator)verifier).Invalidate("operator01");
await verifier.VerifyAsync(Header, CancellationToken.None);
Assert.Equal(3, inner.CallCount);
}
/// <summary>
/// The store decorator coalesces repeated <c>MarkUsed</c> writes for the same key inside the
/// window down to a single forwarded write — the ≤1/min guarantee for <c>last_used_utc</c>.
@@ -154,6 +154,70 @@ public sealed class ConstraintEnforcerTests
Assert.Equal("Read:Secret.Tag", auditEvent.Target);
}
/// <summary>
/// A bulk denial is written as ONE audit event carrying the denied/total counts, the distinct
/// constraint names and a sample capped at <see cref="ConstraintDenialSummary.MaxSampleTargets"/>,
/// however many items were denied.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RecordBulkDenialAsync_WritesOneAggregatedEventWithBoundedSample()
{
ConstraintEnforcer enforcer = CreateEnforcer(out FakeAuditWriter auditWriter);
ConstraintDenialSummary summary = new(totalCount: 1000);
for (int index = 0; index < 999; index++)
{
summary.Add(
$"Area1.Tag{index}",
index % 2 == 0
? new ConstraintFailure("read_scope", "Tag is outside the API key read scope.")
: new ConstraintFailure("tag_metadata", "Tag metadata is not available."));
}
await enforcer.RecordBulkDenialAsync(
CreateIdentity(ApiKeyConstraints.Empty),
"ReadBulk",
summary,
"trace-1",
CancellationToken.None);
AuditEvent auditEvent = Assert.Single(auditWriter.Events);
Assert.Equal("operator01", auditEvent.Actor);
Assert.Equal("constraint-denied", auditEvent.Action);
Assert.Equal(AuditOutcome.Denied, auditEvent.Outcome);
Assert.Equal("ReadBulk:bulk", auditEvent.Target);
Assert.Null(auditEvent.CorrelationId);
using JsonDocument details = JsonDocument.Parse(auditEvent.DetailsJson!);
JsonElement root = details.RootElement;
Assert.Equal(999, root.GetProperty("deniedCount").GetInt32());
Assert.Equal(1000, root.GetProperty("totalCount").GetInt32());
Assert.True(root.GetProperty("sampleTruncated").GetBoolean());
Assert.Equal(ConstraintDenialSummary.MaxSampleTargets, root.GetProperty("sampleTargets").GetArrayLength());
Assert.Equal(
["read_scope", "tag_metadata"],
root.GetProperty("constraints").EnumerateArray().Select(e => e.GetString() ?? string.Empty).ToArray());
Assert.Equal("ReadBulk", root.GetProperty("commandKind").GetString());
Assert.Equal("trace-1", root.GetProperty("clientCorrelationId").GetString());
}
/// <summary>An empty summary (nothing denied) writes no audit event at all.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RecordBulkDenialAsync_WithNoDenials_WritesNothing()
{
ConstraintEnforcer enforcer = CreateEnforcer(out FakeAuditWriter auditWriter);
await enforcer.RecordBulkDenialAsync(
identity: null,
"ReadBulk",
new ConstraintDenialSummary(totalCount: 3),
correlationId: null,
CancellationToken.None);
Assert.Empty(auditWriter.Events);
}
/// <summary>Verifies that historized-only constraint requires historized attribute.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -717,6 +717,151 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests
Assert.Null(GatewayGrpcAuthorizationInterceptor.TryResolveKeyId($"Bearer mxgw_{overCap}_secret"));
}
/// <summary>
/// A client-streaming call is authenticated exactly like a unary call: a missing key is
/// rejected with Unauthenticated and the handler never runs. The base Interceptor passes
/// client- and duplex-streaming calls straight through, so without the override any such RPC
/// added to the service would be reachable unauthenticated.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ClientStreamingServerHandler_MissingApiKey_ReturnsUnauthenticatedWithoutRunningHandler()
{
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(Failure(ApiKeyFailure.MissingOrMalformed)),
new GatewayRequestIdentityAccessor());
bool handlerRan = false;
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => interceptor.ClientStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
new TestServerCallContext([]),
(_, _) =>
{
handlerRan = true;
return Task.FromResult(new MxCommandReply());
}));
Assert.Equal(StatusCode.Unauthenticated, exception.StatusCode);
Assert.False(handlerRan);
}
/// <summary>
/// A client-streaming call has no single request message to derive a scope from, so the
/// interceptor fails closed to the <c>admin</c> scope, the resolver's default for an
/// unrecognized request.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ClientStreamingServerHandler_KeyWithoutAdminScope_ReturnsPermissionDenied()
{
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.InvokeRead, GatewayScopes.InvokeWrite)),
new GatewayRequestIdentityAccessor());
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => interceptor.ClientStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
ContextWithAuthorization("Bearer mxgw_operator01_secret"),
(_, _) => Task.FromResult(new MxCommandReply())));
Assert.Equal(StatusCode.PermissionDenied, exception.StatusCode);
Assert.Contains(GatewayScopes.Admin, exception.Status.Detail, StringComparison.Ordinal);
}
/// <summary>An admin-scoped key reaches the client-streaming handler with its identity pushed.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ClientStreamingServerHandler_AdminScope_RunsHandlerWithIdentity()
{
GatewayRequestIdentityAccessor identityAccessor = new();
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.Admin)),
identityAccessor);
string? keyIdSeenByHandler = null;
await interceptor.ClientStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
ContextWithAuthorization("Bearer mxgw_operator01_secret"),
(_, _) =>
{
keyIdSeenByHandler = identityAccessor.Current?.KeyId;
return Task.FromResult(new MxCommandReply());
});
Assert.Equal("operator01", keyIdSeenByHandler);
Assert.Null(identityAccessor.Current);
}
/// <summary>A duplex-streaming call with no key is rejected before the handler runs.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DuplexStreamingServerHandler_MissingApiKey_ReturnsUnauthenticatedWithoutRunningHandler()
{
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(Failure(ApiKeyFailure.MissingOrMalformed)),
new GatewayRequestIdentityAccessor());
bool handlerRan = false;
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => interceptor.DuplexStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
new RecordingServerStreamWriter<MxCommandReply>(),
new TestServerCallContext([]),
(_, _, _) =>
{
handlerRan = true;
return Task.CompletedTask;
}));
Assert.Equal(StatusCode.Unauthenticated, exception.StatusCode);
Assert.False(handlerRan);
}
/// <summary>A duplex-streaming call fails closed to the admin scope, like client streaming.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DuplexStreamingServerHandler_KeyWithoutAdminScope_ReturnsPermissionDenied()
{
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.EventsRead)),
new GatewayRequestIdentityAccessor());
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => interceptor.DuplexStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
new RecordingServerStreamWriter<MxCommandReply>(),
ContextWithAuthorization("Bearer mxgw_operator01_secret"),
(_, _, _) => Task.CompletedTask));
Assert.Equal(StatusCode.PermissionDenied, exception.StatusCode);
}
/// <summary>An admin-scoped key reaches the duplex handler with its identity pushed.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DuplexStreamingServerHandler_AdminScope_RunsHandlerWithIdentity()
{
GatewayRequestIdentityAccessor identityAccessor = new();
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.Admin)),
identityAccessor);
RecordingServerStreamWriter<MxCommandReply> writer = new();
await interceptor.DuplexStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
writer,
ContextWithAuthorization("Bearer mxgw_operator01_secret"),
async (_, responseStream, _) =>
{
Assert.Equal("operator01", identityAccessor.Current?.KeyId);
await responseStream.WriteAsync(new MxCommandReply { SessionId = "duplex" });
});
Assert.Equal("duplex", Assert.Single(writer.Messages).SessionId);
Assert.Null(identityAccessor.Current);
}
private static MxAccessGatewayService CreateService(
ISessionManager sessionManager,
IGatewayRequestIdentityAccessor identityAccessor)
@@ -933,4 +1078,14 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests
}
}
/// <summary>An <see cref="IAsyncStreamReader{T}"/> that yields no messages.</summary>
/// <typeparam name="T">The streamed message type.</typeparam>
private sealed class EmptyStreamReader<T> : IAsyncStreamReader<T>
{
/// <inheritdoc />
public T Current => throw new InvalidOperationException("The stream is empty.");
/// <inheritdoc />
public Task<bool> MoveNext(CancellationToken cancellationToken) => Task.FromResult(false);
}
}
@@ -0,0 +1,251 @@
using System.Security.Claims;
using System.Text.Encodings.Web;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity;
namespace ZB.MOM.WW.MxGateway.Tests.Security.Authorization;
/// <summary>
/// The optional <c>/metrics</c> authentication (<c>MxGateway:Metrics:RequireAuthentication</c>,
/// SEC-14): the scheme that authenticates a scrape and the requirement that authorizes it.
/// </summary>
public sealed class MetricsAuthorizationTests
{
private const string AdminHeader = "Bearer mxgw_scraper_secret";
/// <summary>A verified API key authenticates and carries its scopes as claims.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Authenticate_WithValidApiKey_SucceedsWithScopeClaims()
{
CountingVerifier verifier = new(Success(GatewayScopes.Admin));
DefaultHttpContext httpContext = ContextWithAuthorization(AdminHeader);
AuthenticateResult result = await AuthenticateAsync(verifier, httpContext);
Assert.True(result.Succeeded);
Assert.Equal("scraper", result.Principal!.Identity!.Name);
Assert.True(result.Principal.HasClaim(MetricsAuthorizationDefaults.ScopeClaimType, GatewayScopes.Admin));
Assert.Equal(AdminHeader, verifier.LastHeader);
}
/// <summary>An invalid key fails authentication with an opaque reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Authenticate_WithInvalidApiKey_Fails()
{
CountingVerifier verifier = new(new ApiKeyVerification(false, null, ApiKeyFailure.SecretMismatch));
AuthenticateResult result = await AuthenticateAsync(verifier, ContextWithAuthorization(AdminHeader));
Assert.False(result.Succeeded);
Assert.NotNull(result.Failure);
Assert.DoesNotContain("secret", result.Failure!.Message, StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// Repeated bad keys from one peer trip the shared failure limiter, after which the verifier is
/// no longer reached and the challenge answers 429 — /metrics is not a bypass of the gRPC throttle.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Authenticate_AfterRepeatedFailures_IsThrottledBeforeVerification()
{
CountingVerifier verifier = new(new ApiKeyVerification(false, null, ApiKeyFailure.SecretMismatch));
ApiKeyFailureLimiter limiter = new(
limit: 2,
TimeSpan.FromMinutes(1),
maxPartitions: 16,
aggregateLimit: 0,
probeInterval: TimeSpan.Zero,
TimeProvider.System);
for (int attempt = 0; attempt < 2; attempt++)
{
await AuthenticateAsync(verifier, ContextWithAuthorization(AdminHeader), limiter);
}
DefaultHttpContext throttled = ContextWithAuthorization(AdminHeader);
MetricsAuthenticationHandler handler = await CreateHandlerAsync(verifier, throttled, limiter);
AuthenticateResult result = await handler.AuthenticateAsync();
await handler.ChallengeAsync(new AuthenticationProperties());
Assert.False(result.Succeeded);
Assert.Equal(2, verifier.CallCount);
Assert.Equal(StatusCodes.Status429TooManyRequests, throttled.Response.StatusCode);
}
/// <summary>An anonymous scrape is challenged with a plain 401 Bearer, not a login redirect.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Challenge_WithoutCredentials_Returns401Bearer()
{
DefaultHttpContext httpContext = new();
MetricsAuthenticationHandler handler = await CreateHandlerAsync(
new CountingVerifier(Success(GatewayScopes.Admin)),
httpContext);
AuthenticateResult result = await handler.AuthenticateAsync();
await handler.ChallengeAsync(new AuthenticationProperties());
Assert.True(result.None);
Assert.Equal(StatusCodes.Status401Unauthorized, httpContext.Response.StatusCode);
Assert.Equal("Bearer", httpContext.Response.Headers.WWWAuthenticate.ToString());
}
/// <summary>An API key holding the admin scope satisfies the metrics requirement.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Requirement_ApiKeyWithAdminScope_Succeeds()
{
Assert.True(await EvaluateAsync(ApiKeyPrincipal(GatewayScopes.Admin)));
}
/// <summary>An API key without the admin scope does not satisfy the requirement.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Requirement_ApiKeyWithoutAdminScope_Fails()
{
Assert.False(await EvaluateAsync(ApiKeyPrincipal(GatewayScopes.EventsRead, GatewayScopes.InvokeRead)));
}
/// <summary>A scope claim counts only on an identity the metrics scheme minted from a verified key.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Requirement_ScopeClaimOnForeignIdentity_Fails()
{
ClaimsPrincipal principal = new(new ClaimsIdentity(
[new Claim(MetricsAuthorizationDefaults.ScopeClaimType, GatewayScopes.Admin)],
DashboardAuthenticationDefaults.AuthenticationScheme));
Assert.False(await EvaluateAsync(principal));
}
/// <summary>A dashboard login holding a real dashboard role satisfies the requirement.</summary>
/// <param name="role">The dashboard role.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData(DashboardRoles.Admin)]
[InlineData(DashboardRoles.Viewer)]
public async Task Requirement_DashboardLoginWithRole_Succeeds(string role)
{
ClaimsPrincipal principal = new(new ClaimsIdentity(
[new Claim(ClaimTypes.Name, "operator"), new Claim(ClaimTypes.Role, role)],
DashboardAuthenticationDefaults.AuthenticationScheme));
Assert.True(await EvaluateAsync(principal));
}
/// <summary>An unauthenticated principal (anonymous localhost included) never satisfies it.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Requirement_Unauthenticated_Fails()
{
ClaimsPrincipal principal = new(new ClaimsIdentity([new Claim(ClaimTypes.Role, DashboardRoles.Admin)]));
Assert.False(await EvaluateAsync(principal));
}
private static async Task<bool> EvaluateAsync(ClaimsPrincipal principal)
{
MetricsAccessRequirement requirement = new();
AuthorizationHandlerContext context = new([requirement], principal, resource: null);
await new MetricsAccessHandler().HandleAsync(context);
return context.HasSucceeded;
}
private static ClaimsPrincipal ApiKeyPrincipal(params string[] scopes)
{
List<Claim> claims = [new Claim(ClaimTypes.Name, "scraper")];
claims.AddRange(scopes.Select(scope => new Claim(MetricsAuthorizationDefaults.ScopeClaimType, scope)));
return new ClaimsPrincipal(new ClaimsIdentity(claims, MetricsAuthorizationDefaults.AuthenticationScheme));
}
private static async Task<AuthenticateResult> AuthenticateAsync(
IApiKeyVerifier verifier,
HttpContext httpContext,
ApiKeyFailureLimiter? limiter = null)
{
MetricsAuthenticationHandler handler = await CreateHandlerAsync(verifier, httpContext, limiter);
return await handler.AuthenticateAsync();
}
private static async Task<MetricsAuthenticationHandler> CreateHandlerAsync(
IApiKeyVerifier verifier,
HttpContext httpContext,
ApiKeyFailureLimiter? limiter = null)
{
MetricsAuthenticationHandler handler = new(
new StaticOptionsMonitor(),
NullLoggerFactory.Instance,
UrlEncoder.Default,
verifier,
limiter ?? new ApiKeyFailureLimiter(1000, TimeSpan.FromMinutes(1), 1024, 0, TimeSpan.Zero, TimeProvider.System),
new AuthenticationSchemeProvider(Options.Create(new Microsoft.AspNetCore.Authentication.AuthenticationOptions())),
Options.Create(new GatewayOptions()),
new GatewayMetrics());
await handler.InitializeAsync(
new AuthenticationScheme(
MetricsAuthorizationDefaults.AuthenticationScheme,
displayName: null,
typeof(MetricsAuthenticationHandler)),
httpContext);
return handler;
}
private static DefaultHttpContext ContextWithAuthorization(string header)
{
DefaultHttpContext context = new();
context.Request.Headers.Authorization = header;
context.Connection.RemoteIpAddress = System.Net.IPAddress.Parse("198.51.100.7");
return context;
}
private static ApiKeyVerification Success(params string[] scopes) => new(
Succeeded: true,
Identity: new LibApiKeyIdentity(
KeyId: "scraper",
DisplayName: "Scraper",
Scopes: new HashSet<string>(scopes, StringComparer.Ordinal),
Constraints: null),
Failure: null);
private sealed class CountingVerifier(ApiKeyVerification result) : IApiKeyVerifier
{
/// <summary>Gets the number of verifications performed.</summary>
public int CallCount { get; private set; }
/// <summary>Gets the last authorization header presented.</summary>
public string? LastHeader { get; private set; }
/// <inheritdoc />
public Task<ApiKeyVerification> VerifyAsync(string authorizationHeader, CancellationToken ct)
{
CallCount++;
LastHeader = authorizationHeader;
return Task.FromResult(result);
}
}
private sealed class StaticOptionsMonitor : IOptionsMonitor<AuthenticationSchemeOptions>
{
/// <inheritdoc />
public AuthenticationSchemeOptions CurrentValue { get; } = new();
/// <inheritdoc />
public AuthenticationSchemeOptions Get(string? name) => CurrentValue;
/// <inheritdoc />
public IDisposable? OnChange(Action<AuthenticationSchemeOptions, string?> listener) => null;
}
}
@@ -11,26 +11,26 @@ namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
public sealed class AllowAllConstraintEnforcer : IConstraintEnforcer
{
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadTagAsync(
public ValueTask<ConstraintFailure?> CheckReadTagAsync(
ApiKeyIdentity? identity,
string tagAddress,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadHandleAsync(
public ValueTask<ConstraintFailure?> CheckReadHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
int itemHandle,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task<ConstraintFailure?> CheckWriteHandleAsync(
public ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
int itemHandle,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task RecordDenialAsync(
@@ -40,4 +40,12 @@ public sealed class AllowAllConstraintEnforcer : IConstraintEnforcer
ConstraintFailure failure,
string? correlationId,
CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task RecordBulkDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
ConstraintDenialSummary summary,
string? correlationId,
CancellationToken cancellationToken) => Task.CompletedTask;
}
@@ -40,8 +40,18 @@ public sealed class FakeGatewayAlarmService : IGatewayAlarmService
yield return new AlarmFeedMessage { SnapshotComplete = true };
await Task.CompletedTask.ConfigureAwait(false);
if (StreamFault is not null)
{
throw StreamFault;
}
}
/// <summary>
/// Gets or sets an exception the feed throws after its snapshot, standing in for the monitor
/// completing a subscriber with a fault (for example a slow-subscriber overflow).
/// </summary>
public Exception? StreamFault { get; set; }
/// <inheritdoc />
public Task<AcknowledgeAlarmReply> AcknowledgeAsync(
AcknowledgeAlarmRequest request,
@@ -26,23 +26,26 @@ public sealed class PredicateConstraintEnforcer : IConstraintEnforcer
/// <summary>Recorded denial messages — (commandKind, target, correlationId) tuples.</summary>
public List<(string CommandKind, string Target, string? CorrelationId)> RecordedDenials { get; } = [];
/// <summary>Recorded aggregated bulk denials — one entry per bulk command that had any item denied.</summary>
public List<(string CommandKind, ConstraintDenialSummary Summary, string? CorrelationId)> RecordedBulkDenials { get; } = [];
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadTagAsync(
public ValueTask<ConstraintFailure?> CheckReadTagAsync(
ApiKeyIdentity? identity,
string tagAddress,
CancellationToken cancellationToken)
{
if (DenyTag(tagAddress))
{
return Task.FromResult<ConstraintFailure?>(
return ValueTask.FromResult<ConstraintFailure?>(
new ConstraintFailure("read-tag", $"Read denied for tag '{tagAddress}'."));
}
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadHandleAsync(
public ValueTask<ConstraintFailure?> CheckReadHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
@@ -51,15 +54,15 @@ public sealed class PredicateConstraintEnforcer : IConstraintEnforcer
{
if (DenyReadHandle(serverHandle, itemHandle))
{
return Task.FromResult<ConstraintFailure?>(
return ValueTask.FromResult<ConstraintFailure?>(
new ConstraintFailure("read-handle", $"Read denied for handle {itemHandle}."));
}
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
/// <inheritdoc />
public Task<ConstraintFailure?> CheckWriteHandleAsync(
public ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
@@ -68,11 +71,11 @@ public sealed class PredicateConstraintEnforcer : IConstraintEnforcer
{
if (DenyWriteHandle(serverHandle, itemHandle))
{
return Task.FromResult<ConstraintFailure?>(
return ValueTask.FromResult<ConstraintFailure?>(
new ConstraintFailure("write-handle", $"Write denied for handle {itemHandle}."));
}
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
/// <inheritdoc />
@@ -87,4 +90,16 @@ public sealed class PredicateConstraintEnforcer : IConstraintEnforcer
RecordedDenials.Add((commandKind, target, correlationId));
return Task.CompletedTask;
}
/// <inheritdoc />
public Task RecordBulkDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
ConstraintDenialSummary summary,
string? correlationId,
CancellationToken cancellationToken)
{
RecordedBulkDenials.Add((commandKind, summary, correlationId));
return Task.CompletedTask;
}
}
@@ -0,0 +1,48 @@
using Microsoft.Extensions.Logging;
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
/// <see cref="ILogger{TCategoryName}"/> that records every entry's level and formatted message,
/// so a test can assert that a path logs (or does not log) at a given level.
/// </summary>
/// <typeparam name="T">The logger category type.</typeparam>
public sealed class RecordingLogger<T> : ILogger<T>
{
private readonly object _syncRoot = new();
private readonly List<(LogLevel Level, string Message, Exception? Exception)> _entries = [];
/// <summary>Gets a snapshot of the recorded entries, in order.</summary>
public IReadOnlyList<(LogLevel Level, string Message, Exception? Exception)> Entries
{
get
{
lock (_syncRoot)
{
return _entries.ToArray();
}
}
}
/// <inheritdoc />
public IDisposable? BeginScope<TState>(TState state)
where TState : notnull => null;
/// <inheritdoc />
public bool IsEnabled(LogLevel logLevel) => true;
/// <inheritdoc />
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
ArgumentNullException.ThrowIfNull(formatter);
lock (_syncRoot)
{
_entries.Add((logLevel, formatter(state, exception), exception));
}
}
}
@@ -29,6 +29,25 @@ public sealed class RecordingServerStreamWriter<T> : IServerStreamWriter<T>
/// <summary>Gets or sets options for writing messages to the stream.</summary>
public WriteOptions? WriteOptions { get; set; }
/// <summary>Gets the cancellation token passed to the most recent token-bearing write.</summary>
public CancellationToken LastWriteCancellationToken { get; private set; }
/// <summary>Records the supplied message unless the token is already cancelled.</summary>
/// <param name="message">The message to record.</param>
/// <param name="cancellationToken">Token that cancels the write.</param>
/// <returns>A completed task.</returns>
/// <remarks>
/// The interface default throws <see cref="NotSupportedException"/> for a cancelable token,
/// and the gateway passes the call's token on every streamed write, so a test writer must
/// override this overload the way the ASP.NET Core writer does.
/// </remarks>
public Task WriteAsync(T message, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
LastWriteCancellationToken = cancellationToken;
return WriteAsync(message);
}
/// <summary>Records the supplied message.</summary>
/// <param name="message">The message to record.</param>
/// <returns>A completed task.</returns>