Merge stream C (gRPC surface: audit aggregation, streaming auth, alarm overflow/snapshot, write cancellation, correlation echo, disconnect reasons, verifier single-flight, opt-in /metrics auth)

This commit is contained in:
Joseph Doherty
2026-10-22 06:47:49 -04:00
39 changed files with 2535 additions and 112 deletions
@@ -372,6 +372,105 @@ public sealed class AlarmTruncationSignalTests
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// A subscriber that stops reading is dropped once its bounded queue fills, and the fault it
/// observes is the typed <see cref="AlarmFeedSubscriberOverflowException"/> — the signal the
/// gRPC layer maps to ResourceExhausted — rather than a bare InvalidOperationException.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SubscriberThatStopsReading_IsDroppedWithTypedOverflowFault()
{
// Comfortably past the monitor's per-subscriber queue capacity (2048).
const int ProbeCount = 2300;
using GatewayMetrics metrics = new();
StubSessionManager sessions = new();
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForReconcileAsync(WaitTimeout);
// The slow subscriber: registered (first frame read) and then never read again.
using CancellationTokenSource slowCts = new(WaitTimeout);
await using IAsyncEnumerator<AlarmFeedMessage> slow =
monitor.StreamAsync(null, slowCts.Token).GetAsyncEnumerator(slowCts.Token);
Assert.True(await slow.MoveNextAsync());
// Raises on distinct references: each one is broadcast (no dedup) without forcing a
// reconcile, which keeps the burst cheap under a loaded parallel test run.
for (ulong sequence = 1; sequence <= ProbeCount; sequence++)
{
sessions.EmitEvent(UniqueRaise(sequence));
}
// The monitor caches each raise and broadcasts it under the same lock, so once the cache
// holds every raise, every broadcast has been offered to the slow subscriber. (A draining
// reader is no barrier here: under a loaded test run it can fall 2048 frames behind the
// burst and be dropped itself.)
await WaitUntilAsync(() => monitor.CurrentAlarms.Count >= ProbeCount, WaitTimeout);
// Drain what the slow subscriber had buffered; the terminal fault is the typed overflow.
await Assert.ThrowsAsync<AlarmFeedSubscriberOverflowException>(async () =>
{
while (await slow.MoveNextAsync())
{
}
});
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
/// <summary>
/// A StreamAsync attach reuses the memoized alarm-set projection that CurrentAlarms serves
/// (no per-attach clone of the whole set under the broadcast lock) and still applies the
/// subscriber's prefix filter to the snapshot it emits.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task SubscriberAttach_ReusesMemoizedProjectionAndAppliesPrefixFilter()
{
using GatewayMetrics metrics = new();
StubSessionManager sessions = new()
{
Snapshots =
[
NewSnapshot("Galaxy!Area.Tank01.Level.HiHi", fromTruncatedSnapshot: false),
NewSnapshot("Galaxy!Area.Pump02.Flow.Lo", fromTruncatedSnapshot: false),
],
};
using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics);
using CancellationTokenSource cts = new();
await monitor.StartAsync(cts.Token);
await sessions.WaitForReconcileAsync(WaitTimeout);
await WaitUntilAsync(() => monitor.CurrentAlarms.Count == 2, WaitTimeout);
IReadOnlyList<ActiveAlarmSnapshot> projection = monitor.CurrentAlarms;
List<ActiveAlarmSnapshot> streamed = [];
using CancellationTokenSource streamCts = new(WaitTimeout);
await foreach (AlarmFeedMessage message in monitor.StreamAsync("Galaxy!Area.Tank01", streamCts.Token))
{
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ActiveAlarm)
{
streamed.Add(message.ActiveAlarm);
}
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.SnapshotComplete)
{
break;
}
}
ActiveAlarmSnapshot only = Assert.Single(streamed);
Assert.Equal("Galaxy!Area.Tank01.Level.HiHi", only.AlarmFullReference);
Assert.Contains(projection, alarm => ReferenceEquals(alarm, only));
await cts.CancelAsync();
await monitor.StopAsync(CancellationToken.None);
}
private static int CountOf(List<AlarmFeedMessage> received, AlarmFeedMessage.PayloadOneofCase payloadCase)
{
lock (received)
@@ -442,6 +541,23 @@ public sealed class AlarmTruncationSignalTests
},
};
// A Raise on a reference unique to the sequence, so the monitor never dedups it.
private static MxEvent UniqueRaise(ulong sequence) => new()
{
Family = MxEventFamily.OnAlarmTransition,
WorkerSequence = sequence,
OnAlarmTransition = new OnAlarmTransitionEvent
{
AlarmFullReference = $"Galaxy!Area.Tank{sequence}.Level.Hi",
SourceObjectReference = $"Tank{sequence}.Level",
AlarmTypeName = "AnalogLimitAlarm.Hi",
TransitionKind = AlarmTransitionKind.Raise,
Severity = 500,
SourceProvider = AlarmProviderMode.Alarmmgr,
TransitionTimestamp = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
},
};
private static ActiveAlarmSnapshot NewSnapshot(string reference, bool fromTruncatedSnapshot)
{
return new ActiveAlarmSnapshot
@@ -263,7 +263,7 @@ public sealed class EventStreamServiceTests
GatewayMetricsSnapshot snapshot = metrics.GetSnapshot();
Assert.Equal(1, snapshot.QueueOverflows);
Assert.Equal(1, snapshot.Faults);
// The finally block in StreamEventsAsync calls StreamDisconnected("Detached") on the
// The finally block in StreamEventsAsync records StreamDisconnected (reason Overflow) on the
// overflow+fault path too; pin it here so a regression removing that call is caught.
Assert.Equal(1, snapshot.StreamDisconnects);
}
@@ -572,6 +572,142 @@ public sealed class EventStreamServiceTests
}
}
/// <summary>
/// A stream that ends because the session's event stream completed records the
/// <c>SessionClosed</c> disconnect reason rather than a blanket constant.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenEventStreamCompletes_RecordsSessionClosedReason()
{
FakeWorkerClient workerClient = new();
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
GatewaySession session = CreateReadySession(workerClient);
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
workerClient.CompleteAfterConfiguredEvents = true;
await CollectEventsAsync(service, session.SessionId);
Assert.Equal([EventStreamDisconnectReasons.SessionClosed], reasons.Reasons);
}
/// <summary>A per-subscriber overflow records the <c>Overflow</c> disconnect reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenStreamQueueOverflows_RecordsOverflowReason()
{
FakeWorkerClient workerClient = new();
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
GatewaySession session = CreateReadySession(
workerClient,
queueCapacity: 1,
metrics: metrics,
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
EventStreamService service = CreateService(
new FakeSessionManager(session),
metrics,
queueCapacity: 1,
backpressurePolicy: EventBackpressurePolicy.DisconnectSubscriber);
for (ulong sequence = 1; sequence <= 50; sequence++)
{
workerClient.Events.Add(CreateWorkerEvent(sequence, MxEventFamily.OnDataChange));
}
workerClient.CompleteAfterConfiguredEvents = true;
await using (IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
.GetAsyncEnumerator())
{
await Assert.ThrowsAsync<SessionManagerException>(
async () =>
{
while (await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout))
{
}
});
}
Assert.Equal([EventStreamDisconnectReasons.Overflow], reasons.Reasons);
}
/// <summary>A terminal worker fault records the <c>WorkerFault</c> disconnect reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenWorkerEventStreamFaults_RecordsWorkerFaultReason()
{
FakeWorkerClient workerClient = new()
{
TerminalException = new WorkerClientException(
WorkerClientErrorCode.WorkerFaulted,
"worker terminal fault"),
};
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
await using (IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
.GetAsyncEnumerator())
{
await Assert.ThrowsAsync<WorkerClientException>(
async () => await subscriber.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
}
Assert.Equal([EventStreamDisconnectReasons.WorkerFault], reasons.Reasons);
}
/// <summary>A client cancellation records the <c>ClientCancelled</c> disconnect reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenCanceled_RecordsClientCancelledReason()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
using CancellationTokenSource cancellationTokenSource = new();
await using (IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, cancellationTokenSource.Token)
.GetAsyncEnumerator(cancellationTokenSource.Token))
{
Task<bool> moveTask = subscriber.MoveNextAsync().AsTask();
await WaitUntilAsync(() => session.ActiveEventSubscriberCount == 1);
await cancellationTokenSource.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(
async () => await moveTask.WaitAsync(TestTimeout));
}
Assert.Equal([EventStreamDisconnectReasons.ClientCancelled], reasons.Reasons);
}
/// <summary>
/// A consumer that stops reading without cancelling (the gRPC writer gave up) keeps the
/// <c>Detached</c> reason.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEventsAsync_WhenConsumerStopsEarly_RecordsDetachedReason()
{
FakeWorkerClient workerClient = new();
GatewaySession session = CreateReadySession(workerClient);
using GatewayMetrics metrics = new();
using DisconnectReasonRecorder reasons = new(metrics);
EventStreamService service = CreateService(new FakeSessionManager(session), metrics);
workerClient.Events.Add(CreateWorkerEvent(sequence: 1, MxEventFamily.OnDataChange));
await using (IAsyncEnumerator<MxEvent> subscriber = service
.StreamEventsAsync(CreateRequest(session.SessionId), callerKeyId: null, CancellationToken.None)
.GetAsyncEnumerator())
{
await ReadNextAsync(subscriber);
}
Assert.Equal([EventStreamDisconnectReasons.Detached], reasons.Reasons);
}
private static async Task<MxEvent> ReadNextAsync(IAsyncEnumerator<MxEvent> enumerator)
{
Assert.True(await enumerator.MoveNextAsync().AsTask().WaitAsync(TestTimeout));
@@ -896,4 +1032,63 @@ public sealed class EventStreamServiceTests
return ValueTask.CompletedTask;
}
}
/// <summary>
/// Captures the <c>reason</c> tag of every <c>mxgateway.grpc.streams.disconnected</c>
/// measurement recorded on one <see cref="GatewayMetrics"/> instance (filtered by meter
/// reference, so parallel tests sharing the meter name cannot cross-talk).
/// </summary>
private sealed class DisconnectReasonRecorder : IDisposable
{
private readonly System.Diagnostics.Metrics.MeterListener _listener = new();
private readonly List<string> _reasons = [];
/// <summary>Initializes a new instance of the <see cref="DisconnectReasonRecorder"/> class.</summary>
/// <param name="metrics">The metrics instance whose meter to observe.</param>
public DisconnectReasonRecorder(GatewayMetrics metrics)
{
_listener.InstrumentPublished = (instrument, listener) =>
{
if (ReferenceEquals(instrument.Meter, metrics.Meter)
&& instrument.Name == "mxgateway.grpc.streams.disconnected")
{
listener.EnableMeasurementEvents(instrument);
}
};
_listener.SetMeasurementEventCallback<long>((instrument, _, tags, _) =>
{
if (!ReferenceEquals(instrument.Meter, metrics.Meter))
{
return;
}
foreach (KeyValuePair<string, object?> tag in tags)
{
if (tag.Key == "reason" && tag.Value is string reason)
{
lock (_reasons)
{
_reasons.Add(reason);
}
}
}
});
_listener.Start();
}
/// <summary>Gets the recorded reasons, in order.</summary>
public string[] Reasons
{
get
{
lock (_reasons)
{
return [.. _reasons];
}
}
}
/// <inheritdoc />
public void Dispose() => _listener.Dispose();
}
}
@@ -87,8 +87,56 @@ public sealed class MxAccessGatewayServiceConstraintTests
Assert.True(merged.Results[3].WasSuccessful);
Assert.Equal("Tank04.Public", merged.Results[3].TagAddress);
Assert.Equal(104, merged.Results[3].ItemHandle);
// Both denied tags recorded.
Assert.Equal(2, enforcer.RecordedDenials.Count);
// Both denied tags recorded in ONE aggregated bulk denial, not one audit call per item.
Assert.Empty(enforcer.RecordedDenials);
(string commandKind, ConstraintDenialSummary summary, _) = Assert.Single(enforcer.RecordedBulkDenials);
Assert.Equal(nameof(MxCommandKind.AddItemBulk), commandKind);
Assert.Equal(2, summary.DeniedCount);
Assert.Equal(4, summary.TotalCount);
Assert.Equal(["Tank01.Locked", "Tank03.Secret"], summary.SampleTargets);
}
/// <summary>
/// Audit-flood guard: a bulk write whose every entry is denied records exactly one
/// aggregated denial carrying the full denied count, the client correlation id and a sample
/// capped at <see cref="ConstraintDenialSummary.MaxSampleTargets"/> — so one low-privilege
/// key cannot push thousands of events into the bounded, drop-on-full audit channel.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Invoke_WriteBulk_WithManyDeniedEntries_RecordsOneAggregatedDenial()
{
const int EntryCount = 500;
PredicateConstraintEnforcer enforcer = new() { DenyWriteHandle = (_, _) => true };
FakeSessionManager sessionManager = CreateSessionManagerWithSeed();
MxAccessGatewayService service = CreateService(sessionManager, enforcer);
WriteBulkCommand payload = new() { ServerHandle = 7 };
for (int index = 0; index < EntryCount; index++)
{
payload.Entries.Add(new WriteBulkEntry { ItemHandle = index + 1 });
}
MxCommandRequest request = new()
{
SessionId = SessionId,
ClientCorrelationId = "bulk-flood-1",
Command = new MxCommand { Kind = MxCommandKind.WriteBulk, WriteBulk = payload },
};
MxCommandReply reply = await service.Invoke(request, new TestServerCallContext());
Assert.Equal(0, sessionManager.InvokeCount);
Assert.Equal(EntryCount, reply.WriteBulk.Results.Count);
Assert.Empty(enforcer.RecordedDenials);
(string commandKind, ConstraintDenialSummary summary, string? correlationId) =
Assert.Single(enforcer.RecordedBulkDenials);
Assert.Equal(nameof(MxCommandKind.WriteBulk), commandKind);
Assert.Equal("bulk-flood-1", correlationId);
Assert.Equal(EntryCount, summary.DeniedCount);
Assert.Equal(EntryCount, summary.TotalCount);
Assert.Equal(ConstraintDenialSummary.MaxSampleTargets, summary.SampleTargets.Count);
Assert.True(summary.SampleTruncated);
Assert.Equal(["write-handle"], summary.ConstraintNames);
}
/// <summary>
@@ -2,9 +2,11 @@ using System.Diagnostics.Metrics;
using System.Runtime.CompilerServices;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Alarms;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
@@ -213,6 +215,8 @@ public sealed class MxAccessGatewayServiceTests
Assert.Equal(42, reply.AddItem.ItemHandle);
Assert.Equal("status detail", Assert.Single(reply.Statuses).DiagnosticText);
Assert.Equal("mxaccess diagnostic", reply.DiagnosticMessage);
// The reply echoes the caller's client_correlation_id, not the internal worker envelope id.
Assert.Equal("client-correlation", reply.CorrelationId);
}
/// <summary>Verifies that StreamEvents writes only events after the specified worker sequence.</summary>
@@ -463,11 +467,195 @@ public sealed class MxAccessGatewayServiceTests
Assert.Contains("server-stream-active-alarms", reply.Capabilities);
}
/// <summary>
/// Every reply echoes the request's <c>client_correlation_id</c>, including when the caller
/// sent none: the internal worker envelope id never reaches the client.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Invoke_WithoutClientCorrelationId_DoesNotLeakWorkerEnvelopeId()
{
FakeSessionManager sessionManager = new()
{
InvokeReply = new WorkerCommandReply
{
Reply = new MxCommandReply
{
SessionId = "session-1",
CorrelationId = "4f1d1b0e-internal-envelope-guid",
Kind = MxCommandKind.AddItem,
ProtocolStatus = MxAccessGrpcMapper.Ok(),
AddItem = new AddItemReply { ItemHandle = 42 },
},
},
};
MxAccessGatewayService service = CreateService(sessionManager);
MxCommandRequest request = new()
{
SessionId = "session-1",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand { ServerHandle = 12, ItemDefinition = "Galaxy.Tag.Value" },
},
};
MxCommandReply reply = await service.Invoke(request, new TestServerCallContext());
Assert.Equal(string.Empty, reply.CorrelationId);
}
/// <summary>
/// Invoke logs the client correlation id alongside the worker envelope id once, at Debug, so
/// a client error report can be matched to gateway and worker logs.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Invoke_LogsClientToWorkerCorrelationPairAtDebug()
{
FakeSessionManager sessionManager = new()
{
InvokeReply = new WorkerCommandReply
{
Reply = new MxCommandReply
{
SessionId = "session-1",
CorrelationId = "envelope-7f3a",
Kind = MxCommandKind.AddItem,
ProtocolStatus = MxAccessGrpcMapper.Ok(),
AddItem = new AddItemReply { ItemHandle = 42 },
},
},
};
RecordingLogger<MxAccessGatewayService> logger = new();
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
MxCommandRequest request = new()
{
SessionId = "session-1",
ClientCorrelationId = "client-trace-9",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand { ServerHandle = 12, ItemDefinition = "Galaxy.Tag.Value" },
},
};
await service.Invoke(request, new TestServerCallContext());
(LogLevel level, string message, _) = Assert.Single(
logger.Entries,
entry => entry.Message.Contains("envelope-7f3a", StringComparison.Ordinal));
Assert.Equal(LogLevel.Debug, level);
Assert.Contains("client-trace-9", message, StringComparison.Ordinal);
}
/// <summary>
/// StreamEvents hands the call's cancellation token to every response write, so a write
/// blocked on a slow or vanished client is released when the call is cancelled.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_PassesCallCancellationTokenToResponseWrites()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
MxAccessGatewayService service = CreateService(sessionManager);
RecordingServerStreamWriter<MxEvent> writer = new();
using CancellationTokenSource callCts = new();
await service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext(cancellationToken: callCts.Token));
Assert.Single(writer.Messages);
Assert.Equal(callCts.Token, writer.LastWriteCancellationToken);
}
/// <summary>
/// A response write that fails with an <see cref="IOException"/> because the client went
/// away (the call token is cancelled) maps to Cancelled and logs no warning: a client
/// disconnect is routine, not a gateway failure.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_WhenWriteFailsAfterClientCancel_ReturnsCancelledWithoutWarning()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
RecordingLogger<MxAccessGatewayService> logger = new();
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
using CancellationTokenSource callCts = new();
ThrowingStreamWriter<MxEvent> writer = new(() =>
{
callCts.Cancel();
return new IOException("The client reset the request stream.");
});
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext(cancellationToken: callCts.Token)));
Assert.Equal(StatusCode.Cancelled, exception.StatusCode);
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
}
/// <summary>
/// The same IOException with a live call token is a genuine transport failure and keeps the
/// existing Unavailable + warning mapping.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_WhenWriteFailsWithoutCancel_KeepsUnavailableMapping()
{
FakeSessionManager sessionManager = new();
sessionManager.Events.Add(CreateWorkerEvent("session-1", workerSequence: 1));
RecordingLogger<MxAccessGatewayService> logger = new();
MxAccessGatewayService service = CreateService(sessionManager, logger: logger);
ThrowingStreamWriter<MxEvent> writer = new(() => new IOException("Transport failure."));
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamEvents(
new StreamEventsRequest { SessionId = "session-1" },
writer,
new TestServerCallContext()));
Assert.Equal(StatusCode.Unavailable, exception.StatusCode);
Assert.Contains(logger.Entries, entry => entry.Level == LogLevel.Warning);
}
/// <summary>
/// A StreamAlarms subscriber the monitor drops for falling behind receives ResourceExhausted
/// (the same status a StreamEvents overflow carries), and the gateway logs no
/// warning-with-stack for what is a client-side slowness condition.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamAlarms_WhenSubscriberOverflows_ReturnsResourceExhaustedWithoutWarning()
{
RecordingLogger<MxAccessGatewayService> logger = new();
FakeGatewayAlarmService alarms = new() { StreamFault = new AlarmFeedSubscriberOverflowException() };
MxAccessGatewayService service = CreateService(new FakeSessionManager(), alarmService: alarms, logger: logger);
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => service.StreamAlarms(
new StreamAlarmsRequest(),
new RecordingServerStreamWriter<AlarmFeedMessage>(),
new TestServerCallContext()));
Assert.Equal(StatusCode.ResourceExhausted, exception.StatusCode);
Assert.DoesNotContain(logger.Entries, entry => entry.Level >= LogLevel.Warning);
}
private static MxAccessGatewayService CreateService(
FakeSessionManager sessionManager,
IGatewayRequestIdentityAccessor? identityAccessor = null,
GatewayMetrics? metrics = null,
IConstraintEnforcer? constraintEnforcer = null)
IConstraintEnforcer? constraintEnforcer = null,
IGatewayAlarmService? alarmService = null,
ILogger<MxAccessGatewayService>? logger = null)
{
return new MxAccessGatewayService(
sessionManager,
@@ -477,8 +665,8 @@ public sealed class MxAccessGatewayServiceTests
new MxAccessGrpcMapper(),
new FakeEventStreamService(sessionManager),
metrics ?? new GatewayMetrics(),
NullLogger<MxAccessGatewayService>.Instance,
new FakeGatewayAlarmService());
logger ?? NullLogger<MxAccessGatewayService>.Instance,
alarmService ?? new FakeGatewayAlarmService());
}
private static ApiKeyIdentity CreateIdentity()
@@ -786,4 +974,18 @@ public sealed class MxAccessGatewayServiceTests
}
}
/// <summary>A stream writer whose token-bearing write throws the supplied exception.</summary>
/// <typeparam name="T">The streamed message type.</typeparam>
private sealed class ThrowingStreamWriter<T>(Func<Exception> createException) : IServerStreamWriter<T>
{
/// <inheritdoc />
public WriteOptions? WriteOptions { get; set; }
/// <inheritdoc />
public Task WriteAsync(T message) => throw createException();
/// <inheritdoc />
public Task WriteAsync(T message, CancellationToken cancellationToken) => throw createException();
}
}
@@ -0,0 +1,182 @@
using System.Net;
using System.Net.Http;
using System.Security.Claims;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication.Cookies;
using Microsoft.AspNetCore.Builder;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Auth.ApiKeys.Admin;
using ZB.MOM.WW.MxGateway.Server;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway;
/// <summary>
/// Host-level coverage of <c>MxGateway:Metrics:RequireAuthentication=true</c> (SEC-14): each case
/// builds the real gateway with <see cref="GatewayApplication.Build"/> and drives the full
/// middleware pipeline (authentication, authorization, the dashboard cookie scheme, the API-key
/// store) over HTTP on an ephemeral loopback port. Requests therefore arrive from loopback, which
/// is exactly the anonymous-localhost case the dashboard Viewer policy would admit and /metrics
/// must not.
/// </summary>
public sealed class MetricsEndpointAuthenticationTests
{
// A fixed test pepper: the configuration-backed pepper provider reads MxGateway:ApiKeyPepper,
// and the auth store itself is isolated per test process by TestHostEnvironmentInitializer.
private const string TestPepper = "metrics-endpoint-tests-pepper-0123456789abcdef";
/// <summary>
/// An anonymous loopback scrape gets a plain 401 Bearer challenge — not a 302 to /login, and not
/// the anonymous-localhost admission the dashboard Viewer policy grants.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task AnonymousLoopbackScrape_Returns401BearerNotLoginRedirect()
{
await using WebApplication app = await StartAsync();
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await client.GetAsync("/metrics");
Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
Assert.Null(response.Headers.Location);
Assert.Contains(response.Headers.WwwAuthenticate, value => value.Scheme == "Bearer");
}
/// <summary>/health/active stays anonymous: the off-repo Overview dashboard polls it without credentials.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task HealthActive_StaysAnonymous()
{
await using WebApplication app = await StartAsync();
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await client.GetAsync("/health/active");
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
/// <summary>A dashboard login cookie holding the Viewer role is accepted.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DashboardViewerCookie_Returns200()
{
await using WebApplication app = await StartAsync();
using HttpClient client = CreateClient(app);
using HttpRequestMessage request = new(HttpMethod.Get, "/metrics");
request.Headers.Add("Cookie", MintDashboardCookie(app, DashboardRoles.Viewer));
using HttpResponseMessage response = await client.SendAsync(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
/// <summary>A real API key with the admin scope is accepted.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task AdminScopedApiKey_Returns200()
{
await using WebApplication app = await StartAsync();
string token = await CreateKeyAsync(app, [GatewayScopes.Admin]);
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await SendWithBearerAsync(client, token);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
/// <summary>A real API key without the admin scope authenticates but is forbidden.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task NonAdminApiKey_Returns403()
{
await using WebApplication app = await StartAsync();
string token = await CreateKeyAsync(app, [GatewayScopes.EventsRead, GatewayScopes.InvokeRead]);
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await SendWithBearerAsync(client, token);
Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode);
}
/// <summary>
/// Dev-only <c>Dashboard:DisableLogin</c> registers an always-succeeding auto-login handler under
/// the cookie scheme name; /metrics must not fall back to it, or the flag would silently open
/// an endpoint the operator explicitly asked to protect.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DisableLogin_DoesNotOpenMetrics()
{
await using WebApplication app = await StartAsync("--MxGateway:Dashboard:DisableLogin=true");
using HttpClient client = CreateClient(app);
using HttpResponseMessage response = await client.GetAsync("/metrics");
Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
}
private static async Task<WebApplication> StartAsync(params string[] extraArgs)
{
// Ephemeral port (:0): xUnit runs collections in parallel, so a fixed port would collide.
WebApplication app = GatewayApplication.Build(
[
"--urls=http://127.0.0.1:0",
"--MxGateway:Metrics:RequireAuthentication=true",
$"--MxGateway:ApiKeyPepper={TestPepper}",
.. extraArgs,
]);
await app.StartAsync();
return app;
}
private static HttpClient CreateClient(WebApplication app) =>
new(new HttpClientHandler { AllowAutoRedirect = false, UseCookies = false })
{
BaseAddress = new Uri(app.Urls.First()),
};
private static async Task<HttpResponseMessage> SendWithBearerAsync(HttpClient client, string token)
{
using HttpRequestMessage request = new(HttpMethod.Get, "/metrics");
request.Headers.TryAddWithoutValidation("Authorization", $"Bearer {token}");
return await client.SendAsync(request);
}
// Creates a key through the same admin command set the CLI and dashboard use. Key ids are unique
// per call because the isolated auth store is shared by every test in the process.
private static async Task<string> CreateKeyAsync(WebApplication app, string[] scopes)
{
ApiKeyAdminCommands commands = app.Services.GetRequiredService<ApiKeyAdminCommands>();
await commands.InitDbAsync(remoteAddress: null, CancellationToken.None);
CreateKeyResult created = await commands.CreateKeyAsync(
$"metrics{Guid.NewGuid():N}",
"Metrics test key",
new HashSet<string>(scopes, StringComparer.Ordinal),
null,
null,
remoteAddress: null,
CancellationToken.None);
return created.Token ?? throw new InvalidOperationException("Key creation returned no token.");
}
// Protects a ticket with the running host's own cookie options (data-protection keys, ticket
// format, cookie name) — the cookie a real dashboard login would issue, minus the LDAP bind.
private static string MintDashboardCookie(WebApplication app, string role)
{
CookieAuthenticationOptions cookieOptions = app.Services
.GetRequiredService<IOptionsMonitor<CookieAuthenticationOptions>>()
.Get(DashboardAuthenticationDefaults.AuthenticationScheme);
ClaimsIdentity identity = new(
[new Claim(ClaimTypes.Name, "viewer-user"), new Claim(ClaimTypes.Role, role)],
DashboardAuthenticationDefaults.AuthenticationScheme,
ClaimTypes.Name,
ClaimTypes.Role);
AuthenticationTicket ticket = new(
new ClaimsPrincipal(identity),
DashboardAuthenticationDefaults.AuthenticationScheme);
return $"{cookieOptions.Cookie.Name}={cookieOptions.TicketDataFormat.Protect(ticket)}";
}
}
@@ -0,0 +1,41 @@
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Metrics;
namespace ZB.MOM.WW.MxGateway.Tests.Metrics;
public sealed class MxEventFamilyNamesTests
{
/// <summary>
/// Every defined family maps to exactly the name <see cref="Enum.ToString()"/> produces, so
/// switching the per-event metric tags to the cached lookup changes no exported tag value.
/// </summary>
[Fact]
public void Get_ForEveryDefinedFamily_MatchesEnumToString()
{
foreach (MxEventFamily family in Enum.GetValues<MxEventFamily>())
{
Assert.Equal(family.ToString(), MxEventFamilyNames.Get(family));
}
}
/// <summary>A defined family returns the same cached string instance on every call (no per-event allocation).</summary>
[Fact]
public void Get_ForDefinedFamily_ReturnsCachedInstance()
{
Assert.Same(
MxEventFamilyNames.Get(MxEventFamily.OnDataChange),
MxEventFamilyNames.Get(MxEventFamily.OnDataChange));
}
/// <summary>
/// A numeric value the contract does not define (a newer worker, or a malformed frame) still
/// produces the <see cref="Enum.ToString()"/> form rather than throwing.
/// </summary>
[Fact]
public void Get_ForUndefinedValue_FallsBackToToString()
{
MxEventFamily undefined = (MxEventFamily)987654;
Assert.Equal(undefined.ToString(), MxEventFamilyNames.Get(undefined));
}
}
@@ -129,6 +129,120 @@ public sealed class CachingApiKeyVerifierTests
Assert.Equal(2, inner.CallCount);
}
/// <summary>
/// Single-flight: concurrent misses for the same token (a TTL expiry under load) share one
/// inner verification instead of each issuing its own SQLite read.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task VerifyAsync_ConcurrentMisses_ShareOneInnerVerification()
{
GatedVerifier inner = new(Success("operator01"));
using MemoryCache cache = NewCache();
CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30));
Task<ApiKeyVerification> first = verifier.VerifyAsync(Header, CancellationToken.None);
await inner.Entered;
Task<ApiKeyVerification>[] joiners = Enumerable
.Range(0, 16)
.Select(_ => verifier.VerifyAsync(Header, CancellationToken.None))
.ToArray();
inner.Release();
ApiKeyVerification[] results = await Task.WhenAll([first, .. joiners]);
Assert.All(results, result => Assert.True(result.Succeeded));
Assert.Equal(1, inner.CallCount);
}
/// <summary>
/// A miss that arrives after <see cref="IApiKeyCacheInvalidator.Invalidate"/> must not join a
/// verification that started before it: that flight may return the pre-revoke identity, so the
/// late caller starts its own flight against the inner verifier.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task VerifyAsync_MissAfterInvalidate_DoesNotJoinPreInvalidateFlight()
{
GatedVerifier inner = new(Success("operator01"));
using MemoryCache cache = NewCache();
CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30));
Task<ApiKeyVerification> preRevoke = verifier.VerifyAsync(Header, CancellationToken.None);
await inner.Entered;
((IApiKeyCacheInvalidator)verifier).Invalidate("operator01");
// The gated first call is still parked; a joined call would park with it.
ApiKeyVerification postRevoke = await verifier
.VerifyAsync(Header, CancellationToken.None)
.WaitAsync(TimeSpan.FromSeconds(5));
Assert.True(postRevoke.Succeeded);
Assert.Equal(2, inner.CallCount);
inner.Release();
await preRevoke;
}
/// <summary>
/// A caller whose own token is cancelled stops waiting, but the shared verification it joined
/// is not cancelled for the other waiters.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task VerifyAsync_CancelledWaiter_DoesNotCancelSharedVerification()
{
GatedVerifier inner = new(Success("operator01"));
using MemoryCache cache = NewCache();
CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30));
using CancellationTokenSource cancelled = new();
Task<ApiKeyVerification> survivor = verifier.VerifyAsync(Header, CancellationToken.None);
await inner.Entered;
Task<ApiKeyVerification> abandoned = verifier.VerifyAsync(Header, cancelled.Token);
await cancelled.CancelAsync();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => abandoned);
inner.Release();
Assert.True((await survivor).Succeeded);
Assert.Equal(1, inner.CallCount);
}
/// <summary>
/// The key-id index is pruned when the cache evicts an entry on its own (expiry, compaction),
/// not only on <see cref="IApiKeyCacheInvalidator.Invalidate"/> — otherwise every token ever
/// verified leaves an index entry behind for the life of the process.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task CacheEviction_PrunesKeyIdIndex()
{
FakeVerifier inner = new(Success("operator01"));
using MemoryCache cache = NewCache();
CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30));
await verifier.VerifyAsync(Header, CancellationToken.None);
Assert.Equal(1, verifier.IndexedCacheKeyCount);
cache.Compact(1.0);
DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
while (verifier.IndexedCacheKeyCount != 0 && DateTime.UtcNow < deadline)
{
await Task.Delay(10);
}
Assert.Equal(0, verifier.IndexedCacheKeyCount);
Assert.Equal(0, verifier.IndexedKeyIdCount);
// The evicted entry is gone, so the next request re-verifies, and Invalidate still reaches
// the repopulated entry through the index.
await verifier.VerifyAsync(Header, CancellationToken.None);
Assert.Equal(2, inner.CallCount);
((IApiKeyCacheInvalidator)verifier).Invalidate("operator01");
await verifier.VerifyAsync(Header, CancellationToken.None);
Assert.Equal(3, inner.CallCount);
}
/// <summary>
/// The store decorator coalesces repeated <c>MarkUsed</c> writes for the same key inside the
/// window down to a single forwarded write — the ≤1/min guarantee for <c>last_used_utc</c>.
@@ -154,6 +154,70 @@ public sealed class ConstraintEnforcerTests
Assert.Equal("Read:Secret.Tag", auditEvent.Target);
}
/// <summary>
/// A bulk denial is written as ONE audit event carrying the denied/total counts, the distinct
/// constraint names and a sample capped at <see cref="ConstraintDenialSummary.MaxSampleTargets"/>,
/// however many items were denied.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RecordBulkDenialAsync_WritesOneAggregatedEventWithBoundedSample()
{
ConstraintEnforcer enforcer = CreateEnforcer(out FakeAuditWriter auditWriter);
ConstraintDenialSummary summary = new(totalCount: 1000);
for (int index = 0; index < 999; index++)
{
summary.Add(
$"Area1.Tag{index}",
index % 2 == 0
? new ConstraintFailure("read_scope", "Tag is outside the API key read scope.")
: new ConstraintFailure("tag_metadata", "Tag metadata is not available."));
}
await enforcer.RecordBulkDenialAsync(
CreateIdentity(ApiKeyConstraints.Empty),
"ReadBulk",
summary,
"trace-1",
CancellationToken.None);
AuditEvent auditEvent = Assert.Single(auditWriter.Events);
Assert.Equal("operator01", auditEvent.Actor);
Assert.Equal("constraint-denied", auditEvent.Action);
Assert.Equal(AuditOutcome.Denied, auditEvent.Outcome);
Assert.Equal("ReadBulk:bulk", auditEvent.Target);
Assert.Null(auditEvent.CorrelationId);
using JsonDocument details = JsonDocument.Parse(auditEvent.DetailsJson!);
JsonElement root = details.RootElement;
Assert.Equal(999, root.GetProperty("deniedCount").GetInt32());
Assert.Equal(1000, root.GetProperty("totalCount").GetInt32());
Assert.True(root.GetProperty("sampleTruncated").GetBoolean());
Assert.Equal(ConstraintDenialSummary.MaxSampleTargets, root.GetProperty("sampleTargets").GetArrayLength());
Assert.Equal(
["read_scope", "tag_metadata"],
root.GetProperty("constraints").EnumerateArray().Select(e => e.GetString() ?? string.Empty).ToArray());
Assert.Equal("ReadBulk", root.GetProperty("commandKind").GetString());
Assert.Equal("trace-1", root.GetProperty("clientCorrelationId").GetString());
}
/// <summary>An empty summary (nothing denied) writes no audit event at all.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RecordBulkDenialAsync_WithNoDenials_WritesNothing()
{
ConstraintEnforcer enforcer = CreateEnforcer(out FakeAuditWriter auditWriter);
await enforcer.RecordBulkDenialAsync(
identity: null,
"ReadBulk",
new ConstraintDenialSummary(totalCount: 3),
correlationId: null,
CancellationToken.None);
Assert.Empty(auditWriter.Events);
}
/// <summary>Verifies that historized-only constraint requires historized attribute.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -717,6 +717,151 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests
Assert.Null(GatewayGrpcAuthorizationInterceptor.TryResolveKeyId($"Bearer mxgw_{overCap}_secret"));
}
/// <summary>
/// A client-streaming call is authenticated exactly like a unary call: a missing key is
/// rejected with Unauthenticated and the handler never runs. The base Interceptor passes
/// client- and duplex-streaming calls straight through, so without the override any such RPC
/// added to the service would be reachable unauthenticated.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ClientStreamingServerHandler_MissingApiKey_ReturnsUnauthenticatedWithoutRunningHandler()
{
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(Failure(ApiKeyFailure.MissingOrMalformed)),
new GatewayRequestIdentityAccessor());
bool handlerRan = false;
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => interceptor.ClientStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
new TestServerCallContext([]),
(_, _) =>
{
handlerRan = true;
return Task.FromResult(new MxCommandReply());
}));
Assert.Equal(StatusCode.Unauthenticated, exception.StatusCode);
Assert.False(handlerRan);
}
/// <summary>
/// A client-streaming call has no single request message to derive a scope from, so the
/// interceptor fails closed to the <c>admin</c> scope, the resolver's default for an
/// unrecognized request.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ClientStreamingServerHandler_KeyWithoutAdminScope_ReturnsPermissionDenied()
{
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.InvokeRead, GatewayScopes.InvokeWrite)),
new GatewayRequestIdentityAccessor());
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => interceptor.ClientStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
ContextWithAuthorization("Bearer mxgw_operator01_secret"),
(_, _) => Task.FromResult(new MxCommandReply())));
Assert.Equal(StatusCode.PermissionDenied, exception.StatusCode);
Assert.Contains(GatewayScopes.Admin, exception.Status.Detail, StringComparison.Ordinal);
}
/// <summary>An admin-scoped key reaches the client-streaming handler with its identity pushed.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ClientStreamingServerHandler_AdminScope_RunsHandlerWithIdentity()
{
GatewayRequestIdentityAccessor identityAccessor = new();
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.Admin)),
identityAccessor);
string? keyIdSeenByHandler = null;
await interceptor.ClientStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
ContextWithAuthorization("Bearer mxgw_operator01_secret"),
(_, _) =>
{
keyIdSeenByHandler = identityAccessor.Current?.KeyId;
return Task.FromResult(new MxCommandReply());
});
Assert.Equal("operator01", keyIdSeenByHandler);
Assert.Null(identityAccessor.Current);
}
/// <summary>A duplex-streaming call with no key is rejected before the handler runs.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DuplexStreamingServerHandler_MissingApiKey_ReturnsUnauthenticatedWithoutRunningHandler()
{
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(Failure(ApiKeyFailure.MissingOrMalformed)),
new GatewayRequestIdentityAccessor());
bool handlerRan = false;
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => interceptor.DuplexStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
new RecordingServerStreamWriter<MxCommandReply>(),
new TestServerCallContext([]),
(_, _, _) =>
{
handlerRan = true;
return Task.CompletedTask;
}));
Assert.Equal(StatusCode.Unauthenticated, exception.StatusCode);
Assert.False(handlerRan);
}
/// <summary>A duplex-streaming call fails closed to the admin scope, like client streaming.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DuplexStreamingServerHandler_KeyWithoutAdminScope_ReturnsPermissionDenied()
{
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.EventsRead)),
new GatewayRequestIdentityAccessor());
RpcException exception = await Assert.ThrowsAsync<RpcException>(
() => interceptor.DuplexStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
new RecordingServerStreamWriter<MxCommandReply>(),
ContextWithAuthorization("Bearer mxgw_operator01_secret"),
(_, _, _) => Task.CompletedTask));
Assert.Equal(StatusCode.PermissionDenied, exception.StatusCode);
}
/// <summary>An admin-scoped key reaches the duplex handler with its identity pushed.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task DuplexStreamingServerHandler_AdminScope_RunsHandlerWithIdentity()
{
GatewayRequestIdentityAccessor identityAccessor = new();
GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor(
new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.Admin)),
identityAccessor);
RecordingServerStreamWriter<MxCommandReply> writer = new();
await interceptor.DuplexStreamingServerHandler<MxCommandRequest, MxCommandReply>(
new EmptyStreamReader<MxCommandRequest>(),
writer,
ContextWithAuthorization("Bearer mxgw_operator01_secret"),
async (_, responseStream, _) =>
{
Assert.Equal("operator01", identityAccessor.Current?.KeyId);
await responseStream.WriteAsync(new MxCommandReply { SessionId = "duplex" });
});
Assert.Equal("duplex", Assert.Single(writer.Messages).SessionId);
Assert.Null(identityAccessor.Current);
}
private static MxAccessGatewayService CreateService(
ISessionManager sessionManager,
IGatewayRequestIdentityAccessor identityAccessor)
@@ -933,4 +1078,14 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests
}
}
/// <summary>An <see cref="IAsyncStreamReader{T}"/> that yields no messages.</summary>
/// <typeparam name="T">The streamed message type.</typeparam>
private sealed class EmptyStreamReader<T> : IAsyncStreamReader<T>
{
/// <inheritdoc />
public T Current => throw new InvalidOperationException("The stream is empty.");
/// <inheritdoc />
public Task<bool> MoveNext(CancellationToken cancellationToken) => Task.FromResult(false);
}
}
@@ -0,0 +1,251 @@
using System.Security.Claims;
using System.Text.Encodings.Web;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.Auth.Abstractions.ApiKeys;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Dashboard;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity;
namespace ZB.MOM.WW.MxGateway.Tests.Security.Authorization;
/// <summary>
/// The optional <c>/metrics</c> authentication (<c>MxGateway:Metrics:RequireAuthentication</c>,
/// SEC-14): the scheme that authenticates a scrape and the requirement that authorizes it.
/// </summary>
public sealed class MetricsAuthorizationTests
{
private const string AdminHeader = "Bearer mxgw_scraper_secret";
/// <summary>A verified API key authenticates and carries its scopes as claims.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Authenticate_WithValidApiKey_SucceedsWithScopeClaims()
{
CountingVerifier verifier = new(Success(GatewayScopes.Admin));
DefaultHttpContext httpContext = ContextWithAuthorization(AdminHeader);
AuthenticateResult result = await AuthenticateAsync(verifier, httpContext);
Assert.True(result.Succeeded);
Assert.Equal("scraper", result.Principal!.Identity!.Name);
Assert.True(result.Principal.HasClaim(MetricsAuthorizationDefaults.ScopeClaimType, GatewayScopes.Admin));
Assert.Equal(AdminHeader, verifier.LastHeader);
}
/// <summary>An invalid key fails authentication with an opaque reason.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Authenticate_WithInvalidApiKey_Fails()
{
CountingVerifier verifier = new(new ApiKeyVerification(false, null, ApiKeyFailure.SecretMismatch));
AuthenticateResult result = await AuthenticateAsync(verifier, ContextWithAuthorization(AdminHeader));
Assert.False(result.Succeeded);
Assert.NotNull(result.Failure);
Assert.DoesNotContain("secret", result.Failure!.Message, StringComparison.OrdinalIgnoreCase);
}
/// <summary>
/// Repeated bad keys from one peer trip the shared failure limiter, after which the verifier is
/// no longer reached and the challenge answers 429 — /metrics is not a bypass of the gRPC throttle.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Authenticate_AfterRepeatedFailures_IsThrottledBeforeVerification()
{
CountingVerifier verifier = new(new ApiKeyVerification(false, null, ApiKeyFailure.SecretMismatch));
ApiKeyFailureLimiter limiter = new(
limit: 2,
TimeSpan.FromMinutes(1),
maxPartitions: 16,
aggregateLimit: 0,
probeInterval: TimeSpan.Zero,
TimeProvider.System);
for (int attempt = 0; attempt < 2; attempt++)
{
await AuthenticateAsync(verifier, ContextWithAuthorization(AdminHeader), limiter);
}
DefaultHttpContext throttled = ContextWithAuthorization(AdminHeader);
MetricsAuthenticationHandler handler = await CreateHandlerAsync(verifier, throttled, limiter);
AuthenticateResult result = await handler.AuthenticateAsync();
await handler.ChallengeAsync(new AuthenticationProperties());
Assert.False(result.Succeeded);
Assert.Equal(2, verifier.CallCount);
Assert.Equal(StatusCodes.Status429TooManyRequests, throttled.Response.StatusCode);
}
/// <summary>An anonymous scrape is challenged with a plain 401 Bearer, not a login redirect.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Challenge_WithoutCredentials_Returns401Bearer()
{
DefaultHttpContext httpContext = new();
MetricsAuthenticationHandler handler = await CreateHandlerAsync(
new CountingVerifier(Success(GatewayScopes.Admin)),
httpContext);
AuthenticateResult result = await handler.AuthenticateAsync();
await handler.ChallengeAsync(new AuthenticationProperties());
Assert.True(result.None);
Assert.Equal(StatusCodes.Status401Unauthorized, httpContext.Response.StatusCode);
Assert.Equal("Bearer", httpContext.Response.Headers.WWWAuthenticate.ToString());
}
/// <summary>An API key holding the admin scope satisfies the metrics requirement.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Requirement_ApiKeyWithAdminScope_Succeeds()
{
Assert.True(await EvaluateAsync(ApiKeyPrincipal(GatewayScopes.Admin)));
}
/// <summary>An API key without the admin scope does not satisfy the requirement.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Requirement_ApiKeyWithoutAdminScope_Fails()
{
Assert.False(await EvaluateAsync(ApiKeyPrincipal(GatewayScopes.EventsRead, GatewayScopes.InvokeRead)));
}
/// <summary>A scope claim counts only on an identity the metrics scheme minted from a verified key.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Requirement_ScopeClaimOnForeignIdentity_Fails()
{
ClaimsPrincipal principal = new(new ClaimsIdentity(
[new Claim(MetricsAuthorizationDefaults.ScopeClaimType, GatewayScopes.Admin)],
DashboardAuthenticationDefaults.AuthenticationScheme));
Assert.False(await EvaluateAsync(principal));
}
/// <summary>A dashboard login holding a real dashboard role satisfies the requirement.</summary>
/// <param name="role">The dashboard role.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData(DashboardRoles.Admin)]
[InlineData(DashboardRoles.Viewer)]
public async Task Requirement_DashboardLoginWithRole_Succeeds(string role)
{
ClaimsPrincipal principal = new(new ClaimsIdentity(
[new Claim(ClaimTypes.Name, "operator"), new Claim(ClaimTypes.Role, role)],
DashboardAuthenticationDefaults.AuthenticationScheme));
Assert.True(await EvaluateAsync(principal));
}
/// <summary>An unauthenticated principal (anonymous localhost included) never satisfies it.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Requirement_Unauthenticated_Fails()
{
ClaimsPrincipal principal = new(new ClaimsIdentity([new Claim(ClaimTypes.Role, DashboardRoles.Admin)]));
Assert.False(await EvaluateAsync(principal));
}
private static async Task<bool> EvaluateAsync(ClaimsPrincipal principal)
{
MetricsAccessRequirement requirement = new();
AuthorizationHandlerContext context = new([requirement], principal, resource: null);
await new MetricsAccessHandler().HandleAsync(context);
return context.HasSucceeded;
}
private static ClaimsPrincipal ApiKeyPrincipal(params string[] scopes)
{
List<Claim> claims = [new Claim(ClaimTypes.Name, "scraper")];
claims.AddRange(scopes.Select(scope => new Claim(MetricsAuthorizationDefaults.ScopeClaimType, scope)));
return new ClaimsPrincipal(new ClaimsIdentity(claims, MetricsAuthorizationDefaults.AuthenticationScheme));
}
private static async Task<AuthenticateResult> AuthenticateAsync(
IApiKeyVerifier verifier,
HttpContext httpContext,
ApiKeyFailureLimiter? limiter = null)
{
MetricsAuthenticationHandler handler = await CreateHandlerAsync(verifier, httpContext, limiter);
return await handler.AuthenticateAsync();
}
private static async Task<MetricsAuthenticationHandler> CreateHandlerAsync(
IApiKeyVerifier verifier,
HttpContext httpContext,
ApiKeyFailureLimiter? limiter = null)
{
MetricsAuthenticationHandler handler = new(
new StaticOptionsMonitor(),
NullLoggerFactory.Instance,
UrlEncoder.Default,
verifier,
limiter ?? new ApiKeyFailureLimiter(1000, TimeSpan.FromMinutes(1), 1024, 0, TimeSpan.Zero, TimeProvider.System),
new AuthenticationSchemeProvider(Options.Create(new Microsoft.AspNetCore.Authentication.AuthenticationOptions())),
Options.Create(new GatewayOptions()),
new GatewayMetrics());
await handler.InitializeAsync(
new AuthenticationScheme(
MetricsAuthorizationDefaults.AuthenticationScheme,
displayName: null,
typeof(MetricsAuthenticationHandler)),
httpContext);
return handler;
}
private static DefaultHttpContext ContextWithAuthorization(string header)
{
DefaultHttpContext context = new();
context.Request.Headers.Authorization = header;
context.Connection.RemoteIpAddress = System.Net.IPAddress.Parse("198.51.100.7");
return context;
}
private static ApiKeyVerification Success(params string[] scopes) => new(
Succeeded: true,
Identity: new LibApiKeyIdentity(
KeyId: "scraper",
DisplayName: "Scraper",
Scopes: new HashSet<string>(scopes, StringComparer.Ordinal),
Constraints: null),
Failure: null);
private sealed class CountingVerifier(ApiKeyVerification result) : IApiKeyVerifier
{
/// <summary>Gets the number of verifications performed.</summary>
public int CallCount { get; private set; }
/// <summary>Gets the last authorization header presented.</summary>
public string? LastHeader { get; private set; }
/// <inheritdoc />
public Task<ApiKeyVerification> VerifyAsync(string authorizationHeader, CancellationToken ct)
{
CallCount++;
LastHeader = authorizationHeader;
return Task.FromResult(result);
}
}
private sealed class StaticOptionsMonitor : IOptionsMonitor<AuthenticationSchemeOptions>
{
/// <inheritdoc />
public AuthenticationSchemeOptions CurrentValue { get; } = new();
/// <inheritdoc />
public AuthenticationSchemeOptions Get(string? name) => CurrentValue;
/// <inheritdoc />
public IDisposable? OnChange(Action<AuthenticationSchemeOptions, string?> listener) => null;
}
}
@@ -11,26 +11,26 @@ namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
public sealed class AllowAllConstraintEnforcer : IConstraintEnforcer
{
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadTagAsync(
public ValueTask<ConstraintFailure?> CheckReadTagAsync(
ApiKeyIdentity? identity,
string tagAddress,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadHandleAsync(
public ValueTask<ConstraintFailure?> CheckReadHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
int itemHandle,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task<ConstraintFailure?> CheckWriteHandleAsync(
public ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
int itemHandle,
CancellationToken cancellationToken) => Task.FromResult<ConstraintFailure?>(null);
CancellationToken cancellationToken) => ValueTask.FromResult<ConstraintFailure?>(null);
/// <inheritdoc />
public Task RecordDenialAsync(
@@ -40,4 +40,12 @@ public sealed class AllowAllConstraintEnforcer : IConstraintEnforcer
ConstraintFailure failure,
string? correlationId,
CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task RecordBulkDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
ConstraintDenialSummary summary,
string? correlationId,
CancellationToken cancellationToken) => Task.CompletedTask;
}
@@ -40,8 +40,18 @@ public sealed class FakeGatewayAlarmService : IGatewayAlarmService
yield return new AlarmFeedMessage { SnapshotComplete = true };
await Task.CompletedTask.ConfigureAwait(false);
if (StreamFault is not null)
{
throw StreamFault;
}
}
/// <summary>
/// Gets or sets an exception the feed throws after its snapshot, standing in for the monitor
/// completing a subscriber with a fault (for example a slow-subscriber overflow).
/// </summary>
public Exception? StreamFault { get; set; }
/// <inheritdoc />
public Task<AcknowledgeAlarmReply> AcknowledgeAsync(
AcknowledgeAlarmRequest request,
@@ -26,23 +26,26 @@ public sealed class PredicateConstraintEnforcer : IConstraintEnforcer
/// <summary>Recorded denial messages — (commandKind, target, correlationId) tuples.</summary>
public List<(string CommandKind, string Target, string? CorrelationId)> RecordedDenials { get; } = [];
/// <summary>Recorded aggregated bulk denials — one entry per bulk command that had any item denied.</summary>
public List<(string CommandKind, ConstraintDenialSummary Summary, string? CorrelationId)> RecordedBulkDenials { get; } = [];
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadTagAsync(
public ValueTask<ConstraintFailure?> CheckReadTagAsync(
ApiKeyIdentity? identity,
string tagAddress,
CancellationToken cancellationToken)
{
if (DenyTag(tagAddress))
{
return Task.FromResult<ConstraintFailure?>(
return ValueTask.FromResult<ConstraintFailure?>(
new ConstraintFailure("read-tag", $"Read denied for tag '{tagAddress}'."));
}
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
/// <inheritdoc />
public Task<ConstraintFailure?> CheckReadHandleAsync(
public ValueTask<ConstraintFailure?> CheckReadHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
@@ -51,15 +54,15 @@ public sealed class PredicateConstraintEnforcer : IConstraintEnforcer
{
if (DenyReadHandle(serverHandle, itemHandle))
{
return Task.FromResult<ConstraintFailure?>(
return ValueTask.FromResult<ConstraintFailure?>(
new ConstraintFailure("read-handle", $"Read denied for handle {itemHandle}."));
}
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
/// <inheritdoc />
public Task<ConstraintFailure?> CheckWriteHandleAsync(
public ValueTask<ConstraintFailure?> CheckWriteHandleAsync(
ApiKeyIdentity? identity,
GatewaySession session,
int serverHandle,
@@ -68,11 +71,11 @@ public sealed class PredicateConstraintEnforcer : IConstraintEnforcer
{
if (DenyWriteHandle(serverHandle, itemHandle))
{
return Task.FromResult<ConstraintFailure?>(
return ValueTask.FromResult<ConstraintFailure?>(
new ConstraintFailure("write-handle", $"Write denied for handle {itemHandle}."));
}
return Task.FromResult<ConstraintFailure?>(null);
return ValueTask.FromResult<ConstraintFailure?>(null);
}
/// <inheritdoc />
@@ -87,4 +90,16 @@ public sealed class PredicateConstraintEnforcer : IConstraintEnforcer
RecordedDenials.Add((commandKind, target, correlationId));
return Task.CompletedTask;
}
/// <inheritdoc />
public Task RecordBulkDenialAsync(
ApiKeyIdentity? identity,
string commandKind,
ConstraintDenialSummary summary,
string? correlationId,
CancellationToken cancellationToken)
{
RecordedBulkDenials.Add((commandKind, summary, correlationId));
return Task.CompletedTask;
}
}
@@ -0,0 +1,48 @@
using Microsoft.Extensions.Logging;
namespace ZB.MOM.WW.MxGateway.Tests.TestSupport;
/// <summary>
/// <see cref="ILogger{TCategoryName}"/> that records every entry's level and formatted message,
/// so a test can assert that a path logs (or does not log) at a given level.
/// </summary>
/// <typeparam name="T">The logger category type.</typeparam>
public sealed class RecordingLogger<T> : ILogger<T>
{
private readonly object _syncRoot = new();
private readonly List<(LogLevel Level, string Message, Exception? Exception)> _entries = [];
/// <summary>Gets a snapshot of the recorded entries, in order.</summary>
public IReadOnlyList<(LogLevel Level, string Message, Exception? Exception)> Entries
{
get
{
lock (_syncRoot)
{
return _entries.ToArray();
}
}
}
/// <inheritdoc />
public IDisposable? BeginScope<TState>(TState state)
where TState : notnull => null;
/// <inheritdoc />
public bool IsEnabled(LogLevel logLevel) => true;
/// <inheritdoc />
public void Log<TState>(
LogLevel logLevel,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
ArgumentNullException.ThrowIfNull(formatter);
lock (_syncRoot)
{
_entries.Add((logLevel, formatter(state, exception), exception));
}
}
}
@@ -29,6 +29,25 @@ public sealed class RecordingServerStreamWriter<T> : IServerStreamWriter<T>
/// <summary>Gets or sets options for writing messages to the stream.</summary>
public WriteOptions? WriteOptions { get; set; }
/// <summary>Gets the cancellation token passed to the most recent token-bearing write.</summary>
public CancellationToken LastWriteCancellationToken { get; private set; }
/// <summary>Records the supplied message unless the token is already cancelled.</summary>
/// <param name="message">The message to record.</param>
/// <param name="cancellationToken">Token that cancels the write.</param>
/// <returns>A completed task.</returns>
/// <remarks>
/// The interface default throws <see cref="NotSupportedException"/> for a cancelable token,
/// and the gateway passes the call's token on every streamed write, so a test writer must
/// override this overload the way the ASP.NET Core writer does.
/// </remarks>
public Task WriteAsync(T message, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
LastWriteCancellationToken = cancellationToken;
return WriteAsync(message);
}
/// <summary>Records the supplied message.</summary>
/// <param name="message">The message to record.</param>
/// <returns>A completed task.</returns>