Merge branch 'fix/archreview-p2' into main (P2 tier: completeness & polish)

# Conflicts:
#	archreview/remediation/00-tracking.md
#	clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli/MxGatewayCliSecretRedactor.cs
#	clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli/MxGatewayClientCli.cs
#	src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs
This commit is contained in:
Joseph Doherty
2026-07-12 22:12:41 -04:00
91 changed files with 7680 additions and 681 deletions
+25
View File
@@ -9,6 +9,31 @@
<Deterministic>true</Deterministic>
</PropertyGroup>
<!-- TST-11: single-source the .NET-side version for Server, Worker, Contracts, and tests
(they otherwise stamp the SDK default 1.0.0, so a deployed gateway cannot be correlated
to a release). Kept at 0.1.2 to match the Contracts package and the aligned Python/Rust/
Go clients; the Java client leads at 0.2.0 after its JDK-17 retarget. The git short SHA is
appended to InformationalVersion (0.1.2+<sha>) so support can map a running binary to a
commit; the query is guarded so a build outside a git checkout still succeeds. -->
<PropertyGroup>
<Version>0.1.2</Version>
</PropertyGroup>
<Target Name="StampSourceRevision"
BeforeTargets="GetAssemblyVersion;GenerateAssemblyInfo"
Condition="'$(SourceRevisionId)' == ''">
<Exec Command="git -C &quot;$(MSBuildThisFileDirectory)&quot; rev-parse --short HEAD"
ConsoleToMSBuild="true"
StandardOutputImportance="Low"
ContinueOnError="true"
IgnoreExitCode="true">
<Output TaskParameter="ConsoleOutput" PropertyName="_StampedGitSha" />
</Exec>
<PropertyGroup>
<SourceRevisionId Condition="'$(_StampedGitSha)' != ''">$(_StampedGitSha.Trim())</SourceRevisionId>
</PropertyGroup>
</Target>
<!-- SQLitePCLRaw.lib.e_sqlite3 2.1.11 (transitive via Microsoft.Data.Sqlite) carries GHSA-2m69-gcr7-jv3q,
which surfaces as NU1903 (warning-as-error). No patched e_sqlite3 release exists yet (2.1.11 is latest),
so this targeted suppression keeps every OTHER transitive package audited. Remove once an upstream fix ships. -->
@@ -35,5 +35,23 @@ public static class DashboardAuthenticationDefaults
public const string LdapGroupClaimType = "mxgateway:ldap_group";
public const string KeyPrefixClaimType = "mxgateway:key_prefix";
/// <summary>
/// Dashboard auth cookie name used when the cookie is not guaranteed to be Secure
/// (<c>RequireHttpsCookie=false</c> → <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.SameAsRequest"/>)
/// or when an explicit <c>MxGateway:Dashboard:CookieName</c> override is absent but the
/// secure default cannot be applied. This plain name carries no browser-enforced guarantees.
/// </summary>
public const string CookieName = "MxGatewayDashboard";
/// <summary>
/// Dashboard auth cookie name applied when the cookie is guaranteed Secure
/// (<c>RequireHttpsCookie=true</c> → <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.Always"/>)
/// and no explicit <c>MxGateway:Dashboard:CookieName</c> override is set. The <c>__Host-</c>
/// prefix instructs browsers to enforce Secure, no <c>Domain</c>, and <c>Path=/</c>; those
/// guarantees only hold for a Secure cookie, so this name must never be applied unless
/// <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.Always"/> is in
/// effect — a <c>__Host-</c> cookie without Secure is silently dropped by browsers.
/// </summary>
public const string SecureCookieName = "__Host-MxGatewayDashboard";
}
@@ -123,13 +123,22 @@ public static class DashboardServiceCollectionExtensions
? CookieSecurePolicy.Always
: CookieSecurePolicy.SameAsRequest;
// Config-driven cookie name (MxGateway:Dashboard:CookieName). Null/blank keeps
// the canonical default set above, so a misconfiguration cannot unname the cookie.
// Config-driven cookie name (MxGateway:Dashboard:CookieName). An explicit override
// always wins. With no override, restore the __Host- prefix when the cookie is
// guaranteed Secure (RequireHttpsCookie true → SecurePolicy Always): the __Host-
// browser guarantees (Secure required, no Domain, Path=/) hold only for a Secure
// cookie, and a __Host- cookie without Secure is silently dropped — so the prefix
// is never applied unless SecurePolicy is Always. Otherwise keep the plain
// canonical default set by AddCookie above, so a misconfiguration cannot unname it.
var cookieName = gatewayOptions.Value.Dashboard.CookieName;
if (!string.IsNullOrWhiteSpace(cookieName))
{
cookieOptions.Cookie.Name = cookieName;
}
else if (cookieOptions.Cookie.SecurePolicy == CookieSecurePolicy.Always)
{
cookieOptions.Cookie.Name = DashboardAuthenticationDefaults.SecureCookieName;
}
});
services.AddAuthorization(authorization =>
@@ -1,5 +1,7 @@
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Configuration;
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
@@ -10,10 +12,22 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
/// Errors are logged once and dropped — keeping the SignalR mirror best-effort
/// preserves the gRPC contract that exists today.
/// </summary>
/// <remarks>
/// When <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), tag
/// values are stripped from a redacted copy of the event before it reaches any
/// dashboard client. The source <see cref="MxEvent"/> is shared with the gRPC
/// event path and the reconnect replay ring, so it is never mutated in place —
/// the redaction is applied to a deep clone. This closes the value-leak seam at
/// the mirror independently of the still-outstanding per-session hub ACL
/// (see <see cref="EventsHub"/>).
/// </remarks>
public sealed class DashboardEventBroadcaster(
IHubContext<EventsHub> hubContext,
IOptions<GatewayOptions> options,
ILogger<DashboardEventBroadcaster> logger) : IDashboardEventBroadcaster
{
private readonly bool _showTagValues = options.Value.Dashboard.ShowTagValues;
/// <inheritdoc />
public void Publish(string sessionId, MxEvent mxEvent)
{
@@ -22,6 +36,8 @@ public sealed class DashboardEventBroadcaster(
return;
}
MxEvent outbound = _showTagValues ? mxEvent : RedactValues(mxEvent);
// Wrap the Task acquisition in a try/catch so a hypothetical synchronous throw
// from SendAsync (e.g. an implementation that throws before returning the Task)
// cannot escape Publish. The interface contract is never-throw; fire-and-forget.
@@ -30,7 +46,7 @@ public sealed class DashboardEventBroadcaster(
{
send = hubContext.Clients
.Group(EventsHub.GroupName(sessionId))
.SendAsync(EventsHub.EventMessage, mxEvent);
.SendAsync(EventsHub.EventMessage, outbound);
}
catch (Exception ex)
{
@@ -51,4 +67,27 @@ public sealed class DashboardEventBroadcaster(
TaskScheduler.Default);
}
}
/// <summary>
/// Produces a deep clone of <paramref name="source"/> with every tag-value
/// field cleared, leaving tag reference, quality, status, and timestamps
/// intact so the dashboard still renders the event without the value. The
/// source event is left untouched because it is shared downstream with the
/// gRPC stream and the replay ring.
/// </summary>
/// <param name="source">The source event to redact a copy of.</param>
/// <returns>A redacted deep clone of the event.</returns>
private static MxEvent RedactValues(MxEvent source)
{
MxEvent redacted = source.Clone();
redacted.Value = null;
if (redacted.BodyCase == MxEvent.BodyOneofCase.OnAlarmTransition)
{
redacted.OnAlarmTransition.CurrentValue = null;
redacted.OnAlarmTransition.LimitValue = null;
}
return redacted;
}
}
@@ -32,15 +32,19 @@ public sealed class EventsHub : Hub
/// the dashboard roles (Admin or Viewer); both roles may subscribe to
/// any session id they choose. This is acceptable today because (a) the
/// dashboard's per-session views show non-secret session metadata that
/// any authenticated dashboard user can already see, and (b) value
/// logging in the source gRPC stream is gated by the same redaction
/// policy that protects logs. The per-session ACL that gates the gRPC
/// any authenticated dashboard user can already see, and (b) tag values
/// are stripped from the mirrored events by
/// <see cref="DashboardEventBroadcaster"/> when
/// <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), so the
/// most sensitive payload cannot leak through this seam regardless of the
/// still-missing ACL. The per-session ACL that gates the gRPC
/// <c>StreamEvents</c> RPC is intentionally not yet mirrored here.
/// TODO(per-session-acl): once a role/scope is introduced that scopes a
/// Viewer to a specific session or tenant, add a session-access check
/// at this seam — either inline (consult the per-user allowed-session
/// set on <c>Context.User</c> claims / <c>Context.Items</c>) or via a
/// dedicated authorization policy applied to the hub method itself.
/// TODO(per-session-acl): tracked as remediation roadmap item 12
/// (SEC-25). Once a role/scope is introduced that scopes a Viewer to a
/// specific session or tenant, add a session-access check at this seam —
/// either inline (consult the per-user allowed-session set on
/// <c>Context.User</c> claims / <c>Context.Items</c>) or via a dedicated
/// authorization policy applied to the hub method itself.
/// </remarks>
/// <param name="sessionId">Session id to subscribe the caller to.</param>
/// <returns>A task representing the subscription operation.</returns>
@@ -106,11 +106,23 @@ public sealed class EventStreamService(
options.Value.Sessions.MaxEventSubscribersPerSession);
}
int streamQueueDepth = 0;
IAsyncEnumerator<MxEvent> reader = subscriber.Reader
.ReadAllAsync(cancellationToken)
.GetAsyncEnumerator(cancellationToken);
// GWC-15: register this subscriber's channel as a live backlog source instead of
// reconciling the queue-depth gauge on every event. The gauge previously read the
// bounded channel's Count (which takes the channel's internal lock) and adjusted the
// metric under its own lock on every streamed event. Now the metric reads Count only
// when it is scraped (ObservableGauge callback) or projected (GetSnapshot), summing the
// live backlog across every registered subscriber — the same "buffered, not yet
// delivered" aggregate the per-event push reported, but with no per-event lock traffic.
// Disposing the registration in the finally removes this subscriber's contribution, so
// the gauge returns to the other subscribers' backlog (zero when none remain) on
// disconnect. CanCount guards a channel that ever cannot report Count (contributes 0).
IDisposable backlogRegistration = metrics.RegisterEventStreamBacklogSource(
() => subscriber.Reader.CanCount ? subscriber.Reader.Count : 0);
try
{
// Emit order for a resume: the ReplayGap sentinel FIRST (only when events were
@@ -169,32 +181,21 @@ public sealed class EventStreamService(
continue;
}
// Queue-depth gauge tracks events the pump has fanned into this subscriber's
// channel but the client has not yet consumed — the same "buffered, not yet
// delivered" quantity the original per-RPC channel reported. The bounded
// subscriber channel supports counting, so reconcile the gauge to the current
// backlog; falling back to a no-op delta if a channel ever cannot count.
int backlog = subscriber.Reader.CanCount ? subscriber.Reader.Count : streamQueueDepth;
int delta = backlog - streamQueueDepth;
if (delta != 0)
{
streamQueueDepth = backlog;
metrics.AdjustGrpcEventStreamQueueDepth(delta);
}
// The queue-depth gauge is maintained lazily via the backlog registration above
// (GWC-15): the metric reads this subscriber's channel Count only when scraped,
// so there is no per-event gauge bookkeeping on this hot path.
yield return mxEvent;
}
}
finally
{
await reader.DisposeAsync().ConfigureAwait(false);
subscriber.Dispose();
if (streamQueueDepth != 0)
{
metrics.AdjustGrpcEventStreamQueueDepth(-streamQueueDepth);
streamQueueDepth = 0;
}
// Remove this subscriber's live backlog contribution before disposing the lease so
// the gauge stops counting a channel that is about to be completed; after this the
// gauge reflects only the remaining subscribers (zero when none remain).
backlogRegistration.Dispose();
subscriber.Dispose();
metrics.StreamDisconnected("Detached");
}
@@ -152,9 +152,15 @@ public sealed class MxAccessGatewayService(
.WithCancellation(context.CancellationToken)
.ConfigureAwait(false))
{
Stopwatch stopwatch = Stopwatch.StartNew();
// GWC-06: measure send latency with the allocation-free timestamp API rather
// than allocating a Stopwatch object per event per subscriber on the highest-
// volume gateway path. Stopwatch.GetTimestamp/GetElapsedTime measure the exact
// same wall-clock span the former Stopwatch.StartNew()/.Elapsed did.
long sendStartTimestamp = Stopwatch.GetTimestamp();
await responseStream.WriteAsync(publicEvent).ConfigureAwait(false);
metrics.RecordEventStreamSend(publicEvent.Family.ToString(), stopwatch.Elapsed);
metrics.RecordEventStreamSend(
publicEvent.Family.ToString(),
Stopwatch.GetElapsedTime(sendStartTimestamp));
}
}
catch (Exception exception) when (exception is not RpcException)
@@ -65,7 +65,16 @@ public sealed class MxAccessGrpcMapper
{
ArgumentNullException.ThrowIfNull(workerEvent);
return workerEvent.Event?.Clone() ?? new MxEvent
// GWC-07 / IPC-05: ownership transfer, not a deep clone. The enclosing WorkerEvent is
// parsed fresh from a single pipe frame in WorkerClient's read loop and is discarded
// immediately after this mapping — the SessionEventDistributor pump is its single
// consumer (GWC-01 claims the worker event channel as single-reader), so nothing else
// aliases or mutates workerEvent.Event. We therefore move the inner MxEvent into the
// outbound graph instead of cloning it. Downstream the pump fans this one MxEvent to
// every subscriber and retains it in the replay ring, but that sharing is READ-ONLY
// (subscribers only yield/filter it), so a single shared instance is safe. If a second
// consumer of WorkerEvent is ever added, restore a .Clone() here to re-isolate.
return workerEvent.Event ?? new MxEvent
{
Family = MxEventFamily.Unspecified,
RawStatus = "Worker event did not contain a public event payload.",
@@ -32,10 +32,17 @@ public sealed class GatewayMetrics : IDisposable
private readonly ConcurrentDictionary<string, long> _eventsBySession = new(StringComparer.Ordinal);
private readonly Dictionary<string, long> _retryAttemptsByArea = new(StringComparer.OrdinalIgnoreCase);
// GWC-15: live backlog sources for the gRPC event-stream queue-depth gauge. Each active
// StreamEvents subscriber registers a delegate returning its channel's current backlog; the
// gauge sums them ONLY when scraped or projected (GetGrpcEventStreamQueueDepth), so nothing is
// read on the per-event streaming hot path. Concurrent because subscribers register/unregister
// from arbitrary gRPC threads while the gauge callback / GetSnapshot enumerate.
private readonly ConcurrentDictionary<long, Func<int>> _eventStreamBacklogSources = new();
private long _nextEventStreamBacklogSourceId;
private int _openSessions;
private int _workersRunning;
private int _workerEventQueueDepth;
private int _grpcEventStreamQueueDepth;
private int _alarmProviderMode;
private long _sessionsOpened;
private long _sessionsClosed;
@@ -292,15 +299,24 @@ public sealed class GatewayMetrics : IDisposable
}
/// <summary>
/// Adjusts the gRPC event stream queue depth by the given delta.
/// Registers a live backlog source for the gRPC event-stream queue-depth gauge and
/// returns a handle that removes it when disposed. The gauge sums every registered
/// source's current value on demand (when scraped or via <see cref="GetSnapshot"/>),
/// rather than tracking a pushed running total, so the streaming hot path does no
/// per-event gauge bookkeeping (GWC-15).
/// </summary>
/// <param name="delta">Amount to adjust the queue depth by.</param>
public void AdjustGrpcEventStreamQueueDepth(int delta)
/// <param name="backlog">
/// Returns this subscriber's current channel backlog. Invoked only at collection time;
/// must be cheap and non-blocking (a bounded channel's <c>Count</c>). Negative returns
/// are clamped to zero when summed.
/// </param>
/// <returns>A handle whose disposal unregisters the source. Safe to dispose more than once.</returns>
public IDisposable RegisterEventStreamBacklogSource(Func<int> backlog)
{
lock (_syncRoot)
{
_grpcEventStreamQueueDepth = Math.Max(0, _grpcEventStreamQueueDepth + delta);
}
ArgumentNullException.ThrowIfNull(backlog);
long id = Interlocked.Increment(ref _nextEventStreamBacklogSourceId);
_eventStreamBacklogSources[id] = backlog;
return new EventStreamBacklogRegistration(this, id);
}
/// <summary>
@@ -429,13 +445,16 @@ public sealed class GatewayMetrics : IDisposable
/// <returns>The current metrics snapshot.</returns>
public GatewayMetricsSnapshot GetSnapshot()
{
// Compute the live gRPC stream backlog outside _syncRoot: the sources are the subscriber
// channels' Count (their own locks) and must not run under this lock. GWC-15.
int grpcEventStreamQueueDepth = GetGrpcEventStreamQueueDepth();
lock (_syncRoot)
{
return new GatewayMetricsSnapshot(
OpenSessions: _openSessions,
WorkersRunning: _workersRunning,
WorkerEventQueueDepth: _workerEventQueueDepth,
GrpcEventStreamQueueDepth: _grpcEventStreamQueueDepth,
GrpcEventStreamQueueDepth: grpcEventStreamQueueDepth,
SessionsOpened: _sessionsOpened,
SessionsClosed: _sessionsClosed,
CommandsStarted: _commandsStarted,
@@ -495,12 +514,28 @@ public sealed class GatewayMetrics : IDisposable
}
}
// Sums the live backlog across every registered event-stream subscriber. Runs at collection
// time (ObservableGauge scrape) or when GetSnapshot projects the value — never on the
// per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent
// register/unregister; a source removed mid-enumeration simply drops from this sample.
private int GetGrpcEventStreamQueueDepth()
{
lock (_syncRoot)
int total = 0;
foreach (Func<int> source in _eventStreamBacklogSources.Values)
{
return _grpcEventStreamQueueDepth;
int value = source();
if (value > 0)
{
total += value;
}
}
return total;
}
private void UnregisterEventStreamBacklogSource(long id)
{
_eventStreamBacklogSources.TryRemove(id, out _);
}
private int GetAlarmProviderMode()
@@ -521,4 +556,19 @@ public sealed class GatewayMetrics : IDisposable
{
values.AddOrUpdate(key, 1, static (_, currentValue) => currentValue + 1);
}
// Handle returned by RegisterEventStreamBacklogSource. Disposal (once) removes the source
// from the gauge's live sum. Idempotent so a double dispose from a stream teardown is safe.
private sealed class EventStreamBacklogRegistration(GatewayMetrics metrics, long id) : IDisposable
{
private int _disposed;
public void Dispose()
{
if (Interlocked.Exchange(ref _disposed, 1) == 0)
{
metrics.UnregisterEventStreamBacklogSource(id);
}
}
}
}
@@ -98,14 +98,21 @@ public sealed class SessionEventDistributor : IAsyncDisposable
private readonly object _lifecycleLock = new();
// Replay ring buffer. Appended on the pump thread and queried from arbitrary
// threads via TryGetReplayFrom, so every access is under _replayLock. The deque
// keeps events in ascending WorkerSequence order (the pump fans in source order),
// so the oldest retained event is always at the front. Capacity == 0 disables
// retention; RetentionSeconds <= 0 disables age-based eviction.
// threads via TryGetReplayFrom, so every access is under _replayLock. Backed by a
// fixed-size circular array preallocated to the capacity so appending a retained
// event allocates no node (the LinkedList this replaced allocated one node per
// event on the fan-out hot path). Events are kept in ascending WorkerSequence order
// (the pump fans in source order): _replayHead is the logical front (oldest retained
// event) and _replayCount entries follow it, wrapping modulo the array length. The
// logical entry at position i is _replayBuffer[(_replayHead + i) % Length]. Capacity
// == 0 disables retention (the array is empty and never indexed); RetentionSeconds
// <= 0 disables age-based eviction.
private readonly int _replayBufferCapacity;
private readonly TimeSpan _replayRetention;
private readonly bool _ageEvictionEnabled;
private readonly LinkedList<ReplayEntry> _replayBuffer = new();
private readonly ReplayEntry[] _replayBuffer;
private int _replayHead;
private int _replayCount;
private readonly object _replayLock = new();
private bool _anyEventSeen;
private ulong _highestSequenceSeen;
@@ -245,6 +252,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
_overflowHandler = overflowHandler;
_shutdownTimeout = DefaultShutdownTimeout;
_replayBufferCapacity = replayBufferCapacity;
_replayBuffer = new ReplayEntry[replayBufferCapacity];
_ageEvictionEnabled = replayRetentionSeconds > 0;
_replayRetention = _ageEvictionEnabled
? TimeSpan.FromSeconds(replayRetentionSeconds)
@@ -452,24 +460,25 @@ public sealed class SessionEventDistributor : IAsyncDisposable
List<MxEvent> newer = [];
ulong highestReplayed = afterSequence;
if (_replayBuffer.Count == 0)
if (_replayCount == 0)
{
gap = _anyEventSeen && afterSequence < _highestSequenceSeen;
oldestAvailableSequence = 0; // meaningful only when gap == true; 0 here since nothing is retained
}
else
{
ulong oldestRetained = _replayBuffer.First!.Value.Event.WorkerSequence;
ulong oldestRetained = ReplayEntryAt(0).Event.WorkerSequence;
gap = oldestRetained > 0 && afterSequence < oldestRetained - 1;
// Per the contract on OldestAvailableSequence: meaningful only when gap == true.
oldestAvailableSequence = gap ? oldestRetained : 0;
foreach (ReplayEntry entry in _replayBuffer)
for (int i = 0; i < _replayCount; i++)
{
if (entry.Event.WorkerSequence > afterSequence)
MxEvent retained = ReplayEntryAt(i).Event;
if (retained.WorkerSequence > afterSequence)
{
newer.Add(entry.Event);
highestReplayed = entry.Event.WorkerSequence;
newer.Add(retained);
highestReplayed = retained.WorkerSequence;
}
}
}
@@ -729,7 +738,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
{
EvictAged();
if (_replayBuffer.Count == 0)
if (_replayCount == 0)
{
events = [];
// Nothing retained. The caller missed events only if it is behind the
@@ -738,7 +747,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
return true;
}
ulong oldestRetained = _replayBuffer.First!.Value.Event.WorkerSequence;
ulong oldestRetained = ReplayEntryAt(0).Event.WorkerSequence;
// A gap exists when at least one event newer than afterSequence was evicted,
// i.e. afterSequence sits below the oldest-retained-minus-one boundary.
@@ -750,11 +759,12 @@ public sealed class SessionEventDistributor : IAsyncDisposable
// O(n) scan over the retained buffer — acceptable because TryGetReplayFrom
// is only called on subscriber reconnect, never on the hot fan-out path.
List<MxEvent> newer = [];
foreach (ReplayEntry entry in _replayBuffer)
for (int i = 0; i < _replayCount; i++)
{
if (entry.Event.WorkerSequence > afterSequence)
MxEvent retained = ReplayEntryAt(i).Event;
if (retained.WorkerSequence > afterSequence)
{
newer.Add(entry.Event);
newer.Add(retained);
}
}
@@ -780,30 +790,43 @@ public sealed class SessionEventDistributor : IAsyncDisposable
return;
}
_replayBuffer.AddLast(new ReplayEntry(mxEvent, _timeProvider.GetUtcNow()));
// Capacity eviction: drop oldest until within bound.
while (_replayBuffer.Count > _replayBufferCapacity)
// Append at the logical tail. When the ring is full the oldest entry is
// overwritten in place (its slot becomes the new tail) and the head advances,
// so the newest _replayBufferCapacity events are retained with no allocation.
ReplayEntry entry = new(mxEvent, _timeProvider.GetUtcNow());
if (_replayCount < _replayBufferCapacity)
{
_replayBuffer.RemoveFirst();
_replayBuffer[(_replayHead + _replayCount) % _replayBufferCapacity] = entry;
_replayCount++;
}
else
{
_replayBuffer[_replayHead] = entry;
_replayHead = (_replayHead + 1) % _replayBufferCapacity;
}
EvictAged();
}
}
// Must be called under _replayLock. Drops entries older than the retention window.
// Returns the logical entry at position i (0 == oldest retained). Must be called
// under _replayLock with 0 <= i < _replayCount (so the array length is nonzero).
private ReplayEntry ReplayEntryAt(int i) => _replayBuffer[(_replayHead + i) % _replayBuffer.Length];
// Must be called under _replayLock. Drops entries older than the retention window
// by advancing the head past them (no dealloc; slots are reused on the next append).
private void EvictAged()
{
if (!_ageEvictionEnabled || _replayBuffer.Count == 0)
if (!_ageEvictionEnabled || _replayCount == 0)
{
return;
}
DateTimeOffset cutoff = _timeProvider.GetUtcNow() - _replayRetention;
while (_replayBuffer.First is { } first && first.Value.RetainedAt < cutoff)
while (_replayCount > 0 && _replayBuffer[_replayHead].RetainedAt < cutoff)
{
_replayBuffer.RemoveFirst();
_replayHead = (_replayHead + 1) % _replayBuffer.Length;
_replayCount--;
}
}
@@ -997,9 +997,14 @@ public sealed class WorkerClient : IWorkerClient
string correlationId,
WorkerCommand command)
{
// IPC-05: no second clone. MxAccessGrpcMapper.MapCommand already deep-cloned the command
// out of the caller-owned gRPC MxCommandRequest, so this WorkerCommand is a fresh graph
// owned by the invoke pipeline and referenced by no other consumer. The envelope is built
// and owned entirely inside WorkerClient and the command is not touched again after this
// point, so transferring it into the envelope introduces no aliasing hazard.
return CreateEnvelope(
correlationId,
envelope => envelope.WorkerCommand = command.Clone());
envelope => envelope.WorkerCommand = command);
}
/// <summary>Creates shutdown envelope.</summary>
@@ -1,3 +1,4 @@
using System.Buffers;
using System.Buffers.Binary;
using Google.Protobuf;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
@@ -47,20 +48,34 @@ public sealed class WorkerFrameReader
$"Worker frame payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes.");
}
byte[] payload = new byte[payloadLength];
await ReadExactlyOrThrowAsync(payload, cancellationToken).ConfigureAwait(false);
// Rent the payload buffer from the shared pool rather than allocating a fresh byte[] per
// frame; large event frames near the cap would otherwise allocate an LOH buffer each time
// (IPC-14). ParseFrom copies whatever it needs into the parsed message, so the rented buffer
// can be returned as soon as parsing completes without the envelope aliasing it. The rented
// buffer may be larger than requested, so the read and parse are bounded to length.
int length = checked((int)payloadLength);
byte[] payload = ArrayPool<byte>.Shared.Rent(length);
WorkerEnvelope envelope;
try
{
envelope = WorkerEnvelope.Parser.ParseFrom(payload);
await ReadExactlyOrThrowAsync(new Memory<byte>(payload, 0, length), cancellationToken)
.ConfigureAwait(false);
try
{
envelope = WorkerEnvelope.Parser.ParseFrom(payload, 0, length);
}
catch (InvalidProtocolBufferException exception)
{
throw new WorkerFrameProtocolException(
WorkerFrameProtocolErrorCode.InvalidEnvelope,
"Worker frame payload is not a valid WorkerEnvelope protobuf message.",
exception);
}
}
catch (InvalidProtocolBufferException exception)
finally
{
throw new WorkerFrameProtocolException(
WorkerFrameProtocolErrorCode.InvalidEnvelope,
"Worker frame payload is not a valid WorkerEnvelope protobuf message.",
exception);
ArrayPool<byte>.Shared.Return(payload);
}
WorkerEnvelopeValidator.Validate(envelope, _options);
@@ -1,3 +1,4 @@
using System.Buffers;
using System.Buffers.Binary;
using Google.Protobuf;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
@@ -53,10 +54,25 @@ public sealed class WorkerFrameWriter
$"Worker envelope payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes.");
}
byte[] lengthPrefix = new byte[sizeof(uint)];
BinaryPrimitives.WriteUInt32LittleEndian(lengthPrefix, (uint)payloadLength);
// Serialize once into a single pooled buffer that carries the 4-byte little-endian length
// prefix followed by the payload, then issue one stream write. This avoids a second
// serialization pass (ToByteArray re-runs CalculateSize), a separate prefix array and its
// own write, and any per-frame heap allocation (GWC-08, IPC-13). The rented buffer may be
// larger than requested, so only the first frameLength bytes are ever written.
int frameLength = sizeof(uint) + payloadLength;
byte[] frame = ArrayPool<byte>.Shared.Rent(frameLength);
try
{
BinaryPrimitives.WriteUInt32LittleEndian(frame, (uint)payloadLength);
envelope.WriteTo(new Span<byte>(frame, sizeof(uint), payloadLength));
await _stream.WriteAsync(lengthPrefix, cancellationToken).ConfigureAwait(false);
await _stream.WriteAsync(envelope.ToByteArray(), cancellationToken).ConfigureAwait(false);
await _stream
.WriteAsync(new ReadOnlyMemory<byte>(frame, 0, frameLength), cancellationToken)
.ConfigureAwait(false);
}
finally
{
ArrayPool<byte>.Shared.Return(frame);
}
}
}
@@ -10,7 +10,12 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
public sealed class DashboardCookieOptionsTests
{
/// <summary>Verifies that the application configures secure dashboard authentication cookies.</summary>
/// <summary>
/// Verifies that the application configures secure dashboard authentication cookies.
/// With <c>RequireHttpsCookie</c> defaulting to <see langword="true"/> and no explicit
/// <c>CookieName</c> override, the cookie is named with the <c>__Host-</c> prefix so
/// browsers enforce the Secure/no-Domain/Path=/ guarantees the prefix promises.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Build_ConfiguresSecureDashboardCookie()
@@ -22,7 +27,7 @@ public sealed class DashboardCookieOptionsTests
CookieAuthenticationOptions options = optionsMonitor.Get(
DashboardAuthenticationDefaults.AuthenticationScheme);
Assert.Equal(DashboardAuthenticationDefaults.CookieName, options.Cookie.Name);
Assert.Equal(DashboardAuthenticationDefaults.SecureCookieName, options.Cookie.Name);
Assert.True(options.Cookie.HttpOnly);
Assert.Equal(CookieSecurePolicy.Always, options.Cookie.SecurePolicy);
Assert.Equal(SameSiteMode.Strict, options.Cookie.SameSite);
@@ -35,11 +40,14 @@ public sealed class DashboardCookieOptionsTests
/// <summary>
/// Verifies that setting <c>MxGateway:Dashboard:RequireHttpsCookie=false</c>
/// relaxes the cookie to <see cref="CookieSecurePolicy.SameAsRequest"/> so
/// the dashboard can be reached over plain HTTP in dev.
/// the dashboard can be reached over plain HTTP in dev, and that the plain
/// <see cref="DashboardAuthenticationDefaults.CookieName"/> is used rather than the
/// <c>__Host-</c> name — a <c>__Host-</c> cookie without a guaranteed Secure flag is
/// silently dropped by browsers.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Build_WithRequireHttpsCookieFalse_UsesSameAsRequest()
public async Task Build_WithRequireHttpsCookieFalse_UsesSameAsRequestAndPlainName()
{
await using WebApplication app = GatewayApplication.Build(
["--MxGateway:Dashboard:RequireHttpsCookie=false"]);
@@ -50,12 +58,14 @@ public sealed class DashboardCookieOptionsTests
DashboardAuthenticationDefaults.AuthenticationScheme);
Assert.Equal(CookieSecurePolicy.SameAsRequest, options.Cookie.SecurePolicy);
Assert.Equal(DashboardAuthenticationDefaults.CookieName, options.Cookie.Name);
}
/// <summary>
/// Verifies that <c>MxGateway:Dashboard:CookieName</c> overrides the dashboard auth
/// cookie name, so a gateway instance sharing a hostname with another can be given a
/// distinct name (browser cookies are scoped by host+path, not port).
/// Verifies that an explicit <c>MxGateway:Dashboard:CookieName</c> override wins over the
/// secure <c>__Host-</c> default (this build leaves <c>RequireHttpsCookie</c> at its
/// <see langword="true"/> default), so a gateway instance sharing a hostname with another
/// can be given a distinct name (browser cookies are scoped by host+path, not port).
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -0,0 +1,171 @@
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
/// <summary>
/// Verifies that <see cref="DashboardEventBroadcaster"/> honours
/// <c>MxGateway:Dashboard:ShowTagValues</c> (SEC-25): tag values are stripped
/// from the mirrored copy when the flag is off, present when it is on, and the
/// shared source event is never mutated.
/// </summary>
public sealed class DashboardEventBroadcasterTests
{
/// <summary>Values are stripped from the mirror when ShowTagValues is off; metadata survives.</summary>
[Fact]
public void Publish_WhenShowTagValuesFalse_RedactsValuesButKeepsMetadata()
{
CapturingHubContext hubContext = new();
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false);
MxEvent source = BuildEventWithValue();
broadcaster.Publish("session-1", source);
MxEvent sent = Assert.IsType<MxEvent>(hubContext.LastArgument);
Assert.Null(sent.Value);
Assert.Null(sent.OnAlarmTransition.CurrentValue);
Assert.Null(sent.OnAlarmTransition.LimitValue);
// Metadata unrelated to the value survives redaction.
Assert.Equal("session-1", sent.SessionId);
Assert.Equal(7, sent.ServerHandle);
Assert.Equal(11, sent.ItemHandle);
Assert.Equal(192, sent.Quality);
Assert.Equal("Tank01.Level.HiHi", sent.OnAlarmTransition.AlarmFullReference);
}
/// <summary>Redaction applies to a clone, so the shared source event keeps its values.</summary>
[Fact]
public void Publish_WhenShowTagValuesFalse_DoesNotMutateSourceEvent()
{
CapturingHubContext hubContext = new();
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false);
MxEvent source = BuildEventWithValue();
broadcaster.Publish("session-1", source);
// The redaction must apply to a clone; the shared source keeps its values.
Assert.NotNull(source.Value);
Assert.Equal(42.5, source.Value.DoubleValue);
Assert.NotNull(source.OnAlarmTransition.CurrentValue);
Assert.NotNull(source.OnAlarmTransition.LimitValue);
Assert.NotSame(source, hubContext.LastArgument);
}
/// <summary>Values pass through unredacted when ShowTagValues is on.</summary>
[Fact]
public void Publish_WhenShowTagValuesTrue_KeepsValues()
{
CapturingHubContext hubContext = new();
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: true);
MxEvent source = BuildEventWithValue();
broadcaster.Publish("session-1", source);
MxEvent sent = Assert.IsType<MxEvent>(hubContext.LastArgument);
Assert.NotNull(sent.Value);
Assert.Equal(42.5, sent.Value.DoubleValue);
Assert.NotNull(sent.OnAlarmTransition.CurrentValue);
Assert.NotNull(sent.OnAlarmTransition.LimitValue);
}
private static DashboardEventBroadcaster Create(CapturingHubContext hubContext, bool showTagValues)
{
GatewayOptions gatewayOptions = new()
{
Dashboard = new DashboardOptions { ShowTagValues = showTagValues },
};
return new DashboardEventBroadcaster(
hubContext,
Options.Create(gatewayOptions),
NullLogger<DashboardEventBroadcaster>.Instance);
}
private static MxEvent BuildEventWithValue()
{
return new MxEvent
{
Family = MxEventFamily.OnAlarmTransition,
SessionId = "session-1",
ServerHandle = 7,
ItemHandle = 11,
Quality = 192,
Value = new MxValue { DataType = MxDataType.Double, DoubleValue = 42.5 },
OnAlarmTransition = new OnAlarmTransitionEvent
{
AlarmFullReference = "Tank01.Level.HiHi",
CurrentValue = new MxValue { DataType = MxDataType.Double, DoubleValue = 88.0 },
LimitValue = new MxValue { DataType = MxDataType.Double, DoubleValue = 90.0 },
},
};
}
private sealed class CapturingHubContext : IHubContext<EventsHub>
{
private readonly CapturingHubClients _clients = new();
/// <summary>Gets the hub clients.</summary>
public IHubClients Clients => _clients;
/// <summary>Gets the group manager.</summary>
public IGroupManager Groups { get; } = new NoopGroupManager();
/// <summary>Gets the first argument of the most recent send call.</summary>
public object? LastArgument => _clients.GroupProxy.LastArgument;
}
private sealed class CapturingHubClients : IHubClients
{
/// <summary>Gets the capturing client proxy shared by this fake.</summary>
public CapturingClientProxy GroupProxy { get; } = new();
public IClientProxy All => GroupProxy;
public IClientProxy AllExcept(IReadOnlyList<string> excludedConnectionIds) => GroupProxy;
public IClientProxy Client(string connectionId) => GroupProxy;
public IClientProxy Clients(IReadOnlyList<string> connectionIds) => GroupProxy;
public IClientProxy Group(string groupName) => GroupProxy;
public IClientProxy GroupExcept(string groupName, IReadOnlyList<string> excludedConnectionIds) => GroupProxy;
public IClientProxy Groups(IReadOnlyList<string> groupNames) => GroupProxy;
public IClientProxy User(string userId) => GroupProxy;
public IClientProxy Users(IReadOnlyList<string> userIds) => GroupProxy;
}
private sealed class CapturingClientProxy : IClientProxy
{
/// <summary>Gets the first argument of the most recent send call.</summary>
public object? LastArgument { get; private set; }
/// <summary>Records the send call arguments and completes synchronously.</summary>
/// <param name="method">The SignalR method name.</param>
/// <param name="args">The method arguments.</param>
/// <param name="cancellationToken">Token to observe for cancellation.</param>
/// <returns>A completed task.</returns>
public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default)
{
LastArgument = args.Length > 0 ? args[0] : null;
return Task.CompletedTask;
}
}
private sealed class NoopGroupManager : IGroupManager
{
public Task AddToGroupAsync(string connectionId, string groupName, CancellationToken cancellationToken = default)
=> Task.CompletedTask;
public Task RemoveFromGroupAsync(string connectionId, string groupName, CancellationToken cancellationToken = default)
=> Task.CompletedTask;
}
}
@@ -0,0 +1,642 @@
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.MxGateway.Contracts;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using ZB.MOM.WW.MxGateway.Server.Configuration;
using ZB.MOM.WW.MxGateway.Server.Grpc;
using ZB.MOM.WW.MxGateway.Server.Metrics;
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
using ZB.MOM.WW.MxGateway.Server.Sessions;
using ZB.MOM.WW.MxGateway.Server.Workers;
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
namespace ZB.MOM.WW.MxGateway.Tests.Gateway;
/// <summary>
/// End-to-end reconnect/replay tests through the real gRPC <c>StreamEvents</c> path via the
/// fake worker harness (TST-01, server half). A single subscriber detaches (cancel + await
/// the stream task so its lease is fully disposed), the session is retained by detach-grace
/// while the distributor pump keeps appending worker events to the replay ring, and a second
/// stream reconnects with <see cref="StreamEventsRequest.AfterWorkerSequence"/>. Covers both
/// the no-gap resume (cursor inside the retained window) and the <c>ReplayGap</c> sentinel
/// (cursor predates the oldest retained event after capacity eviction).
/// </summary>
/// <remarks>
/// These tests run in single-subscriber mode (<c>AllowMultipleEventSubscribers=false</c>).
/// Reconnect works because the first stream is FULLY detached before the reconnect attaches:
/// awaiting the first stream task runs <c>EventStreamService</c>'s finally block, which
/// disposes the subscriber lease and drops the session's active-subscriber count back to
/// zero, so the reconnect's <c>AttachEventSubscriberWithReplay</c> sees an empty slot rather
/// than an "already active" rejection. The distributor (and its replay ring) is created once
/// per session and survives detach, so events emitted while no subscriber is attached are
/// retained for the reconnecting stream.
/// </remarks>
public sealed class GatewayEndToEndReconnectReplayTests
{
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(10);
private const int ServerHandle = 3001;
private const int ItemHandle = 4002;
/// <summary>
/// Reconnecting inside the retained window replays exactly the events newer than the
/// resume cursor — the retained tail of the first batch plus the events emitted while
/// detached — in strictly ascending order, with no duplicates and no <c>ReplayGap</c>
/// sentinel.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_ReconnectInsideRetainedWindow_ReplaysTailNoGap()
{
const int firstBatch = 4;
const int secondBatch = 3;
GatedEventFakeWorkerProcessLauncher launcher = new();
// Capacity 16 retains every event emitted here (7 total), so nothing is evicted and a
// resume from a middle cursor never produces a gap.
await using ReconnectReplayGatewayServiceFixture fixture = new(launcher, replayBufferCapacity: 16);
string sessionId = await OpenSessionAsync(fixture, "reconnect-no-gap");
// ---- first connection: receive the first batch, capture its real sequences ----
using CancellationTokenSource writer1Cts = new();
RecordingServerStreamWriter<MxEvent> writer1 = new();
Task stream1Task = Task.Run(async () =>
await fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
writer1,
new TestServerCallContext(cancellationToken: writer1Cts.Token)));
await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
await WireUpAdviseAsync(fixture, sessionId);
for (int i = 0; i < firstBatch; i++)
{
launcher.AllowNextEvent();
}
IReadOnlyList<MxEvent> batch1 = await writer1.WaitForMessageCountAsync(firstBatch, TestTimeout);
ulong[] batch1Sequences = batch1.Select(e => e.WorkerSequence).ToArray();
// Resume cursor: a middle event of the first batch. Everything strictly newer than this
// must be redelivered to the reconnecting stream.
ulong cursor = batch1Sequences[1];
int retainedTailFromBatch1 = batch1Sequences.Count(s => s > cursor);
Assert.Equal(2, retainedTailFromBatch1); // sanity: events at index 2 and 3
// ---- fully detach writer1 (cancel + await so its lease is disposed) ----
await DetachAsync(writer1Cts, stream1Task);
// ---- emit more events while detached; the ring must retain them for the reconnect ----
for (int i = 0; i < secondBatch; i++)
{
launcher.AllowNextEvent();
}
// ---- reconnect with the middle cursor ----
RecordingServerStreamWriter<MxEvent> writer2 = new();
Task stream2Task = Task.Run(async () =>
await fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId, AfterWorkerSequence = cursor },
writer2,
new TestServerCallContext()));
await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
int expected = retainedTailFromBatch1 + secondBatch;
IReadOnlyList<MxEvent> resumed = await writer2.WaitForMessageCountAsync(expected, TestTimeout);
// ---- tear down before asserting so a hang can't wedge the run ----
launcher.StopEmitting();
await CloseAndDrainAsync(fixture, sessionId, stream2Task, launcher);
// ---- assertions ----
Assert.Equal(expected, resumed.Count);
// No sentinel: every message is a real event, none carries a ReplayGap.
Assert.All(resumed, e => Assert.Null(e.ReplayGap));
// Every replayed/live event is strictly newer than the cursor.
Assert.All(resumed, e => Assert.True(
e.WorkerSequence > cursor,
$"Event sequence {e.WorkerSequence} is not newer than cursor {cursor}."));
// Strictly ascending, no duplicates.
for (int i = 1; i < resumed.Count; i++)
{
Assert.True(
resumed[i].WorkerSequence > resumed[i - 1].WorkerSequence,
$"Sequences must be strictly ascending: {resumed[i - 1].WorkerSequence} then {resumed[i].WorkerSequence}.");
}
Assert.Equal(resumed.Count, resumed.Select(e => e.WorkerSequence).Distinct().Count());
// The retained tail of the first batch (the two events newer than the cursor) is
// replayed first, before the events emitted while detached.
Assert.Equal(batch1Sequences[2], resumed[0].WorkerSequence);
Assert.Equal(batch1Sequences[3], resumed[1].WorkerSequence);
}
/// <summary>
/// Reconnecting with a cursor that predates the oldest retained event (after capacity
/// eviction) yields the <c>ReplayGap</c> sentinel FIRST — family unspecified, no body, no
/// per-item fields, correct requested/oldest sequences — followed by exactly the retained
/// tail, in ascending order.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task StreamEvents_ReconnectWithStaleCursor_EmitsReplayGapSentinelFirst()
{
const int capacity = 3;
const int totalEvents = 6;
GatedEventFakeWorkerProcessLauncher launcher = new();
// Small capacity forces eviction: with 6 events emitted and a 3-slot ring, the first 3
// are evicted and only the newest 3 remain replayable.
await using ReconnectReplayGatewayServiceFixture fixture = new(launcher, replayBufferCapacity: capacity);
string sessionId = await OpenSessionAsync(fixture, "reconnect-gap");
using CancellationTokenSource writer1Cts = new();
RecordingServerStreamWriter<MxEvent> writer1 = new();
Task stream1Task = Task.Run(async () =>
await fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId },
writer1,
new TestServerCallContext(cancellationToken: writer1Cts.Token)));
await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
await WireUpAdviseAsync(fixture, sessionId);
// Emit more events than the ring can hold. Waiting for writer1 to receive all of them
// proves the pump has appended (and evicted) every event, so the ring is settled to its
// final newest-`capacity` contents before we reconnect.
for (int i = 0; i < totalEvents; i++)
{
launcher.AllowNextEvent();
}
IReadOnlyList<MxEvent> all = await writer1.WaitForMessageCountAsync(totalEvents, TestTimeout);
ulong[] sequences = all.Select(e => e.WorkerSequence).ToArray();
// With capacity C and N total events, the newest C are retained; the oldest still
// available is the (N-C+1)th event — index N-C in emission (ascending) order.
ulong oldestAvailable = sequences[totalEvents - capacity];
await DetachAsync(writer1Cts, stream1Task);
// Resume after sequence 1: the real event sequences are envelope-numbered and start well
// above 1 (startup + command-reply envelopes consume the low numbers), so cursor 1
// predates every event and, with the oldest three already evicted, forces a gap.
const ulong staleCursor = 1;
RecordingServerStreamWriter<MxEvent> writer2 = new();
Task stream2Task = Task.Run(async () =>
await fixture.Service.StreamEvents(
new StreamEventsRequest { SessionId = sessionId, AfterWorkerSequence = staleCursor },
writer2,
new TestServerCallContext()));
await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
// sentinel + the retained tail (capacity events, all newer than the stale cursor).
int expected = 1 + capacity;
IReadOnlyList<MxEvent> resumed = await writer2.WaitForMessageCountAsync(expected, TestTimeout);
launcher.StopEmitting();
await CloseAndDrainAsync(fixture, sessionId, stream2Task, launcher);
// ---- the first message is the ReplayGap sentinel ----
Assert.Equal(expected, resumed.Count);
MxEvent sentinel = resumed[0];
Assert.NotNull(sentinel.ReplayGap);
Assert.Equal(MxEventFamily.Unspecified, sentinel.Family);
Assert.Equal(MxEvent.BodyOneofCase.None, sentinel.BodyCase);
Assert.Equal(sessionId, sentinel.SessionId);
Assert.Equal(staleCursor, sentinel.ReplayGap.RequestedAfterSequence);
Assert.Equal(oldestAvailable, sentinel.ReplayGap.OldestAvailableSequence);
// ---- the sentinel is followed by the retained tail, ascending, no further sentinels ----
IReadOnlyList<MxEvent> tail = resumed.Skip(1).ToArray();
Assert.Equal(capacity, tail.Count);
Assert.All(tail, e => Assert.Null(e.ReplayGap));
Assert.All(tail, e => Assert.True(
e.WorkerSequence >= oldestAvailable,
$"Retained event {e.WorkerSequence} is older than the oldest available {oldestAvailable}."));
ulong[] expectedTail = sequences.Skip(totalEvents - capacity).ToArray();
Assert.Equal(expectedTail, tail.Select(e => e.WorkerSequence).ToArray());
}
// ---- shared flow helpers ----
private static async Task<string> OpenSessionAsync(
ReconnectReplayGatewayServiceFixture fixture,
string name)
{
OpenSessionReply openReply = await fixture.Service.OpenSession(
new OpenSessionRequest
{
ClientSessionName = name,
ClientCorrelationId = $"open-{name}",
CommandTimeout = Duration.FromTimeSpan(TestTimeout),
},
new TestServerCallContext()).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
return openReply.SessionId;
}
private static async Task WireUpAdviseAsync(
ReconnectReplayGatewayServiceFixture fixture,
string sessionId)
{
MxCommandReply registerReply = await fixture.Service.Invoke(
CreateRegisterRequest(sessionId),
new TestServerCallContext()).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
MxCommandReply addItemReply = await fixture.Service.Invoke(
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
new TestServerCallContext()).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
MxCommandReply adviseReply = await fixture.Service.Invoke(
CreateAdviseRequest(sessionId, registerReply.Register.ServerHandle, addItemReply.AddItem.ItemHandle),
new TestServerCallContext()).ConfigureAwait(false);
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
}
// Cancels the stream's token and awaits the stream task. Awaiting is load-bearing: it lets
// EventStreamService's finally block dispose the subscriber lease (dropping the session's
// active-subscriber count to zero) BEFORE the reconnect attaches, so the reconnect never
// races a still-registered subscriber.
private static async Task DetachAsync(CancellationTokenSource cts, Task streamTask)
{
await cts.CancelAsync().ConfigureAwait(false);
try
{
await streamTask.WaitAsync(TestTimeout).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
// Expected: the iterator surfaces the cancellation.
}
catch (RpcException rpc) when (rpc.StatusCode == StatusCode.Cancelled)
{
// Also acceptable depending on gRPC exception wrapping.
}
}
private static async Task CloseAndDrainAsync(
ReconnectReplayGatewayServiceFixture fixture,
string sessionId,
Task streamTask,
GatedEventFakeWorkerProcessLauncher launcher)
{
await fixture.Service.CloseSession(
new CloseSessionRequest { SessionId = sessionId, ClientCorrelationId = "close-reconnect" },
new TestServerCallContext()).ConfigureAwait(false);
try
{
await streamTask.WaitAsync(TestTimeout).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
}
await launcher.WorkerTask.WaitAsync(TestTimeout).ConfigureAwait(false);
}
// ---- request builders ----
private static MxCommandRequest CreateRegisterRequest(string sessionId) =>
new()
{
SessionId = sessionId,
ClientCorrelationId = "register-rr",
Command = new MxCommand
{
Kind = MxCommandKind.Register,
Register = new RegisterCommand { ClientName = "reconnect-replay-e2e-client" },
},
};
private static MxCommandRequest CreateAddItemRequest(string sessionId, int serverHandle) =>
new()
{
SessionId = sessionId,
ClientCorrelationId = "add-item-rr",
Command = new MxCommand
{
Kind = MxCommandKind.AddItem,
AddItem = new AddItemCommand
{
ServerHandle = serverHandle,
ItemDefinition = "Galaxy.Tag.Value",
},
},
};
private static MxCommandRequest CreateAdviseRequest(
string sessionId,
int serverHandle,
int itemHandle) =>
new()
{
SessionId = sessionId,
ClientCorrelationId = "advise-rr",
Command = new MxCommand
{
Kind = MxCommandKind.Advise,
Advise = new AdviseCommand { ServerHandle = serverHandle, ItemHandle = itemHandle },
},
};
private static void ConfigureCommandReply(MxCommandReply reply, MxCommandKind kind)
{
switch (kind)
{
case MxCommandKind.Register:
reply.Register = new RegisterReply { ServerHandle = ServerHandle };
break;
case MxCommandKind.AddItem:
reply.AddItem = new AddItemReply { ItemHandle = ItemHandle };
break;
}
}
// ---- fixture ----
/// <summary>
/// Gateway service fixture in single-subscriber mode with a configurable replay ring
/// capacity, so each test can retain the whole event history (no gap) or force capacity
/// eviction (gap).
/// </summary>
private sealed class ReconnectReplayGatewayServiceFixture : IAsyncDisposable
{
private readonly GatewayMetrics _metrics = new();
private readonly SessionRegistry _registry = new();
/// <summary>Initializes a new instance of the <see cref="ReconnectReplayGatewayServiceFixture"/> class.</summary>
/// <param name="launcher">Fake worker process launcher backing the session manager.</param>
/// <param name="replayBufferCapacity">Replay ring capacity for the session's event distributor.</param>
public ReconnectReplayGatewayServiceFixture(
IWorkerProcessLauncher launcher,
int replayBufferCapacity)
{
IOptions<GatewayOptions> options = Options.Create(CreateOptions(replayBufferCapacity));
SessionWorkerClientFactory workerClientFactory = new(
launcher,
options,
_metrics,
NullLoggerFactory.Instance);
SessionManager sessionManager = new(
_registry,
workerClientFactory,
options,
_metrics,
logger: NullLogger<SessionManager>.Instance,
dashboardEventBroadcaster: NullDashboardEventBroadcaster.Instance);
MxAccessGrpcMapper mapper = new();
EventStreamService eventStreamService = new(
sessionManager,
options,
_metrics);
Service = new MxAccessGatewayService(
sessionManager,
new GatewayRequestIdentityAccessor(),
new AllowAllConstraintEnforcer(),
new MxAccessGrpcRequestValidator(),
mapper,
eventStreamService,
_metrics,
NullLogger<MxAccessGatewayService>.Instance,
new FakeGatewayAlarmService());
}
/// <summary>Gets the gateway service under test.</summary>
public MxAccessGatewayService Service { get; }
/// <summary>
/// Polls <see cref="GatewaySession.ActiveEventSubscriberCount"/> for
/// <paramref name="sessionId"/> until it reaches <paramref name="n"/>, bounded by
/// <paramref name="timeout"/>. Fails the test on timeout. This is the deterministic
/// gate that proves the production code has (re)registered a subscriber before the
/// test drives events or reconnects.
/// </summary>
/// <param name="sessionId">Identifier of the session to poll.</param>
/// <param name="n">Target subscriber count to wait for.</param>
/// <param name="timeout">Maximum time to wait before failing the test.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task WaitForSubscriberCountAsync(string sessionId, int n, TimeSpan timeout)
{
using CancellationTokenSource deadlineCts = new(timeout);
while (true)
{
if (_registry.TryGet(sessionId, out GatewaySession? session)
&& session.ActiveEventSubscriberCount >= n)
{
return;
}
if (deadlineCts.IsCancellationRequested)
{
int actual = _registry.TryGet(sessionId, out GatewaySession? s)
? s.ActiveEventSubscriberCount
: -1;
Assert.Fail(
$"Timed out waiting for {n} event subscriber(s) on session {sessionId}. "
+ $"Actual count after {timeout.TotalSeconds:0.#}s: {actual}.");
}
await Task.Delay(millisecondsDelay: 5, deadlineCts.Token).ConfigureAwait(false);
}
}
/// <summary>Disposes every session in the registry and releases the fixture's metrics.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
public async ValueTask DisposeAsync()
{
foreach (GatewaySession session in _registry.Snapshot())
{
await session.DisposeAsync().ConfigureAwait(false);
}
_metrics.Dispose();
}
private static GatewayOptions CreateOptions(int replayBufferCapacity) =>
new()
{
Worker = new WorkerOptions
{
StartupTimeoutSeconds = 5,
ShutdownTimeoutSeconds = 5,
HeartbeatIntervalSeconds = 30,
HeartbeatGraceSeconds = 30,
MaxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
},
Sessions = new SessionOptions
{
DefaultCommandTimeoutSeconds = 5,
MaxSessions = 4,
// Single-subscriber mode: a fully-detached subscriber (stream task awaited)
// frees the sole slot, so the reconnect attaches cleanly without needing
// multi-subscriber fan-out. Detach-grace keeps the session Ready across the
// detach and the fixture runs no lease-reaper, so the session survives to be
// reconnected.
AllowMultipleEventSubscribers = false,
MaxEventSubscribersPerSession = 8,
},
Events = new EventOptions
{
QueueCapacity = 32,
ReplayBufferCapacity = replayBufferCapacity,
// Keep age-eviction effectively off for the duration of a fast test so
// capacity is the only eviction axis under test.
ReplayRetentionSeconds = 300,
},
};
}
// ---- fake worker launcher ----
/// <summary>
/// Fake worker that emits events one at a time, gated by <see cref="AllowNextEvent"/>, so
/// the test drives event timing deterministically. Modeled on the multi-subscriber E2E
/// tests' gated launcher. The worker loop is independent of subscribers, so events emitted
/// while no gRPC stream is attached still flow through the distributor pump into the
/// session's replay ring — exactly the condition the reconnect/replay tests exercise. Call
/// <see cref="StopEmitting"/> before closing the session so the loop exits cleanly and can
/// process the shutdown envelope.
/// </summary>
private sealed class GatedEventFakeWorkerProcessLauncher : IWorkerProcessLauncher
{
public const int ProcessId = 7730;
private readonly FakeWorkerProcess _process = new(ProcessId);
// Capacity 64 so AllowNextEvent can be called ahead of time without blocking.
private readonly SemaphoreSlim _emitGate = new(0, 64);
private volatile bool _stopEmitting;
/// <summary>Gets the task representing the fake worker's running background loop.</summary>
public Task WorkerTask { get; private set; } = Task.CompletedTask;
/// <summary>Releases the gate so the worker emits one event.</summary>
public void AllowNextEvent() => _emitGate.Release();
/// <summary>
/// Signals the worker to stop waiting for the emit gate and process the shutdown
/// envelope. Must be called before <c>CloseSession</c>.
/// </summary>
public void StopEmitting()
{
_stopEmitting = true;
_emitGate.Release(); // unblock a pending gate wait if any
}
/// <inheritdoc />
public Task<WorkerProcessHandle> LaunchAsync(
WorkerProcessLaunchRequest request,
CancellationToken cancellationToken = default)
{
WorkerTask = RunWorkerAsync(request, cancellationToken);
return Task.FromResult(new WorkerProcessHandle(
_process,
new WorkerProcessCommandLine("reconnect-replay-fake-worker.exe", []),
DateTimeOffset.UtcNow));
}
private async Task RunWorkerAsync(
WorkerProcessLaunchRequest request,
CancellationToken cancellationToken)
{
await using FakeWorkerHarness harness = await FakeWorkerHarness.ConnectToGatewayPipeAsync(
request.SessionId,
request.Nonce,
request.PipeName,
request.ProtocolVersion,
cancellationToken: cancellationToken).ConfigureAwait(false);
await harness.CompleteStartupAsync(ProcessId, cancellationToken: cancellationToken).ConfigureAwait(false);
int advisedServerHandle = 0;
int advisedItemHandle = 0;
int emittedCount = 0;
while (!cancellationToken.IsCancellationRequested)
{
// While subscribed and not stopped, emit gated events using a non-blocking peek
// at the gate so incoming envelopes (including shutdown) are never starved.
while (advisedServerHandle != 0
&& !_stopEmitting
&& await _emitGate.WaitAsync(millisecondsTimeout: 0).ConfigureAwait(false))
{
int index = ++emittedCount;
await harness.EmitEventAsync(
MxEventFamily.OnDataChange,
cancellationToken,
mxEvent =>
{
mxEvent.ServerHandle = advisedServerHandle;
mxEvent.ItemHandle = advisedItemHandle;
mxEvent.Quality = 192;
mxEvent.Value = new MxValue
{
DataType = MxDataType.String,
StringValue = $"reconnect-value-{index}",
};
mxEvent.OnDataChange = new OnDataChangeEvent();
}).ConfigureAwait(false);
}
WorkerEnvelope? envelope;
try
{
using CancellationTokenSource readCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
readCts.CancelAfter(TimeSpan.FromMilliseconds(50));
envelope = await harness.ReadGatewayEnvelopeAsync(readCts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
// Timed out waiting for an envelope — loop back to check the gate / emit.
continue;
}
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerShutdown)
{
await harness.SendShutdownAckAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
_process.MarkExited(0);
return;
}
if (envelope.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCommand)
{
throw new InvalidOperationException($"Unexpected envelope {envelope.BodyCase}.");
}
MxCommand command = envelope.WorkerCommand.Command;
await harness.ReplyToCommandAsync(
envelope,
configureReply: reply => ConfigureCommandReply(reply, command.Kind),
cancellationToken: cancellationToken).ConfigureAwait(false);
if (command.Kind == MxCommandKind.Advise)
{
advisedServerHandle = command.Advise.ServerHandle;
advisedItemHandle = command.Advise.ItemHandle;
}
}
}
}
}
@@ -76,4 +76,35 @@ public sealed class MxAccessGrpcMapperTests
Assert.Equal(ProtocolStatusCode.ProtocolViolation, publicReply.ProtocolStatus.Code);
}
/// <summary>
/// Verifies MapEvent transfers ownership of the inner MxEvent (GWC-07 / IPC-05): the
/// returned reference is the same instance carried by the WorkerEvent, not a clone. The
/// WorkerEvent is discarded after mapping and the distributor pump is its single consumer,
/// so moving the inner event out is safe and avoids a per-event deep copy.
/// </summary>
[Fact]
public void MapEvent_TransfersOwnershipOfInnerEventWithoutCloning()
{
MxEvent innerEvent = new()
{
Family = MxEventFamily.OnDataChange,
RawStatus = "OK",
};
WorkerEvent workerEvent = new() { Event = innerEvent };
MxEvent mapped = new MxAccessGrpcMapper().MapEvent(workerEvent);
Assert.Same(innerEvent, mapped);
}
/// <summary>Verifies that a worker event with no public payload maps to the unspecified-family sentinel.</summary>
[Fact]
public void MapEvent_WhenEventMissing_ReturnsUnspecifiedFamilySentinel()
{
MxEvent mapped = new MxAccessGrpcMapper().MapEvent(new WorkerEvent());
Assert.Equal(MxEventFamily.Unspecified, mapped.Family);
Assert.Equal("Worker event did not contain a public event payload.", mapped.RawStatus);
}
}
@@ -185,6 +185,90 @@ public sealed class SessionEventDistributorTests
Assert.Equal(new ulong[] { 3, 4, 5 }, replay.Select(e => e.WorkerSequence));
}
/// <summary>
/// Appending far more events than the capacity wraps the circular buffer's head
/// index past the array boundary multiple times; the retained window stays the
/// newest <c>capacity</c> events, in ascending order, and a replay from before the
/// window reports a gap and returns the whole retained tail.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ReplayBuffer_WrapsRingMultipleTimes_RetainsNewestInAscendingOrder()
{
Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
await using SessionEventDistributor distributor = CreateDistributor(
source.Reader,
replayBufferCapacity: 4,
replayRetentionSeconds: 0);
await distributor.StartAsync(CancellationToken.None);
// 13 events through a capacity-4 ring advances the head index 13 - 4 = 9 slots,
// wrapping the 4-slot array's boundary more than twice.
using IEventSubscriberLease lease = distributor.Register();
for (ulong sequence = 1; sequence <= 13; sequence++)
{
source.Writer.TryWrite(Event(sequence));
}
for (ulong sequence = 1; sequence <= 13; sequence++)
{
MxEvent e = await ReadOneAsync(lease.Reader);
Assert.Equal(sequence, e.WorkerSequence);
}
// Newest four (10, 11, 12, 13) retained in ascending order despite the wraps.
bool found = distributor.TryGetReplayFrom(0, out IReadOnlyList<MxEvent> replay, out bool gap);
Assert.True(found);
Assert.True(gap);
Assert.Equal(new ulong[] { 10, 11, 12, 13 }, replay.Select(e => e.WorkerSequence));
// A replay from inside the wrapped window returns only the newer entries, no gap,
// proving the modular scan reads the logical order and not the physical slots.
bool foundInner = distributor.TryGetReplayFrom(11, out IReadOnlyList<MxEvent> inner, out bool innerGap);
Assert.True(foundInner);
Assert.False(innerGap);
Assert.Equal(new ulong[] { 12, 13 }, inner.Select(e => e.WorkerSequence));
}
/// <summary>
/// A capacity-1 ring retains only the single newest event; each append overwrites
/// the sole slot, and a replay from before it reports a gap.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ReplayBuffer_Capacity1_RetainsOnlyNewest()
{
Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
await using SessionEventDistributor distributor = CreateDistributor(
source.Reader,
replayBufferCapacity: 1,
replayRetentionSeconds: 0);
await distributor.StartAsync(CancellationToken.None);
using IEventSubscriberLease lease = distributor.Register();
for (ulong sequence = 1; sequence <= 3; sequence++)
{
source.Writer.TryWrite(Event(sequence));
_ = await ReadOneAsync(lease.Reader);
}
// Only sequence 3 is retained; a request from 0 missed 1 and 2 => gap.
bool found = distributor.TryGetReplayFrom(0, out IReadOnlyList<MxEvent> replay, out bool gap);
Assert.True(found);
Assert.True(gap);
Assert.Equal(new ulong[] { 3 }, replay.Select(e => e.WorkerSequence));
// A request from exactly the newest retained sequence is caught up: empty, no gap.
bool foundCaughtUp = distributor.TryGetReplayFrom(3, out IReadOnlyList<MxEvent> caughtUp, out bool caughtUpGap);
Assert.True(foundCaughtUp);
Assert.False(caughtUpGap);
Assert.Empty(caughtUp);
}
/// <summary>Requesting replay from a sequence still inside the retained window returns only the newer events, with no gap.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -29,6 +29,32 @@ public sealed class WorkerFrameProtocolTests
Assert.Equal(original, parsed);
}
/// <summary>Verifies that a large payload near the message cap round-trips through the pooled write/read path.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WriteAndReadAsync_WithLargePayloadNearMax_RoundTripsFrame()
{
// A payload comfortably larger than an ArrayPool bucket boundary so the rented buffer is
// larger than the exact frame length, exercising the length-bounded read and parse.
const int maxMessageBytes = 1024 * 1024;
WorkerFrameProtocolOptions options =
new(SessionId, GatewayContractInfo.WorkerProtocolVersion, maxMessageBytes);
await using MemoryStream stream = new();
WorkerEnvelope original = CreateEnvelope();
original.WorkerHello.WorkerVersion = new string('x', maxMessageBytes - 4096);
Assert.InRange(original.CalculateSize(), 1, maxMessageBytes);
WorkerFrameWriter writer = new(stream, options);
await writer.WriteAsync(original);
stream.Position = 0;
WorkerFrameReader reader = new(stream, options);
WorkerEnvelope parsed = await reader.ReadAsync();
Assert.Equal(original, parsed);
}
/// <summary>Verifies that reading a frame with partial reads reassembles the frame correctly.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -20,7 +20,9 @@ public sealed class GatewayMetricsTests
metrics.EventReceived("session-1", "OnDataChange");
metrics.EventReceived("session-1", "OnDataChange");
metrics.SetWorkerEventQueueDepth(7);
metrics.AdjustGrpcEventStreamQueueDepth(3);
// GWC-15: the gRPC stream queue-depth gauge sums live backlog sources at collection time
// rather than tracking a pushed running total. Register a source reporting 3.
using IDisposable backlogSource = metrics.RegisterEventStreamBacklogSource(static () => 3);
metrics.QueueOverflow("session-events");
metrics.Fault("CommandTimeout");
metrics.WorkerKilled("CommandTimeout");
@@ -106,6 +106,35 @@ public sealed class MxStatusProxyConverterTests
Assert.Contains("success", exception.Message);
}
/// <summary>
/// Verifies that repeated conversions of the same status type produce
/// identical results. WRK-06 caches the resolved FieldInfo objects per
/// type after the first conversion; the cached path must yield the same
/// message the uncached first call produced.
/// </summary>
[Fact]
public void Convert_RepeatedForSameType_ProducesIdenticalResults()
{
FakeMxStatusProxy status = new()
{
success = 0,
category = 3,
detectedBy = 1,
detail = 21,
};
// First call populates the per-type FieldInfo cache; the second reuses it.
MxStatusProxy first = _converter.Convert(status);
MxStatusProxy second = _converter.Convert(status);
Assert.Equal(first, second);
Assert.Equal(MxStatusCategory.CommunicationError, second.Category);
Assert.Equal(MxStatusSource.RespondingLmx, second.DetectedBy);
Assert.Equal(0, second.Success);
Assert.Equal(21, second.Detail);
Assert.Equal("Invalid reference", second.DiagnosticText);
}
public struct FakeMxStatusProxy
{
public short success;
@@ -373,6 +373,43 @@ public sealed class WorkerFrameProtocolTests
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, frame3.BodyCase);
}
/// <summary>
/// Verifies the writer coalesces the flush across a batch of frames drained together: four frames
/// queued behind an in-progress write drain in a single pass and share one FlushAsync, not four.
/// Every frame still reaches the wire intact.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task WriteAsync_WhenBatchDrainedTogether_FlushesOnce()
{
WorkerFrameProtocolOptions options = CreateOptions();
using GatedWriteStream stream = new();
WorkerFrameWriter writer = new(stream, options);
// A blocked first write occupies the writer and holds the lock while more frames queue behind it.
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
Task eventWrite1 = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event);
Task eventWrite2 = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event);
Task eventWrite3 = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event);
await Task.Delay(50);
stream.ReleaseFirstWrite();
await AwaitWithTimeoutAsync(Task.WhenAll(firstWrite, eventWrite1, eventWrite2, eventWrite3));
// Four frames written in one drain pass => exactly one flush.
Assert.Equal(1, stream.FlushCount);
stream.Position = 0;
WorkerFrameReader reader = new(stream, options);
for (int index = 0; index < 4; index++)
{
WorkerEnvelope frame = await reader.ReadAsync();
Assert.NotEqual(WorkerEnvelope.BodyOneofCase.None, frame.BodyCase);
}
}
/// <summary>Verifies a zero negotiated frame maximum keeps the constructor default.</summary>
[Fact]
public void AdoptNegotiatedMaxMessageBytes_WithZero_KeepsDefault()
@@ -451,10 +488,14 @@ public sealed class WorkerFrameProtocolTests
private readonly TaskCompletionSource<bool> _firstWriteStarted =
new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
private int _writeCount;
private int _flushCount;
/// <summary>Gets a task that completes once the first <see cref="WriteAsync"/> call has started blocking.</summary>
public Task FirstWriteStarted => _firstWriteStarted.Task;
/// <summary>Gets the number of <see cref="FlushAsync"/> calls observed so far.</summary>
public int FlushCount => Volatile.Read(ref _flushCount);
/// <summary>Releases the first blocked write so it can complete.</summary>
public void ReleaseFirstWrite() => _release.Release();
@@ -470,6 +511,13 @@ public sealed class WorkerFrameProtocolTests
await base.WriteAsync(buffer, offset, count, cancellationToken);
}
/// <inheritdoc />
public override Task FlushAsync(CancellationToken cancellationToken)
{
Interlocked.Increment(ref _flushCount);
return base.FlushAsync(cancellationToken);
}
/// <inheritdoc />
protected override void Dispose(bool disposing)
{
@@ -29,6 +29,26 @@ public sealed class MxAccessEventQueueTests
Assert.False(queue.TryDequeue(out _));
}
/// <summary>
/// Verifies that Enqueue takes ownership of the passed event instead of
/// cloning it: the dequeued instance is the very reference passed in, and
/// the worker sequence/timestamp are stamped on that same instance
/// (WRK-11).
/// </summary>
[Fact]
public void Enqueue_TakesOwnershipOfPassedEventInstance()
{
MxAccessEventQueue queue = new(capacity: 4);
MxEvent original = CreateEvent(MxEventFamily.OnDataChange, itemHandle: 10);
queue.Enqueue(original);
Assert.True(queue.TryDequeue(out WorkerEvent? dequeued));
Assert.Same(original, dequeued?.Event);
Assert.Equal(1UL, original.WorkerSequence);
Assert.NotNull(original.WorkerTimestamp);
}
/// <summary>Verifies that Drain removes at most the requested number of events.</summary>
[Fact]
public void Drain_RemovesAtMostRequestedEvents()
@@ -46,6 +46,37 @@ public sealed class MxAccessValueCacheTests
Assert.Equal(999, other.Value.Int32Value);
}
/// <summary>
/// Verifies that Set stores an independent deep-copied snapshot: mutating
/// the source event's protobuf sub-messages after caching does not alter
/// the cached value. WRK-11 stopped the event sink cloning before enqueue,
/// so the same MxEvent instance now flows to the outbound queue; the cache
/// must own its own copy so the two never share mutable state.
/// </summary>
[Fact]
public void Set_StoresIndependentSnapshot_UnaffectedByLaterEventMutation()
{
MxAccessValueCache cache = new();
Timestamp sourceTimestamp = Timestamp.FromDateTime(new(2026, 5, 19, 9, 0, 0, DateTimeKind.Utc));
MxEvent mxEvent = BuildEvent(serverHandle: 7, itemHandle: 21, intValue: 100, quality: 192, sourceTimestamp);
cache.Set(7, 21, mxEvent);
// Mutate the event in place after it was cached — as if it kept flowing
// through the (unrelated) outbound path. None of this must reach the cache.
mxEvent.Value.Int32Value = 999;
mxEvent.Quality = 0;
mxEvent.SourceTimestamp = Timestamp.FromDateTime(new(2030, 1, 1, 0, 0, 0, DateTimeKind.Utc));
mxEvent.Statuses[0].Category = MxStatusCategory.SecurityError;
Assert.True(cache.TryGet(7, 21, out MxAccessValueCache.CachedValue cached));
Assert.Equal(100, cached.Value.Int32Value);
Assert.Equal(192, cached.Quality);
Assert.Equal(sourceTimestamp, cached.SourceTimestamp);
Assert.Single(cached.Statuses);
Assert.Equal(MxStatusCategory.Ok, cached.Statuses[0].Category);
}
/// <summary>Verifies that TryGet returns false for unknown handles.</summary>
[Fact]
public void TryGet_WithUnknownHandle_ReturnsFalse()
@@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.Reflection;
@@ -9,6 +10,17 @@ namespace ZB.MOM.WW.MxGateway.Worker.Conversion;
/// <summary>Converts MXAccess MXSTATUS_PROXY COM objects to protobuf MxStatusProxy messages.</summary>
public sealed class MxStatusProxyConverter
{
/// <summary>
/// Per-type cache of the four resolved <see cref="FieldInfo"/> objects a
/// status conversion needs. The status type is stable (the interop
/// <c>MXSTATUS_PROXY</c> struct in production; a fixed test double in
/// tests), so the expensive <see cref="Type.GetField(string, BindingFlags)"/>
/// metadata scan is resolved once per type and reused. Keyed by
/// <see cref="Type"/> so a plain-CLR test double and the real interop
/// struct each get their own entry, keeping the converter interop-agnostic.
/// </summary>
private static readonly ConcurrentDictionary<Type, StatusFields> FieldCache = new();
/// <summary>Converts a single status object to a protobuf message, reflecting all fields and diagnostics.</summary>
/// <param name="status">COM status object to convert.</param>
/// <returns>The converted protobuf status message.</returns>
@@ -20,10 +32,11 @@ public sealed class MxStatusProxyConverter
}
Type statusType = status.GetType();
int success = ReadInt32Field(status, statusType, "success");
int rawCategory = ReadInt32Field(status, statusType, "category");
int rawDetectedBy = ReadInt32Field(status, statusType, "detectedBy");
int detail = ReadInt32Field(status, statusType, "detail");
StatusFields fields = GetFields(statusType);
int success = ReadInt32Field(status, statusType, fields.Success);
int rawCategory = ReadInt32Field(status, statusType, fields.Category);
int rawDetectedBy = ReadInt32Field(status, statusType, fields.DetectedBy);
int detail = ReadInt32Field(status, statusType, fields.Detail);
return new MxStatusProxy
{
@@ -82,6 +95,43 @@ public sealed class MxStatusProxyConverter
private static int ReadInt32Field(
object value,
Type valueType,
FieldInfo field)
{
object? fieldValue = field.GetValue(value);
if (fieldValue is null)
{
throw new MxStatusConversionException(
$"Status object field '{field.Name}' on type '{valueType.FullName}' is null.");
}
return System.Convert.ToInt32(fieldValue, CultureInfo.InvariantCulture);
}
/// <summary>
/// Resolves (and caches) the four <see cref="FieldInfo"/> objects for the
/// given status type. The first resolution for a type performs the
/// reflection scan; every subsequent conversion of that type reuses the
/// cached entry. A type missing a required field throws the same
/// <see cref="MxStatusConversionException"/> the per-field lookup used to
/// throw — and, because <see cref="ConcurrentDictionary{TKey,TValue}.GetOrAdd(TKey, Func{TKey, TValue})"/>
/// does not store a value when the factory throws, a bad type keeps
/// failing identically on every call rather than being cached.
/// </summary>
/// <param name="statusType">Runtime type of the status object being converted.</param>
/// <returns>The resolved field set for <paramref name="statusType"/>.</returns>
private static StatusFields GetFields(Type statusType)
{
return FieldCache.GetOrAdd(
statusType,
type => new StatusFields(
ResolveField(type, "success"),
ResolveField(type, "category"),
ResolveField(type, "detectedBy"),
ResolveField(type, "detail")));
}
private static FieldInfo ResolveField(
Type valueType,
string fieldName)
{
@@ -92,14 +142,7 @@ public sealed class MxStatusProxyConverter
$"Status object type '{valueType.FullName}' does not expose required field '{fieldName}'.");
}
object? fieldValue = field.GetValue(value);
if (fieldValue is null)
{
throw new MxStatusConversionException(
$"Status object field '{fieldName}' on type '{valueType.FullName}' is null.");
}
return System.Convert.ToInt32(fieldValue, CultureInfo.InvariantCulture);
return field;
}
private static MxStatusCategory MapCategory(int rawCategory)
@@ -134,4 +177,32 @@ public sealed class MxStatusProxyConverter
_ => MxStatusSource.Unknown,
};
}
/// <summary>
/// The four resolved status fields cached per type. Plain readonly struct
/// (not a record) so it compiles under the worker's net48 target, which
/// lacks <c>IsExternalInit</c>.
/// </summary>
private readonly struct StatusFields
{
public StatusFields(
FieldInfo success,
FieldInfo category,
FieldInfo detectedBy,
FieldInfo detail)
{
Success = success;
Category = category;
DetectedBy = detectedBy;
Detail = detail;
}
public FieldInfo Success { get; }
public FieldInfo Category { get; }
public FieldInfo DetectedBy { get; }
public FieldInfo Detail { get; }
}
}
@@ -116,36 +116,77 @@ public sealed class WorkerFrameWriter
// Runs only under _writeLock. Drains control frames before event frames, stamping and writing each.
// The stream write itself is not cancellable: a frame is written atomically or fails, never left
// half-written on the pipe because a caller gave up waiting.
//
// Flushes are coalesced across the whole drained batch (WRK-12 / IPC-15): each frame is written to
// the stream but not flushed individually; a single FlushAsync runs after the batch, then every
// successfully-written frame is completed. A caller's Completion therefore still signals only after
// its bytes have been written AND flushed, so the "written and flushed" contract is unchanged — but
// a burst of N events now costs one flush syscall instead of N.
private async Task DrainQueuedFramesAsync()
{
List<PendingFrame> written = new List<PendingFrame>();
while (true)
{
PendingFrame? frame = DequeueNext();
if (frame is null)
{
return;
break;
}
try
{
await WriteFrameAsync(frame.Envelope).ConfigureAwait(false);
frame.Completion.TrySetResult(true);
written.Add(frame);
}
catch (WorkerFrameProtocolException exception) when (IsPerFrameRejection(exception))
{
// Validation, empty-payload, and oversized-frame errors are specific to this frame and
// do not damage the stream; fail only this frame and keep draining the rest.
// do not damage the stream; fail only this frame and keep draining the rest. Nothing was
// written for it, so it needs no flush.
frame.Completion.TrySetException(exception);
}
catch (Exception exception)
{
// A stream write/flush failure means the pipe is broken; fail this frame and every frame
// still queued so no caller awaits forever, then stop draining.
// A stream write failure means the pipe is broken; fail this frame, every frame already
// written this batch but not yet flushed, and every frame still queued so no caller
// awaits forever, then stop draining.
frame.Completion.TrySetException(exception);
FailFrames(written, exception);
FailAllQueued(exception);
return;
}
}
if (written.Count == 0)
{
return;
}
try
{
await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false);
}
catch (Exception exception)
{
// The batch reached the stream but the flush that guarantees delivery failed: the pipe is
// broken. Fail every frame in the batch (the queue was already drained) so no caller treats
// an unflushed write as delivered.
FailFrames(written, exception);
return;
}
foreach (PendingFrame frame in written)
{
frame.Completion.TrySetResult(true);
}
}
private static void FailFrames(List<PendingFrame> frames, Exception exception)
{
foreach (PendingFrame frame in frames)
{
frame.Completion.TrySetException(exception);
}
}
private static bool IsPerFrameRejection(WorkerFrameProtocolException exception)
@@ -215,14 +256,14 @@ public sealed class WorkerFrameWriter
// Serialize once into a single buffer that carries the 4-byte length prefix followed by the
// payload, then issue one stream write. This avoids a second serialization pass, a separate
// prefix array, and a separate prefix write.
// prefix array, and a separate prefix write. The flush is deferred to the end of the drained
// batch (see DrainQueuedFramesAsync) so a burst of frames shares one flush.
int frameLength = sizeof(uint) + payloadLength;
byte[] frame = new byte[frameLength];
WriteUInt32LittleEndian(frame, (uint)payloadLength);
envelope.WriteTo(new Span<byte>(frame, sizeof(uint), payloadLength));
await _stream.WriteAsync(frame, 0, frameLength, CancellationToken.None).ConfigureAwait(false);
await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false);
}
private static void WriteUInt32LittleEndian(
@@ -8,6 +8,17 @@ namespace ZB.MOM.WW.MxGateway.Worker.MxAccess;
/// <summary>
/// Thread-safe queue for MxAccess events with capacity overflow and fault tracking.
/// </summary>
/// <remarks>
/// Ownership invariant: <see cref="Enqueue"/> takes ownership of the
/// <see cref="MxEvent"/> passed to it — it stamps the worker sequence and
/// timestamp on that same instance and enqueues it directly, without
/// cloning (WRK-11). Every caller must therefore pass a freshly built
/// <see cref="MxEvent"/> that it does not retain, reuse, or mutate after the
/// call returns. All production callers (MxAccessBaseEventSink,
/// MxAccessAlarmEventSink, AlarmCommandHandler) build a new event per
/// Enqueue via the mapper and satisfy this; the value cache stores its own
/// independent snapshot (see <see cref="MxAccessValueCache.Set"/>).
/// </remarks>
public sealed class MxAccessEventQueue
{
/// <summary>
@@ -110,8 +121,11 @@ public sealed class MxAccessEventQueue
/// <summary>
/// Enqueues an MxAccess event, assigning a sequence number and timestamp.
/// Takes ownership of <paramref name="mxEvent"/>: the sequence and timestamp
/// are stamped on that same instance and it is enqueued without cloning, so
/// the caller must pass a freshly built event it does not reuse afterwards.
/// </summary>
/// <param name="mxEvent">MXAccess event to enqueue.</param>
/// <param name="mxEvent">Freshly built MXAccess event to enqueue; ownership transfers to the queue.</param>
public void Enqueue(MxEvent mxEvent)
{
if (mxEvent is null)
@@ -132,13 +146,17 @@ public sealed class MxAccessEventQueue
throw new MxAccessEventQueueOverflowException(capacity);
}
MxEvent queuedEvent = mxEvent.Clone();
queuedEvent.WorkerSequence = ++lastEventSequence;
queuedEvent.WorkerTimestamp = Timestamp.FromDateTime(DateTime.UtcNow);
// WRK-11: stamp the sequence/timestamp on the caller's own event and
// enqueue that same instance under the lock instead of cloning. See
// the ownership invariant on the class summary — the caller hands the
// event over exclusively, so a defensive Clone() here is pure
// overhead on the hottest path.
mxEvent.WorkerSequence = ++lastEventSequence;
mxEvent.WorkerTimestamp = Timestamp.FromDateTime(DateTime.UtcNow);
WorkerEvent workerEvent = new()
{
Event = queuedEvent,
Event = mxEvent,
};
events.Enqueue(workerEvent);
}
@@ -40,6 +40,20 @@ public sealed class MxAccessValueCache
throw new ArgumentNullException(nameof(mxEvent));
}
// WRK-11: the event sink no longer clones before enqueue, so the passed
// mxEvent is the very instance handed to the outbound queue. Deep-copy
// the value/timestamp/statuses payload we retain here so the cache's
// snapshot stays independent of the enqueued (and later serialized)
// event — the two must never share mutable protobuf sub-messages.
// Value is always set for OnDataChange; SourceTimestamp may be unset when
// the source timestamp could not be parsed, so both are cloned only when
// present. The null-forgiving result matches CachedValue's non-null-
// annotated parameters, which already accepted a runtime-null value or
// timestamp before WRK-11 (the ternary keeps the compiler's null-state
// from poisoning to maybe-null, which a plain null check would do).
MxValue cachedValue = mxEvent.Value is null ? null! : mxEvent.Value.Clone();
Timestamp cachedTimestamp = mxEvent.SourceTimestamp is null ? null! : mxEvent.SourceTimestamp.Clone();
long key = CreateItemKey(serverHandle, itemHandle);
lock (syncRoot)
{
@@ -49,10 +63,10 @@ public sealed class MxAccessValueCache
entries[key] = new CachedValue(
nextVersion,
mxEvent.Value,
cachedValue,
mxEvent.Quality,
mxEvent.SourceTimestamp,
mxEvent.Statuses);
cachedTimestamp,
mxEvent.Statuses.Clone());
}
}