Merge branch 'fix/archreview-p2' into main (P2 tier: completeness & polish)
# Conflicts: # archreview/remediation/00-tracking.md # clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli/MxGatewayCliSecretRedactor.cs # clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli/MxGatewayClientCli.cs # src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs
This commit is contained in:
@@ -9,6 +9,31 @@
|
||||
<Deterministic>true</Deterministic>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- TST-11: single-source the .NET-side version for Server, Worker, Contracts, and tests
|
||||
(they otherwise stamp the SDK default 1.0.0, so a deployed gateway cannot be correlated
|
||||
to a release). Kept at 0.1.2 to match the Contracts package and the aligned Python/Rust/
|
||||
Go clients; the Java client leads at 0.2.0 after its JDK-17 retarget. The git short SHA is
|
||||
appended to InformationalVersion (0.1.2+<sha>) so support can map a running binary to a
|
||||
commit; the query is guarded so a build outside a git checkout still succeeds. -->
|
||||
<PropertyGroup>
|
||||
<Version>0.1.2</Version>
|
||||
</PropertyGroup>
|
||||
|
||||
<Target Name="StampSourceRevision"
|
||||
BeforeTargets="GetAssemblyVersion;GenerateAssemblyInfo"
|
||||
Condition="'$(SourceRevisionId)' == ''">
|
||||
<Exec Command="git -C "$(MSBuildThisFileDirectory)" rev-parse --short HEAD"
|
||||
ConsoleToMSBuild="true"
|
||||
StandardOutputImportance="Low"
|
||||
ContinueOnError="true"
|
||||
IgnoreExitCode="true">
|
||||
<Output TaskParameter="ConsoleOutput" PropertyName="_StampedGitSha" />
|
||||
</Exec>
|
||||
<PropertyGroup>
|
||||
<SourceRevisionId Condition="'$(_StampedGitSha)' != ''">$(_StampedGitSha.Trim())</SourceRevisionId>
|
||||
</PropertyGroup>
|
||||
</Target>
|
||||
|
||||
<!-- SQLitePCLRaw.lib.e_sqlite3 2.1.11 (transitive via Microsoft.Data.Sqlite) carries GHSA-2m69-gcr7-jv3q,
|
||||
which surfaces as NU1903 (warning-as-error). No patched e_sqlite3 release exists yet (2.1.11 is latest),
|
||||
so this targeted suppression keeps every OTHER transitive package audited. Remove once an upstream fix ships. -->
|
||||
|
||||
@@ -35,5 +35,23 @@ public static class DashboardAuthenticationDefaults
|
||||
|
||||
public const string LdapGroupClaimType = "mxgateway:ldap_group";
|
||||
public const string KeyPrefixClaimType = "mxgateway:key_prefix";
|
||||
|
||||
/// <summary>
|
||||
/// Dashboard auth cookie name used when the cookie is not guaranteed to be Secure
|
||||
/// (<c>RequireHttpsCookie=false</c> → <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.SameAsRequest"/>)
|
||||
/// or when an explicit <c>MxGateway:Dashboard:CookieName</c> override is absent but the
|
||||
/// secure default cannot be applied. This plain name carries no browser-enforced guarantees.
|
||||
/// </summary>
|
||||
public const string CookieName = "MxGatewayDashboard";
|
||||
|
||||
/// <summary>
|
||||
/// Dashboard auth cookie name applied when the cookie is guaranteed Secure
|
||||
/// (<c>RequireHttpsCookie=true</c> → <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.Always"/>)
|
||||
/// and no explicit <c>MxGateway:Dashboard:CookieName</c> override is set. The <c>__Host-</c>
|
||||
/// prefix instructs browsers to enforce Secure, no <c>Domain</c>, and <c>Path=/</c>; those
|
||||
/// guarantees only hold for a Secure cookie, so this name must never be applied unless
|
||||
/// <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.Always"/> is in
|
||||
/// effect — a <c>__Host-</c> cookie without Secure is silently dropped by browsers.
|
||||
/// </summary>
|
||||
public const string SecureCookieName = "__Host-MxGatewayDashboard";
|
||||
}
|
||||
|
||||
@@ -123,13 +123,22 @@ public static class DashboardServiceCollectionExtensions
|
||||
? CookieSecurePolicy.Always
|
||||
: CookieSecurePolicy.SameAsRequest;
|
||||
|
||||
// Config-driven cookie name (MxGateway:Dashboard:CookieName). Null/blank keeps
|
||||
// the canonical default set above, so a misconfiguration cannot unname the cookie.
|
||||
// Config-driven cookie name (MxGateway:Dashboard:CookieName). An explicit override
|
||||
// always wins. With no override, restore the __Host- prefix when the cookie is
|
||||
// guaranteed Secure (RequireHttpsCookie true → SecurePolicy Always): the __Host-
|
||||
// browser guarantees (Secure required, no Domain, Path=/) hold only for a Secure
|
||||
// cookie, and a __Host- cookie without Secure is silently dropped — so the prefix
|
||||
// is never applied unless SecurePolicy is Always. Otherwise keep the plain
|
||||
// canonical default set by AddCookie above, so a misconfiguration cannot unname it.
|
||||
var cookieName = gatewayOptions.Value.Dashboard.CookieName;
|
||||
if (!string.IsNullOrWhiteSpace(cookieName))
|
||||
{
|
||||
cookieOptions.Cookie.Name = cookieName;
|
||||
}
|
||||
else if (cookieOptions.Cookie.SecurePolicy == CookieSecurePolicy.Always)
|
||||
{
|
||||
cookieOptions.Cookie.Name = DashboardAuthenticationDefaults.SecureCookieName;
|
||||
}
|
||||
});
|
||||
|
||||
services.AddAuthorization(authorization =>
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
@@ -10,10 +12,22 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// Errors are logged once and dropped — keeping the SignalR mirror best-effort
|
||||
/// preserves the gRPC contract that exists today.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), tag
|
||||
/// values are stripped from a redacted copy of the event before it reaches any
|
||||
/// dashboard client. The source <see cref="MxEvent"/> is shared with the gRPC
|
||||
/// event path and the reconnect replay ring, so it is never mutated in place —
|
||||
/// the redaction is applied to a deep clone. This closes the value-leak seam at
|
||||
/// the mirror independently of the still-outstanding per-session hub ACL
|
||||
/// (see <see cref="EventsHub"/>).
|
||||
/// </remarks>
|
||||
public sealed class DashboardEventBroadcaster(
|
||||
IHubContext<EventsHub> hubContext,
|
||||
IOptions<GatewayOptions> options,
|
||||
ILogger<DashboardEventBroadcaster> logger) : IDashboardEventBroadcaster
|
||||
{
|
||||
private readonly bool _showTagValues = options.Value.Dashboard.ShowTagValues;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Publish(string sessionId, MxEvent mxEvent)
|
||||
{
|
||||
@@ -22,6 +36,8 @@ public sealed class DashboardEventBroadcaster(
|
||||
return;
|
||||
}
|
||||
|
||||
MxEvent outbound = _showTagValues ? mxEvent : RedactValues(mxEvent);
|
||||
|
||||
// Wrap the Task acquisition in a try/catch so a hypothetical synchronous throw
|
||||
// from SendAsync (e.g. an implementation that throws before returning the Task)
|
||||
// cannot escape Publish. The interface contract is never-throw; fire-and-forget.
|
||||
@@ -30,7 +46,7 @@ public sealed class DashboardEventBroadcaster(
|
||||
{
|
||||
send = hubContext.Clients
|
||||
.Group(EventsHub.GroupName(sessionId))
|
||||
.SendAsync(EventsHub.EventMessage, mxEvent);
|
||||
.SendAsync(EventsHub.EventMessage, outbound);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -51,4 +67,27 @@ public sealed class DashboardEventBroadcaster(
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Produces a deep clone of <paramref name="source"/> with every tag-value
|
||||
/// field cleared, leaving tag reference, quality, status, and timestamps
|
||||
/// intact so the dashboard still renders the event without the value. The
|
||||
/// source event is left untouched because it is shared downstream with the
|
||||
/// gRPC stream and the replay ring.
|
||||
/// </summary>
|
||||
/// <param name="source">The source event to redact a copy of.</param>
|
||||
/// <returns>A redacted deep clone of the event.</returns>
|
||||
private static MxEvent RedactValues(MxEvent source)
|
||||
{
|
||||
MxEvent redacted = source.Clone();
|
||||
redacted.Value = null;
|
||||
|
||||
if (redacted.BodyCase == MxEvent.BodyOneofCase.OnAlarmTransition)
|
||||
{
|
||||
redacted.OnAlarmTransition.CurrentValue = null;
|
||||
redacted.OnAlarmTransition.LimitValue = null;
|
||||
}
|
||||
|
||||
return redacted;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,15 +32,19 @@ public sealed class EventsHub : Hub
|
||||
/// the dashboard roles (Admin or Viewer); both roles may subscribe to
|
||||
/// any session id they choose. This is acceptable today because (a) the
|
||||
/// dashboard's per-session views show non-secret session metadata that
|
||||
/// any authenticated dashboard user can already see, and (b) value
|
||||
/// logging in the source gRPC stream is gated by the same redaction
|
||||
/// policy that protects logs. The per-session ACL that gates the gRPC
|
||||
/// any authenticated dashboard user can already see, and (b) tag values
|
||||
/// are stripped from the mirrored events by
|
||||
/// <see cref="DashboardEventBroadcaster"/> when
|
||||
/// <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), so the
|
||||
/// most sensitive payload cannot leak through this seam regardless of the
|
||||
/// still-missing ACL. The per-session ACL that gates the gRPC
|
||||
/// <c>StreamEvents</c> RPC is intentionally not yet mirrored here.
|
||||
/// TODO(per-session-acl): once a role/scope is introduced that scopes a
|
||||
/// Viewer to a specific session or tenant, add a session-access check
|
||||
/// at this seam — either inline (consult the per-user allowed-session
|
||||
/// set on <c>Context.User</c> claims / <c>Context.Items</c>) or via a
|
||||
/// dedicated authorization policy applied to the hub method itself.
|
||||
/// TODO(per-session-acl): tracked as remediation roadmap item 12
|
||||
/// (SEC-25). Once a role/scope is introduced that scopes a Viewer to a
|
||||
/// specific session or tenant, add a session-access check at this seam —
|
||||
/// either inline (consult the per-user allowed-session set on
|
||||
/// <c>Context.User</c> claims / <c>Context.Items</c>) or via a dedicated
|
||||
/// authorization policy applied to the hub method itself.
|
||||
/// </remarks>
|
||||
/// <param name="sessionId">Session id to subscribe the caller to.</param>
|
||||
/// <returns>A task representing the subscription operation.</returns>
|
||||
|
||||
@@ -106,11 +106,23 @@ public sealed class EventStreamService(
|
||||
options.Value.Sessions.MaxEventSubscribersPerSession);
|
||||
}
|
||||
|
||||
int streamQueueDepth = 0;
|
||||
IAsyncEnumerator<MxEvent> reader = subscriber.Reader
|
||||
.ReadAllAsync(cancellationToken)
|
||||
.GetAsyncEnumerator(cancellationToken);
|
||||
|
||||
// GWC-15: register this subscriber's channel as a live backlog source instead of
|
||||
// reconciling the queue-depth gauge on every event. The gauge previously read the
|
||||
// bounded channel's Count (which takes the channel's internal lock) and adjusted the
|
||||
// metric under its own lock on every streamed event. Now the metric reads Count only
|
||||
// when it is scraped (ObservableGauge callback) or projected (GetSnapshot), summing the
|
||||
// live backlog across every registered subscriber — the same "buffered, not yet
|
||||
// delivered" aggregate the per-event push reported, but with no per-event lock traffic.
|
||||
// Disposing the registration in the finally removes this subscriber's contribution, so
|
||||
// the gauge returns to the other subscribers' backlog (zero when none remain) on
|
||||
// disconnect. CanCount guards a channel that ever cannot report Count (contributes 0).
|
||||
IDisposable backlogRegistration = metrics.RegisterEventStreamBacklogSource(
|
||||
() => subscriber.Reader.CanCount ? subscriber.Reader.Count : 0);
|
||||
|
||||
try
|
||||
{
|
||||
// Emit order for a resume: the ReplayGap sentinel FIRST (only when events were
|
||||
@@ -169,32 +181,21 @@ public sealed class EventStreamService(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Queue-depth gauge tracks events the pump has fanned into this subscriber's
|
||||
// channel but the client has not yet consumed — the same "buffered, not yet
|
||||
// delivered" quantity the original per-RPC channel reported. The bounded
|
||||
// subscriber channel supports counting, so reconcile the gauge to the current
|
||||
// backlog; falling back to a no-op delta if a channel ever cannot count.
|
||||
int backlog = subscriber.Reader.CanCount ? subscriber.Reader.Count : streamQueueDepth;
|
||||
int delta = backlog - streamQueueDepth;
|
||||
if (delta != 0)
|
||||
{
|
||||
streamQueueDepth = backlog;
|
||||
metrics.AdjustGrpcEventStreamQueueDepth(delta);
|
||||
}
|
||||
|
||||
// The queue-depth gauge is maintained lazily via the backlog registration above
|
||||
// (GWC-15): the metric reads this subscriber's channel Count only when scraped,
|
||||
// so there is no per-event gauge bookkeeping on this hot path.
|
||||
yield return mxEvent;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await reader.DisposeAsync().ConfigureAwait(false);
|
||||
subscriber.Dispose();
|
||||
|
||||
if (streamQueueDepth != 0)
|
||||
{
|
||||
metrics.AdjustGrpcEventStreamQueueDepth(-streamQueueDepth);
|
||||
streamQueueDepth = 0;
|
||||
}
|
||||
// Remove this subscriber's live backlog contribution before disposing the lease so
|
||||
// the gauge stops counting a channel that is about to be completed; after this the
|
||||
// gauge reflects only the remaining subscribers (zero when none remain).
|
||||
backlogRegistration.Dispose();
|
||||
subscriber.Dispose();
|
||||
|
||||
metrics.StreamDisconnected("Detached");
|
||||
}
|
||||
|
||||
@@ -152,9 +152,15 @@ public sealed class MxAccessGatewayService(
|
||||
.WithCancellation(context.CancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
Stopwatch stopwatch = Stopwatch.StartNew();
|
||||
// GWC-06: measure send latency with the allocation-free timestamp API rather
|
||||
// than allocating a Stopwatch object per event per subscriber on the highest-
|
||||
// volume gateway path. Stopwatch.GetTimestamp/GetElapsedTime measure the exact
|
||||
// same wall-clock span the former Stopwatch.StartNew()/.Elapsed did.
|
||||
long sendStartTimestamp = Stopwatch.GetTimestamp();
|
||||
await responseStream.WriteAsync(publicEvent).ConfigureAwait(false);
|
||||
metrics.RecordEventStreamSend(publicEvent.Family.ToString(), stopwatch.Elapsed);
|
||||
metrics.RecordEventStreamSend(
|
||||
publicEvent.Family.ToString(),
|
||||
Stopwatch.GetElapsedTime(sendStartTimestamp));
|
||||
}
|
||||
}
|
||||
catch (Exception exception) when (exception is not RpcException)
|
||||
|
||||
@@ -65,7 +65,16 @@ public sealed class MxAccessGrpcMapper
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(workerEvent);
|
||||
|
||||
return workerEvent.Event?.Clone() ?? new MxEvent
|
||||
// GWC-07 / IPC-05: ownership transfer, not a deep clone. The enclosing WorkerEvent is
|
||||
// parsed fresh from a single pipe frame in WorkerClient's read loop and is discarded
|
||||
// immediately after this mapping — the SessionEventDistributor pump is its single
|
||||
// consumer (GWC-01 claims the worker event channel as single-reader), so nothing else
|
||||
// aliases or mutates workerEvent.Event. We therefore move the inner MxEvent into the
|
||||
// outbound graph instead of cloning it. Downstream the pump fans this one MxEvent to
|
||||
// every subscriber and retains it in the replay ring, but that sharing is READ-ONLY
|
||||
// (subscribers only yield/filter it), so a single shared instance is safe. If a second
|
||||
// consumer of WorkerEvent is ever added, restore a .Clone() here to re-isolate.
|
||||
return workerEvent.Event ?? new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.Unspecified,
|
||||
RawStatus = "Worker event did not contain a public event payload.",
|
||||
|
||||
@@ -32,10 +32,17 @@ public sealed class GatewayMetrics : IDisposable
|
||||
private readonly ConcurrentDictionary<string, long> _eventsBySession = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _retryAttemptsByArea = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// GWC-15: live backlog sources for the gRPC event-stream queue-depth gauge. Each active
|
||||
// StreamEvents subscriber registers a delegate returning its channel's current backlog; the
|
||||
// gauge sums them ONLY when scraped or projected (GetGrpcEventStreamQueueDepth), so nothing is
|
||||
// read on the per-event streaming hot path. Concurrent because subscribers register/unregister
|
||||
// from arbitrary gRPC threads while the gauge callback / GetSnapshot enumerate.
|
||||
private readonly ConcurrentDictionary<long, Func<int>> _eventStreamBacklogSources = new();
|
||||
private long _nextEventStreamBacklogSourceId;
|
||||
|
||||
private int _openSessions;
|
||||
private int _workersRunning;
|
||||
private int _workerEventQueueDepth;
|
||||
private int _grpcEventStreamQueueDepth;
|
||||
private int _alarmProviderMode;
|
||||
private long _sessionsOpened;
|
||||
private long _sessionsClosed;
|
||||
@@ -292,15 +299,24 @@ public sealed class GatewayMetrics : IDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adjusts the gRPC event stream queue depth by the given delta.
|
||||
/// Registers a live backlog source for the gRPC event-stream queue-depth gauge and
|
||||
/// returns a handle that removes it when disposed. The gauge sums every registered
|
||||
/// source's current value on demand (when scraped or via <see cref="GetSnapshot"/>),
|
||||
/// rather than tracking a pushed running total, so the streaming hot path does no
|
||||
/// per-event gauge bookkeeping (GWC-15).
|
||||
/// </summary>
|
||||
/// <param name="delta">Amount to adjust the queue depth by.</param>
|
||||
public void AdjustGrpcEventStreamQueueDepth(int delta)
|
||||
/// <param name="backlog">
|
||||
/// Returns this subscriber's current channel backlog. Invoked only at collection time;
|
||||
/// must be cheap and non-blocking (a bounded channel's <c>Count</c>). Negative returns
|
||||
/// are clamped to zero when summed.
|
||||
/// </param>
|
||||
/// <returns>A handle whose disposal unregisters the source. Safe to dispose more than once.</returns>
|
||||
public IDisposable RegisterEventStreamBacklogSource(Func<int> backlog)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_grpcEventStreamQueueDepth = Math.Max(0, _grpcEventStreamQueueDepth + delta);
|
||||
}
|
||||
ArgumentNullException.ThrowIfNull(backlog);
|
||||
long id = Interlocked.Increment(ref _nextEventStreamBacklogSourceId);
|
||||
_eventStreamBacklogSources[id] = backlog;
|
||||
return new EventStreamBacklogRegistration(this, id);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -429,13 +445,16 @@ public sealed class GatewayMetrics : IDisposable
|
||||
/// <returns>The current metrics snapshot.</returns>
|
||||
public GatewayMetricsSnapshot GetSnapshot()
|
||||
{
|
||||
// Compute the live gRPC stream backlog outside _syncRoot: the sources are the subscriber
|
||||
// channels' Count (their own locks) and must not run under this lock. GWC-15.
|
||||
int grpcEventStreamQueueDepth = GetGrpcEventStreamQueueDepth();
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return new GatewayMetricsSnapshot(
|
||||
OpenSessions: _openSessions,
|
||||
WorkersRunning: _workersRunning,
|
||||
WorkerEventQueueDepth: _workerEventQueueDepth,
|
||||
GrpcEventStreamQueueDepth: _grpcEventStreamQueueDepth,
|
||||
GrpcEventStreamQueueDepth: grpcEventStreamQueueDepth,
|
||||
SessionsOpened: _sessionsOpened,
|
||||
SessionsClosed: _sessionsClosed,
|
||||
CommandsStarted: _commandsStarted,
|
||||
@@ -495,12 +514,28 @@ public sealed class GatewayMetrics : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
// Sums the live backlog across every registered event-stream subscriber. Runs at collection
|
||||
// time (ObservableGauge scrape) or when GetSnapshot projects the value — never on the
|
||||
// per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent
|
||||
// register/unregister; a source removed mid-enumeration simply drops from this sample.
|
||||
private int GetGrpcEventStreamQueueDepth()
|
||||
{
|
||||
lock (_syncRoot)
|
||||
int total = 0;
|
||||
foreach (Func<int> source in _eventStreamBacklogSources.Values)
|
||||
{
|
||||
return _grpcEventStreamQueueDepth;
|
||||
int value = source();
|
||||
if (value > 0)
|
||||
{
|
||||
total += value;
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
private void UnregisterEventStreamBacklogSource(long id)
|
||||
{
|
||||
_eventStreamBacklogSources.TryRemove(id, out _);
|
||||
}
|
||||
|
||||
private int GetAlarmProviderMode()
|
||||
@@ -521,4 +556,19 @@ public sealed class GatewayMetrics : IDisposable
|
||||
{
|
||||
values.AddOrUpdate(key, 1, static (_, currentValue) => currentValue + 1);
|
||||
}
|
||||
|
||||
// Handle returned by RegisterEventStreamBacklogSource. Disposal (once) removes the source
|
||||
// from the gauge's live sum. Idempotent so a double dispose from a stream teardown is safe.
|
||||
private sealed class EventStreamBacklogRegistration(GatewayMetrics metrics, long id) : IDisposable
|
||||
{
|
||||
private int _disposed;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) == 0)
|
||||
{
|
||||
metrics.UnregisterEventStreamBacklogSource(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,14 +98,21 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
private readonly object _lifecycleLock = new();
|
||||
|
||||
// Replay ring buffer. Appended on the pump thread and queried from arbitrary
|
||||
// threads via TryGetReplayFrom, so every access is under _replayLock. The deque
|
||||
// keeps events in ascending WorkerSequence order (the pump fans in source order),
|
||||
// so the oldest retained event is always at the front. Capacity == 0 disables
|
||||
// retention; RetentionSeconds <= 0 disables age-based eviction.
|
||||
// threads via TryGetReplayFrom, so every access is under _replayLock. Backed by a
|
||||
// fixed-size circular array preallocated to the capacity so appending a retained
|
||||
// event allocates no node (the LinkedList this replaced allocated one node per
|
||||
// event on the fan-out hot path). Events are kept in ascending WorkerSequence order
|
||||
// (the pump fans in source order): _replayHead is the logical front (oldest retained
|
||||
// event) and _replayCount entries follow it, wrapping modulo the array length. The
|
||||
// logical entry at position i is _replayBuffer[(_replayHead + i) % Length]. Capacity
|
||||
// == 0 disables retention (the array is empty and never indexed); RetentionSeconds
|
||||
// <= 0 disables age-based eviction.
|
||||
private readonly int _replayBufferCapacity;
|
||||
private readonly TimeSpan _replayRetention;
|
||||
private readonly bool _ageEvictionEnabled;
|
||||
private readonly LinkedList<ReplayEntry> _replayBuffer = new();
|
||||
private readonly ReplayEntry[] _replayBuffer;
|
||||
private int _replayHead;
|
||||
private int _replayCount;
|
||||
private readonly object _replayLock = new();
|
||||
private bool _anyEventSeen;
|
||||
private ulong _highestSequenceSeen;
|
||||
@@ -245,6 +252,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
_overflowHandler = overflowHandler;
|
||||
_shutdownTimeout = DefaultShutdownTimeout;
|
||||
_replayBufferCapacity = replayBufferCapacity;
|
||||
_replayBuffer = new ReplayEntry[replayBufferCapacity];
|
||||
_ageEvictionEnabled = replayRetentionSeconds > 0;
|
||||
_replayRetention = _ageEvictionEnabled
|
||||
? TimeSpan.FromSeconds(replayRetentionSeconds)
|
||||
@@ -452,24 +460,25 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
List<MxEvent> newer = [];
|
||||
ulong highestReplayed = afterSequence;
|
||||
|
||||
if (_replayBuffer.Count == 0)
|
||||
if (_replayCount == 0)
|
||||
{
|
||||
gap = _anyEventSeen && afterSequence < _highestSequenceSeen;
|
||||
oldestAvailableSequence = 0; // meaningful only when gap == true; 0 here since nothing is retained
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong oldestRetained = _replayBuffer.First!.Value.Event.WorkerSequence;
|
||||
ulong oldestRetained = ReplayEntryAt(0).Event.WorkerSequence;
|
||||
gap = oldestRetained > 0 && afterSequence < oldestRetained - 1;
|
||||
// Per the contract on OldestAvailableSequence: meaningful only when gap == true.
|
||||
oldestAvailableSequence = gap ? oldestRetained : 0;
|
||||
|
||||
foreach (ReplayEntry entry in _replayBuffer)
|
||||
for (int i = 0; i < _replayCount; i++)
|
||||
{
|
||||
if (entry.Event.WorkerSequence > afterSequence)
|
||||
MxEvent retained = ReplayEntryAt(i).Event;
|
||||
if (retained.WorkerSequence > afterSequence)
|
||||
{
|
||||
newer.Add(entry.Event);
|
||||
highestReplayed = entry.Event.WorkerSequence;
|
||||
newer.Add(retained);
|
||||
highestReplayed = retained.WorkerSequence;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -729,7 +738,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
{
|
||||
EvictAged();
|
||||
|
||||
if (_replayBuffer.Count == 0)
|
||||
if (_replayCount == 0)
|
||||
{
|
||||
events = [];
|
||||
// Nothing retained. The caller missed events only if it is behind the
|
||||
@@ -738,7 +747,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
return true;
|
||||
}
|
||||
|
||||
ulong oldestRetained = _replayBuffer.First!.Value.Event.WorkerSequence;
|
||||
ulong oldestRetained = ReplayEntryAt(0).Event.WorkerSequence;
|
||||
|
||||
// A gap exists when at least one event newer than afterSequence was evicted,
|
||||
// i.e. afterSequence sits below the oldest-retained-minus-one boundary.
|
||||
@@ -750,11 +759,12 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
// O(n) scan over the retained buffer — acceptable because TryGetReplayFrom
|
||||
// is only called on subscriber reconnect, never on the hot fan-out path.
|
||||
List<MxEvent> newer = [];
|
||||
foreach (ReplayEntry entry in _replayBuffer)
|
||||
for (int i = 0; i < _replayCount; i++)
|
||||
{
|
||||
if (entry.Event.WorkerSequence > afterSequence)
|
||||
MxEvent retained = ReplayEntryAt(i).Event;
|
||||
if (retained.WorkerSequence > afterSequence)
|
||||
{
|
||||
newer.Add(entry.Event);
|
||||
newer.Add(retained);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -780,30 +790,43 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
return;
|
||||
}
|
||||
|
||||
_replayBuffer.AddLast(new ReplayEntry(mxEvent, _timeProvider.GetUtcNow()));
|
||||
|
||||
// Capacity eviction: drop oldest until within bound.
|
||||
while (_replayBuffer.Count > _replayBufferCapacity)
|
||||
// Append at the logical tail. When the ring is full the oldest entry is
|
||||
// overwritten in place (its slot becomes the new tail) and the head advances,
|
||||
// so the newest _replayBufferCapacity events are retained with no allocation.
|
||||
ReplayEntry entry = new(mxEvent, _timeProvider.GetUtcNow());
|
||||
if (_replayCount < _replayBufferCapacity)
|
||||
{
|
||||
_replayBuffer.RemoveFirst();
|
||||
_replayBuffer[(_replayHead + _replayCount) % _replayBufferCapacity] = entry;
|
||||
_replayCount++;
|
||||
}
|
||||
else
|
||||
{
|
||||
_replayBuffer[_replayHead] = entry;
|
||||
_replayHead = (_replayHead + 1) % _replayBufferCapacity;
|
||||
}
|
||||
|
||||
EvictAged();
|
||||
}
|
||||
}
|
||||
|
||||
// Must be called under _replayLock. Drops entries older than the retention window.
|
||||
// Returns the logical entry at position i (0 == oldest retained). Must be called
|
||||
// under _replayLock with 0 <= i < _replayCount (so the array length is nonzero).
|
||||
private ReplayEntry ReplayEntryAt(int i) => _replayBuffer[(_replayHead + i) % _replayBuffer.Length];
|
||||
|
||||
// Must be called under _replayLock. Drops entries older than the retention window
|
||||
// by advancing the head past them (no dealloc; slots are reused on the next append).
|
||||
private void EvictAged()
|
||||
{
|
||||
if (!_ageEvictionEnabled || _replayBuffer.Count == 0)
|
||||
if (!_ageEvictionEnabled || _replayCount == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
DateTimeOffset cutoff = _timeProvider.GetUtcNow() - _replayRetention;
|
||||
while (_replayBuffer.First is { } first && first.Value.RetainedAt < cutoff)
|
||||
while (_replayCount > 0 && _replayBuffer[_replayHead].RetainedAt < cutoff)
|
||||
{
|
||||
_replayBuffer.RemoveFirst();
|
||||
_replayHead = (_replayHead + 1) % _replayBuffer.Length;
|
||||
_replayCount--;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -997,9 +997,14 @@ public sealed class WorkerClient : IWorkerClient
|
||||
string correlationId,
|
||||
WorkerCommand command)
|
||||
{
|
||||
// IPC-05: no second clone. MxAccessGrpcMapper.MapCommand already deep-cloned the command
|
||||
// out of the caller-owned gRPC MxCommandRequest, so this WorkerCommand is a fresh graph
|
||||
// owned by the invoke pipeline and referenced by no other consumer. The envelope is built
|
||||
// and owned entirely inside WorkerClient and the command is not touched again after this
|
||||
// point, so transferring it into the envelope introduces no aliasing hazard.
|
||||
return CreateEnvelope(
|
||||
correlationId,
|
||||
envelope => envelope.WorkerCommand = command.Clone());
|
||||
envelope => envelope.WorkerCommand = command);
|
||||
}
|
||||
|
||||
/// <summary>Creates shutdown envelope.</summary>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Buffers;
|
||||
using System.Buffers.Binary;
|
||||
using Google.Protobuf;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
@@ -47,20 +48,34 @@ public sealed class WorkerFrameReader
|
||||
$"Worker frame payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes.");
|
||||
}
|
||||
|
||||
byte[] payload = new byte[payloadLength];
|
||||
await ReadExactlyOrThrowAsync(payload, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Rent the payload buffer from the shared pool rather than allocating a fresh byte[] per
|
||||
// frame; large event frames near the cap would otherwise allocate an LOH buffer each time
|
||||
// (IPC-14). ParseFrom copies whatever it needs into the parsed message, so the rented buffer
|
||||
// can be returned as soon as parsing completes without the envelope aliasing it. The rented
|
||||
// buffer may be larger than requested, so the read and parse are bounded to length.
|
||||
int length = checked((int)payloadLength);
|
||||
byte[] payload = ArrayPool<byte>.Shared.Rent(length);
|
||||
WorkerEnvelope envelope;
|
||||
try
|
||||
{
|
||||
envelope = WorkerEnvelope.Parser.ParseFrom(payload);
|
||||
await ReadExactlyOrThrowAsync(new Memory<byte>(payload, 0, length), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
envelope = WorkerEnvelope.Parser.ParseFrom(payload, 0, length);
|
||||
}
|
||||
catch (InvalidProtocolBufferException exception)
|
||||
{
|
||||
throw new WorkerFrameProtocolException(
|
||||
WorkerFrameProtocolErrorCode.InvalidEnvelope,
|
||||
"Worker frame payload is not a valid WorkerEnvelope protobuf message.",
|
||||
exception);
|
||||
}
|
||||
}
|
||||
catch (InvalidProtocolBufferException exception)
|
||||
finally
|
||||
{
|
||||
throw new WorkerFrameProtocolException(
|
||||
WorkerFrameProtocolErrorCode.InvalidEnvelope,
|
||||
"Worker frame payload is not a valid WorkerEnvelope protobuf message.",
|
||||
exception);
|
||||
ArrayPool<byte>.Shared.Return(payload);
|
||||
}
|
||||
|
||||
WorkerEnvelopeValidator.Validate(envelope, _options);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Buffers;
|
||||
using System.Buffers.Binary;
|
||||
using Google.Protobuf;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
@@ -53,10 +54,25 @@ public sealed class WorkerFrameWriter
|
||||
$"Worker envelope payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes.");
|
||||
}
|
||||
|
||||
byte[] lengthPrefix = new byte[sizeof(uint)];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(lengthPrefix, (uint)payloadLength);
|
||||
// Serialize once into a single pooled buffer that carries the 4-byte little-endian length
|
||||
// prefix followed by the payload, then issue one stream write. This avoids a second
|
||||
// serialization pass (ToByteArray re-runs CalculateSize), a separate prefix array and its
|
||||
// own write, and any per-frame heap allocation (GWC-08, IPC-13). The rented buffer may be
|
||||
// larger than requested, so only the first frameLength bytes are ever written.
|
||||
int frameLength = sizeof(uint) + payloadLength;
|
||||
byte[] frame = ArrayPool<byte>.Shared.Rent(frameLength);
|
||||
try
|
||||
{
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(frame, (uint)payloadLength);
|
||||
envelope.WriteTo(new Span<byte>(frame, sizeof(uint), payloadLength));
|
||||
|
||||
await _stream.WriteAsync(lengthPrefix, cancellationToken).ConfigureAwait(false);
|
||||
await _stream.WriteAsync(envelope.ToByteArray(), cancellationToken).ConfigureAwait(false);
|
||||
await _stream
|
||||
.WriteAsync(new ReadOnlyMemory<byte>(frame, 0, frameLength), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,12 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
public sealed class DashboardCookieOptionsTests
|
||||
{
|
||||
/// <summary>Verifies that the application configures secure dashboard authentication cookies.</summary>
|
||||
/// <summary>
|
||||
/// Verifies that the application configures secure dashboard authentication cookies.
|
||||
/// With <c>RequireHttpsCookie</c> defaulting to <see langword="true"/> and no explicit
|
||||
/// <c>CookieName</c> override, the cookie is named with the <c>__Host-</c> prefix so
|
||||
/// browsers enforce the Secure/no-Domain/Path=/ guarantees the prefix promises.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Build_ConfiguresSecureDashboardCookie()
|
||||
@@ -22,7 +27,7 @@ public sealed class DashboardCookieOptionsTests
|
||||
CookieAuthenticationOptions options = optionsMonitor.Get(
|
||||
DashboardAuthenticationDefaults.AuthenticationScheme);
|
||||
|
||||
Assert.Equal(DashboardAuthenticationDefaults.CookieName, options.Cookie.Name);
|
||||
Assert.Equal(DashboardAuthenticationDefaults.SecureCookieName, options.Cookie.Name);
|
||||
Assert.True(options.Cookie.HttpOnly);
|
||||
Assert.Equal(CookieSecurePolicy.Always, options.Cookie.SecurePolicy);
|
||||
Assert.Equal(SameSiteMode.Strict, options.Cookie.SameSite);
|
||||
@@ -35,11 +40,14 @@ public sealed class DashboardCookieOptionsTests
|
||||
/// <summary>
|
||||
/// Verifies that setting <c>MxGateway:Dashboard:RequireHttpsCookie=false</c>
|
||||
/// relaxes the cookie to <see cref="CookieSecurePolicy.SameAsRequest"/> so
|
||||
/// the dashboard can be reached over plain HTTP in dev.
|
||||
/// the dashboard can be reached over plain HTTP in dev, and that the plain
|
||||
/// <see cref="DashboardAuthenticationDefaults.CookieName"/> is used rather than the
|
||||
/// <c>__Host-</c> name — a <c>__Host-</c> cookie without a guaranteed Secure flag is
|
||||
/// silently dropped by browsers.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Build_WithRequireHttpsCookieFalse_UsesSameAsRequest()
|
||||
public async Task Build_WithRequireHttpsCookieFalse_UsesSameAsRequestAndPlainName()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build(
|
||||
["--MxGateway:Dashboard:RequireHttpsCookie=false"]);
|
||||
@@ -50,12 +58,14 @@ public sealed class DashboardCookieOptionsTests
|
||||
DashboardAuthenticationDefaults.AuthenticationScheme);
|
||||
|
||||
Assert.Equal(CookieSecurePolicy.SameAsRequest, options.Cookie.SecurePolicy);
|
||||
Assert.Equal(DashboardAuthenticationDefaults.CookieName, options.Cookie.Name);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <c>MxGateway:Dashboard:CookieName</c> overrides the dashboard auth
|
||||
/// cookie name, so a gateway instance sharing a hostname with another can be given a
|
||||
/// distinct name (browser cookies are scoped by host+path, not port).
|
||||
/// Verifies that an explicit <c>MxGateway:Dashboard:CookieName</c> override wins over the
|
||||
/// secure <c>__Host-</c> default (this build leaves <c>RequireHttpsCookie</c> at its
|
||||
/// <see langword="true"/> default), so a gateway instance sharing a hostname with another
|
||||
/// can be given a distinct name (browser cookies are scoped by host+path, not port).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that <see cref="DashboardEventBroadcaster"/> honours
|
||||
/// <c>MxGateway:Dashboard:ShowTagValues</c> (SEC-25): tag values are stripped
|
||||
/// from the mirrored copy when the flag is off, present when it is on, and the
|
||||
/// shared source event is never mutated.
|
||||
/// </summary>
|
||||
public sealed class DashboardEventBroadcasterTests
|
||||
{
|
||||
/// <summary>Values are stripped from the mirror when ShowTagValues is off; metadata survives.</summary>
|
||||
[Fact]
|
||||
public void Publish_WhenShowTagValuesFalse_RedactsValuesButKeepsMetadata()
|
||||
{
|
||||
CapturingHubContext hubContext = new();
|
||||
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false);
|
||||
MxEvent source = BuildEventWithValue();
|
||||
|
||||
broadcaster.Publish("session-1", source);
|
||||
|
||||
MxEvent sent = Assert.IsType<MxEvent>(hubContext.LastArgument);
|
||||
Assert.Null(sent.Value);
|
||||
Assert.Null(sent.OnAlarmTransition.CurrentValue);
|
||||
Assert.Null(sent.OnAlarmTransition.LimitValue);
|
||||
|
||||
// Metadata unrelated to the value survives redaction.
|
||||
Assert.Equal("session-1", sent.SessionId);
|
||||
Assert.Equal(7, sent.ServerHandle);
|
||||
Assert.Equal(11, sent.ItemHandle);
|
||||
Assert.Equal(192, sent.Quality);
|
||||
Assert.Equal("Tank01.Level.HiHi", sent.OnAlarmTransition.AlarmFullReference);
|
||||
}
|
||||
|
||||
/// <summary>Redaction applies to a clone, so the shared source event keeps its values.</summary>
|
||||
[Fact]
|
||||
public void Publish_WhenShowTagValuesFalse_DoesNotMutateSourceEvent()
|
||||
{
|
||||
CapturingHubContext hubContext = new();
|
||||
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false);
|
||||
MxEvent source = BuildEventWithValue();
|
||||
|
||||
broadcaster.Publish("session-1", source);
|
||||
|
||||
// The redaction must apply to a clone; the shared source keeps its values.
|
||||
Assert.NotNull(source.Value);
|
||||
Assert.Equal(42.5, source.Value.DoubleValue);
|
||||
Assert.NotNull(source.OnAlarmTransition.CurrentValue);
|
||||
Assert.NotNull(source.OnAlarmTransition.LimitValue);
|
||||
Assert.NotSame(source, hubContext.LastArgument);
|
||||
}
|
||||
|
||||
/// <summary>Values pass through unredacted when ShowTagValues is on.</summary>
|
||||
[Fact]
|
||||
public void Publish_WhenShowTagValuesTrue_KeepsValues()
|
||||
{
|
||||
CapturingHubContext hubContext = new();
|
||||
DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: true);
|
||||
MxEvent source = BuildEventWithValue();
|
||||
|
||||
broadcaster.Publish("session-1", source);
|
||||
|
||||
MxEvent sent = Assert.IsType<MxEvent>(hubContext.LastArgument);
|
||||
Assert.NotNull(sent.Value);
|
||||
Assert.Equal(42.5, sent.Value.DoubleValue);
|
||||
Assert.NotNull(sent.OnAlarmTransition.CurrentValue);
|
||||
Assert.NotNull(sent.OnAlarmTransition.LimitValue);
|
||||
}
|
||||
|
||||
private static DashboardEventBroadcaster Create(CapturingHubContext hubContext, bool showTagValues)
|
||||
{
|
||||
GatewayOptions gatewayOptions = new()
|
||||
{
|
||||
Dashboard = new DashboardOptions { ShowTagValues = showTagValues },
|
||||
};
|
||||
|
||||
return new DashboardEventBroadcaster(
|
||||
hubContext,
|
||||
Options.Create(gatewayOptions),
|
||||
NullLogger<DashboardEventBroadcaster>.Instance);
|
||||
}
|
||||
|
||||
private static MxEvent BuildEventWithValue()
|
||||
{
|
||||
return new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.OnAlarmTransition,
|
||||
SessionId = "session-1",
|
||||
ServerHandle = 7,
|
||||
ItemHandle = 11,
|
||||
Quality = 192,
|
||||
Value = new MxValue { DataType = MxDataType.Double, DoubleValue = 42.5 },
|
||||
OnAlarmTransition = new OnAlarmTransitionEvent
|
||||
{
|
||||
AlarmFullReference = "Tank01.Level.HiHi",
|
||||
CurrentValue = new MxValue { DataType = MxDataType.Double, DoubleValue = 88.0 },
|
||||
LimitValue = new MxValue { DataType = MxDataType.Double, DoubleValue = 90.0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class CapturingHubContext : IHubContext<EventsHub>
|
||||
{
|
||||
private readonly CapturingHubClients _clients = new();
|
||||
|
||||
/// <summary>Gets the hub clients.</summary>
|
||||
public IHubClients Clients => _clients;
|
||||
|
||||
/// <summary>Gets the group manager.</summary>
|
||||
public IGroupManager Groups { get; } = new NoopGroupManager();
|
||||
|
||||
/// <summary>Gets the first argument of the most recent send call.</summary>
|
||||
public object? LastArgument => _clients.GroupProxy.LastArgument;
|
||||
}
|
||||
|
||||
private sealed class CapturingHubClients : IHubClients
|
||||
{
|
||||
/// <summary>Gets the capturing client proxy shared by this fake.</summary>
|
||||
public CapturingClientProxy GroupProxy { get; } = new();
|
||||
|
||||
public IClientProxy All => GroupProxy;
|
||||
|
||||
public IClientProxy AllExcept(IReadOnlyList<string> excludedConnectionIds) => GroupProxy;
|
||||
|
||||
public IClientProxy Client(string connectionId) => GroupProxy;
|
||||
|
||||
public IClientProxy Clients(IReadOnlyList<string> connectionIds) => GroupProxy;
|
||||
|
||||
public IClientProxy Group(string groupName) => GroupProxy;
|
||||
|
||||
public IClientProxy GroupExcept(string groupName, IReadOnlyList<string> excludedConnectionIds) => GroupProxy;
|
||||
|
||||
public IClientProxy Groups(IReadOnlyList<string> groupNames) => GroupProxy;
|
||||
|
||||
public IClientProxy User(string userId) => GroupProxy;
|
||||
|
||||
public IClientProxy Users(IReadOnlyList<string> userIds) => GroupProxy;
|
||||
}
|
||||
|
||||
private sealed class CapturingClientProxy : IClientProxy
|
||||
{
|
||||
/// <summary>Gets the first argument of the most recent send call.</summary>
|
||||
public object? LastArgument { get; private set; }
|
||||
|
||||
/// <summary>Records the send call arguments and completes synchronously.</summary>
|
||||
/// <param name="method">The SignalR method name.</param>
|
||||
/// <param name="args">The method arguments.</param>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A completed task.</returns>
|
||||
public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default)
|
||||
{
|
||||
LastArgument = args.Length > 0 ? args[0] : null;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class NoopGroupManager : IGroupManager
|
||||
{
|
||||
public Task AddToGroupAsync(string connectionId, string groupName, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
|
||||
public Task RemoveFromGroupAsync(string connectionId, string groupName, CancellationToken cancellationToken = default)
|
||||
=> Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,642 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
using ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end reconnect/replay tests through the real gRPC <c>StreamEvents</c> path via the
|
||||
/// fake worker harness (TST-01, server half). A single subscriber detaches (cancel + await
|
||||
/// the stream task so its lease is fully disposed), the session is retained by detach-grace
|
||||
/// while the distributor pump keeps appending worker events to the replay ring, and a second
|
||||
/// stream reconnects with <see cref="StreamEventsRequest.AfterWorkerSequence"/>. Covers both
|
||||
/// the no-gap resume (cursor inside the retained window) and the <c>ReplayGap</c> sentinel
|
||||
/// (cursor predates the oldest retained event after capacity eviction).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// These tests run in single-subscriber mode (<c>AllowMultipleEventSubscribers=false</c>).
|
||||
/// Reconnect works because the first stream is FULLY detached before the reconnect attaches:
|
||||
/// awaiting the first stream task runs <c>EventStreamService</c>'s finally block, which
|
||||
/// disposes the subscriber lease and drops the session's active-subscriber count back to
|
||||
/// zero, so the reconnect's <c>AttachEventSubscriberWithReplay</c> sees an empty slot rather
|
||||
/// than an "already active" rejection. The distributor (and its replay ring) is created once
|
||||
/// per session and survives detach, so events emitted while no subscriber is attached are
|
||||
/// retained for the reconnecting stream.
|
||||
/// </remarks>
|
||||
public sealed class GatewayEndToEndReconnectReplayTests
|
||||
{
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(10);
|
||||
private const int ServerHandle = 3001;
|
||||
private const int ItemHandle = 4002;
|
||||
|
||||
/// <summary>
|
||||
/// Reconnecting inside the retained window replays exactly the events newer than the
|
||||
/// resume cursor — the retained tail of the first batch plus the events emitted while
|
||||
/// detached — in strictly ascending order, with no duplicates and no <c>ReplayGap</c>
|
||||
/// sentinel.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEvents_ReconnectInsideRetainedWindow_ReplaysTailNoGap()
|
||||
{
|
||||
const int firstBatch = 4;
|
||||
const int secondBatch = 3;
|
||||
|
||||
GatedEventFakeWorkerProcessLauncher launcher = new();
|
||||
// Capacity 16 retains every event emitted here (7 total), so nothing is evicted and a
|
||||
// resume from a middle cursor never produces a gap.
|
||||
await using ReconnectReplayGatewayServiceFixture fixture = new(launcher, replayBufferCapacity: 16);
|
||||
|
||||
string sessionId = await OpenSessionAsync(fixture, "reconnect-no-gap");
|
||||
|
||||
// ---- first connection: receive the first batch, capture its real sequences ----
|
||||
using CancellationTokenSource writer1Cts = new();
|
||||
RecordingServerStreamWriter<MxEvent> writer1 = new();
|
||||
Task stream1Task = Task.Run(async () =>
|
||||
await fixture.Service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = sessionId },
|
||||
writer1,
|
||||
new TestServerCallContext(cancellationToken: writer1Cts.Token)));
|
||||
|
||||
await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
|
||||
await WireUpAdviseAsync(fixture, sessionId);
|
||||
|
||||
for (int i = 0; i < firstBatch; i++)
|
||||
{
|
||||
launcher.AllowNextEvent();
|
||||
}
|
||||
|
||||
IReadOnlyList<MxEvent> batch1 = await writer1.WaitForMessageCountAsync(firstBatch, TestTimeout);
|
||||
ulong[] batch1Sequences = batch1.Select(e => e.WorkerSequence).ToArray();
|
||||
|
||||
// Resume cursor: a middle event of the first batch. Everything strictly newer than this
|
||||
// must be redelivered to the reconnecting stream.
|
||||
ulong cursor = batch1Sequences[1];
|
||||
int retainedTailFromBatch1 = batch1Sequences.Count(s => s > cursor);
|
||||
Assert.Equal(2, retainedTailFromBatch1); // sanity: events at index 2 and 3
|
||||
|
||||
// ---- fully detach writer1 (cancel + await so its lease is disposed) ----
|
||||
await DetachAsync(writer1Cts, stream1Task);
|
||||
|
||||
// ---- emit more events while detached; the ring must retain them for the reconnect ----
|
||||
for (int i = 0; i < secondBatch; i++)
|
||||
{
|
||||
launcher.AllowNextEvent();
|
||||
}
|
||||
|
||||
// ---- reconnect with the middle cursor ----
|
||||
RecordingServerStreamWriter<MxEvent> writer2 = new();
|
||||
Task stream2Task = Task.Run(async () =>
|
||||
await fixture.Service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = sessionId, AfterWorkerSequence = cursor },
|
||||
writer2,
|
||||
new TestServerCallContext()));
|
||||
|
||||
await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
|
||||
|
||||
int expected = retainedTailFromBatch1 + secondBatch;
|
||||
IReadOnlyList<MxEvent> resumed = await writer2.WaitForMessageCountAsync(expected, TestTimeout);
|
||||
|
||||
// ---- tear down before asserting so a hang can't wedge the run ----
|
||||
launcher.StopEmitting();
|
||||
await CloseAndDrainAsync(fixture, sessionId, stream2Task, launcher);
|
||||
|
||||
// ---- assertions ----
|
||||
Assert.Equal(expected, resumed.Count);
|
||||
|
||||
// No sentinel: every message is a real event, none carries a ReplayGap.
|
||||
Assert.All(resumed, e => Assert.Null(e.ReplayGap));
|
||||
|
||||
// Every replayed/live event is strictly newer than the cursor.
|
||||
Assert.All(resumed, e => Assert.True(
|
||||
e.WorkerSequence > cursor,
|
||||
$"Event sequence {e.WorkerSequence} is not newer than cursor {cursor}."));
|
||||
|
||||
// Strictly ascending, no duplicates.
|
||||
for (int i = 1; i < resumed.Count; i++)
|
||||
{
|
||||
Assert.True(
|
||||
resumed[i].WorkerSequence > resumed[i - 1].WorkerSequence,
|
||||
$"Sequences must be strictly ascending: {resumed[i - 1].WorkerSequence} then {resumed[i].WorkerSequence}.");
|
||||
}
|
||||
|
||||
Assert.Equal(resumed.Count, resumed.Select(e => e.WorkerSequence).Distinct().Count());
|
||||
|
||||
// The retained tail of the first batch (the two events newer than the cursor) is
|
||||
// replayed first, before the events emitted while detached.
|
||||
Assert.Equal(batch1Sequences[2], resumed[0].WorkerSequence);
|
||||
Assert.Equal(batch1Sequences[3], resumed[1].WorkerSequence);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reconnecting with a cursor that predates the oldest retained event (after capacity
|
||||
/// eviction) yields the <c>ReplayGap</c> sentinel FIRST — family unspecified, no body, no
|
||||
/// per-item fields, correct requested/oldest sequences — followed by exactly the retained
|
||||
/// tail, in ascending order.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEvents_ReconnectWithStaleCursor_EmitsReplayGapSentinelFirst()
|
||||
{
|
||||
const int capacity = 3;
|
||||
const int totalEvents = 6;
|
||||
|
||||
GatedEventFakeWorkerProcessLauncher launcher = new();
|
||||
// Small capacity forces eviction: with 6 events emitted and a 3-slot ring, the first 3
|
||||
// are evicted and only the newest 3 remain replayable.
|
||||
await using ReconnectReplayGatewayServiceFixture fixture = new(launcher, replayBufferCapacity: capacity);
|
||||
|
||||
string sessionId = await OpenSessionAsync(fixture, "reconnect-gap");
|
||||
|
||||
using CancellationTokenSource writer1Cts = new();
|
||||
RecordingServerStreamWriter<MxEvent> writer1 = new();
|
||||
Task stream1Task = Task.Run(async () =>
|
||||
await fixture.Service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = sessionId },
|
||||
writer1,
|
||||
new TestServerCallContext(cancellationToken: writer1Cts.Token)));
|
||||
|
||||
await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
|
||||
await WireUpAdviseAsync(fixture, sessionId);
|
||||
|
||||
// Emit more events than the ring can hold. Waiting for writer1 to receive all of them
|
||||
// proves the pump has appended (and evicted) every event, so the ring is settled to its
|
||||
// final newest-`capacity` contents before we reconnect.
|
||||
for (int i = 0; i < totalEvents; i++)
|
||||
{
|
||||
launcher.AllowNextEvent();
|
||||
}
|
||||
|
||||
IReadOnlyList<MxEvent> all = await writer1.WaitForMessageCountAsync(totalEvents, TestTimeout);
|
||||
ulong[] sequences = all.Select(e => e.WorkerSequence).ToArray();
|
||||
|
||||
// With capacity C and N total events, the newest C are retained; the oldest still
|
||||
// available is the (N-C+1)th event — index N-C in emission (ascending) order.
|
||||
ulong oldestAvailable = sequences[totalEvents - capacity];
|
||||
|
||||
await DetachAsync(writer1Cts, stream1Task);
|
||||
|
||||
// Resume after sequence 1: the real event sequences are envelope-numbered and start well
|
||||
// above 1 (startup + command-reply envelopes consume the low numbers), so cursor 1
|
||||
// predates every event and, with the oldest three already evicted, forces a gap.
|
||||
const ulong staleCursor = 1;
|
||||
RecordingServerStreamWriter<MxEvent> writer2 = new();
|
||||
Task stream2Task = Task.Run(async () =>
|
||||
await fixture.Service.StreamEvents(
|
||||
new StreamEventsRequest { SessionId = sessionId, AfterWorkerSequence = staleCursor },
|
||||
writer2,
|
||||
new TestServerCallContext()));
|
||||
|
||||
await fixture.WaitForSubscriberCountAsync(sessionId, n: 1, TestTimeout);
|
||||
|
||||
// sentinel + the retained tail (capacity events, all newer than the stale cursor).
|
||||
int expected = 1 + capacity;
|
||||
IReadOnlyList<MxEvent> resumed = await writer2.WaitForMessageCountAsync(expected, TestTimeout);
|
||||
|
||||
launcher.StopEmitting();
|
||||
await CloseAndDrainAsync(fixture, sessionId, stream2Task, launcher);
|
||||
|
||||
// ---- the first message is the ReplayGap sentinel ----
|
||||
Assert.Equal(expected, resumed.Count);
|
||||
|
||||
MxEvent sentinel = resumed[0];
|
||||
Assert.NotNull(sentinel.ReplayGap);
|
||||
Assert.Equal(MxEventFamily.Unspecified, sentinel.Family);
|
||||
Assert.Equal(MxEvent.BodyOneofCase.None, sentinel.BodyCase);
|
||||
Assert.Equal(sessionId, sentinel.SessionId);
|
||||
Assert.Equal(staleCursor, sentinel.ReplayGap.RequestedAfterSequence);
|
||||
Assert.Equal(oldestAvailable, sentinel.ReplayGap.OldestAvailableSequence);
|
||||
|
||||
// ---- the sentinel is followed by the retained tail, ascending, no further sentinels ----
|
||||
IReadOnlyList<MxEvent> tail = resumed.Skip(1).ToArray();
|
||||
Assert.Equal(capacity, tail.Count);
|
||||
Assert.All(tail, e => Assert.Null(e.ReplayGap));
|
||||
Assert.All(tail, e => Assert.True(
|
||||
e.WorkerSequence >= oldestAvailable,
|
||||
$"Retained event {e.WorkerSequence} is older than the oldest available {oldestAvailable}."));
|
||||
|
||||
ulong[] expectedTail = sequences.Skip(totalEvents - capacity).ToArray();
|
||||
Assert.Equal(expectedTail, tail.Select(e => e.WorkerSequence).ToArray());
|
||||
}
|
||||
|
||||
// ---- shared flow helpers ----
|
||||
|
||||
private static async Task<string> OpenSessionAsync(
|
||||
ReconnectReplayGatewayServiceFixture fixture,
|
||||
string name)
|
||||
{
|
||||
OpenSessionReply openReply = await fixture.Service.OpenSession(
|
||||
new OpenSessionRequest
|
||||
{
|
||||
ClientSessionName = name,
|
||||
ClientCorrelationId = $"open-{name}",
|
||||
CommandTimeout = Duration.FromTimeSpan(TestTimeout),
|
||||
},
|
||||
new TestServerCallContext()).ConfigureAwait(false);
|
||||
|
||||
Assert.Equal(ProtocolStatusCode.Ok, openReply.ProtocolStatus.Code);
|
||||
return openReply.SessionId;
|
||||
}
|
||||
|
||||
private static async Task WireUpAdviseAsync(
|
||||
ReconnectReplayGatewayServiceFixture fixture,
|
||||
string sessionId)
|
||||
{
|
||||
MxCommandReply registerReply = await fixture.Service.Invoke(
|
||||
CreateRegisterRequest(sessionId),
|
||||
new TestServerCallContext()).ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, registerReply.ProtocolStatus.Code);
|
||||
|
||||
MxCommandReply addItemReply = await fixture.Service.Invoke(
|
||||
CreateAddItemRequest(sessionId, registerReply.Register.ServerHandle),
|
||||
new TestServerCallContext()).ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, addItemReply.ProtocolStatus.Code);
|
||||
|
||||
MxCommandReply adviseReply = await fixture.Service.Invoke(
|
||||
CreateAdviseRequest(sessionId, registerReply.Register.ServerHandle, addItemReply.AddItem.ItemHandle),
|
||||
new TestServerCallContext()).ConfigureAwait(false);
|
||||
Assert.Equal(ProtocolStatusCode.Ok, adviseReply.ProtocolStatus.Code);
|
||||
}
|
||||
|
||||
// Cancels the stream's token and awaits the stream task. Awaiting is load-bearing: it lets
|
||||
// EventStreamService's finally block dispose the subscriber lease (dropping the session's
|
||||
// active-subscriber count to zero) BEFORE the reconnect attaches, so the reconnect never
|
||||
// races a still-registered subscriber.
|
||||
private static async Task DetachAsync(CancellationTokenSource cts, Task streamTask)
|
||||
{
|
||||
await cts.CancelAsync().ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await streamTask.WaitAsync(TestTimeout).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected: the iterator surfaces the cancellation.
|
||||
}
|
||||
catch (RpcException rpc) when (rpc.StatusCode == StatusCode.Cancelled)
|
||||
{
|
||||
// Also acceptable depending on gRPC exception wrapping.
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task CloseAndDrainAsync(
|
||||
ReconnectReplayGatewayServiceFixture fixture,
|
||||
string sessionId,
|
||||
Task streamTask,
|
||||
GatedEventFakeWorkerProcessLauncher launcher)
|
||||
{
|
||||
await fixture.Service.CloseSession(
|
||||
new CloseSessionRequest { SessionId = sessionId, ClientCorrelationId = "close-reconnect" },
|
||||
new TestServerCallContext()).ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
await streamTask.WaitAsync(TestTimeout).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
|
||||
await launcher.WorkerTask.WaitAsync(TestTimeout).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ---- request builders ----
|
||||
|
||||
private static MxCommandRequest CreateRegisterRequest(string sessionId) =>
|
||||
new()
|
||||
{
|
||||
SessionId = sessionId,
|
||||
ClientCorrelationId = "register-rr",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.Register,
|
||||
Register = new RegisterCommand { ClientName = "reconnect-replay-e2e-client" },
|
||||
},
|
||||
};
|
||||
|
||||
private static MxCommandRequest CreateAddItemRequest(string sessionId, int serverHandle) =>
|
||||
new()
|
||||
{
|
||||
SessionId = sessionId,
|
||||
ClientCorrelationId = "add-item-rr",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.AddItem,
|
||||
AddItem = new AddItemCommand
|
||||
{
|
||||
ServerHandle = serverHandle,
|
||||
ItemDefinition = "Galaxy.Tag.Value",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
private static MxCommandRequest CreateAdviseRequest(
|
||||
string sessionId,
|
||||
int serverHandle,
|
||||
int itemHandle) =>
|
||||
new()
|
||||
{
|
||||
SessionId = sessionId,
|
||||
ClientCorrelationId = "advise-rr",
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.Advise,
|
||||
Advise = new AdviseCommand { ServerHandle = serverHandle, ItemHandle = itemHandle },
|
||||
},
|
||||
};
|
||||
|
||||
private static void ConfigureCommandReply(MxCommandReply reply, MxCommandKind kind)
|
||||
{
|
||||
switch (kind)
|
||||
{
|
||||
case MxCommandKind.Register:
|
||||
reply.Register = new RegisterReply { ServerHandle = ServerHandle };
|
||||
break;
|
||||
case MxCommandKind.AddItem:
|
||||
reply.AddItem = new AddItemReply { ItemHandle = ItemHandle };
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- fixture ----
|
||||
|
||||
/// <summary>
|
||||
/// Gateway service fixture in single-subscriber mode with a configurable replay ring
|
||||
/// capacity, so each test can retain the whole event history (no gap) or force capacity
|
||||
/// eviction (gap).
|
||||
/// </summary>
|
||||
private sealed class ReconnectReplayGatewayServiceFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly GatewayMetrics _metrics = new();
|
||||
private readonly SessionRegistry _registry = new();
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="ReconnectReplayGatewayServiceFixture"/> class.</summary>
|
||||
/// <param name="launcher">Fake worker process launcher backing the session manager.</param>
|
||||
/// <param name="replayBufferCapacity">Replay ring capacity for the session's event distributor.</param>
|
||||
public ReconnectReplayGatewayServiceFixture(
|
||||
IWorkerProcessLauncher launcher,
|
||||
int replayBufferCapacity)
|
||||
{
|
||||
IOptions<GatewayOptions> options = Options.Create(CreateOptions(replayBufferCapacity));
|
||||
SessionWorkerClientFactory workerClientFactory = new(
|
||||
launcher,
|
||||
options,
|
||||
_metrics,
|
||||
NullLoggerFactory.Instance);
|
||||
SessionManager sessionManager = new(
|
||||
_registry,
|
||||
workerClientFactory,
|
||||
options,
|
||||
_metrics,
|
||||
logger: NullLogger<SessionManager>.Instance,
|
||||
dashboardEventBroadcaster: NullDashboardEventBroadcaster.Instance);
|
||||
MxAccessGrpcMapper mapper = new();
|
||||
EventStreamService eventStreamService = new(
|
||||
sessionManager,
|
||||
options,
|
||||
_metrics);
|
||||
|
||||
Service = new MxAccessGatewayService(
|
||||
sessionManager,
|
||||
new GatewayRequestIdentityAccessor(),
|
||||
new AllowAllConstraintEnforcer(),
|
||||
new MxAccessGrpcRequestValidator(),
|
||||
mapper,
|
||||
eventStreamService,
|
||||
_metrics,
|
||||
NullLogger<MxAccessGatewayService>.Instance,
|
||||
new FakeGatewayAlarmService());
|
||||
}
|
||||
|
||||
/// <summary>Gets the gateway service under test.</summary>
|
||||
public MxAccessGatewayService Service { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Polls <see cref="GatewaySession.ActiveEventSubscriberCount"/> for
|
||||
/// <paramref name="sessionId"/> until it reaches <paramref name="n"/>, bounded by
|
||||
/// <paramref name="timeout"/>. Fails the test on timeout. This is the deterministic
|
||||
/// gate that proves the production code has (re)registered a subscriber before the
|
||||
/// test drives events or reconnects.
|
||||
/// </summary>
|
||||
/// <param name="sessionId">Identifier of the session to poll.</param>
|
||||
/// <param name="n">Target subscriber count to wait for.</param>
|
||||
/// <param name="timeout">Maximum time to wait before failing the test.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task WaitForSubscriberCountAsync(string sessionId, int n, TimeSpan timeout)
|
||||
{
|
||||
using CancellationTokenSource deadlineCts = new(timeout);
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (_registry.TryGet(sessionId, out GatewaySession? session)
|
||||
&& session.ActiveEventSubscriberCount >= n)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (deadlineCts.IsCancellationRequested)
|
||||
{
|
||||
int actual = _registry.TryGet(sessionId, out GatewaySession? s)
|
||||
? s.ActiveEventSubscriberCount
|
||||
: -1;
|
||||
Assert.Fail(
|
||||
$"Timed out waiting for {n} event subscriber(s) on session {sessionId}. "
|
||||
+ $"Actual count after {timeout.TotalSeconds:0.#}s: {actual}.");
|
||||
}
|
||||
|
||||
await Task.Delay(millisecondsDelay: 5, deadlineCts.Token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Disposes every session in the registry and releases the fixture's metrics.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
foreach (GatewaySession session in _registry.Snapshot())
|
||||
{
|
||||
await session.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_metrics.Dispose();
|
||||
}
|
||||
|
||||
private static GatewayOptions CreateOptions(int replayBufferCapacity) =>
|
||||
new()
|
||||
{
|
||||
Worker = new WorkerOptions
|
||||
{
|
||||
StartupTimeoutSeconds = 5,
|
||||
ShutdownTimeoutSeconds = 5,
|
||||
HeartbeatIntervalSeconds = 30,
|
||||
HeartbeatGraceSeconds = 30,
|
||||
MaxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes,
|
||||
},
|
||||
Sessions = new SessionOptions
|
||||
{
|
||||
DefaultCommandTimeoutSeconds = 5,
|
||||
MaxSessions = 4,
|
||||
|
||||
// Single-subscriber mode: a fully-detached subscriber (stream task awaited)
|
||||
// frees the sole slot, so the reconnect attaches cleanly without needing
|
||||
// multi-subscriber fan-out. Detach-grace keeps the session Ready across the
|
||||
// detach and the fixture runs no lease-reaper, so the session survives to be
|
||||
// reconnected.
|
||||
AllowMultipleEventSubscribers = false,
|
||||
MaxEventSubscribersPerSession = 8,
|
||||
},
|
||||
Events = new EventOptions
|
||||
{
|
||||
QueueCapacity = 32,
|
||||
ReplayBufferCapacity = replayBufferCapacity,
|
||||
|
||||
// Keep age-eviction effectively off for the duration of a fast test so
|
||||
// capacity is the only eviction axis under test.
|
||||
ReplayRetentionSeconds = 300,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ---- fake worker launcher ----
|
||||
|
||||
/// <summary>
|
||||
/// Fake worker that emits events one at a time, gated by <see cref="AllowNextEvent"/>, so
|
||||
/// the test drives event timing deterministically. Modeled on the multi-subscriber E2E
|
||||
/// tests' gated launcher. The worker loop is independent of subscribers, so events emitted
|
||||
/// while no gRPC stream is attached still flow through the distributor pump into the
|
||||
/// session's replay ring — exactly the condition the reconnect/replay tests exercise. Call
|
||||
/// <see cref="StopEmitting"/> before closing the session so the loop exits cleanly and can
|
||||
/// process the shutdown envelope.
|
||||
/// </summary>
|
||||
private sealed class GatedEventFakeWorkerProcessLauncher : IWorkerProcessLauncher
|
||||
{
|
||||
public const int ProcessId = 7730;
|
||||
|
||||
private readonly FakeWorkerProcess _process = new(ProcessId);
|
||||
|
||||
// Capacity 64 so AllowNextEvent can be called ahead of time without blocking.
|
||||
private readonly SemaphoreSlim _emitGate = new(0, 64);
|
||||
private volatile bool _stopEmitting;
|
||||
|
||||
/// <summary>Gets the task representing the fake worker's running background loop.</summary>
|
||||
public Task WorkerTask { get; private set; } = Task.CompletedTask;
|
||||
|
||||
/// <summary>Releases the gate so the worker emits one event.</summary>
|
||||
public void AllowNextEvent() => _emitGate.Release();
|
||||
|
||||
/// <summary>
|
||||
/// Signals the worker to stop waiting for the emit gate and process the shutdown
|
||||
/// envelope. Must be called before <c>CloseSession</c>.
|
||||
/// </summary>
|
||||
public void StopEmitting()
|
||||
{
|
||||
_stopEmitting = true;
|
||||
_emitGate.Release(); // unblock a pending gate wait if any
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<WorkerProcessHandle> LaunchAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
WorkerTask = RunWorkerAsync(request, cancellationToken);
|
||||
|
||||
return Task.FromResult(new WorkerProcessHandle(
|
||||
_process,
|
||||
new WorkerProcessCommandLine("reconnect-replay-fake-worker.exe", []),
|
||||
DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
private async Task RunWorkerAsync(
|
||||
WorkerProcessLaunchRequest request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await using FakeWorkerHarness harness = await FakeWorkerHarness.ConnectToGatewayPipeAsync(
|
||||
request.SessionId,
|
||||
request.Nonce,
|
||||
request.PipeName,
|
||||
request.ProtocolVersion,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
await harness.CompleteStartupAsync(ProcessId, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
int advisedServerHandle = 0;
|
||||
int advisedItemHandle = 0;
|
||||
int emittedCount = 0;
|
||||
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// While subscribed and not stopped, emit gated events using a non-blocking peek
|
||||
// at the gate so incoming envelopes (including shutdown) are never starved.
|
||||
while (advisedServerHandle != 0
|
||||
&& !_stopEmitting
|
||||
&& await _emitGate.WaitAsync(millisecondsTimeout: 0).ConfigureAwait(false))
|
||||
{
|
||||
int index = ++emittedCount;
|
||||
await harness.EmitEventAsync(
|
||||
MxEventFamily.OnDataChange,
|
||||
cancellationToken,
|
||||
mxEvent =>
|
||||
{
|
||||
mxEvent.ServerHandle = advisedServerHandle;
|
||||
mxEvent.ItemHandle = advisedItemHandle;
|
||||
mxEvent.Quality = 192;
|
||||
mxEvent.Value = new MxValue
|
||||
{
|
||||
DataType = MxDataType.String,
|
||||
StringValue = $"reconnect-value-{index}",
|
||||
};
|
||||
mxEvent.OnDataChange = new OnDataChangeEvent();
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
WorkerEnvelope? envelope;
|
||||
try
|
||||
{
|
||||
using CancellationTokenSource readCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||
readCts.CancelAfter(TimeSpan.FromMilliseconds(50));
|
||||
envelope = await harness.ReadGatewayEnvelopeAsync(readCts.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Timed out waiting for an envelope — loop back to check the gate / emit.
|
||||
continue;
|
||||
}
|
||||
|
||||
if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerShutdown)
|
||||
{
|
||||
await harness.SendShutdownAckAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
_process.MarkExited(0);
|
||||
return;
|
||||
}
|
||||
|
||||
if (envelope.BodyCase != WorkerEnvelope.BodyOneofCase.WorkerCommand)
|
||||
{
|
||||
throw new InvalidOperationException($"Unexpected envelope {envelope.BodyCase}.");
|
||||
}
|
||||
|
||||
MxCommand command = envelope.WorkerCommand.Command;
|
||||
await harness.ReplyToCommandAsync(
|
||||
envelope,
|
||||
configureReply: reply => ConfigureCommandReply(reply, command.Kind),
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (command.Kind == MxCommandKind.Advise)
|
||||
{
|
||||
advisedServerHandle = command.Advise.ServerHandle;
|
||||
advisedItemHandle = command.Advise.ItemHandle;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,4 +76,35 @@ public sealed class MxAccessGrpcMapperTests
|
||||
|
||||
Assert.Equal(ProtocolStatusCode.ProtocolViolation, publicReply.ProtocolStatus.Code);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies MapEvent transfers ownership of the inner MxEvent (GWC-07 / IPC-05): the
|
||||
/// returned reference is the same instance carried by the WorkerEvent, not a clone. The
|
||||
/// WorkerEvent is discarded after mapping and the distributor pump is its single consumer,
|
||||
/// so moving the inner event out is safe and avoids a per-event deep copy.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void MapEvent_TransfersOwnershipOfInnerEventWithoutCloning()
|
||||
{
|
||||
MxEvent innerEvent = new()
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
RawStatus = "OK",
|
||||
};
|
||||
WorkerEvent workerEvent = new() { Event = innerEvent };
|
||||
|
||||
MxEvent mapped = new MxAccessGrpcMapper().MapEvent(workerEvent);
|
||||
|
||||
Assert.Same(innerEvent, mapped);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a worker event with no public payload maps to the unspecified-family sentinel.</summary>
|
||||
[Fact]
|
||||
public void MapEvent_WhenEventMissing_ReturnsUnspecifiedFamilySentinel()
|
||||
{
|
||||
MxEvent mapped = new MxAccessGrpcMapper().MapEvent(new WorkerEvent());
|
||||
|
||||
Assert.Equal(MxEventFamily.Unspecified, mapped.Family);
|
||||
Assert.Equal("Worker event did not contain a public event payload.", mapped.RawStatus);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,6 +185,90 @@ public sealed class SessionEventDistributorTests
|
||||
Assert.Equal(new ulong[] { 3, 4, 5 }, replay.Select(e => e.WorkerSequence));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appending far more events than the capacity wraps the circular buffer's head
|
||||
/// index past the array boundary multiple times; the retained window stays the
|
||||
/// newest <c>capacity</c> events, in ascending order, and a replay from before the
|
||||
/// window reports a gap and returns the whole retained tail.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReplayBuffer_WrapsRingMultipleTimes_RetainsNewestInAscendingOrder()
|
||||
{
|
||||
Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
|
||||
await using SessionEventDistributor distributor = CreateDistributor(
|
||||
source.Reader,
|
||||
replayBufferCapacity: 4,
|
||||
replayRetentionSeconds: 0);
|
||||
await distributor.StartAsync(CancellationToken.None);
|
||||
|
||||
// 13 events through a capacity-4 ring advances the head index 13 - 4 = 9 slots,
|
||||
// wrapping the 4-slot array's boundary more than twice.
|
||||
using IEventSubscriberLease lease = distributor.Register();
|
||||
for (ulong sequence = 1; sequence <= 13; sequence++)
|
||||
{
|
||||
source.Writer.TryWrite(Event(sequence));
|
||||
}
|
||||
|
||||
for (ulong sequence = 1; sequence <= 13; sequence++)
|
||||
{
|
||||
MxEvent e = await ReadOneAsync(lease.Reader);
|
||||
Assert.Equal(sequence, e.WorkerSequence);
|
||||
}
|
||||
|
||||
// Newest four (10, 11, 12, 13) retained in ascending order despite the wraps.
|
||||
bool found = distributor.TryGetReplayFrom(0, out IReadOnlyList<MxEvent> replay, out bool gap);
|
||||
|
||||
Assert.True(found);
|
||||
Assert.True(gap);
|
||||
Assert.Equal(new ulong[] { 10, 11, 12, 13 }, replay.Select(e => e.WorkerSequence));
|
||||
|
||||
// A replay from inside the wrapped window returns only the newer entries, no gap,
|
||||
// proving the modular scan reads the logical order and not the physical slots.
|
||||
bool foundInner = distributor.TryGetReplayFrom(11, out IReadOnlyList<MxEvent> inner, out bool innerGap);
|
||||
|
||||
Assert.True(foundInner);
|
||||
Assert.False(innerGap);
|
||||
Assert.Equal(new ulong[] { 12, 13 }, inner.Select(e => e.WorkerSequence));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A capacity-1 ring retains only the single newest event; each append overwrites
|
||||
/// the sole slot, and a replay from before it reports a gap.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReplayBuffer_Capacity1_RetainsOnlyNewest()
|
||||
{
|
||||
Channel<MxEvent> source = Channel.CreateUnbounded<MxEvent>();
|
||||
await using SessionEventDistributor distributor = CreateDistributor(
|
||||
source.Reader,
|
||||
replayBufferCapacity: 1,
|
||||
replayRetentionSeconds: 0);
|
||||
await distributor.StartAsync(CancellationToken.None);
|
||||
|
||||
using IEventSubscriberLease lease = distributor.Register();
|
||||
for (ulong sequence = 1; sequence <= 3; sequence++)
|
||||
{
|
||||
source.Writer.TryWrite(Event(sequence));
|
||||
_ = await ReadOneAsync(lease.Reader);
|
||||
}
|
||||
|
||||
// Only sequence 3 is retained; a request from 0 missed 1 and 2 => gap.
|
||||
bool found = distributor.TryGetReplayFrom(0, out IReadOnlyList<MxEvent> replay, out bool gap);
|
||||
|
||||
Assert.True(found);
|
||||
Assert.True(gap);
|
||||
Assert.Equal(new ulong[] { 3 }, replay.Select(e => e.WorkerSequence));
|
||||
|
||||
// A request from exactly the newest retained sequence is caught up: empty, no gap.
|
||||
bool foundCaughtUp = distributor.TryGetReplayFrom(3, out IReadOnlyList<MxEvent> caughtUp, out bool caughtUpGap);
|
||||
|
||||
Assert.True(foundCaughtUp);
|
||||
Assert.False(caughtUpGap);
|
||||
Assert.Empty(caughtUp);
|
||||
}
|
||||
|
||||
/// <summary>Requesting replay from a sequence still inside the retained window returns only the newer events, with no gap.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
|
||||
@@ -29,6 +29,32 @@ public sealed class WorkerFrameProtocolTests
|
||||
Assert.Equal(original, parsed);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a large payload near the message cap round-trips through the pooled write/read path.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteAndReadAsync_WithLargePayloadNearMax_RoundTripsFrame()
|
||||
{
|
||||
// A payload comfortably larger than an ArrayPool bucket boundary so the rented buffer is
|
||||
// larger than the exact frame length, exercising the length-bounded read and parse.
|
||||
const int maxMessageBytes = 1024 * 1024;
|
||||
WorkerFrameProtocolOptions options =
|
||||
new(SessionId, GatewayContractInfo.WorkerProtocolVersion, maxMessageBytes);
|
||||
await using MemoryStream stream = new();
|
||||
|
||||
WorkerEnvelope original = CreateEnvelope();
|
||||
original.WorkerHello.WorkerVersion = new string('x', maxMessageBytes - 4096);
|
||||
Assert.InRange(original.CalculateSize(), 1, maxMessageBytes);
|
||||
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
await writer.WriteAsync(original);
|
||||
stream.Position = 0;
|
||||
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
WorkerEnvelope parsed = await reader.ReadAsync();
|
||||
|
||||
Assert.Equal(original, parsed);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that reading a frame with partial reads reassembles the frame correctly.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
|
||||
@@ -20,7 +20,9 @@ public sealed class GatewayMetricsTests
|
||||
metrics.EventReceived("session-1", "OnDataChange");
|
||||
metrics.EventReceived("session-1", "OnDataChange");
|
||||
metrics.SetWorkerEventQueueDepth(7);
|
||||
metrics.AdjustGrpcEventStreamQueueDepth(3);
|
||||
// GWC-15: the gRPC stream queue-depth gauge sums live backlog sources at collection time
|
||||
// rather than tracking a pushed running total. Register a source reporting 3.
|
||||
using IDisposable backlogSource = metrics.RegisterEventStreamBacklogSource(static () => 3);
|
||||
metrics.QueueOverflow("session-events");
|
||||
metrics.Fault("CommandTimeout");
|
||||
metrics.WorkerKilled("CommandTimeout");
|
||||
|
||||
@@ -106,6 +106,35 @@ public sealed class MxStatusProxyConverterTests
|
||||
Assert.Contains("success", exception.Message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that repeated conversions of the same status type produce
|
||||
/// identical results. WRK-06 caches the resolved FieldInfo objects per
|
||||
/// type after the first conversion; the cached path must yield the same
|
||||
/// message the uncached first call produced.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Convert_RepeatedForSameType_ProducesIdenticalResults()
|
||||
{
|
||||
FakeMxStatusProxy status = new()
|
||||
{
|
||||
success = 0,
|
||||
category = 3,
|
||||
detectedBy = 1,
|
||||
detail = 21,
|
||||
};
|
||||
|
||||
// First call populates the per-type FieldInfo cache; the second reuses it.
|
||||
MxStatusProxy first = _converter.Convert(status);
|
||||
MxStatusProxy second = _converter.Convert(status);
|
||||
|
||||
Assert.Equal(first, second);
|
||||
Assert.Equal(MxStatusCategory.CommunicationError, second.Category);
|
||||
Assert.Equal(MxStatusSource.RespondingLmx, second.DetectedBy);
|
||||
Assert.Equal(0, second.Success);
|
||||
Assert.Equal(21, second.Detail);
|
||||
Assert.Equal("Invalid reference", second.DiagnosticText);
|
||||
}
|
||||
|
||||
public struct FakeMxStatusProxy
|
||||
{
|
||||
public short success;
|
||||
|
||||
@@ -373,6 +373,43 @@ public sealed class WorkerFrameProtocolTests
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, frame3.BodyCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the writer coalesces the flush across a batch of frames drained together: four frames
|
||||
/// queued behind an in-progress write drain in a single pass and share one FlushAsync, not four.
|
||||
/// Every frame still reaches the wire intact.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteAsync_WhenBatchDrainedTogether_FlushesOnce()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using GatedWriteStream stream = new();
|
||||
WorkerFrameWriter writer = new(stream, options);
|
||||
|
||||
// A blocked first write occupies the writer and holds the lock while more frames queue behind it.
|
||||
Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control);
|
||||
await AwaitWithTimeoutAsync(stream.FirstWriteStarted);
|
||||
|
||||
Task eventWrite1 = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event);
|
||||
Task eventWrite2 = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event);
|
||||
Task eventWrite3 = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event);
|
||||
await Task.Delay(50);
|
||||
|
||||
stream.ReleaseFirstWrite();
|
||||
await AwaitWithTimeoutAsync(Task.WhenAll(firstWrite, eventWrite1, eventWrite2, eventWrite3));
|
||||
|
||||
// Four frames written in one drain pass => exactly one flush.
|
||||
Assert.Equal(1, stream.FlushCount);
|
||||
|
||||
stream.Position = 0;
|
||||
WorkerFrameReader reader = new(stream, options);
|
||||
for (int index = 0; index < 4; index++)
|
||||
{
|
||||
WorkerEnvelope frame = await reader.ReadAsync();
|
||||
Assert.NotEqual(WorkerEnvelope.BodyOneofCase.None, frame.BodyCase);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Verifies a zero negotiated frame maximum keeps the constructor default.</summary>
|
||||
[Fact]
|
||||
public void AdoptNegotiatedMaxMessageBytes_WithZero_KeepsDefault()
|
||||
@@ -451,10 +488,14 @@ public sealed class WorkerFrameProtocolTests
|
||||
private readonly TaskCompletionSource<bool> _firstWriteStarted =
|
||||
new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private int _writeCount;
|
||||
private int _flushCount;
|
||||
|
||||
/// <summary>Gets a task that completes once the first <see cref="WriteAsync"/> call has started blocking.</summary>
|
||||
public Task FirstWriteStarted => _firstWriteStarted.Task;
|
||||
|
||||
/// <summary>Gets the number of <see cref="FlushAsync"/> calls observed so far.</summary>
|
||||
public int FlushCount => Volatile.Read(ref _flushCount);
|
||||
|
||||
/// <summary>Releases the first blocked write so it can complete.</summary>
|
||||
public void ReleaseFirstWrite() => _release.Release();
|
||||
|
||||
@@ -470,6 +511,13 @@ public sealed class WorkerFrameProtocolTests
|
||||
await base.WriteAsync(buffer, offset, count, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task FlushAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref _flushCount);
|
||||
return base.FlushAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Dispose(bool disposing)
|
||||
{
|
||||
|
||||
@@ -29,6 +29,26 @@ public sealed class MxAccessEventQueueTests
|
||||
Assert.False(queue.TryDequeue(out _));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that Enqueue takes ownership of the passed event instead of
|
||||
/// cloning it: the dequeued instance is the very reference passed in, and
|
||||
/// the worker sequence/timestamp are stamped on that same instance
|
||||
/// (WRK-11).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Enqueue_TakesOwnershipOfPassedEventInstance()
|
||||
{
|
||||
MxAccessEventQueue queue = new(capacity: 4);
|
||||
MxEvent original = CreateEvent(MxEventFamily.OnDataChange, itemHandle: 10);
|
||||
|
||||
queue.Enqueue(original);
|
||||
|
||||
Assert.True(queue.TryDequeue(out WorkerEvent? dequeued));
|
||||
Assert.Same(original, dequeued?.Event);
|
||||
Assert.Equal(1UL, original.WorkerSequence);
|
||||
Assert.NotNull(original.WorkerTimestamp);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that Drain removes at most the requested number of events.</summary>
|
||||
[Fact]
|
||||
public void Drain_RemovesAtMostRequestedEvents()
|
||||
|
||||
@@ -46,6 +46,37 @@ public sealed class MxAccessValueCacheTests
|
||||
Assert.Equal(999, other.Value.Int32Value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that Set stores an independent deep-copied snapshot: mutating
|
||||
/// the source event's protobuf sub-messages after caching does not alter
|
||||
/// the cached value. WRK-11 stopped the event sink cloning before enqueue,
|
||||
/// so the same MxEvent instance now flows to the outbound queue; the cache
|
||||
/// must own its own copy so the two never share mutable state.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Set_StoresIndependentSnapshot_UnaffectedByLaterEventMutation()
|
||||
{
|
||||
MxAccessValueCache cache = new();
|
||||
Timestamp sourceTimestamp = Timestamp.FromDateTime(new(2026, 5, 19, 9, 0, 0, DateTimeKind.Utc));
|
||||
MxEvent mxEvent = BuildEvent(serverHandle: 7, itemHandle: 21, intValue: 100, quality: 192, sourceTimestamp);
|
||||
|
||||
cache.Set(7, 21, mxEvent);
|
||||
|
||||
// Mutate the event in place after it was cached — as if it kept flowing
|
||||
// through the (unrelated) outbound path. None of this must reach the cache.
|
||||
mxEvent.Value.Int32Value = 999;
|
||||
mxEvent.Quality = 0;
|
||||
mxEvent.SourceTimestamp = Timestamp.FromDateTime(new(2030, 1, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||
mxEvent.Statuses[0].Category = MxStatusCategory.SecurityError;
|
||||
|
||||
Assert.True(cache.TryGet(7, 21, out MxAccessValueCache.CachedValue cached));
|
||||
Assert.Equal(100, cached.Value.Int32Value);
|
||||
Assert.Equal(192, cached.Quality);
|
||||
Assert.Equal(sourceTimestamp, cached.SourceTimestamp);
|
||||
Assert.Single(cached.Statuses);
|
||||
Assert.Equal(MxStatusCategory.Ok, cached.Statuses[0].Category);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that TryGet returns false for unknown handles.</summary>
|
||||
[Fact]
|
||||
public void TryGet_WithUnknownHandle_ReturnsFalse()
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Reflection;
|
||||
@@ -9,6 +10,17 @@ namespace ZB.MOM.WW.MxGateway.Worker.Conversion;
|
||||
/// <summary>Converts MXAccess MXSTATUS_PROXY COM objects to protobuf MxStatusProxy messages.</summary>
|
||||
public sealed class MxStatusProxyConverter
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-type cache of the four resolved <see cref="FieldInfo"/> objects a
|
||||
/// status conversion needs. The status type is stable (the interop
|
||||
/// <c>MXSTATUS_PROXY</c> struct in production; a fixed test double in
|
||||
/// tests), so the expensive <see cref="Type.GetField(string, BindingFlags)"/>
|
||||
/// metadata scan is resolved once per type and reused. Keyed by
|
||||
/// <see cref="Type"/> so a plain-CLR test double and the real interop
|
||||
/// struct each get their own entry, keeping the converter interop-agnostic.
|
||||
/// </summary>
|
||||
private static readonly ConcurrentDictionary<Type, StatusFields> FieldCache = new();
|
||||
|
||||
/// <summary>Converts a single status object to a protobuf message, reflecting all fields and diagnostics.</summary>
|
||||
/// <param name="status">COM status object to convert.</param>
|
||||
/// <returns>The converted protobuf status message.</returns>
|
||||
@@ -20,10 +32,11 @@ public sealed class MxStatusProxyConverter
|
||||
}
|
||||
|
||||
Type statusType = status.GetType();
|
||||
int success = ReadInt32Field(status, statusType, "success");
|
||||
int rawCategory = ReadInt32Field(status, statusType, "category");
|
||||
int rawDetectedBy = ReadInt32Field(status, statusType, "detectedBy");
|
||||
int detail = ReadInt32Field(status, statusType, "detail");
|
||||
StatusFields fields = GetFields(statusType);
|
||||
int success = ReadInt32Field(status, statusType, fields.Success);
|
||||
int rawCategory = ReadInt32Field(status, statusType, fields.Category);
|
||||
int rawDetectedBy = ReadInt32Field(status, statusType, fields.DetectedBy);
|
||||
int detail = ReadInt32Field(status, statusType, fields.Detail);
|
||||
|
||||
return new MxStatusProxy
|
||||
{
|
||||
@@ -82,6 +95,43 @@ public sealed class MxStatusProxyConverter
|
||||
|
||||
private static int ReadInt32Field(
|
||||
object value,
|
||||
Type valueType,
|
||||
FieldInfo field)
|
||||
{
|
||||
object? fieldValue = field.GetValue(value);
|
||||
if (fieldValue is null)
|
||||
{
|
||||
throw new MxStatusConversionException(
|
||||
$"Status object field '{field.Name}' on type '{valueType.FullName}' is null.");
|
||||
}
|
||||
|
||||
return System.Convert.ToInt32(fieldValue, CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves (and caches) the four <see cref="FieldInfo"/> objects for the
|
||||
/// given status type. The first resolution for a type performs the
|
||||
/// reflection scan; every subsequent conversion of that type reuses the
|
||||
/// cached entry. A type missing a required field throws the same
|
||||
/// <see cref="MxStatusConversionException"/> the per-field lookup used to
|
||||
/// throw — and, because <see cref="ConcurrentDictionary{TKey,TValue}.GetOrAdd(TKey, Func{TKey, TValue})"/>
|
||||
/// does not store a value when the factory throws, a bad type keeps
|
||||
/// failing identically on every call rather than being cached.
|
||||
/// </summary>
|
||||
/// <param name="statusType">Runtime type of the status object being converted.</param>
|
||||
/// <returns>The resolved field set for <paramref name="statusType"/>.</returns>
|
||||
private static StatusFields GetFields(Type statusType)
|
||||
{
|
||||
return FieldCache.GetOrAdd(
|
||||
statusType,
|
||||
type => new StatusFields(
|
||||
ResolveField(type, "success"),
|
||||
ResolveField(type, "category"),
|
||||
ResolveField(type, "detectedBy"),
|
||||
ResolveField(type, "detail")));
|
||||
}
|
||||
|
||||
private static FieldInfo ResolveField(
|
||||
Type valueType,
|
||||
string fieldName)
|
||||
{
|
||||
@@ -92,14 +142,7 @@ public sealed class MxStatusProxyConverter
|
||||
$"Status object type '{valueType.FullName}' does not expose required field '{fieldName}'.");
|
||||
}
|
||||
|
||||
object? fieldValue = field.GetValue(value);
|
||||
if (fieldValue is null)
|
||||
{
|
||||
throw new MxStatusConversionException(
|
||||
$"Status object field '{fieldName}' on type '{valueType.FullName}' is null.");
|
||||
}
|
||||
|
||||
return System.Convert.ToInt32(fieldValue, CultureInfo.InvariantCulture);
|
||||
return field;
|
||||
}
|
||||
|
||||
private static MxStatusCategory MapCategory(int rawCategory)
|
||||
@@ -134,4 +177,32 @@ public sealed class MxStatusProxyConverter
|
||||
_ => MxStatusSource.Unknown,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The four resolved status fields cached per type. Plain readonly struct
|
||||
/// (not a record) so it compiles under the worker's net48 target, which
|
||||
/// lacks <c>IsExternalInit</c>.
|
||||
/// </summary>
|
||||
private readonly struct StatusFields
|
||||
{
|
||||
public StatusFields(
|
||||
FieldInfo success,
|
||||
FieldInfo category,
|
||||
FieldInfo detectedBy,
|
||||
FieldInfo detail)
|
||||
{
|
||||
Success = success;
|
||||
Category = category;
|
||||
DetectedBy = detectedBy;
|
||||
Detail = detail;
|
||||
}
|
||||
|
||||
public FieldInfo Success { get; }
|
||||
|
||||
public FieldInfo Category { get; }
|
||||
|
||||
public FieldInfo DetectedBy { get; }
|
||||
|
||||
public FieldInfo Detail { get; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,36 +116,77 @@ public sealed class WorkerFrameWriter
|
||||
// Runs only under _writeLock. Drains control frames before event frames, stamping and writing each.
|
||||
// The stream write itself is not cancellable: a frame is written atomically or fails, never left
|
||||
// half-written on the pipe because a caller gave up waiting.
|
||||
//
|
||||
// Flushes are coalesced across the whole drained batch (WRK-12 / IPC-15): each frame is written to
|
||||
// the stream but not flushed individually; a single FlushAsync runs after the batch, then every
|
||||
// successfully-written frame is completed. A caller's Completion therefore still signals only after
|
||||
// its bytes have been written AND flushed, so the "written and flushed" contract is unchanged — but
|
||||
// a burst of N events now costs one flush syscall instead of N.
|
||||
private async Task DrainQueuedFramesAsync()
|
||||
{
|
||||
List<PendingFrame> written = new List<PendingFrame>();
|
||||
while (true)
|
||||
{
|
||||
PendingFrame? frame = DequeueNext();
|
||||
if (frame is null)
|
||||
{
|
||||
return;
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await WriteFrameAsync(frame.Envelope).ConfigureAwait(false);
|
||||
frame.Completion.TrySetResult(true);
|
||||
written.Add(frame);
|
||||
}
|
||||
catch (WorkerFrameProtocolException exception) when (IsPerFrameRejection(exception))
|
||||
{
|
||||
// Validation, empty-payload, and oversized-frame errors are specific to this frame and
|
||||
// do not damage the stream; fail only this frame and keep draining the rest.
|
||||
// do not damage the stream; fail only this frame and keep draining the rest. Nothing was
|
||||
// written for it, so it needs no flush.
|
||||
frame.Completion.TrySetException(exception);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// A stream write/flush failure means the pipe is broken; fail this frame and every frame
|
||||
// still queued so no caller awaits forever, then stop draining.
|
||||
// A stream write failure means the pipe is broken; fail this frame, every frame already
|
||||
// written this batch but not yet flushed, and every frame still queued so no caller
|
||||
// awaits forever, then stop draining.
|
||||
frame.Completion.TrySetException(exception);
|
||||
FailFrames(written, exception);
|
||||
FailAllQueued(exception);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (written.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// The batch reached the stream but the flush that guarantees delivery failed: the pipe is
|
||||
// broken. Fail every frame in the batch (the queue was already drained) so no caller treats
|
||||
// an unflushed write as delivered.
|
||||
FailFrames(written, exception);
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (PendingFrame frame in written)
|
||||
{
|
||||
frame.Completion.TrySetResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
private static void FailFrames(List<PendingFrame> frames, Exception exception)
|
||||
{
|
||||
foreach (PendingFrame frame in frames)
|
||||
{
|
||||
frame.Completion.TrySetException(exception);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsPerFrameRejection(WorkerFrameProtocolException exception)
|
||||
@@ -215,14 +256,14 @@ public sealed class WorkerFrameWriter
|
||||
|
||||
// Serialize once into a single buffer that carries the 4-byte length prefix followed by the
|
||||
// payload, then issue one stream write. This avoids a second serialization pass, a separate
|
||||
// prefix array, and a separate prefix write.
|
||||
// prefix array, and a separate prefix write. The flush is deferred to the end of the drained
|
||||
// batch (see DrainQueuedFramesAsync) so a burst of frames shares one flush.
|
||||
int frameLength = sizeof(uint) + payloadLength;
|
||||
byte[] frame = new byte[frameLength];
|
||||
WriteUInt32LittleEndian(frame, (uint)payloadLength);
|
||||
envelope.WriteTo(new Span<byte>(frame, sizeof(uint), payloadLength));
|
||||
|
||||
await _stream.WriteAsync(frame, 0, frameLength, CancellationToken.None).ConfigureAwait(false);
|
||||
await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static void WriteUInt32LittleEndian(
|
||||
|
||||
@@ -8,6 +8,17 @@ namespace ZB.MOM.WW.MxGateway.Worker.MxAccess;
|
||||
/// <summary>
|
||||
/// Thread-safe queue for MxAccess events with capacity overflow and fault tracking.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Ownership invariant: <see cref="Enqueue"/> takes ownership of the
|
||||
/// <see cref="MxEvent"/> passed to it — it stamps the worker sequence and
|
||||
/// timestamp on that same instance and enqueues it directly, without
|
||||
/// cloning (WRK-11). Every caller must therefore pass a freshly built
|
||||
/// <see cref="MxEvent"/> that it does not retain, reuse, or mutate after the
|
||||
/// call returns. All production callers (MxAccessBaseEventSink,
|
||||
/// MxAccessAlarmEventSink, AlarmCommandHandler) build a new event per
|
||||
/// Enqueue via the mapper and satisfy this; the value cache stores its own
|
||||
/// independent snapshot (see <see cref="MxAccessValueCache.Set"/>).
|
||||
/// </remarks>
|
||||
public sealed class MxAccessEventQueue
|
||||
{
|
||||
/// <summary>
|
||||
@@ -110,8 +121,11 @@ public sealed class MxAccessEventQueue
|
||||
|
||||
/// <summary>
|
||||
/// Enqueues an MxAccess event, assigning a sequence number and timestamp.
|
||||
/// Takes ownership of <paramref name="mxEvent"/>: the sequence and timestamp
|
||||
/// are stamped on that same instance and it is enqueued without cloning, so
|
||||
/// the caller must pass a freshly built event it does not reuse afterwards.
|
||||
/// </summary>
|
||||
/// <param name="mxEvent">MXAccess event to enqueue.</param>
|
||||
/// <param name="mxEvent">Freshly built MXAccess event to enqueue; ownership transfers to the queue.</param>
|
||||
public void Enqueue(MxEvent mxEvent)
|
||||
{
|
||||
if (mxEvent is null)
|
||||
@@ -132,13 +146,17 @@ public sealed class MxAccessEventQueue
|
||||
throw new MxAccessEventQueueOverflowException(capacity);
|
||||
}
|
||||
|
||||
MxEvent queuedEvent = mxEvent.Clone();
|
||||
queuedEvent.WorkerSequence = ++lastEventSequence;
|
||||
queuedEvent.WorkerTimestamp = Timestamp.FromDateTime(DateTime.UtcNow);
|
||||
// WRK-11: stamp the sequence/timestamp on the caller's own event and
|
||||
// enqueue that same instance under the lock instead of cloning. See
|
||||
// the ownership invariant on the class summary — the caller hands the
|
||||
// event over exclusively, so a defensive Clone() here is pure
|
||||
// overhead on the hottest path.
|
||||
mxEvent.WorkerSequence = ++lastEventSequence;
|
||||
mxEvent.WorkerTimestamp = Timestamp.FromDateTime(DateTime.UtcNow);
|
||||
|
||||
WorkerEvent workerEvent = new()
|
||||
{
|
||||
Event = queuedEvent,
|
||||
Event = mxEvent,
|
||||
};
|
||||
events.Enqueue(workerEvent);
|
||||
}
|
||||
|
||||
@@ -40,6 +40,20 @@ public sealed class MxAccessValueCache
|
||||
throw new ArgumentNullException(nameof(mxEvent));
|
||||
}
|
||||
|
||||
// WRK-11: the event sink no longer clones before enqueue, so the passed
|
||||
// mxEvent is the very instance handed to the outbound queue. Deep-copy
|
||||
// the value/timestamp/statuses payload we retain here so the cache's
|
||||
// snapshot stays independent of the enqueued (and later serialized)
|
||||
// event — the two must never share mutable protobuf sub-messages.
|
||||
// Value is always set for OnDataChange; SourceTimestamp may be unset when
|
||||
// the source timestamp could not be parsed, so both are cloned only when
|
||||
// present. The null-forgiving result matches CachedValue's non-null-
|
||||
// annotated parameters, which already accepted a runtime-null value or
|
||||
// timestamp before WRK-11 (the ternary keeps the compiler's null-state
|
||||
// from poisoning to maybe-null, which a plain null check would do).
|
||||
MxValue cachedValue = mxEvent.Value is null ? null! : mxEvent.Value.Clone();
|
||||
Timestamp cachedTimestamp = mxEvent.SourceTimestamp is null ? null! : mxEvent.SourceTimestamp.Clone();
|
||||
|
||||
long key = CreateItemKey(serverHandle, itemHandle);
|
||||
lock (syncRoot)
|
||||
{
|
||||
@@ -49,10 +63,10 @@ public sealed class MxAccessValueCache
|
||||
|
||||
entries[key] = new CachedValue(
|
||||
nextVersion,
|
||||
mxEvent.Value,
|
||||
cachedValue,
|
||||
mxEvent.Quality,
|
||||
mxEvent.SourceTimestamp,
|
||||
mxEvent.Statuses);
|
||||
cachedTimestamp,
|
||||
mxEvent.Statuses.Clone());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user