Merge branch 'fix/archreview-p2' into main (P2 tier: completeness & polish)
# Conflicts: # archreview/remediation/00-tracking.md # clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli/MxGatewayCliSecretRedactor.cs # clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli/MxGatewayClientCli.cs # src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs
This commit is contained in:
@@ -35,5 +35,23 @@ public static class DashboardAuthenticationDefaults
|
||||
|
||||
public const string LdapGroupClaimType = "mxgateway:ldap_group";
|
||||
public const string KeyPrefixClaimType = "mxgateway:key_prefix";
|
||||
|
||||
/// <summary>
|
||||
/// Dashboard auth cookie name used when the cookie is not guaranteed to be Secure
|
||||
/// (<c>RequireHttpsCookie=false</c> → <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.SameAsRequest"/>)
|
||||
/// or when an explicit <c>MxGateway:Dashboard:CookieName</c> override is absent but the
|
||||
/// secure default cannot be applied. This plain name carries no browser-enforced guarantees.
|
||||
/// </summary>
|
||||
public const string CookieName = "MxGatewayDashboard";
|
||||
|
||||
/// <summary>
|
||||
/// Dashboard auth cookie name applied when the cookie is guaranteed Secure
|
||||
/// (<c>RequireHttpsCookie=true</c> → <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.Always"/>)
|
||||
/// and no explicit <c>MxGateway:Dashboard:CookieName</c> override is set. The <c>__Host-</c>
|
||||
/// prefix instructs browsers to enforce Secure, no <c>Domain</c>, and <c>Path=/</c>; those
|
||||
/// guarantees only hold for a Secure cookie, so this name must never be applied unless
|
||||
/// <see cref="Microsoft.AspNetCore.Authentication.Cookies.CookieSecurePolicy.Always"/> is in
|
||||
/// effect — a <c>__Host-</c> cookie without Secure is silently dropped by browsers.
|
||||
/// </summary>
|
||||
public const string SecureCookieName = "__Host-MxGatewayDashboard";
|
||||
}
|
||||
|
||||
@@ -123,13 +123,22 @@ public static class DashboardServiceCollectionExtensions
|
||||
? CookieSecurePolicy.Always
|
||||
: CookieSecurePolicy.SameAsRequest;
|
||||
|
||||
// Config-driven cookie name (MxGateway:Dashboard:CookieName). Null/blank keeps
|
||||
// the canonical default set above, so a misconfiguration cannot unname the cookie.
|
||||
// Config-driven cookie name (MxGateway:Dashboard:CookieName). An explicit override
|
||||
// always wins. With no override, restore the __Host- prefix when the cookie is
|
||||
// guaranteed Secure (RequireHttpsCookie true → SecurePolicy Always): the __Host-
|
||||
// browser guarantees (Secure required, no Domain, Path=/) hold only for a Secure
|
||||
// cookie, and a __Host- cookie without Secure is silently dropped — so the prefix
|
||||
// is never applied unless SecurePolicy is Always. Otherwise keep the plain
|
||||
// canonical default set by AddCookie above, so a misconfiguration cannot unname it.
|
||||
var cookieName = gatewayOptions.Value.Dashboard.CookieName;
|
||||
if (!string.IsNullOrWhiteSpace(cookieName))
|
||||
{
|
||||
cookieOptions.Cookie.Name = cookieName;
|
||||
}
|
||||
else if (cookieOptions.Cookie.SecurePolicy == CookieSecurePolicy.Always)
|
||||
{
|
||||
cookieOptions.Cookie.Name = DashboardAuthenticationDefaults.SecureCookieName;
|
||||
}
|
||||
});
|
||||
|
||||
services.AddAuthorization(authorization =>
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
@@ -10,10 +12,22 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// Errors are logged once and dropped — keeping the SignalR mirror best-effort
|
||||
/// preserves the gRPC contract that exists today.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), tag
|
||||
/// values are stripped from a redacted copy of the event before it reaches any
|
||||
/// dashboard client. The source <see cref="MxEvent"/> is shared with the gRPC
|
||||
/// event path and the reconnect replay ring, so it is never mutated in place —
|
||||
/// the redaction is applied to a deep clone. This closes the value-leak seam at
|
||||
/// the mirror independently of the still-outstanding per-session hub ACL
|
||||
/// (see <see cref="EventsHub"/>).
|
||||
/// </remarks>
|
||||
public sealed class DashboardEventBroadcaster(
|
||||
IHubContext<EventsHub> hubContext,
|
||||
IOptions<GatewayOptions> options,
|
||||
ILogger<DashboardEventBroadcaster> logger) : IDashboardEventBroadcaster
|
||||
{
|
||||
private readonly bool _showTagValues = options.Value.Dashboard.ShowTagValues;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Publish(string sessionId, MxEvent mxEvent)
|
||||
{
|
||||
@@ -22,6 +36,8 @@ public sealed class DashboardEventBroadcaster(
|
||||
return;
|
||||
}
|
||||
|
||||
MxEvent outbound = _showTagValues ? mxEvent : RedactValues(mxEvent);
|
||||
|
||||
// Wrap the Task acquisition in a try/catch so a hypothetical synchronous throw
|
||||
// from SendAsync (e.g. an implementation that throws before returning the Task)
|
||||
// cannot escape Publish. The interface contract is never-throw; fire-and-forget.
|
||||
@@ -30,7 +46,7 @@ public sealed class DashboardEventBroadcaster(
|
||||
{
|
||||
send = hubContext.Clients
|
||||
.Group(EventsHub.GroupName(sessionId))
|
||||
.SendAsync(EventsHub.EventMessage, mxEvent);
|
||||
.SendAsync(EventsHub.EventMessage, outbound);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -51,4 +67,27 @@ public sealed class DashboardEventBroadcaster(
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Produces a deep clone of <paramref name="source"/> with every tag-value
|
||||
/// field cleared, leaving tag reference, quality, status, and timestamps
|
||||
/// intact so the dashboard still renders the event without the value. The
|
||||
/// source event is left untouched because it is shared downstream with the
|
||||
/// gRPC stream and the replay ring.
|
||||
/// </summary>
|
||||
/// <param name="source">The source event to redact a copy of.</param>
|
||||
/// <returns>A redacted deep clone of the event.</returns>
|
||||
private static MxEvent RedactValues(MxEvent source)
|
||||
{
|
||||
MxEvent redacted = source.Clone();
|
||||
redacted.Value = null;
|
||||
|
||||
if (redacted.BodyCase == MxEvent.BodyOneofCase.OnAlarmTransition)
|
||||
{
|
||||
redacted.OnAlarmTransition.CurrentValue = null;
|
||||
redacted.OnAlarmTransition.LimitValue = null;
|
||||
}
|
||||
|
||||
return redacted;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,15 +32,19 @@ public sealed class EventsHub : Hub
|
||||
/// the dashboard roles (Admin or Viewer); both roles may subscribe to
|
||||
/// any session id they choose. This is acceptable today because (a) the
|
||||
/// dashboard's per-session views show non-secret session metadata that
|
||||
/// any authenticated dashboard user can already see, and (b) value
|
||||
/// logging in the source gRPC stream is gated by the same redaction
|
||||
/// policy that protects logs. The per-session ACL that gates the gRPC
|
||||
/// any authenticated dashboard user can already see, and (b) tag values
|
||||
/// are stripped from the mirrored events by
|
||||
/// <see cref="DashboardEventBroadcaster"/> when
|
||||
/// <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), so the
|
||||
/// most sensitive payload cannot leak through this seam regardless of the
|
||||
/// still-missing ACL. The per-session ACL that gates the gRPC
|
||||
/// <c>StreamEvents</c> RPC is intentionally not yet mirrored here.
|
||||
/// TODO(per-session-acl): once a role/scope is introduced that scopes a
|
||||
/// Viewer to a specific session or tenant, add a session-access check
|
||||
/// at this seam — either inline (consult the per-user allowed-session
|
||||
/// set on <c>Context.User</c> claims / <c>Context.Items</c>) or via a
|
||||
/// dedicated authorization policy applied to the hub method itself.
|
||||
/// TODO(per-session-acl): tracked as remediation roadmap item 12
|
||||
/// (SEC-25). Once a role/scope is introduced that scopes a Viewer to a
|
||||
/// specific session or tenant, add a session-access check at this seam —
|
||||
/// either inline (consult the per-user allowed-session set on
|
||||
/// <c>Context.User</c> claims / <c>Context.Items</c>) or via a dedicated
|
||||
/// authorization policy applied to the hub method itself.
|
||||
/// </remarks>
|
||||
/// <param name="sessionId">Session id to subscribe the caller to.</param>
|
||||
/// <returns>A task representing the subscription operation.</returns>
|
||||
|
||||
@@ -106,11 +106,23 @@ public sealed class EventStreamService(
|
||||
options.Value.Sessions.MaxEventSubscribersPerSession);
|
||||
}
|
||||
|
||||
int streamQueueDepth = 0;
|
||||
IAsyncEnumerator<MxEvent> reader = subscriber.Reader
|
||||
.ReadAllAsync(cancellationToken)
|
||||
.GetAsyncEnumerator(cancellationToken);
|
||||
|
||||
// GWC-15: register this subscriber's channel as a live backlog source instead of
|
||||
// reconciling the queue-depth gauge on every event. The gauge previously read the
|
||||
// bounded channel's Count (which takes the channel's internal lock) and adjusted the
|
||||
// metric under its own lock on every streamed event. Now the metric reads Count only
|
||||
// when it is scraped (ObservableGauge callback) or projected (GetSnapshot), summing the
|
||||
// live backlog across every registered subscriber — the same "buffered, not yet
|
||||
// delivered" aggregate the per-event push reported, but with no per-event lock traffic.
|
||||
// Disposing the registration in the finally removes this subscriber's contribution, so
|
||||
// the gauge returns to the other subscribers' backlog (zero when none remain) on
|
||||
// disconnect. CanCount guards a channel that ever cannot report Count (contributes 0).
|
||||
IDisposable backlogRegistration = metrics.RegisterEventStreamBacklogSource(
|
||||
() => subscriber.Reader.CanCount ? subscriber.Reader.Count : 0);
|
||||
|
||||
try
|
||||
{
|
||||
// Emit order for a resume: the ReplayGap sentinel FIRST (only when events were
|
||||
@@ -169,32 +181,21 @@ public sealed class EventStreamService(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Queue-depth gauge tracks events the pump has fanned into this subscriber's
|
||||
// channel but the client has not yet consumed — the same "buffered, not yet
|
||||
// delivered" quantity the original per-RPC channel reported. The bounded
|
||||
// subscriber channel supports counting, so reconcile the gauge to the current
|
||||
// backlog; falling back to a no-op delta if a channel ever cannot count.
|
||||
int backlog = subscriber.Reader.CanCount ? subscriber.Reader.Count : streamQueueDepth;
|
||||
int delta = backlog - streamQueueDepth;
|
||||
if (delta != 0)
|
||||
{
|
||||
streamQueueDepth = backlog;
|
||||
metrics.AdjustGrpcEventStreamQueueDepth(delta);
|
||||
}
|
||||
|
||||
// The queue-depth gauge is maintained lazily via the backlog registration above
|
||||
// (GWC-15): the metric reads this subscriber's channel Count only when scraped,
|
||||
// so there is no per-event gauge bookkeeping on this hot path.
|
||||
yield return mxEvent;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await reader.DisposeAsync().ConfigureAwait(false);
|
||||
subscriber.Dispose();
|
||||
|
||||
if (streamQueueDepth != 0)
|
||||
{
|
||||
metrics.AdjustGrpcEventStreamQueueDepth(-streamQueueDepth);
|
||||
streamQueueDepth = 0;
|
||||
}
|
||||
// Remove this subscriber's live backlog contribution before disposing the lease so
|
||||
// the gauge stops counting a channel that is about to be completed; after this the
|
||||
// gauge reflects only the remaining subscribers (zero when none remain).
|
||||
backlogRegistration.Dispose();
|
||||
subscriber.Dispose();
|
||||
|
||||
metrics.StreamDisconnected("Detached");
|
||||
}
|
||||
|
||||
@@ -152,9 +152,15 @@ public sealed class MxAccessGatewayService(
|
||||
.WithCancellation(context.CancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
Stopwatch stopwatch = Stopwatch.StartNew();
|
||||
// GWC-06: measure send latency with the allocation-free timestamp API rather
|
||||
// than allocating a Stopwatch object per event per subscriber on the highest-
|
||||
// volume gateway path. Stopwatch.GetTimestamp/GetElapsedTime measure the exact
|
||||
// same wall-clock span the former Stopwatch.StartNew()/.Elapsed did.
|
||||
long sendStartTimestamp = Stopwatch.GetTimestamp();
|
||||
await responseStream.WriteAsync(publicEvent).ConfigureAwait(false);
|
||||
metrics.RecordEventStreamSend(publicEvent.Family.ToString(), stopwatch.Elapsed);
|
||||
metrics.RecordEventStreamSend(
|
||||
publicEvent.Family.ToString(),
|
||||
Stopwatch.GetElapsedTime(sendStartTimestamp));
|
||||
}
|
||||
}
|
||||
catch (Exception exception) when (exception is not RpcException)
|
||||
|
||||
@@ -65,7 +65,16 @@ public sealed class MxAccessGrpcMapper
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(workerEvent);
|
||||
|
||||
return workerEvent.Event?.Clone() ?? new MxEvent
|
||||
// GWC-07 / IPC-05: ownership transfer, not a deep clone. The enclosing WorkerEvent is
|
||||
// parsed fresh from a single pipe frame in WorkerClient's read loop and is discarded
|
||||
// immediately after this mapping — the SessionEventDistributor pump is its single
|
||||
// consumer (GWC-01 claims the worker event channel as single-reader), so nothing else
|
||||
// aliases or mutates workerEvent.Event. We therefore move the inner MxEvent into the
|
||||
// outbound graph instead of cloning it. Downstream the pump fans this one MxEvent to
|
||||
// every subscriber and retains it in the replay ring, but that sharing is READ-ONLY
|
||||
// (subscribers only yield/filter it), so a single shared instance is safe. If a second
|
||||
// consumer of WorkerEvent is ever added, restore a .Clone() here to re-isolate.
|
||||
return workerEvent.Event ?? new MxEvent
|
||||
{
|
||||
Family = MxEventFamily.Unspecified,
|
||||
RawStatus = "Worker event did not contain a public event payload.",
|
||||
|
||||
@@ -32,10 +32,17 @@ public sealed class GatewayMetrics : IDisposable
|
||||
private readonly ConcurrentDictionary<string, long> _eventsBySession = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _retryAttemptsByArea = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// GWC-15: live backlog sources for the gRPC event-stream queue-depth gauge. Each active
|
||||
// StreamEvents subscriber registers a delegate returning its channel's current backlog; the
|
||||
// gauge sums them ONLY when scraped or projected (GetGrpcEventStreamQueueDepth), so nothing is
|
||||
// read on the per-event streaming hot path. Concurrent because subscribers register/unregister
|
||||
// from arbitrary gRPC threads while the gauge callback / GetSnapshot enumerate.
|
||||
private readonly ConcurrentDictionary<long, Func<int>> _eventStreamBacklogSources = new();
|
||||
private long _nextEventStreamBacklogSourceId;
|
||||
|
||||
private int _openSessions;
|
||||
private int _workersRunning;
|
||||
private int _workerEventQueueDepth;
|
||||
private int _grpcEventStreamQueueDepth;
|
||||
private int _alarmProviderMode;
|
||||
private long _sessionsOpened;
|
||||
private long _sessionsClosed;
|
||||
@@ -292,15 +299,24 @@ public sealed class GatewayMetrics : IDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adjusts the gRPC event stream queue depth by the given delta.
|
||||
/// Registers a live backlog source for the gRPC event-stream queue-depth gauge and
|
||||
/// returns a handle that removes it when disposed. The gauge sums every registered
|
||||
/// source's current value on demand (when scraped or via <see cref="GetSnapshot"/>),
|
||||
/// rather than tracking a pushed running total, so the streaming hot path does no
|
||||
/// per-event gauge bookkeeping (GWC-15).
|
||||
/// </summary>
|
||||
/// <param name="delta">Amount to adjust the queue depth by.</param>
|
||||
public void AdjustGrpcEventStreamQueueDepth(int delta)
|
||||
/// <param name="backlog">
|
||||
/// Returns this subscriber's current channel backlog. Invoked only at collection time;
|
||||
/// must be cheap and non-blocking (a bounded channel's <c>Count</c>). Negative returns
|
||||
/// are clamped to zero when summed.
|
||||
/// </param>
|
||||
/// <returns>A handle whose disposal unregisters the source. Safe to dispose more than once.</returns>
|
||||
public IDisposable RegisterEventStreamBacklogSource(Func<int> backlog)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_grpcEventStreamQueueDepth = Math.Max(0, _grpcEventStreamQueueDepth + delta);
|
||||
}
|
||||
ArgumentNullException.ThrowIfNull(backlog);
|
||||
long id = Interlocked.Increment(ref _nextEventStreamBacklogSourceId);
|
||||
_eventStreamBacklogSources[id] = backlog;
|
||||
return new EventStreamBacklogRegistration(this, id);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -429,13 +445,16 @@ public sealed class GatewayMetrics : IDisposable
|
||||
/// <returns>The current metrics snapshot.</returns>
|
||||
public GatewayMetricsSnapshot GetSnapshot()
|
||||
{
|
||||
// Compute the live gRPC stream backlog outside _syncRoot: the sources are the subscriber
|
||||
// channels' Count (their own locks) and must not run under this lock. GWC-15.
|
||||
int grpcEventStreamQueueDepth = GetGrpcEventStreamQueueDepth();
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return new GatewayMetricsSnapshot(
|
||||
OpenSessions: _openSessions,
|
||||
WorkersRunning: _workersRunning,
|
||||
WorkerEventQueueDepth: _workerEventQueueDepth,
|
||||
GrpcEventStreamQueueDepth: _grpcEventStreamQueueDepth,
|
||||
GrpcEventStreamQueueDepth: grpcEventStreamQueueDepth,
|
||||
SessionsOpened: _sessionsOpened,
|
||||
SessionsClosed: _sessionsClosed,
|
||||
CommandsStarted: _commandsStarted,
|
||||
@@ -495,12 +514,28 @@ public sealed class GatewayMetrics : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
// Sums the live backlog across every registered event-stream subscriber. Runs at collection
|
||||
// time (ObservableGauge scrape) or when GetSnapshot projects the value — never on the
|
||||
// per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent
|
||||
// register/unregister; a source removed mid-enumeration simply drops from this sample.
|
||||
private int GetGrpcEventStreamQueueDepth()
|
||||
{
|
||||
lock (_syncRoot)
|
||||
int total = 0;
|
||||
foreach (Func<int> source in _eventStreamBacklogSources.Values)
|
||||
{
|
||||
return _grpcEventStreamQueueDepth;
|
||||
int value = source();
|
||||
if (value > 0)
|
||||
{
|
||||
total += value;
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
private void UnregisterEventStreamBacklogSource(long id)
|
||||
{
|
||||
_eventStreamBacklogSources.TryRemove(id, out _);
|
||||
}
|
||||
|
||||
private int GetAlarmProviderMode()
|
||||
@@ -521,4 +556,19 @@ public sealed class GatewayMetrics : IDisposable
|
||||
{
|
||||
values.AddOrUpdate(key, 1, static (_, currentValue) => currentValue + 1);
|
||||
}
|
||||
|
||||
// Handle returned by RegisterEventStreamBacklogSource. Disposal (once) removes the source
|
||||
// from the gauge's live sum. Idempotent so a double dispose from a stream teardown is safe.
|
||||
private sealed class EventStreamBacklogRegistration(GatewayMetrics metrics, long id) : IDisposable
|
||||
{
|
||||
private int _disposed;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) == 0)
|
||||
{
|
||||
metrics.UnregisterEventStreamBacklogSource(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,14 +98,21 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
private readonly object _lifecycleLock = new();
|
||||
|
||||
// Replay ring buffer. Appended on the pump thread and queried from arbitrary
|
||||
// threads via TryGetReplayFrom, so every access is under _replayLock. The deque
|
||||
// keeps events in ascending WorkerSequence order (the pump fans in source order),
|
||||
// so the oldest retained event is always at the front. Capacity == 0 disables
|
||||
// retention; RetentionSeconds <= 0 disables age-based eviction.
|
||||
// threads via TryGetReplayFrom, so every access is under _replayLock. Backed by a
|
||||
// fixed-size circular array preallocated to the capacity so appending a retained
|
||||
// event allocates no node (the LinkedList this replaced allocated one node per
|
||||
// event on the fan-out hot path). Events are kept in ascending WorkerSequence order
|
||||
// (the pump fans in source order): _replayHead is the logical front (oldest retained
|
||||
// event) and _replayCount entries follow it, wrapping modulo the array length. The
|
||||
// logical entry at position i is _replayBuffer[(_replayHead + i) % Length]. Capacity
|
||||
// == 0 disables retention (the array is empty and never indexed); RetentionSeconds
|
||||
// <= 0 disables age-based eviction.
|
||||
private readonly int _replayBufferCapacity;
|
||||
private readonly TimeSpan _replayRetention;
|
||||
private readonly bool _ageEvictionEnabled;
|
||||
private readonly LinkedList<ReplayEntry> _replayBuffer = new();
|
||||
private readonly ReplayEntry[] _replayBuffer;
|
||||
private int _replayHead;
|
||||
private int _replayCount;
|
||||
private readonly object _replayLock = new();
|
||||
private bool _anyEventSeen;
|
||||
private ulong _highestSequenceSeen;
|
||||
@@ -245,6 +252,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
_overflowHandler = overflowHandler;
|
||||
_shutdownTimeout = DefaultShutdownTimeout;
|
||||
_replayBufferCapacity = replayBufferCapacity;
|
||||
_replayBuffer = new ReplayEntry[replayBufferCapacity];
|
||||
_ageEvictionEnabled = replayRetentionSeconds > 0;
|
||||
_replayRetention = _ageEvictionEnabled
|
||||
? TimeSpan.FromSeconds(replayRetentionSeconds)
|
||||
@@ -452,24 +460,25 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
List<MxEvent> newer = [];
|
||||
ulong highestReplayed = afterSequence;
|
||||
|
||||
if (_replayBuffer.Count == 0)
|
||||
if (_replayCount == 0)
|
||||
{
|
||||
gap = _anyEventSeen && afterSequence < _highestSequenceSeen;
|
||||
oldestAvailableSequence = 0; // meaningful only when gap == true; 0 here since nothing is retained
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong oldestRetained = _replayBuffer.First!.Value.Event.WorkerSequence;
|
||||
ulong oldestRetained = ReplayEntryAt(0).Event.WorkerSequence;
|
||||
gap = oldestRetained > 0 && afterSequence < oldestRetained - 1;
|
||||
// Per the contract on OldestAvailableSequence: meaningful only when gap == true.
|
||||
oldestAvailableSequence = gap ? oldestRetained : 0;
|
||||
|
||||
foreach (ReplayEntry entry in _replayBuffer)
|
||||
for (int i = 0; i < _replayCount; i++)
|
||||
{
|
||||
if (entry.Event.WorkerSequence > afterSequence)
|
||||
MxEvent retained = ReplayEntryAt(i).Event;
|
||||
if (retained.WorkerSequence > afterSequence)
|
||||
{
|
||||
newer.Add(entry.Event);
|
||||
highestReplayed = entry.Event.WorkerSequence;
|
||||
newer.Add(retained);
|
||||
highestReplayed = retained.WorkerSequence;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -729,7 +738,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
{
|
||||
EvictAged();
|
||||
|
||||
if (_replayBuffer.Count == 0)
|
||||
if (_replayCount == 0)
|
||||
{
|
||||
events = [];
|
||||
// Nothing retained. The caller missed events only if it is behind the
|
||||
@@ -738,7 +747,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
return true;
|
||||
}
|
||||
|
||||
ulong oldestRetained = _replayBuffer.First!.Value.Event.WorkerSequence;
|
||||
ulong oldestRetained = ReplayEntryAt(0).Event.WorkerSequence;
|
||||
|
||||
// A gap exists when at least one event newer than afterSequence was evicted,
|
||||
// i.e. afterSequence sits below the oldest-retained-minus-one boundary.
|
||||
@@ -750,11 +759,12 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
// O(n) scan over the retained buffer — acceptable because TryGetReplayFrom
|
||||
// is only called on subscriber reconnect, never on the hot fan-out path.
|
||||
List<MxEvent> newer = [];
|
||||
foreach (ReplayEntry entry in _replayBuffer)
|
||||
for (int i = 0; i < _replayCount; i++)
|
||||
{
|
||||
if (entry.Event.WorkerSequence > afterSequence)
|
||||
MxEvent retained = ReplayEntryAt(i).Event;
|
||||
if (retained.WorkerSequence > afterSequence)
|
||||
{
|
||||
newer.Add(entry.Event);
|
||||
newer.Add(retained);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -780,30 +790,43 @@ public sealed class SessionEventDistributor : IAsyncDisposable
|
||||
return;
|
||||
}
|
||||
|
||||
_replayBuffer.AddLast(new ReplayEntry(mxEvent, _timeProvider.GetUtcNow()));
|
||||
|
||||
// Capacity eviction: drop oldest until within bound.
|
||||
while (_replayBuffer.Count > _replayBufferCapacity)
|
||||
// Append at the logical tail. When the ring is full the oldest entry is
|
||||
// overwritten in place (its slot becomes the new tail) and the head advances,
|
||||
// so the newest _replayBufferCapacity events are retained with no allocation.
|
||||
ReplayEntry entry = new(mxEvent, _timeProvider.GetUtcNow());
|
||||
if (_replayCount < _replayBufferCapacity)
|
||||
{
|
||||
_replayBuffer.RemoveFirst();
|
||||
_replayBuffer[(_replayHead + _replayCount) % _replayBufferCapacity] = entry;
|
||||
_replayCount++;
|
||||
}
|
||||
else
|
||||
{
|
||||
_replayBuffer[_replayHead] = entry;
|
||||
_replayHead = (_replayHead + 1) % _replayBufferCapacity;
|
||||
}
|
||||
|
||||
EvictAged();
|
||||
}
|
||||
}
|
||||
|
||||
// Must be called under _replayLock. Drops entries older than the retention window.
|
||||
// Returns the logical entry at position i (0 == oldest retained). Must be called
|
||||
// under _replayLock with 0 <= i < _replayCount (so the array length is nonzero).
|
||||
private ReplayEntry ReplayEntryAt(int i) => _replayBuffer[(_replayHead + i) % _replayBuffer.Length];
|
||||
|
||||
// Must be called under _replayLock. Drops entries older than the retention window
|
||||
// by advancing the head past them (no dealloc; slots are reused on the next append).
|
||||
private void EvictAged()
|
||||
{
|
||||
if (!_ageEvictionEnabled || _replayBuffer.Count == 0)
|
||||
if (!_ageEvictionEnabled || _replayCount == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
DateTimeOffset cutoff = _timeProvider.GetUtcNow() - _replayRetention;
|
||||
while (_replayBuffer.First is { } first && first.Value.RetainedAt < cutoff)
|
||||
while (_replayCount > 0 && _replayBuffer[_replayHead].RetainedAt < cutoff)
|
||||
{
|
||||
_replayBuffer.RemoveFirst();
|
||||
_replayHead = (_replayHead + 1) % _replayBuffer.Length;
|
||||
_replayCount--;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -997,9 +997,14 @@ public sealed class WorkerClient : IWorkerClient
|
||||
string correlationId,
|
||||
WorkerCommand command)
|
||||
{
|
||||
// IPC-05: no second clone. MxAccessGrpcMapper.MapCommand already deep-cloned the command
|
||||
// out of the caller-owned gRPC MxCommandRequest, so this WorkerCommand is a fresh graph
|
||||
// owned by the invoke pipeline and referenced by no other consumer. The envelope is built
|
||||
// and owned entirely inside WorkerClient and the command is not touched again after this
|
||||
// point, so transferring it into the envelope introduces no aliasing hazard.
|
||||
return CreateEnvelope(
|
||||
correlationId,
|
||||
envelope => envelope.WorkerCommand = command.Clone());
|
||||
envelope => envelope.WorkerCommand = command);
|
||||
}
|
||||
|
||||
/// <summary>Creates shutdown envelope.</summary>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Buffers;
|
||||
using System.Buffers.Binary;
|
||||
using Google.Protobuf;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
@@ -47,20 +48,34 @@ public sealed class WorkerFrameReader
|
||||
$"Worker frame payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes.");
|
||||
}
|
||||
|
||||
byte[] payload = new byte[payloadLength];
|
||||
await ReadExactlyOrThrowAsync(payload, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// Rent the payload buffer from the shared pool rather than allocating a fresh byte[] per
|
||||
// frame; large event frames near the cap would otherwise allocate an LOH buffer each time
|
||||
// (IPC-14). ParseFrom copies whatever it needs into the parsed message, so the rented buffer
|
||||
// can be returned as soon as parsing completes without the envelope aliasing it. The rented
|
||||
// buffer may be larger than requested, so the read and parse are bounded to length.
|
||||
int length = checked((int)payloadLength);
|
||||
byte[] payload = ArrayPool<byte>.Shared.Rent(length);
|
||||
WorkerEnvelope envelope;
|
||||
try
|
||||
{
|
||||
envelope = WorkerEnvelope.Parser.ParseFrom(payload);
|
||||
await ReadExactlyOrThrowAsync(new Memory<byte>(payload, 0, length), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
envelope = WorkerEnvelope.Parser.ParseFrom(payload, 0, length);
|
||||
}
|
||||
catch (InvalidProtocolBufferException exception)
|
||||
{
|
||||
throw new WorkerFrameProtocolException(
|
||||
WorkerFrameProtocolErrorCode.InvalidEnvelope,
|
||||
"Worker frame payload is not a valid WorkerEnvelope protobuf message.",
|
||||
exception);
|
||||
}
|
||||
}
|
||||
catch (InvalidProtocolBufferException exception)
|
||||
finally
|
||||
{
|
||||
throw new WorkerFrameProtocolException(
|
||||
WorkerFrameProtocolErrorCode.InvalidEnvelope,
|
||||
"Worker frame payload is not a valid WorkerEnvelope protobuf message.",
|
||||
exception);
|
||||
ArrayPool<byte>.Shared.Return(payload);
|
||||
}
|
||||
|
||||
WorkerEnvelopeValidator.Validate(envelope, _options);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Buffers;
|
||||
using System.Buffers.Binary;
|
||||
using Google.Protobuf;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
@@ -53,10 +54,25 @@ public sealed class WorkerFrameWriter
|
||||
$"Worker envelope payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes.");
|
||||
}
|
||||
|
||||
byte[] lengthPrefix = new byte[sizeof(uint)];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(lengthPrefix, (uint)payloadLength);
|
||||
// Serialize once into a single pooled buffer that carries the 4-byte little-endian length
|
||||
// prefix followed by the payload, then issue one stream write. This avoids a second
|
||||
// serialization pass (ToByteArray re-runs CalculateSize), a separate prefix array and its
|
||||
// own write, and any per-frame heap allocation (GWC-08, IPC-13). The rented buffer may be
|
||||
// larger than requested, so only the first frameLength bytes are ever written.
|
||||
int frameLength = sizeof(uint) + payloadLength;
|
||||
byte[] frame = ArrayPool<byte>.Shared.Rent(frameLength);
|
||||
try
|
||||
{
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(frame, (uint)payloadLength);
|
||||
envelope.WriteTo(new Span<byte>(frame, sizeof(uint), payloadLength));
|
||||
|
||||
await _stream.WriteAsync(lengthPrefix, cancellationToken).ConfigureAwait(false);
|
||||
await _stream.WriteAsync(envelope.ToByteArray(), cancellationToken).ConfigureAwait(false);
|
||||
await _stream
|
||||
.WriteAsync(new ReadOnlyMemory<byte>(frame, 0, frameLength), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user