Merge remote-tracking branch 'origin/fix/wrk-21-drain-cluster'
# Conflicts: # archreview/2026-07-12/remediation/00-tracking.md # docs/MxAccessWorkerInstanceDesign.md
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -16,14 +16,14 @@ members, no positional records). The worker builds and tests only on the Windows
|
|||||||
|
|
||||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||||
|----|-----|------|-----|-----|--------|-------|
|
|----|-----|------|-----|-----|--------|-------|
|
||||||
| WRK-21 | Medium | P0 | M | IPC-23 (same defect, fix owned here); WRK-28 (same lines) | Not started | DrainEvents bound is count-based only; an oversized reply still kills the session and loses the drained events |
|
| WRK-21 | Medium | P0 | M | IPC-23 (same defect, fix owned here); WRK-28 (same lines) | Done | DrainEvents bound is count-based only; an oversized reply still kills the session and loses the drained events |
|
||||||
| WRK-22 | Low | — | S | IPC-26 (same defect, fix owned here) | Not started | Cancelled `WriteAsync` leaves its frame queued; it is still written later |
|
| WRK-22 | Low | — | S | IPC-26 (same defect, fix owned here) | Not started | Cancelled `WriteAsync` leaves its frame queued; it is still written later |
|
||||||
| WRK-23 | Low | — | S | WRK-21 (rejection path becomes backstop-only) | Not started | Rejected frames consume sequence numbers, producing wire gaps |
|
| WRK-23 | Low | — | S | WRK-21 (rejection path becomes backstop-only) | Done | Rejected frames consume sequence numbers, producing wire gaps |
|
||||||
| WRK-24 | Low | — | S | — | Not started | `AdoptNegotiatedMaxMessageBytes` has no lower-bound sanity check |
|
| WRK-24 | Low | — | S | — | Not started | `AdoptNegotiatedMaxMessageBytes` has no lower-bound sanity check |
|
||||||
| WRK-25 | Low | P2 | S | WRK-22 (both touch enqueue/dequeue) | Not started | WRK-12 flush coalescing never engages on the event hot path |
|
| WRK-25 | Low | P2 | S | WRK-22 (both touch enqueue/dequeue) | Not started | WRK-12 flush coalescing never engages on the event hot path |
|
||||||
| WRK-26 | Low | P1 | S | WRK-23 (soft — sequence prose); discharges IPC-29 | Not started | Write-priority and overflow doc drift from the WRK-07 change |
|
| WRK-26 | Low | P1 | S | WRK-23 (soft — sequence prose); discharges IPC-29 | Not started | Write-priority and overflow doc drift from the WRK-07 change |
|
||||||
| WRK-27 | Low | — | S | — | Not started | Alarm poll bypasses the watchdog's in-flight suppression (15 s vs 75 s) |
|
| WRK-27 | Low | — | S | — | Not started | Alarm poll bypasses the watchdog's in-flight suppression (15 s vs 75 s) |
|
||||||
| WRK-28 | Low | — | S | WRK-21 (land in the same commit cluster) | Not started | 10,000 drain cap is a duplicated magic constant with a comment-only sync contract |
|
| WRK-28 | Low | — | S | WRK-21 (land in the same commit cluster) | Done | 10,000 drain cap is a duplicated magic constant with a comment-only sync contract |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -12,14 +12,14 @@ All `path:line` citations were re-verified against the working tree at `4f5371f`
|
|||||||
|
|
||||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||||
|----|-----|------|-----|-----|--------|-------|
|
|----|-----|------|-----|-----|--------|-------|
|
||||||
| IPC-23 | Medium | P0 | S¹ | WRK-21 | Not started | DrainEvents bound is count-based only; byte-heavy queue still builds a session-killing reply frame (contract requirements here; fix mechanics in WRK-21) |
|
| IPC-23 | Medium | P0 | S¹ | WRK-21 | In progress — mechanics landed with WRK-21; proto-comment/doc wave pending | DrainEvents bound is count-based only; byte-heavy queue still builds a session-killing reply frame (contract requirements here; fix mechanics in WRK-21) |
|
||||||
| IPC-24 | Medium | P0 | S | — | Not started | CI's unconditional Java churn-revert masks real generated-code drift for message-level proto changes |
|
| IPC-24 | Medium | P0 | S | — | Not started | CI's unconditional Java churn-revert masks real generated-code drift for message-level proto changes |
|
||||||
| IPC-25 | Medium | P0 | M | — | Not started | Committed Go/Python worker bindings are stale at HEAD; no guard covers them |
|
| IPC-25 | Medium | P0 | M | — | Not started | Committed Go/Python worker bindings are stale at HEAD; no guard covers them |
|
||||||
| IPC-26 | Low | P2 | S¹ | WRK-22 | Not started | Cancelled write leaves a ghost frame that is still written (contract requirement here; fix mechanics in WRK-22) |
|
| IPC-26 | Low | P2 | S¹ | WRK-22 | Not started | Cancelled write leaves a ghost frame that is still written (contract requirement here; fix mechanics in WRK-22) |
|
||||||
| IPC-27 | Low | P2 | S | — | Not started | Descriptor freshness test blind to enums, enum values, services/methods, and the Galaxy contract |
|
| IPC-27 | Low | P2 | S | — | Not started | Descriptor freshness test blind to enums, enum values, services/methods, and the Galaxy contract |
|
||||||
| IPC-28 | Low | — | S | — | Not started | `docs/Grpc.md` omits the `CommandTooLarge` → `ResourceExhausted` mapping |
|
| IPC-28 | Low | — | S | — | Not started | `docs/Grpc.md` omits the `CommandTooLarge` → `ResourceExhausted` mapping |
|
||||||
| IPC-29 | Low | — | S | — | Not started | Worker writer priority scheduling and write-time sequence stamping undocumented in the frame-protocol doc |
|
| IPC-29 | Low | — | S | — | Not started | Worker writer priority scheduling and write-time sequence stamping undocumented in the frame-protocol doc |
|
||||||
| IPC-30 | Low | P0 | M | WRK-21 (same file/batch) | Not started | Oversized worker→gateway event frame is session-fatal — make the death deliberate, structured, and diagnosable |
|
| IPC-30 | Low | P0 | M | WRK-21 (same file/batch) | Done | Oversized worker→gateway event frame is session-fatal — make the death deliberate, structured, and diagnosable |
|
||||||
| IPC-31 | Info | — | — | — | N/A | Gateway stamps sequence at creation, worker at write — accepted divergence; sequence is documented diagnostic-only (`gateway.md:328-330`); revisit only if sequence ever becomes load-bearing |
|
| IPC-31 | Info | — | — | — | N/A | Gateway stamps sequence at creation, worker at write — accepted divergence; sequence is documented diagnostic-only (`gateway.md:328-330`); revisit only if sequence ever becomes load-bearing |
|
||||||
| IPC-32 | Info | — | S | IPC-25 | Not started | `check-codegen.ps1` check labels miscounted (folded into the IPC-25 script edit) |
|
| IPC-32 | Info | — | S | IPC-25 | Not started | `check-codegen.ps1` check labels miscounted (folded into the IPC-25 script edit) |
|
||||||
|
|
||||||
|
|||||||
@@ -378,6 +378,20 @@ If event conversion throws, catch it inside the event handler, record a
|
|||||||
structured `WorkerFault`, and keep the worker alive only if the fault policy
|
structured `WorkerFault`, and keep the worker alive only if the fault policy
|
||||||
allows it.
|
allows it.
|
||||||
|
|
||||||
|
The event drain loop streams queued events as `WorkerEvent` frames. A single
|
||||||
|
event whose envelope exceeds the negotiated frame maximum is **undeliverable end
|
||||||
|
to end** — the pipe maximum sits only the envelope-overhead reserve above the
|
||||||
|
public gRPC cap, so a frame the pipe rejects would also be rejected on the
|
||||||
|
client-facing stream. The session therefore faults on it rather than dropping it
|
||||||
|
(a silent drop makes the event stream unfaithful, and a synthesized placeholder
|
||||||
|
is barred by the no-synthesized-events rule), but the death is structured: the
|
||||||
|
worker logs the event's identity — family, handles, worker sequence, and sizes,
|
||||||
|
never the value — writes a `WorkerFault` with category `ProtocolViolation` and
|
||||||
|
command method `EventDrain` carrying the same identity, and only then exits.
|
||||||
|
Operator remediation is configuration: raise `MxGateway:Worker:MaxMessageBytes`
|
||||||
|
for that workload. Other per-frame rejection codes keep their previous behavior
|
||||||
|
because they indicate worker bugs, not workload size.
|
||||||
|
|
||||||
## Command Queue
|
## Command Queue
|
||||||
|
|
||||||
The pipe reader converts `WorkerCommand` messages into `StaCommand` entries.
|
The pipe reader converts `WorkerCommand` messages into `StaCommand` entries.
|
||||||
@@ -440,6 +454,29 @@ Diagnostics:
|
|||||||
- `DrainEvents`
|
- `DrainEvents`
|
||||||
- `ShutdownWorker`
|
- `ShutdownWorker`
|
||||||
|
|
||||||
|
`DrainEvents` is answered on the message-loop thread, not the STA, and its reply
|
||||||
|
is bounded on **two** axes because no diagnostics command may be session-fatal:
|
||||||
|
|
||||||
|
- **Count** — `GatewayContractInfo.MaxDrainEventsPerCommand` (10,000) is the
|
||||||
|
single home of the ceiling, shared by the gateway's request validator (which
|
||||||
|
rejects a larger `max_events` at the public boundary) and this worker clamp
|
||||||
|
(which also interprets `max_events = 0`, "as many as available").
|
||||||
|
- **Bytes** — the count cap alone is not sufficient: byte-heavy events (large
|
||||||
|
string or array `MxValue`s) overshoot the negotiated frame maximum long before
|
||||||
|
10,000 events. The drain is therefore byte-budgeted against the negotiated
|
||||||
|
maximum less a 64 KiB envelope/reply-wrapper reserve, and the size decision
|
||||||
|
happens inside the event queue's lock, so an event is dequeued only once it is
|
||||||
|
known to fit. An event that does not fit stays at the head of the queue and is
|
||||||
|
never lost.
|
||||||
|
|
||||||
|
Truncation is reported in the reply's existing `DiagnosticMessage`
|
||||||
|
("N events returned, M remain; repeat DrainEvents for the rest") rather than in a
|
||||||
|
new field, so the contract is unchanged and callers drain iteratively until a
|
||||||
|
reply comes back empty. In the degenerate case where the head event alone exceeds
|
||||||
|
the budget, the reply returns whatever fit before it (possibly nothing) and names
|
||||||
|
the blocked event's worker sequence so an operator can find the offending tag;
|
||||||
|
that event needs a larger `MxGateway:Worker:MaxMessageBytes` to move at all.
|
||||||
|
|
||||||
Implement method-specific dispatch instead of a generic string method invoker.
|
Implement method-specific dispatch instead of a generic string method invoker.
|
||||||
Parity tests need stable command-specific request and reply shapes.
|
Parity tests need stable command-specific request and reply shapes.
|
||||||
|
|
||||||
@@ -634,6 +671,13 @@ consumer that merely drains slower than this worker produces (the staging bound,
|
|||||||
metric `QueueOverflow("worker-event-staging")`). See
|
metric `QueueOverflow("worker-event-staging")`). See
|
||||||
`docs/GatewayProcessDesign.md`. A worker that outruns its consumer therefore
|
`docs/GatewayProcessDesign.md`. A worker that outruns its consumer therefore
|
||||||
dies loudly rather than growing gateway memory silently.
|
dies loudly rather than growing gateway memory silently.
|
||||||
|
No control reply is session-fatal on size. Reply builders size their payloads
|
||||||
|
against the negotiated frame maximum, and the two reply-write seams (the
|
||||||
|
control-command path and the STA command path) additionally catch a
|
||||||
|
`MessageTooLarge` per-frame rejection and answer that correlation with a small
|
||||||
|
`InvalidRequest` reply instead of unwinding the session. Oversized *event*
|
||||||
|
frames keep the opposite policy — see Event Sink — because an event above the
|
||||||
|
frame maximum cannot be delivered to the client at all.
|
||||||
|
|
||||||
## Heartbeat And Watchdog
|
## Heartbeat And Watchdog
|
||||||
|
|
||||||
|
|||||||
@@ -29,6 +29,29 @@ default. A `max_frame_bytes` of 0 (an older gateway that never set the field)
|
|||||||
means "use the worker's built-in default". This keeps both ends framing to the
|
means "use the worker's built-in default". This keeps both ends framing to the
|
||||||
same limit rather than depending on matched compile-time constants.
|
same limit rather than depending on matched compile-time constants.
|
||||||
|
|
||||||
|
Every worker-to-gateway frame must serialize within this limit, control replies
|
||||||
|
included, so reply builders truncate to fit rather than emit a frame the writer
|
||||||
|
will reject. `WorkerPipeSession` pre-sizes a `DrainEvents` reply below the
|
||||||
|
negotiated maximum (less a fixed envelope/reply-wrapper reserve) and reports the
|
||||||
|
truncation in the reply's `DiagnosticMessage`; the caller contract is to repeat
|
||||||
|
`DrainEvents` until it returns an empty reply. Should a reply still overshoot,
|
||||||
|
`MessageTooLarge` at the reply-write seam is answered with a small
|
||||||
|
`InvalidRequest` reply for that correlation, not with session teardown — no
|
||||||
|
diagnostics command may kill a session.
|
||||||
|
|
||||||
|
An oversized *event* frame is the deliberate exception. Such an event is
|
||||||
|
undeliverable end to end (the pipe maximum sits only the envelope-overhead
|
||||||
|
reserve above the public gRPC cap), so the session faults: the worker logs the
|
||||||
|
event's identity and sizes — never its value — writes a `WorkerFault` with
|
||||||
|
category `ProtocolViolation` and command method `EventDrain`, then exits.
|
||||||
|
Remediation is raising `MxGateway:Worker:MaxMessageBytes` for that workload.
|
||||||
|
|
||||||
|
A per-frame rejection does not consume an envelope `sequence`. The writer stamps
|
||||||
|
a candidate sequence, runs the empty-payload and size checks against the stamped
|
||||||
|
envelope, and commits the counter only immediately before the stream write, so
|
||||||
|
the sequences observed on the wire stay contiguous across rejections and an
|
||||||
|
operator reading a pipe capture never sees a phantom gap.
|
||||||
|
|
||||||
## Envelope Validation
|
## Envelope Validation
|
||||||
|
|
||||||
`WorkerFrameReader` and `WorkerFrameWriter` validate each envelope against the
|
`WorkerFrameReader` and `WorkerFrameWriter` validate each envelope against the
|
||||||
|
|||||||
+8
-4
@@ -474,10 +474,14 @@ Optional diagnostics:
|
|||||||
- `Ping`
|
- `Ping`
|
||||||
- `GetSessionState`
|
- `GetSessionState`
|
||||||
- `GetWorkerInfo`
|
- `GetWorkerInfo`
|
||||||
- `DrainEvents` — diagnostic; `max_events` is bounded (the gateway rejects requests
|
- `DrainEvents` — diagnostic; the reply is bounded on two axes so one drain cannot
|
||||||
above a public ceiling, and the worker caps each reply at its own per-reply limit,
|
pack an unbounded, session-killing reply frame. By **count**: the gateway rejects
|
||||||
treating `max_events = 0` as "the default cap") so one drain cannot pack an
|
requests above the shared ceiling `GatewayContractInfo.MaxDrainEventsPerCommand`
|
||||||
unbounded, session-killing reply frame.
|
and the worker clamps to the same value, treating `max_events = 0` as "the default
|
||||||
|
cap". By **bytes**: the worker sizes the reply while draining, against the
|
||||||
|
negotiated worker-frame maximum, so a byte-heavy queue truncates instead of
|
||||||
|
overshooting and events that do not fit stay queued. Truncation is reported in the
|
||||||
|
reply's `DiagnosticMessage`; callers drain iteratively until an empty reply.
|
||||||
- `ShutdownWorker`
|
- `ShutdownWorker`
|
||||||
|
|
||||||
Do not compress MXAccess semantics into generic verbs too early. A command enum
|
Do not compress MXAccess semantics into generic verbs too early. A command enum
|
||||||
|
|||||||
@@ -17,6 +17,21 @@ public static class GatewayContractInfo
|
|||||||
/// <summary>Default backend name identifying the MXAccess worker process type.</summary>
|
/// <summary>Default backend name identifying the MXAccess worker process type.</summary>
|
||||||
public const string DefaultBackendName = "mxaccess-worker";
|
public const string DefaultBackendName = "mxaccess-worker";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Ceiling on how many events one <c>DrainEvents</c> command may move in a single reply.
|
||||||
|
/// Shared so the gateway's request-validation ceiling
|
||||||
|
/// (<c>MxAccessGrpcRequestValidator</c>, which rejects a larger <c>max_events</c> loudly at
|
||||||
|
/// the public boundary) and the worker's per-reply clamp
|
||||||
|
/// (<c>WorkerPipeSession.CreateDrainEventsReply</c>, the backstop that also interprets
|
||||||
|
/// <c>max_events = 0</c>) cannot drift apart. A count cap alone is necessary but not
|
||||||
|
/// sufficient: the worker additionally caps the reply by serialized bytes against the
|
||||||
|
/// negotiated frame maximum (WRK-21), so a reply may carry fewer events than this ceiling
|
||||||
|
/// and fewer than are queued. Callers drain iteratively until an empty reply.
|
||||||
|
/// This is a documented behavioral bound, not wire schema — it is deliberately a C#
|
||||||
|
/// constant and not a <c>.proto</c> field.
|
||||||
|
/// </summary>
|
||||||
|
public const uint MaxDrainEventsPerCommand = 10_000;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Environment variable name that opts an xUnit suite into running live
|
/// Environment variable name that opts an xUnit suite into running live
|
||||||
/// MXAccess COM tests. Single source of truth shared by both
|
/// MXAccess COM tests. Single source of truth shared by both
|
||||||
|
|||||||
@@ -1,17 +1,11 @@
|
|||||||
using Grpc.Core;
|
using Grpc.Core;
|
||||||
|
using ZB.MOM.WW.MxGateway.Contracts;
|
||||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.MxGateway.Server.Grpc;
|
namespace ZB.MOM.WW.MxGateway.Server.Grpc;
|
||||||
|
|
||||||
public sealed class MxAccessGrpcRequestValidator
|
public sealed class MxAccessGrpcRequestValidator
|
||||||
{
|
{
|
||||||
// Upper bound on a single DrainEvents request. DrainEvents is a diagnostics RPC that returns
|
|
||||||
// buffered events in one non-streaming reply, so an unbounded max_events could pack the whole
|
|
||||||
// queue into a session-killing frame. The worker independently caps each reply at its
|
|
||||||
// own MaxDrainEventsPerReply; this public bound rejects an obviously-abusive request loudly at
|
|
||||||
// the boundary. max_events = 0 is allowed and means "the worker's default batch cap".
|
|
||||||
private const uint MaxDrainEventsPerRequest = 10_000;
|
|
||||||
|
|
||||||
/// <summary>Validates an open session request.</summary>
|
/// <summary>Validates an open session request.</summary>
|
||||||
/// <param name="request">The request to validate.</param>
|
/// <param name="request">The request to validate.</param>
|
||||||
public void ValidateOpenSession(OpenSessionRequest request)
|
public void ValidateOpenSession(OpenSessionRequest request)
|
||||||
@@ -78,10 +72,18 @@ public sealed class MxAccessGrpcRequestValidator
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The payload case now matches the kind, so command.DrainEvents is non-null here.
|
// The payload case now matches the kind, so command.DrainEvents is non-null here.
|
||||||
if (command.Kind is MxCommandKind.DrainEvents && command.DrainEvents.MaxEvents > MaxDrainEventsPerRequest)
|
// DrainEvents is a diagnostics RPC that returns buffered events in one non-streaming
|
||||||
|
// reply, so an unbounded max_events could pack the whole queue into a session-killing
|
||||||
|
// frame. The worker independently clamps every reply to the same shared ceiling and
|
||||||
|
// additionally caps it by serialized bytes; this public bound rejects an obviously-abusive
|
||||||
|
// request loudly at the boundary. max_events = 0 is allowed and means "the worker's
|
||||||
|
// default batch cap".
|
||||||
|
if (command.Kind is MxCommandKind.DrainEvents
|
||||||
|
&& command.DrainEvents.MaxEvents > GatewayContractInfo.MaxDrainEventsPerCommand)
|
||||||
{
|
{
|
||||||
throw InvalidArgument(
|
throw InvalidArgument(
|
||||||
$"DrainEvents max_events ({command.DrainEvents.MaxEvents}) must not exceed {MaxDrainEventsPerRequest}; "
|
$"DrainEvents max_events ({command.DrainEvents.MaxEvents}) must not exceed "
|
||||||
|
+ $"{GatewayContractInfo.MaxDrainEventsPerCommand}; "
|
||||||
+ "use 0 to request the worker default batch cap.");
|
+ "use 0 to request the worker default batch cap.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -410,6 +410,48 @@ public sealed class WorkerFrameProtocolTests
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies a per-frame rejection does not burn a sequence number (WRK-23). The sequence is a
|
||||||
|
/// diagnostic counter, so a gap breaks nothing functionally — but an operator correlating a pipe
|
||||||
|
/// capture reads a gap as a lost frame and chases a bug that does not exist, and the gap-free
|
||||||
|
/// guarantee the concurrent-write test asserts would otherwise only hold until the first
|
||||||
|
/// rejection.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task WriteAsync_PerFrameRejection_DoesNotConsumeSequence()
|
||||||
|
{
|
||||||
|
const int maxMessageBytes = 512;
|
||||||
|
WorkerFrameProtocolOptions options = new(
|
||||||
|
SessionId,
|
||||||
|
GatewayContractInfo.WorkerProtocolVersion,
|
||||||
|
Nonce,
|
||||||
|
maxMessageBytes);
|
||||||
|
using MemoryStream stream = new();
|
||||||
|
WorkerFrameWriter writer = new(stream, options);
|
||||||
|
|
||||||
|
await writer.WriteAsync(CreateEventEnvelope());
|
||||||
|
|
||||||
|
WorkerEnvelope oversized = CreateGatewayHelloEnvelope();
|
||||||
|
oversized.GatewayHello.GatewayVersion = new string('x', maxMessageBytes * 2);
|
||||||
|
WorkerFrameProtocolException exception =
|
||||||
|
await Assert.ThrowsAsync<WorkerFrameProtocolException>(
|
||||||
|
async () => await writer.WriteAsync(oversized));
|
||||||
|
Assert.Equal(WorkerFrameProtocolErrorCode.MessageTooLarge, exception.ErrorCode);
|
||||||
|
|
||||||
|
await writer.WriteAsync(CreateEventEnvelope());
|
||||||
|
|
||||||
|
stream.Position = 0;
|
||||||
|
WorkerFrameReader reader = new(stream, options);
|
||||||
|
WorkerEnvelope first = await reader.ReadAsync();
|
||||||
|
WorkerEnvelope second = await reader.ReadAsync();
|
||||||
|
|
||||||
|
// Two frames reached the wire; the rejected frame in between left no gap.
|
||||||
|
Assert.Equal(1UL, first.Sequence);
|
||||||
|
Assert.Equal(2UL, second.Sequence);
|
||||||
|
Assert.Equal(stream.Length, stream.Position);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Verifies a zero negotiated frame maximum keeps the constructor default.</summary>
|
/// <summary>Verifies a zero negotiated frame maximum keeps the constructor default.</summary>
|
||||||
[Fact]
|
[Fact]
|
||||||
public void AdoptNegotiatedMaxMessageBytes_WithZero_KeepsDefault()
|
public void AdoptNegotiatedMaxMessageBytes_WithZero_KeepsDefault()
|
||||||
|
|||||||
@@ -19,6 +19,21 @@ public sealed class WorkerPipeSessionTests
|
|||||||
private const string SessionId = "session-1";
|
private const string SessionId = "session-1";
|
||||||
private const string Nonce = "nonce-secret";
|
private const string Nonce = "nonce-secret";
|
||||||
|
|
||||||
|
// Byte-heavy drain fixture (WRK-21). 10,000 events at ~1.7 KiB each is ~17 MB of queue — far
|
||||||
|
// more than one frame — so DrainEvents must truncate.
|
||||||
|
//
|
||||||
|
// Two limits below are harness accommodations, not properties of the fix. PipePair runs both
|
||||||
|
// ends of a duplex pipe inside one process, with no continuous read pump and with blocking
|
||||||
|
// FlushFileBuffers under every frame write, so it tolerates neither multi-megabyte frames nor
|
||||||
|
// hundreds of large round trips before both ends wedge waiting on each other. Hence a small
|
||||||
|
// negotiated frame maximum, and a smaller queue for the drain-to-empty walk. The byte cap
|
||||||
|
// behaves identically at any frame size; exhaustive no-loss over the full 10,000 events is
|
||||||
|
// covered without a pipe by MxAccessEventQueueTests.
|
||||||
|
private const int ByteHeavyEventCount = 10_000;
|
||||||
|
private const int RepeatedDrainEventCount = 1_000;
|
||||||
|
private const int ByteHeavyEventPayloadBytes = 1_800;
|
||||||
|
private const uint NegotiatedMaxFrameBytes = 128 * 1024;
|
||||||
|
|
||||||
/// <summary>Verifies that valid gateway hello triggers worker hello and ready responses.</summary>
|
/// <summary>Verifies that valid gateway hello triggers worker hello and ready responses.</summary>
|
||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -487,6 +502,361 @@ public sealed class WorkerPipeSessionTests
|
|||||||
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The WRK-21 repro. A queue full of byte-heavy events (large string values — the payload
|
||||||
|
/// profile this gateway exists for) used to make <c>DrainEvents max_events = 0</c> build a
|
||||||
|
/// reply above the negotiated frame maximum: the writer rejected the frame, the exception
|
||||||
|
/// unwound the session, and the already-dequeued events were gone. The drain is now
|
||||||
|
/// byte-budgeted, so the reply fits, the truncation is reported in the reply's diagnostic
|
||||||
|
/// message, and the session keeps serving.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task DrainEvents_ByteHeavyQueue_ReplyIsBoundedAndSessionSurvives()
|
||||||
|
{
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(60));
|
||||||
|
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||||
|
FakeRuntimeSession runtime = new()
|
||||||
|
{
|
||||||
|
SuppressDrainForBatchSize = 128,
|
||||||
|
BackingQueue = CreateByteHeavyQueue(ByteHeavyEventCount, ByteHeavyEventPayloadBytes),
|
||||||
|
};
|
||||||
|
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
|
||||||
|
Task runTask = session.RunAsync(cancellation.Token);
|
||||||
|
await CompleteGatewayHandshakeAsync(pipePair, NegotiatedMaxFrameBytes, cancellation.Token);
|
||||||
|
|
||||||
|
await pipePair.GatewayWriter
|
||||||
|
.WriteAsync(
|
||||||
|
CreateControlCommandEnvelope(
|
||||||
|
"drain-heavy-1",
|
||||||
|
MxCommandKind.DrainEvents,
|
||||||
|
command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }),
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
WorkerEnvelope replyEnvelope = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply;
|
||||||
|
Assert.Equal(ProtocolStatusCode.Ok, reply.ProtocolStatus.Code);
|
||||||
|
|
||||||
|
// The whole queue is far larger than one frame, so the reply is a strict subset that fits.
|
||||||
|
Assert.True(
|
||||||
|
replyEnvelope.CalculateSize() <= NegotiatedMaxFrameBytes,
|
||||||
|
$"DrainEvents reply serialized to {replyEnvelope.CalculateSize()} bytes, above the negotiated {NegotiatedMaxFrameBytes}.");
|
||||||
|
Assert.InRange(reply.DrainEvents.Events.Count, 1, ByteHeavyEventCount - 1);
|
||||||
|
Assert.Contains("remain", reply.DiagnosticMessage);
|
||||||
|
Assert.Contains("repeat DrainEvents", reply.DiagnosticMessage);
|
||||||
|
|
||||||
|
// The session is alive: it still answers a ping, and RunAsync has not unwound.
|
||||||
|
await pipePair.GatewayWriter
|
||||||
|
.WriteAsync(CreatePingCommandEnvelope("ping-after-drain", "still-here"), cancellation.Token);
|
||||||
|
WorkerEnvelope pingReply = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||||
|
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "ping-after-drain",
|
||||||
|
cancellation.Token);
|
||||||
|
Assert.Equal("still-here", pingReply.WorkerCommandReply.Reply.DiagnosticMessage);
|
||||||
|
Assert.False(runTask.IsCompleted, "The session must survive a byte-heavy DrainEvents.");
|
||||||
|
|
||||||
|
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies the byte-budgeted drain loses nothing: repeating DrainEvents until it comes back
|
||||||
|
/// empty recovers every enqueued event exactly once, in order, across the split replies. The
|
||||||
|
/// pre-fix drain removed events from the queue before sizing the reply, so a rejected frame
|
||||||
|
/// destroyed them — no-loss is the half of the P0 criterion a catch-only fix cannot deliver.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task DrainEvents_RepeatedCalls_RecoverAllEventsWithoutLoss()
|
||||||
|
{
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(90));
|
||||||
|
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||||
|
FakeRuntimeSession runtime = new()
|
||||||
|
{
|
||||||
|
SuppressDrainForBatchSize = 128,
|
||||||
|
BackingQueue = CreateByteHeavyQueue(RepeatedDrainEventCount, ByteHeavyEventPayloadBytes),
|
||||||
|
};
|
||||||
|
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
|
||||||
|
Task runTask = session.RunAsync(cancellation.Token);
|
||||||
|
await CompleteGatewayHandshakeAsync(pipePair, NegotiatedMaxFrameBytes, cancellation.Token);
|
||||||
|
|
||||||
|
List<ulong> recovered = new();
|
||||||
|
int replyCount = 0;
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
string correlationId = $"drain-loop-{replyCount}";
|
||||||
|
await pipePair.GatewayWriter
|
||||||
|
.WriteAsync(
|
||||||
|
CreateControlCommandEnvelope(
|
||||||
|
correlationId,
|
||||||
|
MxCommandKind.DrainEvents,
|
||||||
|
command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }),
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
WorkerEnvelope replyEnvelope = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||||
|
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == correlationId,
|
||||||
|
cancellation.Token);
|
||||||
|
replyCount++;
|
||||||
|
|
||||||
|
MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply;
|
||||||
|
Assert.True(
|
||||||
|
replyEnvelope.CalculateSize() <= NegotiatedMaxFrameBytes,
|
||||||
|
$"DrainEvents reply {replyCount} serialized to {replyEnvelope.CalculateSize()} bytes.");
|
||||||
|
if (reply.DrainEvents.Events.Count == 0)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (MxEvent drained in reply.DrainEvents.Events)
|
||||||
|
{
|
||||||
|
recovered.Add(drained.WorkerSequence);
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.True(replyCount < 200, "DrainEvents made no progress across 200 replies.");
|
||||||
|
}
|
||||||
|
|
||||||
|
// More than one reply proves the drain really split; every event came back exactly once, in
|
||||||
|
// enqueue order.
|
||||||
|
Assert.True(replyCount > 2, $"Expected the byte cap to split the drain, saw {replyCount} replies.");
|
||||||
|
Assert.Equal(RepeatedDrainEventCount, recovered.Count);
|
||||||
|
for (int index = 0; index < recovered.Count; index++)
|
||||||
|
{
|
||||||
|
Assert.Equal((ulong)(index + 1), recovered[index]);
|
||||||
|
}
|
||||||
|
|
||||||
|
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Regression for the reserve-boundary budget bug. The gateway validator accepts a Worker
|
||||||
|
/// frame maximum as low as 1024 + 64 KiB, and just above that boundary a naive
|
||||||
|
/// subtract-then-guard budget collapses to ~1024 bytes — too small to move even one
|
||||||
|
/// byte-heavy event, so every drain reports truncation with the same head blocked and the
|
||||||
|
/// drain-until-empty loop never terminates. The budget is now a floor (never below half the
|
||||||
|
/// negotiated maximum), so a byte-heavy queue drains to empty even at the validator floor.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task DrainEvents_AtValidatorFloorFrameMax_MakesProgressAndTerminates()
|
||||||
|
{
|
||||||
|
// The lowest Worker.MaxMessageBytes GatewayOptionsValidator permits: the public gRPC floor
|
||||||
|
// (1024) plus the 64 KiB envelope-overhead reserve. The naive budget would be exactly 1024
|
||||||
|
// here; the floored budget is half of the frame max (~33 KiB).
|
||||||
|
const uint validatorFloorFrameMax = 1024 + (64 * 1024);
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(60));
|
||||||
|
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||||
|
FakeRuntimeSession runtime = new()
|
||||||
|
{
|
||||||
|
SuppressDrainForBatchSize = 128,
|
||||||
|
BackingQueue = CreateByteHeavyQueue(200, ByteHeavyEventPayloadBytes),
|
||||||
|
};
|
||||||
|
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
|
||||||
|
Task runTask = session.RunAsync(cancellation.Token);
|
||||||
|
await CompleteGatewayHandshakeAsync(pipePair, validatorFloorFrameMax, cancellation.Token);
|
||||||
|
|
||||||
|
int recovered = 0;
|
||||||
|
int replyCount = 0;
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
string correlationId = $"floor-drain-{replyCount}";
|
||||||
|
await pipePair.GatewayWriter
|
||||||
|
.WriteAsync(
|
||||||
|
CreateControlCommandEnvelope(
|
||||||
|
correlationId,
|
||||||
|
MxCommandKind.DrainEvents,
|
||||||
|
command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }),
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
WorkerEnvelope replyEnvelope = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||||
|
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == correlationId,
|
||||||
|
cancellation.Token);
|
||||||
|
replyCount++;
|
||||||
|
|
||||||
|
MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply;
|
||||||
|
Assert.True(
|
||||||
|
replyEnvelope.CalculateSize() <= validatorFloorFrameMax,
|
||||||
|
$"reply {replyCount} serialized to {replyEnvelope.CalculateSize()} bytes.");
|
||||||
|
|
||||||
|
int drainedThisReply = reply.DrainEvents.Events.Count;
|
||||||
|
if (drainedThisReply == 0)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The head is never reported as oversized at this frame max: the ~33 KiB floored budget
|
||||||
|
// comfortably fits the ~1.8 KiB events, so each reply makes real progress.
|
||||||
|
Assert.DoesNotContain("alone exceeds", reply.DiagnosticMessage);
|
||||||
|
recovered += drainedThisReply;
|
||||||
|
Assert.True(replyCount < 200, "DrainEvents made no progress at the validator floor frame max.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.Equal(200, recovered);
|
||||||
|
|
||||||
|
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies the control-reply write seam is not session-fatal on size (WRK-21 backstop). The
|
||||||
|
/// reply builders size their payloads, so this path needs a deliberately budget-blind drain
|
||||||
|
/// to reach — but that is the point: a future command or a sizing bug must degrade to an
|
||||||
|
/// error reply for that correlation, never to a dead session.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task ControlReplyTooLarge_WritesErrorReplyInsteadOfDying()
|
||||||
|
{
|
||||||
|
const uint tinyMaxFrameBytes = 4096;
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(15));
|
||||||
|
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||||
|
FakeRuntimeSession runtime = new()
|
||||||
|
{
|
||||||
|
SuppressDrainForBatchSize = 128,
|
||||||
|
BackingQueue = CreateByteHeavyQueue(eventCount: 1, payloadBytes: 16 * 1024),
|
||||||
|
IgnoreDrainByteBudget = true,
|
||||||
|
};
|
||||||
|
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
|
||||||
|
Task runTask = session.RunAsync(cancellation.Token);
|
||||||
|
await CompleteGatewayHandshakeAsync(pipePair, tinyMaxFrameBytes, cancellation.Token);
|
||||||
|
|
||||||
|
await pipePair.GatewayWriter
|
||||||
|
.WriteAsync(
|
||||||
|
CreateControlCommandEnvelope(
|
||||||
|
"drain-too-large",
|
||||||
|
MxCommandKind.DrainEvents,
|
||||||
|
command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }),
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
WorkerEnvelope replyEnvelope = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply;
|
||||||
|
Assert.Equal("drain-too-large", reply.CorrelationId);
|
||||||
|
Assert.Equal(MxCommandKind.DrainEvents, reply.Kind);
|
||||||
|
Assert.Equal(ProtocolStatusCode.InvalidRequest, reply.ProtocolStatus.Code);
|
||||||
|
Assert.Contains("frame maximum", reply.ProtocolStatus.Message);
|
||||||
|
Assert.False(runTask.IsCompleted, "An oversized control reply must not end the session.");
|
||||||
|
|
||||||
|
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies the same backstop on the STA command path: an oversized reply from a dispatched
|
||||||
|
/// command answers its correlation with an error reply instead of falling into the generic
|
||||||
|
/// catch that faults the whole session with MxaccessCommandFailed.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task CommandReplyTooLarge_WritesErrorReplyInsteadOfFaulting()
|
||||||
|
{
|
||||||
|
const uint tinyMaxFrameBytes = 4096;
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(15));
|
||||||
|
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||||
|
FakeRuntimeSession runtime = new()
|
||||||
|
{
|
||||||
|
DispatchReplyDiagnosticMessage = new string('x', 16 * 1024),
|
||||||
|
};
|
||||||
|
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
|
||||||
|
Task runTask = session.RunAsync(cancellation.Token);
|
||||||
|
await CompleteGatewayHandshakeAsync(pipePair, tinyMaxFrameBytes, cancellation.Token);
|
||||||
|
|
||||||
|
await pipePair.GatewayWriter
|
||||||
|
.WriteAsync(CreateCommandEnvelope("command-too-large"), cancellation.Token);
|
||||||
|
|
||||||
|
WorkerEnvelope replyEnvelope = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||||
|
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "command-too-large",
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
MxCommandReply reply = replyEnvelope.WorkerCommandReply.Reply;
|
||||||
|
Assert.Equal(MxCommandKind.Register, reply.Kind);
|
||||||
|
Assert.Equal(ProtocolStatusCode.InvalidRequest, reply.ProtocolStatus.Code);
|
||||||
|
|
||||||
|
// No fault, and the session still reports itself Ready rather than Faulted.
|
||||||
|
await pipePair.GatewayWriter
|
||||||
|
.WriteAsync(
|
||||||
|
CreateControlCommandEnvelope(
|
||||||
|
"state-after-too-large",
|
||||||
|
MxCommandKind.GetSessionState,
|
||||||
|
command => command.GetSessionState = new GetSessionStateCommand()),
|
||||||
|
cancellation.Token);
|
||||||
|
WorkerEnvelope stateEnvelope = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerCommandReply,
|
||||||
|
envelope => envelope.WorkerCommandReply.Reply.CorrelationId == "state-after-too-large",
|
||||||
|
cancellation.Token);
|
||||||
|
Assert.Equal(SessionState.Ready, stateEnvelope.WorkerCommandReply.Reply.SessionState.State);
|
||||||
|
Assert.False(runTask.IsCompleted, "An oversized command reply must not fault the session.");
|
||||||
|
|
||||||
|
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// IPC-30. An event above the negotiated frame maximum is undeliverable end to end (the pipe
|
||||||
|
/// maximum sits only an envelope reserve above the public gRPC cap), so the session stays
|
||||||
|
/// fatal by design — but the death must be structured: a WorkerFault naming the event, with
|
||||||
|
/// no value payload in it, before the process exits. Silently dropping the event or
|
||||||
|
/// synthesizing a placeholder were both rejected.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
|
[Fact]
|
||||||
|
public async Task RunAsync_EventFrameTooLarge_WritesStructuredFaultThenEndsSession()
|
||||||
|
{
|
||||||
|
const uint tinyMaxFrameBytes = 4096;
|
||||||
|
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(15));
|
||||||
|
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||||
|
FakeRuntimeSession runtime = new();
|
||||||
|
RecordingWorkerLogger logger = new();
|
||||||
|
WorkerPipeSession session = CreatePipeSession(
|
||||||
|
pipePair.WorkerStream,
|
||||||
|
runtime,
|
||||||
|
new WorkerPipeSessionOptions
|
||||||
|
{
|
||||||
|
HeartbeatInterval = TimeSpan.FromMilliseconds(100),
|
||||||
|
HeartbeatGrace = TimeSpan.FromSeconds(5),
|
||||||
|
},
|
||||||
|
logger);
|
||||||
|
runtime.EnqueueEvent(CreateOversizedWorkerEvent(sequence: 77, payloadBytes: 16 * 1024));
|
||||||
|
Task runTask = session.RunAsync(cancellation.Token);
|
||||||
|
await CompleteGatewayHandshakeAsync(pipePair, tinyMaxFrameBytes, cancellation.Token);
|
||||||
|
|
||||||
|
WorkerEnvelope faultEnvelope = await ReadUntilAsync(
|
||||||
|
pipePair.GatewayReader,
|
||||||
|
WorkerEnvelope.BodyOneofCase.WorkerFault,
|
||||||
|
cancellation.Token);
|
||||||
|
|
||||||
|
WorkerFault fault = faultEnvelope.WorkerFault;
|
||||||
|
Assert.Equal(WorkerFaultCategory.ProtocolViolation, fault.Category);
|
||||||
|
Assert.Equal("EventDrain", fault.CommandMethod);
|
||||||
|
Assert.Contains("77", fault.DiagnosticMessage);
|
||||||
|
Assert.Contains("MaxMessageBytes", fault.DiagnosticMessage);
|
||||||
|
// The identity is reported; the value payload never is.
|
||||||
|
Assert.DoesNotContain(new string('x', 64), fault.DiagnosticMessage);
|
||||||
|
|
||||||
|
Assert.Contains(
|
||||||
|
logger.Events,
|
||||||
|
entry => entry.EventName == "WorkerEventFrameTooLarge"
|
||||||
|
&& entry.Fields.TryGetValue("worker_sequence", out object? sequence)
|
||||||
|
&& sequence is ulong sequenceValue
|
||||||
|
&& sequenceValue == 77UL);
|
||||||
|
|
||||||
|
// The session ends, and the fault frame parsed cleanly off the same stream above — the
|
||||||
|
// rejected event never corrupted the wire.
|
||||||
|
Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token));
|
||||||
|
Assert.Same(runTask, completedTask);
|
||||||
|
await Assert.ThrowsAsync<InvalidOperationException>(async () => await runTask);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Verifies that ShutdownWorker returns its OK reply BEFORE the graceful
|
/// Verifies that ShutdownWorker returns its OK reply BEFORE the graceful
|
||||||
/// shutdown runs and disposes the runtime session, and that the message
|
/// shutdown runs and disposes the runtime session, and that the message
|
||||||
@@ -1241,6 +1611,15 @@ public sealed class WorkerPipeSessionTests
|
|||||||
Stream stream,
|
Stream stream,
|
||||||
FakeRuntimeSession runtime,
|
FakeRuntimeSession runtime,
|
||||||
WorkerPipeSessionOptions sessionOptions)
|
WorkerPipeSessionOptions sessionOptions)
|
||||||
|
{
|
||||||
|
return CreatePipeSession(stream, runtime, sessionOptions, logger: null);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static WorkerPipeSession CreatePipeSession(
|
||||||
|
Stream stream,
|
||||||
|
FakeRuntimeSession runtime,
|
||||||
|
WorkerPipeSessionOptions sessionOptions,
|
||||||
|
ZB.MOM.WW.MxGateway.Worker.Bootstrap.IWorkerLogger? logger)
|
||||||
{
|
{
|
||||||
WorkerFrameProtocolOptions options = CreateOptions();
|
WorkerFrameProtocolOptions options = CreateOptions();
|
||||||
return new WorkerPipeSession(
|
return new WorkerPipeSession(
|
||||||
@@ -1249,7 +1628,8 @@ public sealed class WorkerPipeSessionTests
|
|||||||
options,
|
options,
|
||||||
() => 1234,
|
() => 1234,
|
||||||
sessionOptions,
|
sessionOptions,
|
||||||
() => runtime);
|
() => runtime,
|
||||||
|
logger);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static WorkerFrameProtocolOptions CreateOptions()
|
private static WorkerFrameProtocolOptions CreateOptions()
|
||||||
@@ -1270,7 +1650,8 @@ public sealed class WorkerPipeSessionTests
|
|||||||
private static WorkerEnvelope CreateGatewayHelloEnvelope(
|
private static WorkerEnvelope CreateGatewayHelloEnvelope(
|
||||||
string nonce = Nonce,
|
string nonce = Nonce,
|
||||||
uint supportedProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
uint supportedProtocolVersion = GatewayContractInfo.WorkerProtocolVersion,
|
||||||
ulong sequence = 1)
|
ulong sequence = 1,
|
||||||
|
uint maxFrameBytes = 0)
|
||||||
{
|
{
|
||||||
return new WorkerEnvelope
|
return new WorkerEnvelope
|
||||||
{
|
{
|
||||||
@@ -1282,6 +1663,10 @@ public sealed class WorkerPipeSessionTests
|
|||||||
SupportedProtocolVersion = supportedProtocolVersion,
|
SupportedProtocolVersion = supportedProtocolVersion,
|
||||||
Nonce = nonce,
|
Nonce = nonce,
|
||||||
GatewayVersion = "test-gateway",
|
GatewayVersion = "test-gateway",
|
||||||
|
// 0 leaves the worker on its compile-time default; a non-zero value is adopted
|
||||||
|
// during the handshake and becomes the frame maximum every later assertion is
|
||||||
|
// measured against.
|
||||||
|
MaxFrameBytes = maxFrameBytes,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -1392,12 +1777,55 @@ public sealed class WorkerPipeSessionTests
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task CompleteGatewayHandshakeAsync(
|
private static WorkerEvent CreateOversizedWorkerEvent(ulong sequence, int payloadBytes)
|
||||||
|
{
|
||||||
|
WorkerEvent workerEvent = CreateWorkerEvent(sequence);
|
||||||
|
workerEvent.Event.ItemHandle = 42;
|
||||||
|
workerEvent.Event.RawStatus = new string('x', payloadBytes);
|
||||||
|
return workerEvent;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Fills a real event queue with byte-heavy events — a large string field stands in for the
|
||||||
|
/// array/string <c>MxValue</c> payloads that make a count-capped drain overshoot the frame
|
||||||
|
/// maximum. The queue is real (not the fake's plain list) so the production byte-budgeting runs.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="eventCount">Number of events to enqueue.</param>
|
||||||
|
/// <param name="payloadBytes">Size of each event's raw-status payload string.</param>
|
||||||
|
/// <returns>The populated queue.</returns>
|
||||||
|
private static MxAccessEventQueue CreateByteHeavyQueue(int eventCount, int payloadBytes)
|
||||||
|
{
|
||||||
|
MxAccessEventQueue queue = new(Math.Max(eventCount, 1));
|
||||||
|
string payload = new string('x', payloadBytes);
|
||||||
|
for (int index = 0; index < eventCount; index++)
|
||||||
|
{
|
||||||
|
queue.Enqueue(new MxEvent
|
||||||
|
{
|
||||||
|
SessionId = SessionId,
|
||||||
|
Family = MxEventFamily.OnDataChange,
|
||||||
|
ItemHandle = index,
|
||||||
|
RawStatus = payload,
|
||||||
|
OnDataChange = new OnDataChangeEvent(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return queue;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Task CompleteGatewayHandshakeAsync(
|
||||||
PipePair pipePair,
|
PipePair pipePair,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
return CompleteGatewayHandshakeAsync(pipePair, maxFrameBytes: 0, cancellationToken);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task CompleteGatewayHandshakeAsync(
|
||||||
|
PipePair pipePair,
|
||||||
|
uint maxFrameBytes,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
await pipePair.GatewayWriter
|
await pipePair.GatewayWriter
|
||||||
.WriteAsync(CreateGatewayHelloEnvelope(), cancellationToken)
|
.WriteAsync(CreateGatewayHelloEnvelope(maxFrameBytes: maxFrameBytes), cancellationToken)
|
||||||
.ConfigureAwait(false);
|
.ConfigureAwait(false);
|
||||||
|
|
||||||
WorkerEnvelope hello = await pipePair.GatewayReader.ReadAsync(cancellationToken).ConfigureAwait(false);
|
WorkerEnvelope hello = await pipePair.GatewayReader.ReadAsync(cancellationToken).ConfigureAwait(false);
|
||||||
|
|||||||
@@ -98,6 +98,157 @@ public sealed class MxAccessEventQueueTests
|
|||||||
Assert.Equal(0, queue.Count);
|
Assert.Equal(0, queue.Count);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies the byte-budgeted drain stops before the budget is exceeded, leaves the
|
||||||
|
/// remainder queued in order, and reports the exact remaining count (WRK-21). Events that
|
||||||
|
/// do not fit must never be dequeued — dequeuing them is how the pre-fix drain lost events
|
||||||
|
/// when the reply frame was rejected.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Drain_ByteBudget_StopsBeforeBudgetAndLeavesRemainderQueued()
|
||||||
|
{
|
||||||
|
MxAccessEventQueue queue = new(capacity: 8);
|
||||||
|
for (int itemHandle = 0; itemHandle < 5; itemHandle++)
|
||||||
|
{
|
||||||
|
queue.Enqueue(CreateEventWithPayload(itemHandle, payloadLength: 512));
|
||||||
|
}
|
||||||
|
|
||||||
|
int perEventCost = MeasureDrainCost(payloadLength: 512);
|
||||||
|
|
||||||
|
// Budget for exactly two events (plus a sliver too small for a third).
|
||||||
|
IReadOnlyList<WorkerEvent> drained =
|
||||||
|
queue.Drain(maxEvents: 0, maxTotalBytes: (perEventCost * 2) + (perEventCost / 2)).Events;
|
||||||
|
|
||||||
|
Assert.Equal(2, drained.Count);
|
||||||
|
Assert.Equal(0, drained[0].Event.ItemHandle);
|
||||||
|
Assert.Equal(1, drained[1].Event.ItemHandle);
|
||||||
|
Assert.Equal(3, queue.Count);
|
||||||
|
|
||||||
|
// The undrained remainder is still present, still in order.
|
||||||
|
IReadOnlyList<WorkerEvent> rest = queue.Drain(maxEvents: 0);
|
||||||
|
Assert.Equal(new[] { 2, 3, 4 }, new[] { rest[0].Event.ItemHandle, rest[1].Event.ItemHandle, rest[2].Event.ItemHandle });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies the byte-budgeted drain reports truncation and the exact remaining count so the
|
||||||
|
/// DrainEvents reply can tell the caller to drain again.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Drain_ByteBudget_ReportsTruncationAndRemainingCount()
|
||||||
|
{
|
||||||
|
MxAccessEventQueue queue = new(capacity: 8);
|
||||||
|
for (int itemHandle = 0; itemHandle < 4; itemHandle++)
|
||||||
|
{
|
||||||
|
queue.Enqueue(CreateEventWithPayload(itemHandle, payloadLength: 256));
|
||||||
|
}
|
||||||
|
|
||||||
|
// One-and-a-half events' worth of budget: the head fits, the next does not, and the next is
|
||||||
|
// comfortably smaller than the whole budget so it is a plain truncation rather than the
|
||||||
|
// oversized-head case.
|
||||||
|
WorkerEventDrainResult result = queue.Drain(
|
||||||
|
maxEvents: 0,
|
||||||
|
maxTotalBytes: MeasureDrainCost(payloadLength: 256) * 3 / 2);
|
||||||
|
|
||||||
|
Assert.Single(result.Events);
|
||||||
|
Assert.True(result.TruncatedBySize);
|
||||||
|
Assert.Equal(3, result.RemainingCount);
|
||||||
|
Assert.Equal(0UL, result.OversizedHeadSequence);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies the degenerate case: a head event whose own serialized size exceeds the whole
|
||||||
|
/// budget is not drained (draining it would build an oversized reply or lose the event) and
|
||||||
|
/// its worker sequence is reported so an operator can find the offending tag.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Drain_ByteBudget_OversizedHead_DrainsNothingAndReportsHeadSequence()
|
||||||
|
{
|
||||||
|
MxAccessEventQueue queue = new(capacity: 8);
|
||||||
|
queue.Enqueue(CreateEventWithPayload(itemHandle: 0, payloadLength: 4096));
|
||||||
|
queue.Enqueue(CreateEventWithPayload(itemHandle: 1, payloadLength: 8));
|
||||||
|
|
||||||
|
WorkerEventDrainResult result = queue.Drain(maxEvents: 0, maxTotalBytes: 1024);
|
||||||
|
|
||||||
|
Assert.Empty(result.Events);
|
||||||
|
Assert.True(result.TruncatedBySize);
|
||||||
|
Assert.Equal(2, result.RemainingCount);
|
||||||
|
Assert.Equal(1UL, result.OversizedHeadSequence);
|
||||||
|
|
||||||
|
// The blocked event is still queued — it was never removed.
|
||||||
|
Assert.Equal(2, queue.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The no-loss half of the WRK-21 acceptance criterion, at full scale. Draining the review's
|
||||||
|
/// 10,000 byte-heavy events under a budget that fits only a fraction of them per call must
|
||||||
|
/// return every event exactly once and in order: the pre-fix drain removed events from the
|
||||||
|
/// queue before the reply was sized, so a rejected frame destroyed them. This runs at the
|
||||||
|
/// queue layer because the property is the queue's, and because the pipe harness that covers
|
||||||
|
/// the same walk end to end cannot sustain hundreds of large round trips.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Drain_ByteBudget_RepeatedCalls_RecoverAllEventsInOrderWithoutLoss()
|
||||||
|
{
|
||||||
|
const int eventCount = 10_000;
|
||||||
|
const int payloadLength = 1_800;
|
||||||
|
MxAccessEventQueue queue = new(eventCount);
|
||||||
|
for (int index = 0; index < eventCount; index++)
|
||||||
|
{
|
||||||
|
queue.Enqueue(CreateEventWithPayload(index, payloadLength));
|
||||||
|
}
|
||||||
|
|
||||||
|
// A budget that fits roughly 35 events, so the walk takes hundreds of calls.
|
||||||
|
int budget = MeasureDrainCost(payloadLength) * 35;
|
||||||
|
List<ulong> recovered = new();
|
||||||
|
int calls = 0;
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
WorkerEventDrainResult result = queue.Drain(maxEvents: 0, maxTotalBytes: budget);
|
||||||
|
calls++;
|
||||||
|
if (result.Events.Count == 0)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (WorkerEvent drained in result.Events)
|
||||||
|
{
|
||||||
|
recovered.Add(drained.Event.WorkerSequence);
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.Equal(eventCount - recovered.Count, result.RemainingCount);
|
||||||
|
Assert.True(calls < eventCount, "Drain made no progress.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.True(calls > 100, $"Expected the byte budget to split the drain, saw {calls} calls.");
|
||||||
|
Assert.Equal(eventCount, recovered.Count);
|
||||||
|
for (int index = 0; index < recovered.Count; index++)
|
||||||
|
{
|
||||||
|
Assert.Equal((ulong)(index + 1), recovered[index]);
|
||||||
|
}
|
||||||
|
|
||||||
|
Assert.Equal(0, queue.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies the count cap still binds when the byte budget is generous: the byte cap is an
|
||||||
|
/// additional bound, not a replacement.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Drain_ByteBudget_CountCapStillBinds()
|
||||||
|
{
|
||||||
|
MxAccessEventQueue queue = new(capacity: 8);
|
||||||
|
for (int itemHandle = 0; itemHandle < 5; itemHandle++)
|
||||||
|
{
|
||||||
|
queue.Enqueue(CreateEventWithPayload(itemHandle, payloadLength: 16));
|
||||||
|
}
|
||||||
|
|
||||||
|
WorkerEventDrainResult result = queue.Drain(maxEvents: 2, maxTotalBytes: 1024 * 1024);
|
||||||
|
|
||||||
|
Assert.Equal(2, result.Events.Count);
|
||||||
|
Assert.False(result.TruncatedBySize);
|
||||||
|
Assert.Equal(3, result.RemainingCount);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>Verifies that Enqueue is rejected after a fault is recorded manually.</summary>
|
/// <summary>Verifies that Enqueue is rejected after a fault is recorded manually.</summary>
|
||||||
[Fact]
|
[Fact]
|
||||||
public void Enqueue_AfterRecordFault_ThrowsInvalidOperationException()
|
public void Enqueue_AfterRecordFault_ThrowsInvalidOperationException()
|
||||||
@@ -149,6 +300,42 @@ public sealed class MxAccessEventQueueTests
|
|||||||
Assert.Equal(WorkerFaultCategory.MxaccessEventConversionFailed, queue.Fault?.Category);
|
Assert.Equal(WorkerFaultCategory.MxaccessEventConversionFailed, queue.Fault?.Category);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Mirrors MxAccessEventQueue's per-event repeated-field allowance. Kept local rather than made
|
||||||
|
// public on the queue: the byte-budget tests state their budgets in units of that charge, so a
|
||||||
|
// change to it should surface here as a failing bound instead of silently moving with the code.
|
||||||
|
private const int RepeatedFieldOverheadBytes = 8;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Measures what the queue charges one event of the given payload size against the byte budget:
|
||||||
|
/// the serialized <see cref="WorkerEvent"/> as it exists after Enqueue (sequence and timestamp
|
||||||
|
/// stamped) plus the repeated-field allowance. The probe uses item handle 0, a proto3 default
|
||||||
|
/// that is not serialized, so this is a lower bound on the fixtures' real per-event cost — the
|
||||||
|
/// budgets above carry slack rather than assuming byte equality.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="payloadLength">Length of the event's raw-status payload string.</param>
|
||||||
|
/// <returns>The per-event byte cost.</returns>
|
||||||
|
private static int MeasureDrainCost(int payloadLength)
|
||||||
|
{
|
||||||
|
MxAccessEventQueue probe = new(capacity: 1);
|
||||||
|
probe.Enqueue(CreateEventWithPayload(0, payloadLength));
|
||||||
|
Assert.True(probe.TryDequeue(out WorkerEvent? probeEvent));
|
||||||
|
return probeEvent!.CalculateSize() + RepeatedFieldOverheadBytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Builds a byte-heavy event: a large string field is the cheapest stand-in for the array/string
|
||||||
|
/// <see cref="MxValue"/> payloads that make a count-capped drain overshoot the frame maximum.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="itemHandle">Item handle identifying the event in assertions.</param>
|
||||||
|
/// <param name="payloadLength">Length of the raw-status payload string.</param>
|
||||||
|
/// <returns>The constructed event.</returns>
|
||||||
|
private static MxEvent CreateEventWithPayload(int itemHandle, int payloadLength)
|
||||||
|
{
|
||||||
|
MxEvent mxEvent = CreateEvent(MxEventFamily.OnDataChange, itemHandle);
|
||||||
|
mxEvent.RawStatus = new string('x', payloadLength);
|
||||||
|
return mxEvent;
|
||||||
|
}
|
||||||
|
|
||||||
private static MxEvent CreateEvent(
|
private static MxEvent CreateEvent(
|
||||||
MxEventFamily family,
|
MxEventFamily family,
|
||||||
int itemHandle)
|
int itemHandle)
|
||||||
|
|||||||
@@ -43,6 +43,13 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
|||||||
/// <summary>Gets or sets whether ShutdownGracefullyAsync throws a TimeoutException.</summary>
|
/// <summary>Gets or sets whether ShutdownGracefullyAsync throws a TimeoutException.</summary>
|
||||||
public bool ThrowTimeoutOnShutdown { get; set; }
|
public bool ThrowTimeoutOnShutdown { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Optional diagnostic message stuffed into every dispatched command reply. A long value
|
||||||
|
/// pushes the STA command reply past a small negotiated frame maximum, which is how a test
|
||||||
|
/// drives the <c>ProcessCommandAsync</c> reply-size backstop.
|
||||||
|
/// </summary>
|
||||||
|
public string? DispatchReplyDiagnosticMessage { get; set; }
|
||||||
|
|
||||||
/// <summary>Gets a value indicating whether Dispose was called.</summary>
|
/// <summary>Gets a value indicating whether Dispose was called.</summary>
|
||||||
public bool Disposed { get; private set; }
|
public bool Disposed { get; private set; }
|
||||||
|
|
||||||
@@ -92,7 +99,7 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
|||||||
throw new InvalidOperationException("Command failed after shutdown started.");
|
throw new InvalidOperationException("Command failed after shutdown started.");
|
||||||
}
|
}
|
||||||
|
|
||||||
return new MxCommandReply
|
MxCommandReply reply = new()
|
||||||
{
|
{
|
||||||
SessionId = command.SessionId,
|
SessionId = command.SessionId,
|
||||||
CorrelationId = command.CorrelationId,
|
CorrelationId = command.CorrelationId,
|
||||||
@@ -103,6 +110,13 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
|||||||
Message = "OK",
|
Message = "OK",
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (DispatchReplyDiagnosticMessage is not null)
|
||||||
|
{
|
||||||
|
reply.DiagnosticMessage = DispatchReplyDiagnosticMessage;
|
||||||
|
}
|
||||||
|
|
||||||
|
return reply;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,6 +147,27 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public uint? LastDrainMaxEvents { get; private set; }
|
public uint? LastDrainMaxEvents { get; private set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Optional real event queue backing the drain paths. When set, both
|
||||||
|
/// <see cref="DrainEvents(uint)"/> and <see cref="DrainEvents(uint, int)"/> delegate to it
|
||||||
|
/// so a test can exercise the production byte-budgeting logic behind the fake session.
|
||||||
|
/// </summary>
|
||||||
|
public MxAccessEventQueue? BackingQueue { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// When set, <see cref="DrainEvents(uint, int)"/> ignores the byte budget and drains purely
|
||||||
|
/// by count. Simulates the "sizing bug or future command" case the control-reply size
|
||||||
|
/// backstop exists for, so a test can drive an oversized reply without a real budgeting
|
||||||
|
/// defect.
|
||||||
|
/// </summary>
|
||||||
|
public bool IgnoreDrainByteBudget { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Records the <c>maxTotalBytes</c> argument of the most recent byte-budgeted
|
||||||
|
/// <see cref="DrainEvents(uint, int)"/> call.
|
||||||
|
/// </summary>
|
||||||
|
public int? LastDrainMaxTotalBytes { get; private set; }
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public IReadOnlyList<WorkerEvent> DrainEvents(uint maxEvents)
|
public IReadOnlyList<WorkerEvent> DrainEvents(uint maxEvents)
|
||||||
{
|
{
|
||||||
@@ -143,6 +178,76 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
|||||||
|
|
||||||
LastDrainMaxEvents = maxEvents;
|
LastDrainMaxEvents = maxEvents;
|
||||||
|
|
||||||
|
if (BackingQueue is not null)
|
||||||
|
{
|
||||||
|
return BackingQueue.Drain(maxEvents);
|
||||||
|
}
|
||||||
|
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
int drainCount = maxEvents == 0
|
||||||
|
? events.Count
|
||||||
|
: Math.Min(events.Count, checked((int)Math.Min(maxEvents, int.MaxValue)));
|
||||||
|
List<WorkerEvent> drained = new(drainCount);
|
||||||
|
for (int index = 0; index < drainCount; index++)
|
||||||
|
{
|
||||||
|
drained.Add(events.Dequeue());
|
||||||
|
}
|
||||||
|
|
||||||
|
return drained;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public WorkerEventDrainResult DrainEvents(uint maxEvents, int maxTotalBytes)
|
||||||
|
{
|
||||||
|
if (SuppressDrainForBatchSize is uint suppressed && maxEvents == suppressed)
|
||||||
|
{
|
||||||
|
return new WorkerEventDrainResult(
|
||||||
|
Array.Empty<WorkerEvent>(),
|
||||||
|
truncatedBySize: false,
|
||||||
|
remainingCount: PendingEventCount,
|
||||||
|
oversizedHeadSequence: 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
LastDrainMaxEvents = maxEvents;
|
||||||
|
LastDrainMaxTotalBytes = maxTotalBytes;
|
||||||
|
|
||||||
|
if (BackingQueue is not null && !IgnoreDrainByteBudget)
|
||||||
|
{
|
||||||
|
return BackingQueue.Drain(maxEvents, maxTotalBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count-only drain: either no backing queue (the simple fakes) or a deliberately
|
||||||
|
// budget-blind drain used to exercise the reply-size backstop.
|
||||||
|
IReadOnlyList<WorkerEvent> drained = BackingQueue is not null
|
||||||
|
? BackingQueue.Drain(maxEvents)
|
||||||
|
: DrainByCount(maxEvents);
|
||||||
|
return new WorkerEventDrainResult(
|
||||||
|
drained,
|
||||||
|
truncatedBySize: false,
|
||||||
|
remainingCount: PendingEventCount,
|
||||||
|
oversizedHeadSequence: 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
private int PendingEventCount
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
if (BackingQueue is not null)
|
||||||
|
{
|
||||||
|
return BackingQueue.Count;
|
||||||
|
}
|
||||||
|
|
||||||
|
lock (gate)
|
||||||
|
{
|
||||||
|
return events.Count;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private IReadOnlyList<WorkerEvent> DrainByCount(uint maxEvents)
|
||||||
|
{
|
||||||
lock (gate)
|
lock (gate)
|
||||||
{
|
{
|
||||||
int drainCount = maxEvents == 0
|
int drainCount = maxEvents == 0
|
||||||
|
|||||||
@@ -43,8 +43,9 @@ public sealed class WorkerFrameWriter
|
|||||||
private readonly Queue<PendingFrame> _eventFrames = new Queue<PendingFrame>();
|
private readonly Queue<PendingFrame> _eventFrames = new Queue<PendingFrame>();
|
||||||
|
|
||||||
// Only ever read/written by the current write-lock holder while draining, so no interlock is
|
// Only ever read/written by the current write-lock holder while draining, so no interlock is
|
||||||
// needed. Starts at 0 and is pre-incremented, so the first written frame carries sequence 1
|
// needed. Starts at 0 and is committed only immediately before the stream write, so the first
|
||||||
// (matching the previous behaviour).
|
// written frame carries sequence 1 and a per-frame rejection leaves the counter untouched —
|
||||||
|
// the next accepted frame reuses the number and the wire sequence stays contiguous.
|
||||||
private ulong _nextSequence;
|
private ulong _nextSequence;
|
||||||
|
|
||||||
/// <summary>Initializes a new instance of the WorkerFrameWriter class.</summary>
|
/// <summary>Initializes a new instance of the WorkerFrameWriter class.</summary>
|
||||||
@@ -237,7 +238,14 @@ public sealed class WorkerFrameWriter
|
|||||||
|
|
||||||
// Stamp the sequence at the actual point of writing, under the write lock, so the wire order
|
// Stamp the sequence at the actual point of writing, under the write lock, so the wire order
|
||||||
// and the stamped sequence agree regardless of caller concurrency or priority.
|
// and the stamped sequence agree regardless of caller concurrency or priority.
|
||||||
envelope.Sequence = unchecked(++_nextSequence);
|
//
|
||||||
|
// Peek-stamp-commit (WRK-23): the sequence participates in CalculateSize() (varint width),
|
||||||
|
// so it must be stamped before the size checks — but a per-frame rejection must not burn a
|
||||||
|
// number, or the wire shows phantom gaps that an operator reads as lost frames. Stamp a
|
||||||
|
// candidate, validate the stamped envelope, and commit the counter only once the frame is
|
||||||
|
// certain to be written.
|
||||||
|
ulong candidateSequence = unchecked(_nextSequence + 1);
|
||||||
|
envelope.Sequence = candidateSequence;
|
||||||
|
|
||||||
int payloadLength = envelope.CalculateSize();
|
int payloadLength = envelope.CalculateSize();
|
||||||
if (payloadLength == 0)
|
if (payloadLength == 0)
|
||||||
@@ -254,6 +262,8 @@ public sealed class WorkerFrameWriter
|
|||||||
$"Worker envelope payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes.");
|
$"Worker envelope payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_nextSequence = candidateSequence;
|
||||||
|
|
||||||
// Serialize once into a single buffer that carries the 4-byte length prefix followed by the
|
// Serialize once into a single buffer that carries the 4-byte length prefix followed by the
|
||||||
// payload, then issue one stream write. This avoids a second serialization pass, a separate
|
// payload, then issue one stream write. This avoids a second serialization pass, a separate
|
||||||
// prefix array, and a separate prefix write. The flush is deferred to the end of the drained
|
// prefix array, and a separate prefix write. The flush is deferred to the end of the drained
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ using System.IO;
|
|||||||
using System.Threading;
|
using System.Threading;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using Google.Protobuf.WellKnownTypes;
|
using Google.Protobuf.WellKnownTypes;
|
||||||
|
using ZB.MOM.WW.MxGateway.Contracts;
|
||||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||||
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
|
using ZB.MOM.WW.MxGateway.Worker.Bootstrap;
|
||||||
using ZB.MOM.WW.MxGateway.Worker.MxAccess;
|
using ZB.MOM.WW.MxGateway.Worker.MxAccess;
|
||||||
@@ -18,12 +19,11 @@ public sealed class WorkerPipeSession
|
|||||||
private static readonly TimeSpan BackgroundTaskStopTimeout = TimeSpan.FromSeconds(1);
|
private static readonly TimeSpan BackgroundTaskStopTimeout = TimeSpan.FromSeconds(1);
|
||||||
private const uint EventDrainBatchSize = 128;
|
private const uint EventDrainBatchSize = 128;
|
||||||
|
|
||||||
// Hard cap on how many events a single DrainEvents diagnostic reply may carry. DrainEvents is a
|
// Headroom subtracted from the negotiated frame maximum when budgeting a DrainEvents reply. It
|
||||||
// non-streaming control command, so an unbounded drain (including the max_events = 0 "as many as
|
// covers the WorkerEnvelope/WorkerCommandReply/MxCommandReply wrapper the drained events are
|
||||||
// available" request) could pack the whole queue into one session-killing reply frame.
|
// packed into — the same envelope-overhead reserve rationale docs/WorkerFrameProtocol.md
|
||||||
// The gateway request validator rejects requests above its public ceiling; this worker-side cap is
|
// records for the frame max itself.
|
||||||
// the backstop and defines the effective per-reply maximum. Kept in step with that public ceiling.
|
private const int DrainReplyFrameHeadroomBytes = 64 * 1024;
|
||||||
private const uint MaxDrainEventsPerReply = 10_000;
|
|
||||||
|
|
||||||
private readonly WorkerFrameProtocolOptions _options;
|
private readonly WorkerFrameProtocolOptions _options;
|
||||||
private readonly Func<int> _processIdProvider;
|
private readonly Func<int> _processIdProvider;
|
||||||
@@ -376,13 +376,83 @@ public sealed class WorkerPipeSession
|
|||||||
// Events are the low-priority frame class: the writer holds them behind any pending
|
// Events are the low-priority frame class: the writer holds them behind any pending
|
||||||
// control frame (reply, fault, heartbeat, shutdown ack) so those are not delayed
|
// control frame (reply, fault, heartbeat, shutdown ack) so those are not delayed
|
||||||
// behind an event backlog.
|
// behind an event backlog.
|
||||||
await _writer
|
try
|
||||||
.WriteAsync(CreateEnvelope(workerEvent), WorkerFrameWritePriority.Event, cancellationToken)
|
{
|
||||||
.ConfigureAwait(false);
|
await _writer
|
||||||
|
.WriteAsync(CreateEnvelope(workerEvent), WorkerFrameWritePriority.Event, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (WorkerFrameProtocolException exception)
|
||||||
|
when (exception.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge)
|
||||||
|
{
|
||||||
|
await FaultOnOversizedEventAsync(workerEvent, exception, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Ends the session on an event that cannot be framed, but deliberately and diagnosably
|
||||||
|
/// (IPC-30). An event above the negotiated frame maximum is undeliverable end to end — the
|
||||||
|
/// pipe maximum sits only an envelope reserve above the public gRPC cap — so dropping it
|
||||||
|
/// would silently make the event stream unfaithful, and synthesizing a placeholder is barred
|
||||||
|
/// by the no-synthesized-events rule. Instead the worker records which event blocked (never
|
||||||
|
/// its value: the redaction rule), writes a structured fault the gateway and dashboard can
|
||||||
|
/// surface, and then exits as it did before. Remediation is configuration:
|
||||||
|
/// <c>MxGateway:Worker:MaxMessageBytes</c>. Other per-frame rejection codes keep the previous
|
||||||
|
/// behavior — they indicate worker bugs, not workload size.
|
||||||
|
/// </summary>
|
||||||
|
private async Task FaultOnOversizedEventAsync(
|
||||||
|
WorkerEvent workerEvent,
|
||||||
|
WorkerFrameProtocolException exception,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
MxEvent? mxEvent = workerEvent.Event;
|
||||||
|
string family = (mxEvent?.Family ?? MxEventFamily.Unspecified).ToString();
|
||||||
|
ulong workerSequence = mxEvent?.WorkerSequence ?? 0;
|
||||||
|
int serverHandle = mxEvent?.ServerHandle ?? 0;
|
||||||
|
int itemHandle = mxEvent?.ItemHandle ?? 0;
|
||||||
|
|
||||||
|
_logger?.Error(
|
||||||
|
"WorkerEventFrameTooLarge",
|
||||||
|
new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["session_id"] = _options.SessionId,
|
||||||
|
["event_family"] = family,
|
||||||
|
["worker_sequence"] = workerSequence,
|
||||||
|
["server_handle"] = serverHandle,
|
||||||
|
["item_handle"] = itemHandle,
|
||||||
|
["max_message_bytes"] = _options.MaxMessageBytes,
|
||||||
|
// Sizes only — the event value never reaches the log.
|
||||||
|
["reason"] = exception.Message,
|
||||||
|
});
|
||||||
|
|
||||||
|
string diagnosticMessage =
|
||||||
|
$"{family} event for server handle {serverHandle}, item handle {itemHandle} "
|
||||||
|
+ $"(worker sequence {workerSequence}) exceeds the negotiated frame maximum of "
|
||||||
|
+ $"{_options.MaxMessageBytes} bytes and cannot be delivered; raise "
|
||||||
|
+ "MxGateway:Worker:MaxMessageBytes for this workload.";
|
||||||
|
|
||||||
|
_state = WorkerState.Faulted;
|
||||||
|
await TryWriteFaultAsync(
|
||||||
|
new WorkerFault
|
||||||
|
{
|
||||||
|
Category = WorkerFaultCategory.ProtocolViolation,
|
||||||
|
CommandMethod = "EventDrain",
|
||||||
|
ExceptionType = exception.GetType().FullName ?? string.Empty,
|
||||||
|
DiagnosticMessage = diagnosticMessage,
|
||||||
|
ProtocolStatus = new ProtocolStatus
|
||||||
|
{
|
||||||
|
Code = ProtocolStatusCode.ProtocolViolation,
|
||||||
|
Message = diagnosticMessage,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
cancellationToken).ConfigureAwait(false);
|
||||||
|
|
||||||
|
throw new InvalidOperationException(diagnosticMessage, exception);
|
||||||
|
}
|
||||||
|
|
||||||
private async Task<bool> DispatchGatewayEnvelopeAsync(
|
private async Task<bool> DispatchGatewayEnvelopeAsync(
|
||||||
WorkerEnvelope envelope,
|
WorkerEnvelope envelope,
|
||||||
CancellationToken cancellationToken)
|
CancellationToken cancellationToken)
|
||||||
@@ -478,7 +548,8 @@ public sealed class WorkerPipeSession
|
|||||||
_ => CreateControlOkReply(correlationId, command.Kind),
|
_ => CreateControlOkReply(correlationId, command.Kind),
|
||||||
};
|
};
|
||||||
|
|
||||||
await WriteControlReplyAsync(reply, cancellationToken).ConfigureAwait(false);
|
await WriteControlReplyWithSizeBackstopAsync(reply, correlationId, command.Kind, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -495,6 +566,105 @@ public sealed class WorkerPipeSession
|
|||||||
cancellationToken);
|
cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes a control reply, answering the correlation with a small error reply instead of
|
||||||
|
/// unwinding the session if the reply does not fit the negotiated frame maximum. Reply
|
||||||
|
/// builders already size their payloads (see <see cref="CreateDrainEventsReply"/>), so this
|
||||||
|
/// is a backstop against a future command or a sizing bug — but without it a single
|
||||||
|
/// oversized diagnostic reply is session-fatal, which no diagnostics command may be.
|
||||||
|
/// </summary>
|
||||||
|
private async Task WriteControlReplyWithSizeBackstopAsync(
|
||||||
|
MxCommandReply reply,
|
||||||
|
string correlationId,
|
||||||
|
MxCommandKind kind,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await WriteControlReplyAsync(reply, cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (WorkerFrameProtocolException exception)
|
||||||
|
when (exception.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge)
|
||||||
|
{
|
||||||
|
LogControlReplyTooLarge(correlationId, kind, exception);
|
||||||
|
await WriteReplyTooLargeFallbackAsync(correlationId, kind, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes the small <c>InvalidRequest</c> reply that answers a correlation whose real reply
|
||||||
|
/// overshot the frame maximum. The fallback itself is a handful of bytes, so it fits any
|
||||||
|
/// sane negotiated maximum; the only way it can also throw <c>MessageTooLarge</c> is a
|
||||||
|
/// pathologically tiny negotiated maximum below the gateway's validation floor — the
|
||||||
|
/// pre-existing WRK-24 gap, which adds the negotiated-max lower bound that makes this
|
||||||
|
/// unreachable. Until then, a defensive swallow keeps the "no diagnostics command is
|
||||||
|
/// session-fatal" invariant true even in that degenerate config: the correlation goes
|
||||||
|
/// unanswered and the gateway's own per-command timeout covers it, but the session lives.
|
||||||
|
/// </summary>
|
||||||
|
private async Task WriteReplyTooLargeFallbackAsync(
|
||||||
|
string correlationId,
|
||||||
|
MxCommandKind kind,
|
||||||
|
CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await WriteControlReplyAsync(
|
||||||
|
CreateReplyTooLargeReply(correlationId, kind),
|
||||||
|
cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (WorkerFrameProtocolException exception)
|
||||||
|
when (exception.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge)
|
||||||
|
{
|
||||||
|
_logger?.Error(
|
||||||
|
"WorkerControlReplyFallbackTooLarge",
|
||||||
|
new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["correlation_id"] = correlationId,
|
||||||
|
["command_kind"] = kind.ToString(),
|
||||||
|
["max_message_bytes"] = _options.MaxMessageBytes,
|
||||||
|
["reason"] = exception.Message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void LogControlReplyTooLarge(
|
||||||
|
string correlationId,
|
||||||
|
MxCommandKind kind,
|
||||||
|
WorkerFrameProtocolException exception)
|
||||||
|
{
|
||||||
|
_logger?.Error(
|
||||||
|
"WorkerControlReplyTooLarge",
|
||||||
|
new Dictionary<string, object?>
|
||||||
|
{
|
||||||
|
["correlation_id"] = correlationId,
|
||||||
|
["command_kind"] = kind.ToString(),
|
||||||
|
["max_message_bytes"] = _options.MaxMessageBytes,
|
||||||
|
// The writer's message carries the rejected payload length; it names sizes only,
|
||||||
|
// never reply content.
|
||||||
|
["reason"] = exception.Message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private MxCommandReply CreateReplyTooLargeReply(string correlationId, MxCommandKind kind)
|
||||||
|
{
|
||||||
|
const string message =
|
||||||
|
"Worker reply exceeded the negotiated frame maximum; retry with a smaller request.";
|
||||||
|
return new MxCommandReply
|
||||||
|
{
|
||||||
|
SessionId = _options.SessionId,
|
||||||
|
CorrelationId = correlationId,
|
||||||
|
Kind = kind,
|
||||||
|
Hresult = 0,
|
||||||
|
DiagnosticMessage = message,
|
||||||
|
ProtocolStatus = new ProtocolStatus
|
||||||
|
{
|
||||||
|
Code = ProtocolStatusCode.InvalidRequest,
|
||||||
|
Message = message,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private MxCommandReply CreatePingReply(string correlationId, MxCommand command)
|
private MxCommandReply CreatePingReply(string correlationId, MxCommand command)
|
||||||
{
|
{
|
||||||
MxCommandReply reply = CreateControlOkReply(correlationId, command.Kind);
|
MxCommandReply reply = CreateControlOkReply(correlationId, command.Kind);
|
||||||
@@ -543,24 +713,77 @@ public sealed class WorkerPipeSession
|
|||||||
if (runtimeSession is not null)
|
if (runtimeSession is not null)
|
||||||
{
|
{
|
||||||
// Bound the diagnostic drain so max_events = 0 ("as many as available") or an over-large
|
// Bound the diagnostic drain so max_events = 0 ("as many as available") or an over-large
|
||||||
// request cannot pack the whole queue into one session-killing reply frame.
|
// request cannot pack the whole queue into one session-killing reply frame. The count cap
|
||||||
|
// alone is not enough: byte-heavy events overshoot the negotiated frame maximum long
|
||||||
|
// before the count ceiling, so the drain is also byte-budgeted and sizes the reply while
|
||||||
|
// draining — an event that does not fit is left queued rather than dequeued and lost.
|
||||||
uint requested = command.DrainEvents?.MaxEvents ?? 0;
|
uint requested = command.DrainEvents?.MaxEvents ?? 0;
|
||||||
uint maxEvents = requested == 0 || requested > MaxDrainEventsPerReply
|
uint maxEvents = requested == 0 || requested > GatewayContractInfo.MaxDrainEventsPerCommand
|
||||||
? MaxDrainEventsPerReply
|
? GatewayContractInfo.MaxDrainEventsPerCommand
|
||||||
: requested;
|
: requested;
|
||||||
foreach (WorkerEvent workerEvent in runtimeSession.DrainEvents(maxEvents))
|
WorkerEventDrainResult drainResult = runtimeSession.DrainEvents(
|
||||||
|
maxEvents,
|
||||||
|
ResolveDrainReplyByteBudget());
|
||||||
|
foreach (WorkerEvent workerEvent in drainResult.Events)
|
||||||
{
|
{
|
||||||
if (workerEvent.Event is not null)
|
if (workerEvent.Event is not null)
|
||||||
{
|
{
|
||||||
drainReply.Events.Add(workerEvent.Event);
|
drainReply.Events.Add(workerEvent.Event);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (drainResult.TruncatedBySize)
|
||||||
|
{
|
||||||
|
// DrainEventsReply has no truncation field, and adding one would regenerate every
|
||||||
|
// language client for a diagnostic nicety. The reply's existing DiagnosticMessage
|
||||||
|
// carries the same information at zero contract cost; the caller contract is to
|
||||||
|
// repeat DrainEvents until it comes back empty.
|
||||||
|
reply.DiagnosticMessage = CreateDrainTruncationMessage(
|
||||||
|
drainReply.Events.Count,
|
||||||
|
drainResult);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
reply.DrainEvents = drainReply;
|
reply.DrainEvents = drainReply;
|
||||||
return reply;
|
return reply;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Byte budget for the events packed into one DrainEvents reply: the negotiated frame
|
||||||
|
/// maximum less a fixed reserve for the envelope/reply wrapper, but never below half the
|
||||||
|
/// negotiated maximum. The lower bound must be a floor, not a step: a bare
|
||||||
|
/// <c>subtract-then-guard-positive</c> collapses the budget to a handful of bytes just above
|
||||||
|
/// the reserve (e.g. at the validator-permitted floor MaxMessageBytes = 1024 + 64 KiB the
|
||||||
|
/// subtraction leaves 1024, too small to move even one byte-heavy event, so every drain
|
||||||
|
/// truncates and the drain-until-empty caller never terminates). Taking the max with
|
||||||
|
/// half the negotiated maximum keeps the budget monotonic across the reserve boundary while
|
||||||
|
/// still leaving the full reserve for the wrapper whenever the frame max is large enough
|
||||||
|
/// that the reserve is the smaller subtraction — which is every configuration above 128 KiB.
|
||||||
|
/// </summary>
|
||||||
|
private int ResolveDrainReplyByteBudget()
|
||||||
|
{
|
||||||
|
return Math.Max(
|
||||||
|
_options.MaxMessageBytes - DrainReplyFrameHeadroomBytes,
|
||||||
|
_options.MaxMessageBytes / 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string CreateDrainTruncationMessage(
|
||||||
|
int returnedCount,
|
||||||
|
WorkerEventDrainResult drainResult)
|
||||||
|
{
|
||||||
|
string message =
|
||||||
|
$"{returnedCount} events returned, {drainResult.RemainingCount} remain; "
|
||||||
|
+ "repeat DrainEvents for the rest.";
|
||||||
|
if (drainResult.OversizedHeadSequence != 0)
|
||||||
|
{
|
||||||
|
message +=
|
||||||
|
$" The next event (worker sequence {drainResult.OversizedHeadSequence}) alone exceeds "
|
||||||
|
+ "the negotiated frame maximum and cannot be drained; raise MxGateway:Worker:MaxMessageBytes.";
|
||||||
|
}
|
||||||
|
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
private MxCommandReply CreateControlOkReply(string correlationId, MxCommandKind kind)
|
private MxCommandReply CreateControlOkReply(string correlationId, MxCommandKind kind)
|
||||||
{
|
{
|
||||||
return new MxCommandReply
|
return new MxCommandReply
|
||||||
@@ -627,15 +850,30 @@ public sealed class WorkerPipeSession
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await _writer
|
try
|
||||||
.WriteAsync(
|
{
|
||||||
CreateEnvelope(new WorkerCommandReply
|
await _writer
|
||||||
{
|
.WriteAsync(
|
||||||
Reply = reply,
|
CreateEnvelope(new WorkerCommandReply
|
||||||
CompletedTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
{
|
||||||
}),
|
Reply = reply,
|
||||||
cancellationToken)
|
CompletedTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||||
.ConfigureAwait(false);
|
}),
|
||||||
|
cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (WorkerFrameProtocolException sizeException)
|
||||||
|
when (sizeException.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge)
|
||||||
|
{
|
||||||
|
// An oversized STA command reply is a property of that one command, not of the
|
||||||
|
// session. Answer the correlation with an error reply instead of falling into the
|
||||||
|
// generic catch below, which would fault the whole session for it. The fallback
|
||||||
|
// write is itself size-guarded (see WriteReplyTooLargeFallbackAsync) so a degenerate
|
||||||
|
// negotiated maximum cannot make even this backstop session-fatal.
|
||||||
|
LogControlReplyTooLarge(envelope.CorrelationId, command.Kind, sizeException);
|
||||||
|
await WriteReplyTooLargeFallbackAsync(envelope.CorrelationId, command.Kind, cancellationToken)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -44,6 +44,19 @@ public interface IWorkerRuntimeSession : IDisposable
|
|||||||
/// <returns>List of drained events.</returns>
|
/// <returns>List of drained events.</returns>
|
||||||
IReadOnlyList<WorkerEvent> DrainEvents(uint maxEvents);
|
IReadOnlyList<WorkerEvent> DrainEvents(uint maxEvents);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Drains pending events bounded by both a count cap and a byte budget, so a caller building a
|
||||||
|
/// single reply frame never removes an event it cannot ship.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Declared as a second method rather than a default interface method: the worker targets
|
||||||
|
/// .NET Framework 4.8, which has no runtime support for default interface members.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="maxEvents">Maximum number of events to drain; 0 means "no count limit".</param>
|
||||||
|
/// <param name="maxTotalBytes">Byte budget for the drained events' estimated serialized size.</param>
|
||||||
|
/// <returns>The drained events and the truncation facts describing what stayed queued.</returns>
|
||||||
|
WorkerEventDrainResult DrainEvents(uint maxEvents, int maxTotalBytes);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Drains a pending fault from the queue, if any.
|
/// Drains a pending fault from the queue, if any.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
@@ -26,6 +26,15 @@ public sealed class MxAccessEventQueue
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public const int DefaultCapacity = 10000;
|
public const int DefaultCapacity = 10000;
|
||||||
|
|
||||||
|
// Extra per-event slack added to WorkerEvent.CalculateSize() when charging the byte budget in
|
||||||
|
// Drain(maxEvents, maxTotalBytes). CalculateSize() already accounts for the event's own tag and
|
||||||
|
// length-delimiter (the WorkerEvent wrapper serializes MxEvent as field 1, and the reply packs
|
||||||
|
// each MxEvent as DrainEventsReply.events field 1 with the identical tag+length shape), so this
|
||||||
|
// is a pure safety margin over an already-conservative estimate — not compensation for a missing
|
||||||
|
// wrapper. It keeps the running total strictly ahead of the true serialized size so a rounding
|
||||||
|
// edge can never push the packed reply past the frame maximum.
|
||||||
|
private const int RepeatedFieldOverheadBytes = 8;
|
||||||
|
|
||||||
private readonly int capacity;
|
private readonly int capacity;
|
||||||
private readonly Queue<WorkerEvent> events;
|
private readonly Queue<WorkerEvent> events;
|
||||||
private readonly object syncRoot = new();
|
private readonly object syncRoot = new();
|
||||||
@@ -209,6 +218,64 @@ public sealed class MxAccessEventQueue
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Drains from the head while both the count cap and a byte budget allow it, so the caller can
|
||||||
|
/// build a reply frame that is guaranteed to fit the negotiated frame maximum.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The size decision happens inside the queue lock, so an event is dequeued only once it is
|
||||||
|
/// known to fit: an event that does not fit stays at the head for the next call and is never
|
||||||
|
/// lost (WRK-21). Per-event cost is <c>WorkerEvent.CalculateSize()</c> — which already
|
||||||
|
/// includes the event's own tag and length prefix, the same shape the reply's
|
||||||
|
/// <c>events</c> repeated field packs it into — plus <see cref="RepeatedFieldOverheadBytes"/>
|
||||||
|
/// of pure slack, so the running total stays strictly ahead of the true serialized size and
|
||||||
|
/// the estimate errs on the safe side.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="maxEvents">Maximum number of events to drain; 0 means "no count limit".</param>
|
||||||
|
/// <param name="maxTotalBytes">Byte budget for the drained events' estimated serialized size.</param>
|
||||||
|
/// <returns>The drained events plus the truncation facts the caller reports to the gateway.</returns>
|
||||||
|
public WorkerEventDrainResult Drain(uint maxEvents, int maxTotalBytes)
|
||||||
|
{
|
||||||
|
lock (syncRoot)
|
||||||
|
{
|
||||||
|
int countLimit = maxEvents == 0
|
||||||
|
? int.MaxValue
|
||||||
|
: checked((int)Math.Min(maxEvents, int.MaxValue));
|
||||||
|
List<WorkerEvent> drained = new();
|
||||||
|
int remainingBudget = maxTotalBytes;
|
||||||
|
bool truncatedBySize = false;
|
||||||
|
ulong oversizedHeadSequence = 0;
|
||||||
|
|
||||||
|
while (drained.Count < countLimit && events.Count > 0)
|
||||||
|
{
|
||||||
|
WorkerEvent head = events.Peek();
|
||||||
|
int cost = head.CalculateSize() + RepeatedFieldOverheadBytes;
|
||||||
|
if (cost > remainingBudget)
|
||||||
|
{
|
||||||
|
truncatedBySize = true;
|
||||||
|
if (cost > maxTotalBytes)
|
||||||
|
{
|
||||||
|
// The head alone cannot fit this budget, so repeating the call will not
|
||||||
|
// move it either. Report its sequence instead of silently stalling; the
|
||||||
|
// events that did fit are still returned.
|
||||||
|
oversizedHeadSequence = head.Event?.WorkerSequence ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
remainingBudget -= cost;
|
||||||
|
drained.Add(events.Dequeue());
|
||||||
|
}
|
||||||
|
|
||||||
|
return new WorkerEventDrainResult(
|
||||||
|
drained,
|
||||||
|
truncatedBySize,
|
||||||
|
events.Count,
|
||||||
|
oversizedHeadSequence);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a fault if one has not already been recorded.
|
/// Records a fault if one has not already been recorded.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
@@ -392,6 +392,12 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession
|
|||||||
return eventQueue.Drain(maxEvents);
|
return eventQueue.Drain(maxEvents);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public WorkerEventDrainResult DrainEvents(uint maxEvents, int maxTotalBytes)
|
||||||
|
{
|
||||||
|
return eventQueue.Drain(maxEvents, maxTotalBytes);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public WorkerFault? DrainFault()
|
public WorkerFault? DrainFault()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
using System.Collections.Generic;
|
||||||
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.MxGateway.Worker.MxAccess;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Outcome of a byte-budgeted drain from the MXAccess outbound event queue.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// A count cap alone cannot keep a <c>DrainEvents</c> reply inside the negotiated frame
|
||||||
|
/// maximum: byte-heavy events (large string or array <c>MxValue</c>s) overshoot the frame max
|
||||||
|
/// long before the count ceiling is reached, and the writer's per-frame rejection then
|
||||||
|
/// destroys events that were already removed from the queue. The byte-budgeted drain sizes
|
||||||
|
/// the reply while draining, so an event that does not fit is never dequeued (WRK-21), and
|
||||||
|
/// this result carries the truncation facts the reply's <c>DiagnosticMessage</c> reports —
|
||||||
|
/// no contract change is needed to express them.
|
||||||
|
/// Plain constructor and get-only properties: the worker targets .NET Framework 4.8, which
|
||||||
|
/// has no init-only members or positional records.
|
||||||
|
/// </remarks>
|
||||||
|
public sealed class WorkerEventDrainResult
|
||||||
|
{
|
||||||
|
/// <summary>Initializes a new instance of the <see cref="WorkerEventDrainResult"/> class.</summary>
|
||||||
|
/// <param name="events">Events removed from the queue, in enqueue order.</param>
|
||||||
|
/// <param name="truncatedBySize">Whether the byte budget, not the count cap, ended the drain.</param>
|
||||||
|
/// <param name="remainingCount">Number of events still queued after the drain.</param>
|
||||||
|
/// <param name="oversizedHeadSequence">
|
||||||
|
/// Worker sequence of a head event whose own serialized size exceeds the whole budget, so
|
||||||
|
/// no future call of the same budget can ship it; 0 when there is no such event.
|
||||||
|
/// </param>
|
||||||
|
public WorkerEventDrainResult(
|
||||||
|
IReadOnlyList<WorkerEvent> events,
|
||||||
|
bool truncatedBySize,
|
||||||
|
int remainingCount,
|
||||||
|
ulong oversizedHeadSequence)
|
||||||
|
{
|
||||||
|
Events = events;
|
||||||
|
TruncatedBySize = truncatedBySize;
|
||||||
|
RemainingCount = remainingCount;
|
||||||
|
OversizedHeadSequence = oversizedHeadSequence;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Gets the events removed from the queue, in enqueue order.</summary>
|
||||||
|
public IReadOnlyList<WorkerEvent> Events { get; }
|
||||||
|
|
||||||
|
/// <summary>Gets a value indicating whether the byte budget ended the drain early.</summary>
|
||||||
|
public bool TruncatedBySize { get; }
|
||||||
|
|
||||||
|
/// <summary>Gets the number of events still queued after the drain.</summary>
|
||||||
|
public int RemainingCount { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the worker sequence of the head event that alone exceeds the byte budget, or 0 when
|
||||||
|
/// no single event blocks the drain. Naming it lets an operator find the offending tag.
|
||||||
|
/// </summary>
|
||||||
|
public ulong OversizedHeadSequence { get; }
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user