feat(comm): batch site→central stream events over gRPC (R2)
Every AttributeValueChanged/AlarmStateChanged rode its own gRPC message on
SiteStreamService; at target scale that is ~37.5k messages/s/site of pure
framing overhead. Coalesce them, additively, with no new RPC.
Wire (sitestream.proto, regenerated via docker/regen-proto.sh sitestream):
InstanceStreamRequest.batching_supported = 3
SiteStreamRequest.batching_supported = 2
SiteStreamEvent.batch = 4 (new oneof case)
SiteStreamEventBatch { repeated SiteStreamEvent events = 1 }
The proto3 default of batching_supported IS the negotiation, and it is
load-bearing: a batch frame reaches a pre-R2 central as EventOneofCase.None,
whose ConvertToDomainEvent returns null — the whole batch would vanish with
no error anywhere. An old central cannot set the flag so it never receives
one; an old site ignores the unknown request field and keeps sending
per-event frames, which the new client's ForEachEvent handles as the
single-event case. Both skew directions are covered by tests that go through
a real proto serialize/parse round-trip.
Server: SiteStreamEventBatcher, a per-subscriber pump replacing the handler's
await-foreach/WriteAsync loop and byte-identical to it at a cap of 1. It
never delays a lone event — it drains the already-queued backlog for free and
lingers only once a backlog is proven — emits a single-event buffer as a
plain frame, preserves order and per-event Timestamps exactly, and flushes
what is buffered when the send channel's writer completes. It sits DOWNSTREAM
of StreamRelayActor's bounded DropOldest channel, so it changes framing only
and does not move the burst ceiling (deferred register row 31, which lives in
the shared publish stage upstream of the BroadcastHub).
Client: sets the flag on both subscriptions and unpacks in order into the
existing per-event pipeline, so SiteAlarmAggregatorActor,
DebugStreamBridgeActor, consumer-keepalive/orphan logic,
reconnect-on-graceful-completion, generation fencing, the (siteId, endpoint)
factory key and IsLive semantics are untouched.
Options (validated): GrpcStreamBatchMaxEvents 100 (1 disables),
GrpcStreamBatchWindow 25 ms — validated strictly under 250 ms, the load
test's end-to-end P99 threshold. Measured worst case (trickle-with-backlog,
window-bound rather than cap-bound): P50 13.8 ms, P99 25.4 ms, max 25.8 ms;
cap-bound case sent 600 queued events in 6 frames.
Telemetry: histogram scadabridge.site.stream.batch_size tagged by stream
kind, recorded only on negotiated streams (per-event otherwise). It rides
ScadaBridgeTelemetry.MeterName, already in the ObservedMeters allowlist.
Docs: Component-Communication.md gains an Event Batching section; CLAUDE.md's
gRPC streaming bullet records the wire shape and the negotiation rationale.
This commit is contained in:
Joseph Doherty
@@ -90,6 +90,73 @@ public class CommunicationOptionsValidatorTests
|
||||
Assert.Contains("GrpcMaxConcurrentStreams", result.FailureMessage);
|
||||
}
|
||||
|
||||
// ── R2: site→central stream event batching ──────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void DefaultStreamBatchOptions_AreValid()
|
||||
{
|
||||
var options = new CommunicationOptions();
|
||||
Assert.Equal(100, options.GrpcStreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.FromMilliseconds(25), options.GrpcStreamBatchWindow);
|
||||
Assert.True(Validate(options).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StreamBatchMaxEventsOfOne_IsValid_AndMeansBatchingDisabled()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchMaxEvents = 1 });
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonPositiveStreamBatchMaxEvents_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchMaxEvents = 0 });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("GrpcStreamBatchMaxEvents", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroStreamBatchWindow_IsValid()
|
||||
{
|
||||
// Zero = "pack only what is already queued, never wait" — a legitimate posture for
|
||||
// a latency-critical deployment that still wants the framing saving.
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchWindow = TimeSpan.Zero });
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegativeStreamBatchWindow_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchWindow = TimeSpan.FromMilliseconds(-1)
|
||||
});
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("GrpcStreamBatchWindow", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StreamBatchWindowAtOrAboveTheLatencyBudget_IsRejected()
|
||||
{
|
||||
// The coalescing window is the only latency batching adds and the target-scale
|
||||
// load test holds end-to-end stream latency to a 250 ms P99 — a window that could
|
||||
// spend the whole budget must not boot.
|
||||
foreach (var window in new[] { TimeSpan.FromMilliseconds(250), TimeSpan.FromSeconds(1) })
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchWindow = window });
|
||||
Assert.True(result.Failed, $"{window} was accepted");
|
||||
Assert.Contains("GrpcStreamBatchWindow", result.FailureMessage);
|
||||
}
|
||||
|
||||
// Just inside the ceiling is accepted — the bound is exclusive, not a round-down.
|
||||
Assert.True(Validate(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchWindow = CommunicationOptionsValidator.StreamBatchWindowCeiling
|
||||
- TimeSpan.FromMilliseconds(1)
|
||||
}).Succeeded);
|
||||
}
|
||||
|
||||
// ── Aggregated live alarm cache options (plan #10, Task 6) ───────────────────
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -20,6 +20,17 @@ public class ProtoContractTests
|
||||
SiteStreamEvent.EventOneofCase.AlarmChanged
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Oneof variants that are NOT per-event payloads and so are deliberately absent from
|
||||
/// <see cref="HandledCases"/>. <c>Batch</c> (R2) is a framing envelope: it is unpacked
|
||||
/// by <see cref="SiteStreamGrpcClient.ForEachEvent"/> into the per-event cases above
|
||||
/// BEFORE conversion, and never reaches <c>ConvertToDomainEvent</c> as a whole frame.
|
||||
/// </summary>
|
||||
private static readonly SiteStreamEvent.EventOneofCase[] FramingCases =
|
||||
[
|
||||
SiteStreamEvent.EventOneofCase.Batch
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void AllOneofVariants_HaveConversionHandlers()
|
||||
{
|
||||
@@ -27,9 +38,37 @@ public class ProtoContractTests
|
||||
.Where(c => c != SiteStreamEvent.EventOneofCase.None)
|
||||
.ToArray();
|
||||
|
||||
Assert.Equal(allCases.Length, HandledCases.Length);
|
||||
var accountedFor = HandledCases.Concat(FramingCases).ToArray();
|
||||
Assert.Equal(allCases.Length, accountedFor.Length);
|
||||
foreach (var c in allCases)
|
||||
Assert.Contains(c, HandledCases);
|
||||
Assert.Contains(c, accountedFor);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BatchFrame_IsUnpackedIntoPerEventCases_NotConvertedWhole()
|
||||
{
|
||||
// The framing case's contract: ForEachEvent hands the per-event cases to the
|
||||
// handler in order, and ConvertToDomainEvent is never asked to make sense of the
|
||||
// envelope itself (it would return null, silently dropping the whole batch).
|
||||
var inner = new[]
|
||||
{
|
||||
CreateTestEvent(SiteStreamEvent.EventOneofCase.AttributeChanged),
|
||||
CreateTestEvent(SiteStreamEvent.EventOneofCase.AlarmChanged)
|
||||
};
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "test",
|
||||
Batch = new SiteStreamEventBatch { Events = { inner } }
|
||||
};
|
||||
|
||||
Assert.Null(SiteStreamGrpcClient.ConvertToDomainEvent(frame));
|
||||
|
||||
var seen = new List<SiteStreamEvent.EventOneofCase>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, e => seen.Add(e.EventCase));
|
||||
|
||||
Assert.Equal(
|
||||
[SiteStreamEvent.EventOneofCase.AttributeChanged, SiteStreamEvent.EventOneofCase.AlarmChanged],
|
||||
seen);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for the per-subscriber coalescing pump behind R2 (gRPC event batching).
|
||||
/// The pump is the only new behaviour on the site→central hot path, so its contract is
|
||||
/// pinned directly rather than only through the server: the size cap, the time cap, the
|
||||
/// flush when the channel writer completes, the "never reorder" guarantee, and the
|
||||
/// un-negotiated (maxBatchEvents == 1) shape that keeps an older central working.
|
||||
/// </summary>
|
||||
public class SiteStreamEventBatcherTests
|
||||
{
|
||||
private const string Corr = "corr-batch";
|
||||
|
||||
private static SiteStreamEvent Event(int seq) => new()
|
||||
{
|
||||
CorrelationId = Corr,
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = "SiteA.Pump01",
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Seq",
|
||||
Value = seq.ToString(),
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UnixEpoch.AddSeconds(seq))
|
||||
}
|
||||
};
|
||||
|
||||
/// <summary>Flattens an emitted frame into the sequence numbers it carried, in order.</summary>
|
||||
private static IEnumerable<int> Seqs(SiteStreamEvent frame)
|
||||
{
|
||||
if (frame.EventCase == SiteStreamEvent.EventOneofCase.Batch)
|
||||
{
|
||||
foreach (var inner in frame.Batch.Events)
|
||||
yield return int.Parse(inner.AttributeChanged.Value);
|
||||
yield break;
|
||||
}
|
||||
|
||||
yield return int.Parse(frame.AttributeChanged.Value);
|
||||
}
|
||||
|
||||
private sealed record PumpRun(List<SiteStreamEvent> Frames, List<int> FrameSizes, Task Completion);
|
||||
|
||||
private static PumpRun StartPump(
|
||||
ChannelReader<SiteStreamEvent> reader,
|
||||
int maxBatchEvents,
|
||||
TimeSpan window,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var frames = new List<SiteStreamEvent>();
|
||||
var sizes = new List<int>();
|
||||
var task = SiteStreamEventBatcher.PumpAsync(
|
||||
reader,
|
||||
Corr,
|
||||
maxBatchEvents,
|
||||
window,
|
||||
(evt, _) =>
|
||||
{
|
||||
lock (frames) { frames.Add(evt); }
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
size => { lock (frames) { sizes.Add(size); } },
|
||||
ct);
|
||||
return new PumpRun(frames, sizes, task);
|
||||
}
|
||||
|
||||
// ── Size cap ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task SizeCap_SplitsABacklogIntoFramesOfAtMostMaxEvents()
|
||||
{
|
||||
// A backlog already sitting in the channel is drained without waiting, but never
|
||||
// beyond the size cap — 250 queued events at a cap of 100 must come out as
|
||||
// 100 + 100 + 50, in order, with nothing lost or duplicated.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 250; i++)
|
||||
Assert.True(channel.Writer.TryWrite(Event(i)));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
Assert.All(run.FrameSizes, s => Assert.True(s <= 100, $"frame carried {s} events (cap 100)"));
|
||||
Assert.Equal([100, 100, 50], run.FrameSizes);
|
||||
Assert.Equal(Enumerable.Range(0, 250), run.Frames.SelectMany(Seqs));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SizeCapOfOne_EmitsPlainPerEventFrames_TheUnnegotiatedShape()
|
||||
{
|
||||
// maxBatchEvents == 1 is what an un-negotiated subscription (an older central)
|
||||
// gets. Every event must ride its own plain frame — never a Batch case, which
|
||||
// that central's generated code cannot parse.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 5; i++)
|
||||
channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 1, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(5, run.Frames.Count);
|
||||
Assert.All(run.Frames, f =>
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, f.EventCase));
|
||||
Assert.All(run.Frames, f => Assert.Equal(Corr, f.CorrelationId));
|
||||
Assert.Equal(Enumerable.Range(0, 5), run.Frames.SelectMany(Seqs));
|
||||
}
|
||||
|
||||
// ── Time cap ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TimeCap_ClosesAnUnderfullBatchWhenTheWindowElapses()
|
||||
{
|
||||
// Two events arrive (a backlog, so the pump lingers), then the source goes quiet
|
||||
// well short of the size cap. The window — not the cap — must close the batch,
|
||||
// and it must do so within a bounded time rather than waiting for a 100th event
|
||||
// that never comes.
|
||||
var window = TimeSpan.FromMilliseconds(120);
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
channel.Writer.TryWrite(Event(0));
|
||||
channel.Writer.TryWrite(Event(1));
|
||||
|
||||
var started = Stopwatch.GetTimestamp();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window);
|
||||
|
||||
SiteStreamEvent frame;
|
||||
while (true)
|
||||
{
|
||||
lock (run.Frames)
|
||||
{
|
||||
if (run.Frames.Count > 0) { frame = run.Frames[0]; break; }
|
||||
}
|
||||
Assert.True(Stopwatch.GetElapsedTime(started) < TimeSpan.FromSeconds(5),
|
||||
"the window never closed the underfull batch");
|
||||
await Task.Delay(5);
|
||||
}
|
||||
|
||||
var elapsed = Stopwatch.GetElapsedTime(started);
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.Batch, frame.EventCase);
|
||||
Assert.Equal([0, 1], Seqs(frame));
|
||||
// The batch waited (it did not close instantly on the two queued events) and it
|
||||
// closed on the window, not on a cap it never reached.
|
||||
Assert.True(elapsed >= window - TimeSpan.FromMilliseconds(20),
|
||||
$"batch closed after {elapsed.TotalMilliseconds:0.0} ms, before the {window.TotalMilliseconds:0} ms window");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoneEventOnAQuietStream_IsNeverDelayedByTheWindow()
|
||||
{
|
||||
// The latency contract: the window applies only AFTER a backlog has been observed.
|
||||
// A single event on an idle stream must be emitted immediately as a plain frame,
|
||||
// so per-event latency on a quiet site is unchanged by batching.
|
||||
var window = TimeSpan.FromSeconds(5);
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window);
|
||||
|
||||
var started = Stopwatch.GetTimestamp();
|
||||
channel.Writer.TryWrite(Event(7));
|
||||
|
||||
while (true)
|
||||
{
|
||||
lock (run.Frames)
|
||||
{
|
||||
if (run.Frames.Count > 0) break;
|
||||
}
|
||||
Assert.True(Stopwatch.GetElapsedTime(started) < TimeSpan.FromSeconds(3),
|
||||
"a lone event was held by the coalescing window");
|
||||
await Task.Delay(2);
|
||||
}
|
||||
|
||||
var elapsed = Stopwatch.GetElapsedTime(started);
|
||||
channel.Writer.Complete();
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, run.Frames[0].EventCase);
|
||||
Assert.True(elapsed < TimeSpan.FromSeconds(1),
|
||||
$"lone event took {elapsed.TotalMilliseconds:0.0} ms against a {window.TotalSeconds:0} s window");
|
||||
}
|
||||
|
||||
// ── Flush on stream close ───────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task WriterCompletion_FlushesTheInFlightBatchBeforeReturning()
|
||||
{
|
||||
// The channel writer completing mid-window (the site stopping the relay actor and
|
||||
// calling channel.Writer.TryComplete()) must flush what is already buffered rather
|
||||
// than silently discarding it while waiting out the window.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
channel.Writer.TryWrite(Event(0));
|
||||
channel.Writer.TryWrite(Event(1));
|
||||
|
||||
// A long window guarantees the pump is lingering, not already past the emit.
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromSeconds(30));
|
||||
|
||||
await Task.Delay(100);
|
||||
lock (run.Frames)
|
||||
{
|
||||
Assert.Empty(run.Frames); // still lingering
|
||||
}
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Single(run.Frames);
|
||||
Assert.Equal([0, 1], Seqs(run.Frames[0]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriterCompletionWithNothingBuffered_ReturnsWithoutEmitting()
|
||||
{
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Empty(run.Frames);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancellation_EndsThePumpWithOperationCanceled()
|
||||
{
|
||||
// Client disconnect / duplicate replacement / site shutdown. The pump must
|
||||
// surface OperationCanceledException exactly as the pre-batching await-foreach
|
||||
// loop did, so SiteStreamGrpcServer's existing catch and finally are unchanged.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
using var cts = new CancellationTokenSource();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25), cts.Token);
|
||||
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
() => run.Completion.WaitAsync(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
|
||||
// ── Ordering ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Ordering_IsPreservedAcrossManyBatchesUnderAProducerRace()
|
||||
{
|
||||
// Batching is a framing change and nothing else: with a producer writing
|
||||
// concurrently with the pump, the flattened output must be the exact input
|
||||
// sequence — no reordering, no loss, no duplication, across many frames.
|
||||
const int total = 5_000;
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 32, window: TimeSpan.FromMilliseconds(5));
|
||||
|
||||
var producer = Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
channel.Writer.TryWrite(Event(i));
|
||||
if (i % 250 == 0) await Task.Yield();
|
||||
}
|
||||
channel.Writer.Complete();
|
||||
});
|
||||
|
||||
await producer;
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
|
||||
Assert.Equal(Enumerable.Range(0, total), run.Frames.SelectMany(Seqs));
|
||||
Assert.All(run.FrameSizes, s => Assert.InRange(s, 1, 32));
|
||||
Assert.Equal(total, run.FrameSizes.Sum());
|
||||
}
|
||||
|
||||
// ── Frame shape ─────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task BatchFrame_CarriesTheCorrelationIdOnceAndBlanksItOnInnerEvents()
|
||||
{
|
||||
// The byte saving batching exists for: the correlation id is stamped once on the
|
||||
// enclosing frame, not repeated on every packed event. No consumer reads the
|
||||
// inner value (SiteStreamGrpcClient.ForEachEvent ignores it).
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 4; i++) channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
var frame = Assert.Single(run.Frames);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.Batch, frame.EventCase);
|
||||
Assert.Equal(Corr, frame.CorrelationId);
|
||||
Assert.All(frame.Batch.Events, e => Assert.Equal(string.Empty, e.CorrelationId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PerEventTimestampsSurviveBatching()
|
||||
{
|
||||
// End-to-end latency measurement rides the per-event Timestamp; coalescing must
|
||||
// not rewrite it to a single frame-level stamp.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 3; i++) channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
var frame = Assert.Single(run.Frames);
|
||||
Assert.Equal(
|
||||
[
|
||||
DateTimeOffset.UnixEpoch,
|
||||
DateTimeOffset.UnixEpoch.AddSeconds(1),
|
||||
DateTimeOffset.UnixEpoch.AddSeconds(2)
|
||||
],
|
||||
frame.Batch.Events.Select(e => e.AttributeChanged.Timestamp.ToDateTimeOffset()));
|
||||
}
|
||||
}
|
||||
@@ -508,6 +508,125 @@ public class SiteStreamGrpcClientTests
|
||||
}
|
||||
}
|
||||
|
||||
// ── R2: batch unpacking on the client ───────────────────────────────────────
|
||||
|
||||
private static SiteStreamEvent Attr(string value, DateTimeOffset ts) => new()
|
||||
{
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = "SiteA.Pump01",
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Seq",
|
||||
Value = value,
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(ts)
|
||||
}
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_PlainFrame_IsDeliveredAsIs()
|
||||
{
|
||||
// An OLD SITE (or any un-negotiated stream) sends one event per frame. The new
|
||||
// client's unpack path must pass it straight through — this is the new-central ↔
|
||||
// old-site skew direction.
|
||||
var frame = Attr("1", DateTimeOffset.UnixEpoch);
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Same(frame, Assert.Single(seen));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_BatchFrame_UnpacksInOrderPreservingPerEventTimestamps()
|
||||
{
|
||||
// Order and per-event Timestamp fidelity are the two properties the downstream
|
||||
// consumers (SiteAlarmAggregatorActor, DebugStreamBridgeActor) and the end-to-end
|
||||
// latency measurement depend on.
|
||||
var t0 = new DateTimeOffset(2026, 8, 15, 9, 0, 0, TimeSpan.Zero);
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-batch",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events =
|
||||
{
|
||||
Attr("0", t0),
|
||||
Attr("1", t0.AddMilliseconds(3)),
|
||||
Attr("2", t0.AddMilliseconds(11))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Equal(["0", "1", "2"], seen.Select(e => e.AttributeChanged.Value));
|
||||
Assert.Equal(
|
||||
[t0, t0.AddMilliseconds(3), t0.AddMilliseconds(11)],
|
||||
seen.Select(e => e.AttributeChanged.Timestamp.ToDateTimeOffset()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_IgnoresNestedAndUnknownInnerCases()
|
||||
{
|
||||
// The server never nests a batch inside a batch. A nested (or empty) inner frame
|
||||
// from a malformed or hostile peer must be skipped, not followed — unpacking is
|
||||
// deliberately non-recursive so a crafted frame cannot drive unbounded recursion.
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-nested",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events =
|
||||
{
|
||||
Attr("0", DateTimeOffset.UnixEpoch),
|
||||
new SiteStreamEvent { Batch = new SiteStreamEventBatch { Events = { Attr("hidden", DateTimeOffset.UnixEpoch) } } },
|
||||
new SiteStreamEvent(),
|
||||
Attr("1", DateTimeOffset.UnixEpoch)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Equal(["0", "1"], seen.Select(e => e.AttributeChanged.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_MixedBatchedAndPlainFrames_DeliverEveryEventInOrder()
|
||||
{
|
||||
// A reconnect can straddle a site upgrade, so one stream may legitimately carry
|
||||
// both frame shapes. Driving the real ConsumeStreamAsync with the real unpack
|
||||
// proves the combination is flat and ordered from the consumer's point of view.
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var delivered = new List<string>();
|
||||
|
||||
void Deliver(SiteStreamEvent e) => delivered.Add(e.AttributeChanged.Value);
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-mixed",
|
||||
cts,
|
||||
() => FakeCall(new StubStreamReader(
|
||||
Attr("0", DateTimeOffset.UnixEpoch),
|
||||
new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-mixed",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events = { Attr("1", DateTimeOffset.UnixEpoch), Attr("2", DateTimeOffset.UnixEpoch) }
|
||||
}
|
||||
},
|
||||
Attr("3", DateTimeOffset.UnixEpoch))),
|
||||
frame => SiteStreamGrpcClient.ForEachEvent(frame, Deliver),
|
||||
_ => { },
|
||||
() => { });
|
||||
|
||||
Assert.Equal(["0", "1", "2", "3"], delivered);
|
||||
}
|
||||
|
||||
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(StubStreamReader reader) =>
|
||||
FakeCall(reader, Task.FromResult(new Metadata()));
|
||||
|
||||
|
||||
@@ -580,4 +580,216 @@ public class SiteStreamGrpcServerTests : TestKit
|
||||
var server = CreateServer();
|
||||
Assert.Equal(0, server.DroppedStreamEventCount);
|
||||
}
|
||||
|
||||
// ── R2: gRPC event batching, and its negotiation ────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void BatchOptions_AreBoundFromOptions_AndClampDegenerateValues()
|
||||
{
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new CommunicationOptions());
|
||||
var server = new SiteStreamGrpcServer(_subscriber, _logger, options);
|
||||
|
||||
Assert.Equal(100, server.StreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.FromMilliseconds(25), server.StreamBatchWindow);
|
||||
|
||||
// CommunicationOptionsValidator fails the boot on these, but a host composed
|
||||
// without validation must not blow up deep inside a live RPC.
|
||||
var degenerate = new SiteStreamGrpcServer(_subscriber, _logger,
|
||||
Microsoft.Extensions.Options.Options.Create(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchMaxEvents = 0,
|
||||
GrpcStreamBatchWindow = TimeSpan.FromMilliseconds(-5),
|
||||
}));
|
||||
Assert.Equal(1, degenerate.StreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.Zero, degenerate.StreamBatchWindow);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnnegotiatedSubscription_NeverEmitsABatchFrame()
|
||||
{
|
||||
// OLD-CENTRAL ↔ NEW-SITE skew. proto3 defaults batching_supported to false, which
|
||||
// is exactly what a central built before R2 sends. The site must then keep to one
|
||||
// event per frame — a Batch frame would arrive at that central as
|
||||
// EventOneofCase.None and be silently dropped by its ConvertToDomainEvent.
|
||||
var (server, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: false);
|
||||
|
||||
for (var i = 0; i < 50; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= 50, 10_000);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
lock (capture)
|
||||
{
|
||||
Assert.All(capture, f => Assert.NotEqual(SiteStreamEvent.EventOneofCase.Batch, f.EventCase));
|
||||
Assert.Equal(50, capture.Count);
|
||||
}
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NegotiatedSubscription_CoalescesABacklogIntoFewerFramesThanEvents()
|
||||
{
|
||||
// NEW-CENTRAL ↔ NEW-SITE. A burst pushed at the relay faster than the pump drains
|
||||
// it must come out in strictly fewer frames than events, with every event
|
||||
// preserved in order.
|
||||
const int burst = 400;
|
||||
var (server, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: true);
|
||||
|
||||
for (var i = 0; i < burst; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= burst, 15_000);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
List<SiteStreamEvent> frames;
|
||||
lock (capture) { frames = [.. capture]; }
|
||||
|
||||
Assert.Equal(burst, frames.Sum(CountFrameEvents));
|
||||
Assert.True(frames.Count < burst,
|
||||
$"batching produced {frames.Count} frames for {burst} events — no coalescing happened");
|
||||
Assert.Contains(frames, f => f.EventCase == SiteStreamEvent.EventOneofCase.Batch);
|
||||
|
||||
// Order is preserved end to end: the values arrive 0..burst-1 exactly once each.
|
||||
var values = frames.SelectMany(FlattenAttributeValues).ToArray();
|
||||
Assert.Equal(Enumerable.Range(0, burst).Select(i => i.ToString()), values);
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BatchSizeHistogram_IsRecordedOnlyForNegotiatedStreams()
|
||||
{
|
||||
// scadabridge.site.stream.batch_size rides ScadaBridgeTelemetry.MeterName, which is
|
||||
// already in SiteServiceRegistration.ObservedMeters — an unlisted meter exports
|
||||
// nothing, silently. Assert the instrument actually fires, and that it does NOT
|
||||
// fire on an un-negotiated stream (where it would degenerate into a per-event
|
||||
// instrument on the hottest path in the product).
|
||||
var measurements = new List<int>();
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName &&
|
||||
instrument.Name == "scadabridge.site.stream.batch_size")
|
||||
{
|
||||
l.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<int>((_, m, _, _) =>
|
||||
{
|
||||
lock (measurements) { measurements.Add(m); }
|
||||
});
|
||||
listener.Start();
|
||||
|
||||
// Un-negotiated: no measurements at all.
|
||||
var (_, plainCapture, plainCts, plainTask, plainRelay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: false, correlationId: "corr-hist-off");
|
||||
plainRelay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", 1, "Good", DateTimeOffset.UtcNow));
|
||||
await WaitForConditionAsync(() => CountEvents(plainCapture) >= 1);
|
||||
plainCts.Cancel();
|
||||
await plainTask;
|
||||
|
||||
lock (measurements) { Assert.Empty(measurements); }
|
||||
|
||||
// Negotiated: one measurement per emitted frame, each within the size cap.
|
||||
var (_, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: true, correlationId: "corr-hist-on");
|
||||
for (var i = 0; i < 20; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= 20, 10_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
lock (measurements)
|
||||
{
|
||||
Assert.NotEmpty(measurements);
|
||||
Assert.Equal(20, measurements.Sum());
|
||||
Assert.All(measurements, m => Assert.InRange(m, 1, SiteStreamGrpcServer.DefaultStreamBatchMaxEvents));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Total events carried across all captured frames (unpacking batch frames).</summary>
|
||||
private static int CountEvents(List<SiteStreamEvent> capture)
|
||||
{
|
||||
lock (capture) { return capture.Sum(CountFrameEvents); }
|
||||
}
|
||||
|
||||
private static int CountFrameEvents(SiteStreamEvent frame) =>
|
||||
frame.EventCase == SiteStreamEvent.EventOneofCase.Batch ? frame.Batch.Events.Count : 1;
|
||||
|
||||
private static IEnumerable<string> FlattenAttributeValues(SiteStreamEvent frame)
|
||||
{
|
||||
if (frame.EventCase == SiteStreamEvent.EventOneofCase.Batch)
|
||||
{
|
||||
foreach (var inner in frame.Batch.Events)
|
||||
yield return inner.AttributeChanged.Value;
|
||||
yield break;
|
||||
}
|
||||
|
||||
yield return frame.AttributeChanged.Value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a SubscribeInstance stream with the given batch negotiation, capturing every
|
||||
/// written frame and handing back the relay actor so the test can drive domain events.
|
||||
/// </summary>
|
||||
private async Task<(SiteStreamGrpcServer Server, List<SiteStreamEvent> Capture,
|
||||
CancellationTokenSource Cts, Task StreamTask, IActorRef Relay)>
|
||||
StartCapturingStreamAsync(bool batchingSupported, string correlationId = "corr-batch")
|
||||
{
|
||||
IActorRef? capturedActor = null;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
capturedActor = ci.Arg<IActorRef>();
|
||||
return "sub-batch";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(subscriber, _logger,
|
||||
Microsoft.Extensions.Options.Options.Create(new CommunicationOptions()));
|
||||
server.SetReady(Sys);
|
||||
|
||||
var capture = new List<SiteStreamEvent>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci =>
|
||||
{
|
||||
var frame = ci.Arg<SiteStreamEvent>();
|
||||
lock (capture) { capture.Add(frame); }
|
||||
});
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
|
||||
var request = new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = correlationId,
|
||||
InstanceUniqueName = "Site1.Pump01",
|
||||
BatchingSupported = batchingSupported
|
||||
};
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
await WaitForConditionAsync(() => capturedActor != null);
|
||||
|
||||
return (server, capture, cts, streamTask, capturedActor!);
|
||||
}
|
||||
}
|
||||
|
||||
+436
@@ -0,0 +1,436 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Google.Protobuf;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using Xunit.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end coverage for R2 — gRPC event batching on the site→central
|
||||
/// <c>SiteStreamService</c> stream.
|
||||
///
|
||||
/// <para>
|
||||
/// The chain assembled here is the real one, mocking only the HTTP/2 transport:
|
||||
/// domain event → real <see cref="SiteStreamManager"/> broadcast → real
|
||||
/// <see cref="SiteStreamGrpcServer"/> handler → real <c>StreamRelayActor</c> → real
|
||||
/// coalescing pump → <b>proto serialize/parse round-trip</b> (what the wire actually
|
||||
/// carries) → real <see cref="SiteStreamGrpcClient.ForEachEvent"/> unpack → real
|
||||
/// <c>ConvertToDomainEvent</c>. The serialize/parse step is what makes these
|
||||
/// version-skew claims real rather than in-memory object graph assertions.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Version skew is covered in both directions.</b> Negotiation is a single additive
|
||||
/// request field (<c>batching_supported</c>), whose proto3 default of false IS the
|
||||
/// compatibility mechanism: an old central cannot set it, so a new site never sends it a
|
||||
/// frame case its generated code would drop; a new central always sets it, and an old
|
||||
/// site ignores the unknown field and keeps sending per-event frames the new client
|
||||
/// accepts unchanged.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class GrpcStreamBatchingIntegrationTests(ITestOutputHelper output) : TestKit
|
||||
{
|
||||
private const string Instance = "SiteA.Pump01";
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end latency threshold the target-scale load test asserts a P99 against
|
||||
/// (measured P99 there: 4.57 ms). The coalescing window is the only latency batching
|
||||
/// can add, so the batched pipe must stay comfortably inside the same budget.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan LatencyThreshold = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
// ── Round trip: batched frames deliver every event, in order ────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task NegotiatedStream_RoundTripsEveryEventThroughTheWire_InOrder()
|
||||
{
|
||||
var (server, manager, frames, cts, streamTask) = await StartAsync(batchingSupported: true);
|
||||
|
||||
const int total = 600;
|
||||
var t0 = new DateTimeOffset(2026, 8, 15, 12, 0, 0, TimeSpan.Zero);
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
manager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
Instance, "Modules.IO", "Seq", i, "Good", t0.AddMilliseconds(i)));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => TotalEvents(frames) >= total, 30_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
var wire = SnapshotThroughTheWire(frames);
|
||||
|
||||
// Batching actually happened — otherwise this test proves nothing about batching.
|
||||
Assert.Contains(wire, f => f.EventCase == SiteStreamEvent.EventOneofCase.Batch);
|
||||
Assert.True(wire.Count < total,
|
||||
$"{wire.Count} frames for {total} events — no coalescing happened");
|
||||
|
||||
var delivered = Unpack(wire);
|
||||
Assert.Equal(total, delivered.Count);
|
||||
|
||||
// Every event, exactly once, in the order the site produced it — and with its OWN
|
||||
// timestamp, not a frame-level one (end-to-end latency measurement rides it).
|
||||
Assert.Equal(
|
||||
Enumerable.Range(0, total).Select(i => i.ToString()),
|
||||
delivered.Select(e => e.Value));
|
||||
Assert.Equal(
|
||||
Enumerable.Range(0, total).Select(i => t0.AddMilliseconds(i)),
|
||||
delivered.Select(e => e.Timestamp));
|
||||
|
||||
output.WriteLine($"round-trip: {total} events in {wire.Count} frames " +
|
||||
$"(mean {(double)total / wire.Count:0.0} events/frame)");
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
// ── Latency cost of the default window ─────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task DefaultWindow_KeepsPerEventLatencyFarBelowTheThreshold()
|
||||
{
|
||||
// A trickle-with-backlog workload is the case the coalescing window actually
|
||||
// bites on: each burst is far short of the 100-event size cap, so the batch is
|
||||
// closed by the 25 ms window rather than by the cap. That makes this the WORST
|
||||
// case for added latency, not the best.
|
||||
var options = new CommunicationOptions();
|
||||
var (server, manager, frames, cts, streamTask) = await StartAsync(
|
||||
batchingSupported: true, options: options);
|
||||
|
||||
const int bursts = 150;
|
||||
const int perBurst = 8;
|
||||
var stamps = new Dictionary<int, DateTimeOffset>();
|
||||
|
||||
var seq = 0;
|
||||
for (var b = 0; b < bursts; b++)
|
||||
{
|
||||
for (var i = 0; i < perBurst; i++)
|
||||
{
|
||||
var ts = DateTimeOffset.UtcNow;
|
||||
stamps[seq] = ts;
|
||||
manager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
Instance, "Modules.IO", "Seq", seq, "Good", ts));
|
||||
seq++;
|
||||
}
|
||||
await Task.Delay(5);
|
||||
}
|
||||
|
||||
var total = seq;
|
||||
await WaitForConditionAsync(() => TotalEvents(frames) >= total, 60_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
// Latency = the event's own site-side timestamp → the instant the frame carrying
|
||||
// it was handed to the response stream. That interval contains the coalescing
|
||||
// window and nothing else the pre-batching pipe did not already have.
|
||||
var latencies = new List<double>();
|
||||
lock (frames)
|
||||
{
|
||||
foreach (var (frame, writtenAt) in frames)
|
||||
{
|
||||
foreach (var evt in Flatten(frame))
|
||||
{
|
||||
var s = int.Parse(evt.AttributeChanged.Value);
|
||||
latencies.Add((writtenAt - stamps[s]).TotalMilliseconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
latencies.Sort();
|
||||
var p50 = latencies[(int)(latencies.Count * 0.50)];
|
||||
var p99 = latencies[(int)(latencies.Count * 0.99)];
|
||||
var max = latencies[^1];
|
||||
|
||||
output.WriteLine(
|
||||
$"window={options.GrpcStreamBatchWindow.TotalMilliseconds:0} ms cap={options.GrpcStreamBatchMaxEvents} " +
|
||||
$"events={latencies.Count} P50={p50:0.00} ms P99={p99:0.00} ms max={max:0.00} ms");
|
||||
|
||||
Assert.Equal(total, latencies.Count);
|
||||
Assert.True(p99 < LatencyThreshold.TotalMilliseconds,
|
||||
$"P99 {p99:0.00} ms exceeded the {LatencyThreshold.TotalMilliseconds:0} ms end-to-end threshold");
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
// ── Version skew: OLD central ↔ NEW site ───────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task OldCentral_AgainstNewSite_NeverReceivesABatchFrame()
|
||||
{
|
||||
// An old central's InstanceStreamRequest bytes simply have no field 3 — build
|
||||
// exactly those bytes and let the NEW site parse them, so the negotiation default
|
||||
// is exercised off the wire rather than asserted on an object.
|
||||
var oldCentralBytes = BuildLegacyInstanceRequest("corr-old-central", Instance);
|
||||
var request = InstanceStreamRequest.Parser.ParseFrom(oldCentralBytes);
|
||||
Assert.False(request.BatchingSupported);
|
||||
|
||||
var (server, manager, frames, cts, streamTask) = await StartAsync(request);
|
||||
|
||||
const int total = 300;
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
manager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
Instance, "Modules.IO", "Seq", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => TotalEvents(frames) >= total, 30_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
var wire = SnapshotThroughTheWire(frames);
|
||||
|
||||
// One event per frame, and — checked at the byte level, since that is what the
|
||||
// old peer's parser sees — never the field-4 batch tag.
|
||||
Assert.Equal(total, wire.Count);
|
||||
Assert.All(wire, f =>
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, f.EventCase));
|
||||
Assert.All(wire, f => Assert.DoesNotContain(4, FieldNumbers(f)));
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BatchFrameRidesFieldFour_WhichAPreBatchingParserWouldDropSilently()
|
||||
{
|
||||
// WHY negotiation is mandatory rather than "just send batches". A batch frame is a
|
||||
// length-delimited field 4: an older generated parser skips it into unknown fields
|
||||
// and reports EventOneofCase.None, whose ConvertToDomainEvent returns null — the
|
||||
// whole batch would vanish with no error anywhere. The proto3 default on
|
||||
// batching_supported is what guarantees such a peer never receives one.
|
||||
var batch = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-shape",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events = { MakeAttributeEvent(1), MakeAttributeEvent(2) }
|
||||
}
|
||||
};
|
||||
|
||||
var fields = FieldNumbers(batch);
|
||||
Assert.Contains(4, fields);
|
||||
Assert.DoesNotContain(2, fields);
|
||||
Assert.DoesNotContain(3, fields);
|
||||
|
||||
// Field 4 is length-delimited (wire type 2) — the shape an unknown-field-tolerant
|
||||
// parser can skip without corrupting the rest of the message.
|
||||
Assert.Equal(2u, WireTypeOfField(batch, 4));
|
||||
|
||||
// And the per-event frames a pre-batching site emits still parse and convert on the
|
||||
// NEW client (the other skew direction, at the same byte level).
|
||||
var plain = SiteStreamEvent.Parser.ParseFrom(MakeAttributeEvent(7).ToByteArray());
|
||||
Assert.NotNull(SiteStreamGrpcClient.ConvertToDomainEvent(plain));
|
||||
}
|
||||
|
||||
// ── Version skew: NEW central ↔ OLD site ───────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task NewCentral_AgainstOldSite_StillReceivesEveryEvent()
|
||||
{
|
||||
// An old site ignores batching_supported and emits per-event frames. That emission
|
||||
// shape is exactly what the current server produces with batching off, so drive
|
||||
// the real server that way and feed the result through the NEW client's unpack —
|
||||
// which must handle the single-event case identically to before R2.
|
||||
var (server, manager, frames, cts, streamTask) = await StartAsync(batchingSupported: false);
|
||||
|
||||
const int total = 200;
|
||||
var t0 = new DateTimeOffset(2026, 8, 15, 13, 0, 0, TimeSpan.Zero);
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
manager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
Instance, "Modules.IO", "Seq", i, "Good", t0.AddMilliseconds(i)));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => TotalEvents(frames) >= total, 30_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
var wire = SnapshotThroughTheWire(frames);
|
||||
Assert.Equal(total, wire.Count);
|
||||
|
||||
var delivered = Unpack(wire);
|
||||
Assert.Equal(
|
||||
Enumerable.Range(0, total).Select(i => i.ToString()),
|
||||
delivered.Select(e => e.Value));
|
||||
Assert.Equal(
|
||||
Enumerable.Range(0, total).Select(i => t0.AddMilliseconds(i)),
|
||||
delivered.Select(e => e.Timestamp));
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
private static SiteStreamEvent MakeAttributeEvent(int seq) => new()
|
||||
{
|
||||
CorrelationId = "corr-shape",
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = Instance,
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Seq",
|
||||
Value = seq.ToString(),
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(DateTimeOffset.UnixEpoch)
|
||||
}
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Serializes an <c>InstanceStreamRequest</c> the way a central built BEFORE R2 would:
|
||||
/// fields 1 and 2 only, with no <c>batching_supported</c> on the wire at all.
|
||||
/// </summary>
|
||||
private static byte[] BuildLegacyInstanceRequest(string correlationId, string instance)
|
||||
{
|
||||
using var ms = new MemoryStream();
|
||||
var output = new CodedOutputStream(ms);
|
||||
output.WriteTag(1, WireFormat.WireType.LengthDelimited);
|
||||
output.WriteString(correlationId);
|
||||
output.WriteTag(2, WireFormat.WireType.LengthDelimited);
|
||||
output.WriteString(instance);
|
||||
output.Flush();
|
||||
return ms.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>Top-level field numbers present in a serialized message.</summary>
|
||||
private static HashSet<int> FieldNumbers(IMessage message)
|
||||
{
|
||||
var fields = new HashSet<int>();
|
||||
var input = new CodedInputStream(message.ToByteArray());
|
||||
uint tag;
|
||||
while ((tag = input.ReadTag()) != 0)
|
||||
{
|
||||
fields.Add(WireFormat.GetTagFieldNumber(tag));
|
||||
input.SkipLastField();
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
/// <summary>Wire type of the given top-level field number in a serialized message.</summary>
|
||||
private static uint WireTypeOfField(IMessage message, int fieldNumber)
|
||||
{
|
||||
var input = new CodedInputStream(message.ToByteArray());
|
||||
uint tag;
|
||||
while ((tag = input.ReadTag()) != 0)
|
||||
{
|
||||
if (WireFormat.GetTagFieldNumber(tag) == fieldNumber)
|
||||
return (uint)WireFormat.GetTagWireType(tag);
|
||||
input.SkipLastField();
|
||||
}
|
||||
throw new InvalidOperationException($"field {fieldNumber} not present");
|
||||
}
|
||||
|
||||
private static IEnumerable<SiteStreamEvent> Flatten(SiteStreamEvent frame)
|
||||
{
|
||||
if (frame.EventCase == SiteStreamEvent.EventOneofCase.Batch)
|
||||
{
|
||||
foreach (var inner in frame.Batch.Events) yield return inner;
|
||||
yield break;
|
||||
}
|
||||
yield return frame;
|
||||
}
|
||||
|
||||
private static int TotalEvents(List<(SiteStreamEvent Frame, DateTimeOffset WrittenAt)> frames)
|
||||
{
|
||||
lock (frames) { return frames.Sum(f => Flatten(f.Frame).Count()); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Takes the captured frames through a real protobuf serialize/parse round-trip — the
|
||||
/// step that makes every claim in this file about wire compatibility a wire claim.
|
||||
/// </summary>
|
||||
private static List<SiteStreamEvent> SnapshotThroughTheWire(
|
||||
List<(SiteStreamEvent Frame, DateTimeOffset WrittenAt)> frames)
|
||||
{
|
||||
lock (frames)
|
||||
{
|
||||
return [.. frames.Select(f => SiteStreamEvent.Parser.ParseFrom(f.Frame.ToByteArray()))];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Unpacks wire frames through the REAL client path into domain events.</summary>
|
||||
private static List<AttributeValueChanged> Unpack(IEnumerable<SiteStreamEvent> wire)
|
||||
{
|
||||
var delivered = new List<AttributeValueChanged>();
|
||||
foreach (var frame in wire)
|
||||
{
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, e =>
|
||||
{
|
||||
if (SiteStreamGrpcClient.ConvertToDomainEvent(e) is AttributeValueChanged a)
|
||||
delivered.Add(a);
|
||||
});
|
||||
}
|
||||
return delivered;
|
||||
}
|
||||
|
||||
private Task<(SiteStreamGrpcServer Server, SiteStreamManager Manager,
|
||||
List<(SiteStreamEvent Frame, DateTimeOffset WrittenAt)> Frames,
|
||||
CancellationTokenSource Cts, Task StreamTask)>
|
||||
StartAsync(bool batchingSupported, CommunicationOptions? options = null)
|
||||
=> StartAsync(new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = "corr-batching",
|
||||
InstanceUniqueName = Instance,
|
||||
BatchingSupported = batchingSupported
|
||||
}, options);
|
||||
|
||||
/// <summary>
|
||||
/// Brings up a real site broadcast hub + real gRPC server handler for the supplied
|
||||
/// subscription request, capturing every written frame with the instant it was written.
|
||||
/// </summary>
|
||||
private async Task<(SiteStreamGrpcServer Server, SiteStreamManager Manager,
|
||||
List<(SiteStreamEvent Frame, DateTimeOffset WrittenAt)> Frames,
|
||||
CancellationTokenSource Cts, Task StreamTask)>
|
||||
StartAsync(InstanceStreamRequest request, CommunicationOptions? options = null)
|
||||
{
|
||||
var manager = new SiteStreamManager(
|
||||
new SiteRuntimeOptions { StreamBufferSize = 4096 },
|
||||
NullLogger<SiteStreamManager>.Instance);
|
||||
manager.Initialize(Sys);
|
||||
|
||||
var server = new SiteStreamGrpcServer(
|
||||
manager,
|
||||
NullLogger<SiteStreamGrpcServer>.Instance,
|
||||
Options.Create(options ?? new CommunicationOptions()));
|
||||
server.SetReady(Sys);
|
||||
|
||||
var frames = new List<(SiteStreamEvent, DateTimeOffset)>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci =>
|
||||
{
|
||||
var frame = ci.Arg<SiteStreamEvent>();
|
||||
var at = DateTimeOffset.UtcNow;
|
||||
lock (frames) { frames.Add((frame, at)); }
|
||||
});
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = Substitute.For<ServerCallContext>();
|
||||
context.CancellationToken.Returns(cts.Token);
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
|
||||
// The publish must not race the materialized subscription.
|
||||
await WaitForConditionAsync(() => manager.SubscriptionCount == 1);
|
||||
|
||||
return (server, manager, frames, cts, streamTask);
|
||||
}
|
||||
|
||||
private static async Task WaitForConditionAsync(Func<bool> condition, int timeoutMs = 5000)
|
||||
{
|
||||
var started = Stopwatch.GetTimestamp();
|
||||
while (!condition() && Stopwatch.GetElapsedTime(started) < TimeSpan.FromMilliseconds(timeoutMs))
|
||||
{
|
||||
await Task.Delay(10);
|
||||
}
|
||||
Assert.True(condition(), $"Condition not met within {timeoutMs}ms");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user