Merge branch 'grpc-event-batching' — additive site-stream event batching, negotiated, 100ev/25ms window (residual #3 / R2)
This commit is contained in:
Joseph Doherty
@@ -90,6 +90,73 @@ public class CommunicationOptionsValidatorTests
|
||||
Assert.Contains("GrpcMaxConcurrentStreams", result.FailureMessage);
|
||||
}
|
||||
|
||||
// ── R2: site→central stream event batching ──────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void DefaultStreamBatchOptions_AreValid()
|
||||
{
|
||||
var options = new CommunicationOptions();
|
||||
Assert.Equal(100, options.GrpcStreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.FromMilliseconds(25), options.GrpcStreamBatchWindow);
|
||||
Assert.True(Validate(options).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StreamBatchMaxEventsOfOne_IsValid_AndMeansBatchingDisabled()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchMaxEvents = 1 });
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonPositiveStreamBatchMaxEvents_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchMaxEvents = 0 });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("GrpcStreamBatchMaxEvents", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroStreamBatchWindow_IsValid()
|
||||
{
|
||||
// Zero = "pack only what is already queued, never wait" — a legitimate posture for
|
||||
// a latency-critical deployment that still wants the framing saving.
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchWindow = TimeSpan.Zero });
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegativeStreamBatchWindow_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchWindow = TimeSpan.FromMilliseconds(-1)
|
||||
});
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("GrpcStreamBatchWindow", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StreamBatchWindowAtOrAboveTheLatencyBudget_IsRejected()
|
||||
{
|
||||
// The coalescing window is the only latency batching adds and the target-scale
|
||||
// load test holds end-to-end stream latency to a 250 ms P99 — a window that could
|
||||
// spend the whole budget must not boot.
|
||||
foreach (var window in new[] { TimeSpan.FromMilliseconds(250), TimeSpan.FromSeconds(1) })
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchWindow = window });
|
||||
Assert.True(result.Failed, $"{window} was accepted");
|
||||
Assert.Contains("GrpcStreamBatchWindow", result.FailureMessage);
|
||||
}
|
||||
|
||||
// Just inside the ceiling is accepted — the bound is exclusive, not a round-down.
|
||||
Assert.True(Validate(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchWindow = CommunicationOptionsValidator.StreamBatchWindowCeiling
|
||||
- TimeSpan.FromMilliseconds(1)
|
||||
}).Succeeded);
|
||||
}
|
||||
|
||||
// ── Aggregated live alarm cache options (plan #10, Task 6) ───────────────────
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -20,6 +20,17 @@ public class ProtoContractTests
|
||||
SiteStreamEvent.EventOneofCase.AlarmChanged
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Oneof variants that are NOT per-event payloads and so are deliberately absent from
|
||||
/// <see cref="HandledCases"/>. <c>Batch</c> (R2) is a framing envelope: it is unpacked
|
||||
/// by <see cref="SiteStreamGrpcClient.ForEachEvent"/> into the per-event cases above
|
||||
/// BEFORE conversion, and never reaches <c>ConvertToDomainEvent</c> as a whole frame.
|
||||
/// </summary>
|
||||
private static readonly SiteStreamEvent.EventOneofCase[] FramingCases =
|
||||
[
|
||||
SiteStreamEvent.EventOneofCase.Batch
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void AllOneofVariants_HaveConversionHandlers()
|
||||
{
|
||||
@@ -27,9 +38,37 @@ public class ProtoContractTests
|
||||
.Where(c => c != SiteStreamEvent.EventOneofCase.None)
|
||||
.ToArray();
|
||||
|
||||
Assert.Equal(allCases.Length, HandledCases.Length);
|
||||
var accountedFor = HandledCases.Concat(FramingCases).ToArray();
|
||||
Assert.Equal(allCases.Length, accountedFor.Length);
|
||||
foreach (var c in allCases)
|
||||
Assert.Contains(c, HandledCases);
|
||||
Assert.Contains(c, accountedFor);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BatchFrame_IsUnpackedIntoPerEventCases_NotConvertedWhole()
|
||||
{
|
||||
// The framing case's contract: ForEachEvent hands the per-event cases to the
|
||||
// handler in order, and ConvertToDomainEvent is never asked to make sense of the
|
||||
// envelope itself (it would return null, silently dropping the whole batch).
|
||||
var inner = new[]
|
||||
{
|
||||
CreateTestEvent(SiteStreamEvent.EventOneofCase.AttributeChanged),
|
||||
CreateTestEvent(SiteStreamEvent.EventOneofCase.AlarmChanged)
|
||||
};
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "test",
|
||||
Batch = new SiteStreamEventBatch { Events = { inner } }
|
||||
};
|
||||
|
||||
Assert.Null(SiteStreamGrpcClient.ConvertToDomainEvent(frame));
|
||||
|
||||
var seen = new List<SiteStreamEvent.EventOneofCase>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, e => seen.Add(e.EventCase));
|
||||
|
||||
Assert.Equal(
|
||||
[SiteStreamEvent.EventOneofCase.AttributeChanged, SiteStreamEvent.EventOneofCase.AlarmChanged],
|
||||
seen);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for the per-subscriber coalescing pump behind R2 (gRPC event batching).
|
||||
/// The pump is the only new behaviour on the site→central hot path, so its contract is
|
||||
/// pinned directly rather than only through the server: the size cap, the time cap, the
|
||||
/// flush when the channel writer completes, the "never reorder" guarantee, and the
|
||||
/// un-negotiated (maxBatchEvents == 1) shape that keeps an older central working.
|
||||
/// </summary>
|
||||
public class SiteStreamEventBatcherTests
|
||||
{
|
||||
private const string Corr = "corr-batch";
|
||||
|
||||
private static SiteStreamEvent Event(int seq) => new()
|
||||
{
|
||||
CorrelationId = Corr,
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = "SiteA.Pump01",
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Seq",
|
||||
Value = seq.ToString(),
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UnixEpoch.AddSeconds(seq))
|
||||
}
|
||||
};
|
||||
|
||||
/// <summary>Flattens an emitted frame into the sequence numbers it carried, in order.</summary>
|
||||
private static IEnumerable<int> Seqs(SiteStreamEvent frame)
|
||||
{
|
||||
if (frame.EventCase == SiteStreamEvent.EventOneofCase.Batch)
|
||||
{
|
||||
foreach (var inner in frame.Batch.Events)
|
||||
yield return int.Parse(inner.AttributeChanged.Value);
|
||||
yield break;
|
||||
}
|
||||
|
||||
yield return int.Parse(frame.AttributeChanged.Value);
|
||||
}
|
||||
|
||||
private sealed record PumpRun(List<SiteStreamEvent> Frames, List<int> FrameSizes, Task Completion);
|
||||
|
||||
private static PumpRun StartPump(
|
||||
ChannelReader<SiteStreamEvent> reader,
|
||||
int maxBatchEvents,
|
||||
TimeSpan window,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var frames = new List<SiteStreamEvent>();
|
||||
var sizes = new List<int>();
|
||||
var task = SiteStreamEventBatcher.PumpAsync(
|
||||
reader,
|
||||
Corr,
|
||||
maxBatchEvents,
|
||||
window,
|
||||
(evt, _) =>
|
||||
{
|
||||
lock (frames) { frames.Add(evt); }
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
size => { lock (frames) { sizes.Add(size); } },
|
||||
ct);
|
||||
return new PumpRun(frames, sizes, task);
|
||||
}
|
||||
|
||||
// ── Size cap ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task SizeCap_SplitsABacklogIntoFramesOfAtMostMaxEvents()
|
||||
{
|
||||
// A backlog already sitting in the channel is drained without waiting, but never
|
||||
// beyond the size cap — 250 queued events at a cap of 100 must come out as
|
||||
// 100 + 100 + 50, in order, with nothing lost or duplicated.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 250; i++)
|
||||
Assert.True(channel.Writer.TryWrite(Event(i)));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
Assert.All(run.FrameSizes, s => Assert.True(s <= 100, $"frame carried {s} events (cap 100)"));
|
||||
Assert.Equal([100, 100, 50], run.FrameSizes);
|
||||
Assert.Equal(Enumerable.Range(0, 250), run.Frames.SelectMany(Seqs));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SizeCapOfOne_EmitsPlainPerEventFrames_TheUnnegotiatedShape()
|
||||
{
|
||||
// maxBatchEvents == 1 is what an un-negotiated subscription (an older central)
|
||||
// gets. Every event must ride its own plain frame — never a Batch case, which
|
||||
// that central's generated code cannot parse.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 5; i++)
|
||||
channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 1, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(5, run.Frames.Count);
|
||||
Assert.All(run.Frames, f =>
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, f.EventCase));
|
||||
Assert.All(run.Frames, f => Assert.Equal(Corr, f.CorrelationId));
|
||||
Assert.Equal(Enumerable.Range(0, 5), run.Frames.SelectMany(Seqs));
|
||||
}
|
||||
|
||||
// ── Time cap ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TimeCap_ClosesAnUnderfullBatchWhenTheWindowElapses()
|
||||
{
|
||||
// Two events arrive (a backlog, so the pump lingers), then the source goes quiet
|
||||
// well short of the size cap. The window — not the cap — must close the batch,
|
||||
// and it must do so within a bounded time rather than waiting for a 100th event
|
||||
// that never comes.
|
||||
var window = TimeSpan.FromMilliseconds(120);
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
channel.Writer.TryWrite(Event(0));
|
||||
channel.Writer.TryWrite(Event(1));
|
||||
|
||||
var started = Stopwatch.GetTimestamp();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window);
|
||||
|
||||
SiteStreamEvent frame;
|
||||
while (true)
|
||||
{
|
||||
lock (run.Frames)
|
||||
{
|
||||
if (run.Frames.Count > 0) { frame = run.Frames[0]; break; }
|
||||
}
|
||||
Assert.True(Stopwatch.GetElapsedTime(started) < TimeSpan.FromSeconds(5),
|
||||
"the window never closed the underfull batch");
|
||||
await Task.Delay(5);
|
||||
}
|
||||
|
||||
var elapsed = Stopwatch.GetElapsedTime(started);
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.Batch, frame.EventCase);
|
||||
Assert.Equal([0, 1], Seqs(frame));
|
||||
// The batch waited (it did not close instantly on the two queued events) and it
|
||||
// closed on the window, not on a cap it never reached.
|
||||
Assert.True(elapsed >= window - TimeSpan.FromMilliseconds(20),
|
||||
$"batch closed after {elapsed.TotalMilliseconds:0.0} ms, before the {window.TotalMilliseconds:0} ms window");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoneEventOnAQuietStream_IsNeverDelayedByTheWindow()
|
||||
{
|
||||
// The latency contract: the window applies only AFTER a backlog has been observed.
|
||||
// A single event on an idle stream must be emitted immediately as a plain frame,
|
||||
// so per-event latency on a quiet site is unchanged by batching.
|
||||
var window = TimeSpan.FromSeconds(5);
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window);
|
||||
|
||||
var started = Stopwatch.GetTimestamp();
|
||||
channel.Writer.TryWrite(Event(7));
|
||||
|
||||
while (true)
|
||||
{
|
||||
lock (run.Frames)
|
||||
{
|
||||
if (run.Frames.Count > 0) break;
|
||||
}
|
||||
Assert.True(Stopwatch.GetElapsedTime(started) < TimeSpan.FromSeconds(3),
|
||||
"a lone event was held by the coalescing window");
|
||||
await Task.Delay(2);
|
||||
}
|
||||
|
||||
var elapsed = Stopwatch.GetElapsedTime(started);
|
||||
channel.Writer.Complete();
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, run.Frames[0].EventCase);
|
||||
Assert.True(elapsed < TimeSpan.FromSeconds(1),
|
||||
$"lone event took {elapsed.TotalMilliseconds:0.0} ms against a {window.TotalSeconds:0} s window");
|
||||
}
|
||||
|
||||
// ── Flush on stream close ───────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task WriterCompletion_FlushesTheInFlightBatchBeforeReturning()
|
||||
{
|
||||
// The channel writer completing mid-window (the site stopping the relay actor and
|
||||
// calling channel.Writer.TryComplete()) must flush what is already buffered rather
|
||||
// than silently discarding it while waiting out the window.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
channel.Writer.TryWrite(Event(0));
|
||||
channel.Writer.TryWrite(Event(1));
|
||||
|
||||
// A long window guarantees the pump is lingering, not already past the emit.
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromSeconds(30));
|
||||
|
||||
await Task.Delay(100);
|
||||
lock (run.Frames)
|
||||
{
|
||||
Assert.Empty(run.Frames); // still lingering
|
||||
}
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Single(run.Frames);
|
||||
Assert.Equal([0, 1], Seqs(run.Frames[0]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriterCompletionWithNothingBuffered_ReturnsWithoutEmitting()
|
||||
{
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Empty(run.Frames);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancellation_EndsThePumpWithOperationCanceled()
|
||||
{
|
||||
// Client disconnect / duplicate replacement / site shutdown. The pump must
|
||||
// surface OperationCanceledException exactly as the pre-batching await-foreach
|
||||
// loop did, so SiteStreamGrpcServer's existing catch and finally are unchanged.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
using var cts = new CancellationTokenSource();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25), cts.Token);
|
||||
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
() => run.Completion.WaitAsync(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
|
||||
// ── Ordering ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Ordering_IsPreservedAcrossManyBatchesUnderAProducerRace()
|
||||
{
|
||||
// Batching is a framing change and nothing else: with a producer writing
|
||||
// concurrently with the pump, the flattened output must be the exact input
|
||||
// sequence — no reordering, no loss, no duplication, across many frames.
|
||||
const int total = 5_000;
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 32, window: TimeSpan.FromMilliseconds(5));
|
||||
|
||||
var producer = Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
channel.Writer.TryWrite(Event(i));
|
||||
if (i % 250 == 0) await Task.Yield();
|
||||
}
|
||||
channel.Writer.Complete();
|
||||
});
|
||||
|
||||
await producer;
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
|
||||
Assert.Equal(Enumerable.Range(0, total), run.Frames.SelectMany(Seqs));
|
||||
Assert.All(run.FrameSizes, s => Assert.InRange(s, 1, 32));
|
||||
Assert.Equal(total, run.FrameSizes.Sum());
|
||||
}
|
||||
|
||||
// ── Frame shape ─────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task BatchFrame_CarriesTheCorrelationIdOnceAndBlanksItOnInnerEvents()
|
||||
{
|
||||
// The byte saving batching exists for: the correlation id is stamped once on the
|
||||
// enclosing frame, not repeated on every packed event. No consumer reads the
|
||||
// inner value (SiteStreamGrpcClient.ForEachEvent ignores it).
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 4; i++) channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
var frame = Assert.Single(run.Frames);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.Batch, frame.EventCase);
|
||||
Assert.Equal(Corr, frame.CorrelationId);
|
||||
Assert.All(frame.Batch.Events, e => Assert.Equal(string.Empty, e.CorrelationId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PerEventTimestampsSurviveBatching()
|
||||
{
|
||||
// End-to-end latency measurement rides the per-event Timestamp; coalescing must
|
||||
// not rewrite it to a single frame-level stamp.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 3; i++) channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
var frame = Assert.Single(run.Frames);
|
||||
Assert.Equal(
|
||||
[
|
||||
DateTimeOffset.UnixEpoch,
|
||||
DateTimeOffset.UnixEpoch.AddSeconds(1),
|
||||
DateTimeOffset.UnixEpoch.AddSeconds(2)
|
||||
],
|
||||
frame.Batch.Events.Select(e => e.AttributeChanged.Timestamp.ToDateTimeOffset()));
|
||||
}
|
||||
}
|
||||
@@ -508,6 +508,125 @@ public class SiteStreamGrpcClientTests
|
||||
}
|
||||
}
|
||||
|
||||
// ── R2: batch unpacking on the client ───────────────────────────────────────
|
||||
|
||||
private static SiteStreamEvent Attr(string value, DateTimeOffset ts) => new()
|
||||
{
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = "SiteA.Pump01",
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Seq",
|
||||
Value = value,
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(ts)
|
||||
}
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_PlainFrame_IsDeliveredAsIs()
|
||||
{
|
||||
// An OLD SITE (or any un-negotiated stream) sends one event per frame. The new
|
||||
// client's unpack path must pass it straight through — this is the new-central ↔
|
||||
// old-site skew direction.
|
||||
var frame = Attr("1", DateTimeOffset.UnixEpoch);
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Same(frame, Assert.Single(seen));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_BatchFrame_UnpacksInOrderPreservingPerEventTimestamps()
|
||||
{
|
||||
// Order and per-event Timestamp fidelity are the two properties the downstream
|
||||
// consumers (SiteAlarmAggregatorActor, DebugStreamBridgeActor) and the end-to-end
|
||||
// latency measurement depend on.
|
||||
var t0 = new DateTimeOffset(2026, 8, 15, 9, 0, 0, TimeSpan.Zero);
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-batch",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events =
|
||||
{
|
||||
Attr("0", t0),
|
||||
Attr("1", t0.AddMilliseconds(3)),
|
||||
Attr("2", t0.AddMilliseconds(11))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Equal(["0", "1", "2"], seen.Select(e => e.AttributeChanged.Value));
|
||||
Assert.Equal(
|
||||
[t0, t0.AddMilliseconds(3), t0.AddMilliseconds(11)],
|
||||
seen.Select(e => e.AttributeChanged.Timestamp.ToDateTimeOffset()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_IgnoresNestedAndUnknownInnerCases()
|
||||
{
|
||||
// The server never nests a batch inside a batch. A nested (or empty) inner frame
|
||||
// from a malformed or hostile peer must be skipped, not followed — unpacking is
|
||||
// deliberately non-recursive so a crafted frame cannot drive unbounded recursion.
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-nested",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events =
|
||||
{
|
||||
Attr("0", DateTimeOffset.UnixEpoch),
|
||||
new SiteStreamEvent { Batch = new SiteStreamEventBatch { Events = { Attr("hidden", DateTimeOffset.UnixEpoch) } } },
|
||||
new SiteStreamEvent(),
|
||||
Attr("1", DateTimeOffset.UnixEpoch)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Equal(["0", "1"], seen.Select(e => e.AttributeChanged.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_MixedBatchedAndPlainFrames_DeliverEveryEventInOrder()
|
||||
{
|
||||
// A reconnect can straddle a site upgrade, so one stream may legitimately carry
|
||||
// both frame shapes. Driving the real ConsumeStreamAsync with the real unpack
|
||||
// proves the combination is flat and ordered from the consumer's point of view.
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var delivered = new List<string>();
|
||||
|
||||
void Deliver(SiteStreamEvent e) => delivered.Add(e.AttributeChanged.Value);
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-mixed",
|
||||
cts,
|
||||
() => FakeCall(new StubStreamReader(
|
||||
Attr("0", DateTimeOffset.UnixEpoch),
|
||||
new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-mixed",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events = { Attr("1", DateTimeOffset.UnixEpoch), Attr("2", DateTimeOffset.UnixEpoch) }
|
||||
}
|
||||
},
|
||||
Attr("3", DateTimeOffset.UnixEpoch))),
|
||||
frame => SiteStreamGrpcClient.ForEachEvent(frame, Deliver),
|
||||
_ => { },
|
||||
() => { });
|
||||
|
||||
Assert.Equal(["0", "1", "2", "3"], delivered);
|
||||
}
|
||||
|
||||
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(StubStreamReader reader) =>
|
||||
FakeCall(reader, Task.FromResult(new Metadata()));
|
||||
|
||||
|
||||
@@ -580,4 +580,277 @@ public class SiteStreamGrpcServerTests : TestKit
|
||||
var server = CreateServer();
|
||||
Assert.Equal(0, server.DroppedStreamEventCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DuplicateReplacement_CancelsTheReplacedStream_WithoutDisposingItsCts()
|
||||
{
|
||||
// Regression: the duplicate-replacement path used to Cancel AND Dispose the
|
||||
// replaced stream's CancellationTokenSource. That CTS belongs to the replaced
|
||||
// handler's own `using var streamCts`, which is still running and still has to
|
||||
// read `streamCts.Token` — so the Dispose raced that read and escaped the RPC as
|
||||
// an unhandled ObjectDisposedException. It surfaced only under full-suite load
|
||||
// (GrpcStreamIntegrationTests.Pipeline_DuplicateCorrelationId_ReplacesStream) and
|
||||
// predates R2: the same Dispose and the same first-token-read relationship existed
|
||||
// when the handler still used `ReadAllAsync(streamCts.Token)`.
|
||||
//
|
||||
// The race is made DETERMINISTIC here by gating the first stream inside its setup
|
||||
// window (its _activeStreams entry is registered before Subscribe is called), so
|
||||
// the replacement always lands before the first stream reads its token.
|
||||
using var gate = new ManualResetEventSlim(false);
|
||||
var calls = 0;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
var n = Interlocked.Increment(ref calls);
|
||||
if (n == 1)
|
||||
gate.Wait(TimeSpan.FromSeconds(15));
|
||||
return $"sub-dup-race-{n}";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(subscriber, _logger);
|
||||
server.SetReady(Sys);
|
||||
|
||||
using var cts1 = new CancellationTokenSource();
|
||||
var stream1 = Task.Run(() => server.SubscribeInstance(
|
||||
MakeRequest("corr-dup-race"),
|
||||
Substitute.For<IServerStreamWriter<SiteStreamEvent>>(),
|
||||
CreateMockContext(cts1.Token)));
|
||||
|
||||
await WaitForConditionAsync(() => server.ActiveStreamCount == 1);
|
||||
await WaitForConditionAsync(() => Volatile.Read(ref calls) == 1);
|
||||
|
||||
using var cts2 = new CancellationTokenSource();
|
||||
var stream2 = Task.Run(() => server.SubscribeInstance(
|
||||
MakeRequest("corr-dup-race"),
|
||||
Substitute.For<IServerStreamWriter<SiteStreamEvent>>(),
|
||||
CreateMockContext(cts2.Token)));
|
||||
|
||||
// The replacement has taken the slot (and cancelled stream 1's CTS) by the time
|
||||
// its own Subscribe has been called.
|
||||
await WaitForConditionAsync(() => Volatile.Read(ref calls) == 2);
|
||||
|
||||
gate.Set();
|
||||
|
||||
// Pre-fix this threw ObjectDisposedException out of the RPC. Post-fix the replaced
|
||||
// stream observes a plain cancellation and unwinds through its normal finally.
|
||||
await stream1;
|
||||
|
||||
cts2.Cancel();
|
||||
await stream2;
|
||||
|
||||
Assert.Equal(0, server.ActiveStreamCount);
|
||||
}
|
||||
|
||||
// ── R2: gRPC event batching, and its negotiation ────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void BatchOptions_AreBoundFromOptions_AndClampDegenerateValues()
|
||||
{
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new CommunicationOptions());
|
||||
var server = new SiteStreamGrpcServer(_subscriber, _logger, options);
|
||||
|
||||
Assert.Equal(100, server.StreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.FromMilliseconds(25), server.StreamBatchWindow);
|
||||
|
||||
// CommunicationOptionsValidator fails the boot on these, but a host composed
|
||||
// without validation must not blow up deep inside a live RPC.
|
||||
var degenerate = new SiteStreamGrpcServer(_subscriber, _logger,
|
||||
Microsoft.Extensions.Options.Options.Create(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchMaxEvents = 0,
|
||||
GrpcStreamBatchWindow = TimeSpan.FromMilliseconds(-5),
|
||||
}));
|
||||
Assert.Equal(1, degenerate.StreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.Zero, degenerate.StreamBatchWindow);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnnegotiatedSubscription_NeverEmitsABatchFrame()
|
||||
{
|
||||
// OLD-CENTRAL ↔ NEW-SITE skew. proto3 defaults batching_supported to false, which
|
||||
// is exactly what a central built before R2 sends. The site must then keep to one
|
||||
// event per frame — a Batch frame would arrive at that central as
|
||||
// EventOneofCase.None and be silently dropped by its ConvertToDomainEvent.
|
||||
var (server, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: false);
|
||||
|
||||
for (var i = 0; i < 50; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= 50, 10_000);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
lock (capture)
|
||||
{
|
||||
Assert.All(capture, f => Assert.NotEqual(SiteStreamEvent.EventOneofCase.Batch, f.EventCase));
|
||||
Assert.Equal(50, capture.Count);
|
||||
}
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NegotiatedSubscription_CoalescesABacklogIntoFewerFramesThanEvents()
|
||||
{
|
||||
// NEW-CENTRAL ↔ NEW-SITE. A burst pushed at the relay faster than the pump drains
|
||||
// it must come out in strictly fewer frames than events, with every event
|
||||
// preserved in order.
|
||||
const int burst = 400;
|
||||
var (server, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: true);
|
||||
|
||||
for (var i = 0; i < burst; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= burst, 15_000);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
List<SiteStreamEvent> frames;
|
||||
lock (capture) { frames = [.. capture]; }
|
||||
|
||||
Assert.Equal(burst, frames.Sum(CountFrameEvents));
|
||||
Assert.True(frames.Count < burst,
|
||||
$"batching produced {frames.Count} frames for {burst} events — no coalescing happened");
|
||||
Assert.Contains(frames, f => f.EventCase == SiteStreamEvent.EventOneofCase.Batch);
|
||||
|
||||
// Order is preserved end to end: the values arrive 0..burst-1 exactly once each.
|
||||
var values = frames.SelectMany(FlattenAttributeValues).ToArray();
|
||||
Assert.Equal(Enumerable.Range(0, burst).Select(i => i.ToString()), values);
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BatchSizeHistogram_IsRecordedOnlyForNegotiatedStreams()
|
||||
{
|
||||
// scadabridge.site.stream.batch_size rides ScadaBridgeTelemetry.MeterName, which is
|
||||
// already in SiteServiceRegistration.ObservedMeters — an unlisted meter exports
|
||||
// nothing, silently. Assert the instrument actually fires, and that it does NOT
|
||||
// fire on an un-negotiated stream (where it would degenerate into a per-event
|
||||
// instrument on the hottest path in the product).
|
||||
var measurements = new List<int>();
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName &&
|
||||
instrument.Name == "scadabridge.site.stream.batch_size")
|
||||
{
|
||||
l.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<int>((_, m, _, _) =>
|
||||
{
|
||||
lock (measurements) { measurements.Add(m); }
|
||||
});
|
||||
listener.Start();
|
||||
|
||||
// Un-negotiated: no measurements at all.
|
||||
var (_, plainCapture, plainCts, plainTask, plainRelay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: false, correlationId: "corr-hist-off");
|
||||
plainRelay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", 1, "Good", DateTimeOffset.UtcNow));
|
||||
await WaitForConditionAsync(() => CountEvents(plainCapture) >= 1);
|
||||
plainCts.Cancel();
|
||||
await plainTask;
|
||||
|
||||
lock (measurements) { Assert.Empty(measurements); }
|
||||
|
||||
// Negotiated: one measurement per emitted frame, each within the size cap.
|
||||
var (_, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: true, correlationId: "corr-hist-on");
|
||||
for (var i = 0; i < 20; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= 20, 10_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
lock (measurements)
|
||||
{
|
||||
Assert.NotEmpty(measurements);
|
||||
Assert.Equal(20, measurements.Sum());
|
||||
Assert.All(measurements, m => Assert.InRange(m, 1, SiteStreamGrpcServer.DefaultStreamBatchMaxEvents));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Total events carried across all captured frames (unpacking batch frames).</summary>
|
||||
private static int CountEvents(List<SiteStreamEvent> capture)
|
||||
{
|
||||
lock (capture) { return capture.Sum(CountFrameEvents); }
|
||||
}
|
||||
|
||||
private static int CountFrameEvents(SiteStreamEvent frame) =>
|
||||
frame.EventCase == SiteStreamEvent.EventOneofCase.Batch ? frame.Batch.Events.Count : 1;
|
||||
|
||||
private static IEnumerable<string> FlattenAttributeValues(SiteStreamEvent frame)
|
||||
{
|
||||
if (frame.EventCase == SiteStreamEvent.EventOneofCase.Batch)
|
||||
{
|
||||
foreach (var inner in frame.Batch.Events)
|
||||
yield return inner.AttributeChanged.Value;
|
||||
yield break;
|
||||
}
|
||||
|
||||
yield return frame.AttributeChanged.Value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a SubscribeInstance stream with the given batch negotiation, capturing every
|
||||
/// written frame and handing back the relay actor so the test can drive domain events.
|
||||
/// </summary>
|
||||
private async Task<(SiteStreamGrpcServer Server, List<SiteStreamEvent> Capture,
|
||||
CancellationTokenSource Cts, Task StreamTask, IActorRef Relay)>
|
||||
StartCapturingStreamAsync(bool batchingSupported, string correlationId = "corr-batch")
|
||||
{
|
||||
IActorRef? capturedActor = null;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
capturedActor = ci.Arg<IActorRef>();
|
||||
return "sub-batch";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(subscriber, _logger,
|
||||
Microsoft.Extensions.Options.Options.Create(new CommunicationOptions()));
|
||||
server.SetReady(Sys);
|
||||
|
||||
var capture = new List<SiteStreamEvent>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci =>
|
||||
{
|
||||
var frame = ci.Arg<SiteStreamEvent>();
|
||||
lock (capture) { capture.Add(frame); }
|
||||
});
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
|
||||
var request = new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = correlationId,
|
||||
InstanceUniqueName = "Site1.Pump01",
|
||||
BatchingSupported = batchingSupported
|
||||
};
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
await WaitForConditionAsync(() => capturedActor != null);
|
||||
|
||||
return (server, capture, cts, streamTask, capturedActor!);
|
||||
}
|
||||
}
|
||||
|
||||
+436
@@ -0,0 +1,436 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Google.Protobuf;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using Xunit.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Streaming;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end coverage for R2 — gRPC event batching on the site→central
|
||||
/// <c>SiteStreamService</c> stream.
|
||||
///
|
||||
/// <para>
|
||||
/// The chain assembled here is the real one, mocking only the HTTP/2 transport:
|
||||
/// domain event → real <see cref="SiteStreamManager"/> broadcast → real
|
||||
/// <see cref="SiteStreamGrpcServer"/> handler → real <c>StreamRelayActor</c> → real
|
||||
/// coalescing pump → <b>proto serialize/parse round-trip</b> (what the wire actually
|
||||
/// carries) → real <see cref="SiteStreamGrpcClient.ForEachEvent"/> unpack → real
|
||||
/// <c>ConvertToDomainEvent</c>. The serialize/parse step is what makes these
|
||||
/// version-skew claims real rather than in-memory object graph assertions.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Version skew is covered in both directions.</b> Negotiation is a single additive
|
||||
/// request field (<c>batching_supported</c>), whose proto3 default of false IS the
|
||||
/// compatibility mechanism: an old central cannot set it, so a new site never sends it a
|
||||
/// frame case its generated code would drop; a new central always sets it, and an old
|
||||
/// site ignores the unknown field and keeps sending per-event frames the new client
|
||||
/// accepts unchanged.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class GrpcStreamBatchingIntegrationTests(ITestOutputHelper output) : TestKit
|
||||
{
|
||||
private const string Instance = "SiteA.Pump01";
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end latency threshold the target-scale load test asserts a P99 against
|
||||
/// (measured P99 there: 4.57 ms). The coalescing window is the only latency batching
|
||||
/// can add, so the batched pipe must stay comfortably inside the same budget.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan LatencyThreshold = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
// ── Round trip: batched frames deliver every event, in order ────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task NegotiatedStream_RoundTripsEveryEventThroughTheWire_InOrder()
|
||||
{
|
||||
var (server, manager, frames, cts, streamTask) = await StartAsync(batchingSupported: true);
|
||||
|
||||
const int total = 600;
|
||||
var t0 = new DateTimeOffset(2026, 8, 15, 12, 0, 0, TimeSpan.Zero);
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
manager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
Instance, "Modules.IO", "Seq", i, "Good", t0.AddMilliseconds(i)));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => TotalEvents(frames) >= total, 30_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
var wire = SnapshotThroughTheWire(frames);
|
||||
|
||||
// Batching actually happened — otherwise this test proves nothing about batching.
|
||||
Assert.Contains(wire, f => f.EventCase == SiteStreamEvent.EventOneofCase.Batch);
|
||||
Assert.True(wire.Count < total,
|
||||
$"{wire.Count} frames for {total} events — no coalescing happened");
|
||||
|
||||
var delivered = Unpack(wire);
|
||||
Assert.Equal(total, delivered.Count);
|
||||
|
||||
// Every event, exactly once, in the order the site produced it — and with its OWN
|
||||
// timestamp, not a frame-level one (end-to-end latency measurement rides it).
|
||||
Assert.Equal(
|
||||
Enumerable.Range(0, total).Select(i => i.ToString()),
|
||||
delivered.Select(e => e.Value));
|
||||
Assert.Equal(
|
||||
Enumerable.Range(0, total).Select(i => t0.AddMilliseconds(i)),
|
||||
delivered.Select(e => e.Timestamp));
|
||||
|
||||
output.WriteLine($"round-trip: {total} events in {wire.Count} frames " +
|
||||
$"(mean {(double)total / wire.Count:0.0} events/frame)");
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
// ── Latency cost of the default window ─────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task DefaultWindow_KeepsPerEventLatencyFarBelowTheThreshold()
|
||||
{
|
||||
// A trickle-with-backlog workload is the case the coalescing window actually
|
||||
// bites on: each burst is far short of the 100-event size cap, so the batch is
|
||||
// closed by the 25 ms window rather than by the cap. That makes this the WORST
|
||||
// case for added latency, not the best.
|
||||
var options = new CommunicationOptions();
|
||||
var (server, manager, frames, cts, streamTask) = await StartAsync(
|
||||
batchingSupported: true, options: options);
|
||||
|
||||
const int bursts = 150;
|
||||
const int perBurst = 8;
|
||||
var stamps = new Dictionary<int, DateTimeOffset>();
|
||||
|
||||
var seq = 0;
|
||||
for (var b = 0; b < bursts; b++)
|
||||
{
|
||||
for (var i = 0; i < perBurst; i++)
|
||||
{
|
||||
var ts = DateTimeOffset.UtcNow;
|
||||
stamps[seq] = ts;
|
||||
manager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
Instance, "Modules.IO", "Seq", seq, "Good", ts));
|
||||
seq++;
|
||||
}
|
||||
await Task.Delay(5);
|
||||
}
|
||||
|
||||
var total = seq;
|
||||
await WaitForConditionAsync(() => TotalEvents(frames) >= total, 60_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
// Latency = the event's own site-side timestamp → the instant the frame carrying
|
||||
// it was handed to the response stream. That interval contains the coalescing
|
||||
// window and nothing else the pre-batching pipe did not already have.
|
||||
var latencies = new List<double>();
|
||||
lock (frames)
|
||||
{
|
||||
foreach (var (frame, writtenAt) in frames)
|
||||
{
|
||||
foreach (var evt in Flatten(frame))
|
||||
{
|
||||
var s = int.Parse(evt.AttributeChanged.Value);
|
||||
latencies.Add((writtenAt - stamps[s]).TotalMilliseconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
latencies.Sort();
|
||||
var p50 = latencies[(int)(latencies.Count * 0.50)];
|
||||
var p99 = latencies[(int)(latencies.Count * 0.99)];
|
||||
var max = latencies[^1];
|
||||
|
||||
output.WriteLine(
|
||||
$"window={options.GrpcStreamBatchWindow.TotalMilliseconds:0} ms cap={options.GrpcStreamBatchMaxEvents} " +
|
||||
$"events={latencies.Count} P50={p50:0.00} ms P99={p99:0.00} ms max={max:0.00} ms");
|
||||
|
||||
Assert.Equal(total, latencies.Count);
|
||||
Assert.True(p99 < LatencyThreshold.TotalMilliseconds,
|
||||
$"P99 {p99:0.00} ms exceeded the {LatencyThreshold.TotalMilliseconds:0} ms end-to-end threshold");
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
// ── Version skew: OLD central ↔ NEW site ───────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task OldCentral_AgainstNewSite_NeverReceivesABatchFrame()
|
||||
{
|
||||
// An old central's InstanceStreamRequest bytes simply have no field 3 — build
|
||||
// exactly those bytes and let the NEW site parse them, so the negotiation default
|
||||
// is exercised off the wire rather than asserted on an object.
|
||||
var oldCentralBytes = BuildLegacyInstanceRequest("corr-old-central", Instance);
|
||||
var request = InstanceStreamRequest.Parser.ParseFrom(oldCentralBytes);
|
||||
Assert.False(request.BatchingSupported);
|
||||
|
||||
var (server, manager, frames, cts, streamTask) = await StartAsync(request);
|
||||
|
||||
const int total = 300;
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
manager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
Instance, "Modules.IO", "Seq", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => TotalEvents(frames) >= total, 30_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
var wire = SnapshotThroughTheWire(frames);
|
||||
|
||||
// One event per frame, and — checked at the byte level, since that is what the
|
||||
// old peer's parser sees — never the field-4 batch tag.
|
||||
Assert.Equal(total, wire.Count);
|
||||
Assert.All(wire, f =>
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, f.EventCase));
|
||||
Assert.All(wire, f => Assert.DoesNotContain(4, FieldNumbers(f)));
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BatchFrameRidesFieldFour_WhichAPreBatchingParserWouldDropSilently()
|
||||
{
|
||||
// WHY negotiation is mandatory rather than "just send batches". A batch frame is a
|
||||
// length-delimited field 4: an older generated parser skips it into unknown fields
|
||||
// and reports EventOneofCase.None, whose ConvertToDomainEvent returns null — the
|
||||
// whole batch would vanish with no error anywhere. The proto3 default on
|
||||
// batching_supported is what guarantees such a peer never receives one.
|
||||
var batch = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-shape",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events = { MakeAttributeEvent(1), MakeAttributeEvent(2) }
|
||||
}
|
||||
};
|
||||
|
||||
var fields = FieldNumbers(batch);
|
||||
Assert.Contains(4, fields);
|
||||
Assert.DoesNotContain(2, fields);
|
||||
Assert.DoesNotContain(3, fields);
|
||||
|
||||
// Field 4 is length-delimited (wire type 2) — the shape an unknown-field-tolerant
|
||||
// parser can skip without corrupting the rest of the message.
|
||||
Assert.Equal(2u, WireTypeOfField(batch, 4));
|
||||
|
||||
// And the per-event frames a pre-batching site emits still parse and convert on the
|
||||
// NEW client (the other skew direction, at the same byte level).
|
||||
var plain = SiteStreamEvent.Parser.ParseFrom(MakeAttributeEvent(7).ToByteArray());
|
||||
Assert.NotNull(SiteStreamGrpcClient.ConvertToDomainEvent(plain));
|
||||
}
|
||||
|
||||
// ── Version skew: NEW central ↔ OLD site ───────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task NewCentral_AgainstOldSite_StillReceivesEveryEvent()
|
||||
{
|
||||
// An old site ignores batching_supported and emits per-event frames. That emission
|
||||
// shape is exactly what the current server produces with batching off, so drive
|
||||
// the real server that way and feed the result through the NEW client's unpack —
|
||||
// which must handle the single-event case identically to before R2.
|
||||
var (server, manager, frames, cts, streamTask) = await StartAsync(batchingSupported: false);
|
||||
|
||||
const int total = 200;
|
||||
var t0 = new DateTimeOffset(2026, 8, 15, 13, 0, 0, TimeSpan.Zero);
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
manager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
Instance, "Modules.IO", "Seq", i, "Good", t0.AddMilliseconds(i)));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => TotalEvents(frames) >= total, 30_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
var wire = SnapshotThroughTheWire(frames);
|
||||
Assert.Equal(total, wire.Count);
|
||||
|
||||
var delivered = Unpack(wire);
|
||||
Assert.Equal(
|
||||
Enumerable.Range(0, total).Select(i => i.ToString()),
|
||||
delivered.Select(e => e.Value));
|
||||
Assert.Equal(
|
||||
Enumerable.Range(0, total).Select(i => t0.AddMilliseconds(i)),
|
||||
delivered.Select(e => e.Timestamp));
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
private static SiteStreamEvent MakeAttributeEvent(int seq) => new()
|
||||
{
|
||||
CorrelationId = "corr-shape",
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = Instance,
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Seq",
|
||||
Value = seq.ToString(),
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(DateTimeOffset.UnixEpoch)
|
||||
}
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Serializes an <c>InstanceStreamRequest</c> the way a central built BEFORE R2 would:
|
||||
/// fields 1 and 2 only, with no <c>batching_supported</c> on the wire at all.
|
||||
/// </summary>
|
||||
private static byte[] BuildLegacyInstanceRequest(string correlationId, string instance)
|
||||
{
|
||||
using var ms = new MemoryStream();
|
||||
var output = new CodedOutputStream(ms);
|
||||
output.WriteTag(1, WireFormat.WireType.LengthDelimited);
|
||||
output.WriteString(correlationId);
|
||||
output.WriteTag(2, WireFormat.WireType.LengthDelimited);
|
||||
output.WriteString(instance);
|
||||
output.Flush();
|
||||
return ms.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>Top-level field numbers present in a serialized message.</summary>
|
||||
private static HashSet<int> FieldNumbers(IMessage message)
|
||||
{
|
||||
var fields = new HashSet<int>();
|
||||
var input = new CodedInputStream(message.ToByteArray());
|
||||
uint tag;
|
||||
while ((tag = input.ReadTag()) != 0)
|
||||
{
|
||||
fields.Add(WireFormat.GetTagFieldNumber(tag));
|
||||
input.SkipLastField();
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
/// <summary>Wire type of the given top-level field number in a serialized message.</summary>
|
||||
private static uint WireTypeOfField(IMessage message, int fieldNumber)
|
||||
{
|
||||
var input = new CodedInputStream(message.ToByteArray());
|
||||
uint tag;
|
||||
while ((tag = input.ReadTag()) != 0)
|
||||
{
|
||||
if (WireFormat.GetTagFieldNumber(tag) == fieldNumber)
|
||||
return (uint)WireFormat.GetTagWireType(tag);
|
||||
input.SkipLastField();
|
||||
}
|
||||
throw new InvalidOperationException($"field {fieldNumber} not present");
|
||||
}
|
||||
|
||||
private static IEnumerable<SiteStreamEvent> Flatten(SiteStreamEvent frame)
|
||||
{
|
||||
if (frame.EventCase == SiteStreamEvent.EventOneofCase.Batch)
|
||||
{
|
||||
foreach (var inner in frame.Batch.Events) yield return inner;
|
||||
yield break;
|
||||
}
|
||||
yield return frame;
|
||||
}
|
||||
|
||||
private static int TotalEvents(List<(SiteStreamEvent Frame, DateTimeOffset WrittenAt)> frames)
|
||||
{
|
||||
lock (frames) { return frames.Sum(f => Flatten(f.Frame).Count()); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Takes the captured frames through a real protobuf serialize/parse round-trip — the
|
||||
/// step that makes every claim in this file about wire compatibility a wire claim.
|
||||
/// </summary>
|
||||
private static List<SiteStreamEvent> SnapshotThroughTheWire(
|
||||
List<(SiteStreamEvent Frame, DateTimeOffset WrittenAt)> frames)
|
||||
{
|
||||
lock (frames)
|
||||
{
|
||||
return [.. frames.Select(f => SiteStreamEvent.Parser.ParseFrom(f.Frame.ToByteArray()))];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Unpacks wire frames through the REAL client path into domain events.</summary>
|
||||
private static List<AttributeValueChanged> Unpack(IEnumerable<SiteStreamEvent> wire)
|
||||
{
|
||||
var delivered = new List<AttributeValueChanged>();
|
||||
foreach (var frame in wire)
|
||||
{
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, e =>
|
||||
{
|
||||
if (SiteStreamGrpcClient.ConvertToDomainEvent(e) is AttributeValueChanged a)
|
||||
delivered.Add(a);
|
||||
});
|
||||
}
|
||||
return delivered;
|
||||
}
|
||||
|
||||
private Task<(SiteStreamGrpcServer Server, SiteStreamManager Manager,
|
||||
List<(SiteStreamEvent Frame, DateTimeOffset WrittenAt)> Frames,
|
||||
CancellationTokenSource Cts, Task StreamTask)>
|
||||
StartAsync(bool batchingSupported, CommunicationOptions? options = null)
|
||||
=> StartAsync(new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = "corr-batching",
|
||||
InstanceUniqueName = Instance,
|
||||
BatchingSupported = batchingSupported
|
||||
}, options);
|
||||
|
||||
/// <summary>
|
||||
/// Brings up a real site broadcast hub + real gRPC server handler for the supplied
|
||||
/// subscription request, capturing every written frame with the instant it was written.
|
||||
/// </summary>
|
||||
private async Task<(SiteStreamGrpcServer Server, SiteStreamManager Manager,
|
||||
List<(SiteStreamEvent Frame, DateTimeOffset WrittenAt)> Frames,
|
||||
CancellationTokenSource Cts, Task StreamTask)>
|
||||
StartAsync(InstanceStreamRequest request, CommunicationOptions? options = null)
|
||||
{
|
||||
var manager = new SiteStreamManager(
|
||||
new SiteRuntimeOptions { StreamBufferSize = 4096 },
|
||||
NullLogger<SiteStreamManager>.Instance);
|
||||
manager.Initialize(Sys);
|
||||
|
||||
var server = new SiteStreamGrpcServer(
|
||||
manager,
|
||||
NullLogger<SiteStreamGrpcServer>.Instance,
|
||||
Options.Create(options ?? new CommunicationOptions()));
|
||||
server.SetReady(Sys);
|
||||
|
||||
var frames = new List<(SiteStreamEvent, DateTimeOffset)>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci =>
|
||||
{
|
||||
var frame = ci.Arg<SiteStreamEvent>();
|
||||
var at = DateTimeOffset.UtcNow;
|
||||
lock (frames) { frames.Add((frame, at)); }
|
||||
});
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = Substitute.For<ServerCallContext>();
|
||||
context.CancellationToken.Returns(cts.Token);
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
|
||||
// The publish must not race the materialized subscription.
|
||||
await WaitForConditionAsync(() => manager.SubscriptionCount == 1);
|
||||
|
||||
return (server, manager, frames, cts, streamTask);
|
||||
}
|
||||
|
||||
private static async Task WaitForConditionAsync(Func<bool> condition, int timeoutMs = 5000)
|
||||
{
|
||||
var started = Stopwatch.GetTimestamp();
|
||||
while (!condition() && Stopwatch.GetElapsedTime(started) < TimeSpan.FromMilliseconds(timeoutMs))
|
||||
{
|
||||
await Task.Delay(10);
|
||||
}
|
||||
Assert.True(condition(), $"Condition not met within {timeoutMs}ms");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user