Merge branch 'grpc-event-batching' — additive site-stream event batching, negotiated, 100ev/25ms window (residual #3 / R2)
This commit is contained in:
Joseph Doherty
@@ -90,6 +90,73 @@ public class CommunicationOptionsValidatorTests
|
||||
Assert.Contains("GrpcMaxConcurrentStreams", result.FailureMessage);
|
||||
}
|
||||
|
||||
// ── R2: site→central stream event batching ──────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void DefaultStreamBatchOptions_AreValid()
|
||||
{
|
||||
var options = new CommunicationOptions();
|
||||
Assert.Equal(100, options.GrpcStreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.FromMilliseconds(25), options.GrpcStreamBatchWindow);
|
||||
Assert.True(Validate(options).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StreamBatchMaxEventsOfOne_IsValid_AndMeansBatchingDisabled()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchMaxEvents = 1 });
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonPositiveStreamBatchMaxEvents_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchMaxEvents = 0 });
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("GrpcStreamBatchMaxEvents", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ZeroStreamBatchWindow_IsValid()
|
||||
{
|
||||
// Zero = "pack only what is already queued, never wait" — a legitimate posture for
|
||||
// a latency-critical deployment that still wants the framing saving.
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchWindow = TimeSpan.Zero });
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NegativeStreamBatchWindow_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchWindow = TimeSpan.FromMilliseconds(-1)
|
||||
});
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("GrpcStreamBatchWindow", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StreamBatchWindowAtOrAboveTheLatencyBudget_IsRejected()
|
||||
{
|
||||
// The coalescing window is the only latency batching adds and the target-scale
|
||||
// load test holds end-to-end stream latency to a 250 ms P99 — a window that could
|
||||
// spend the whole budget must not boot.
|
||||
foreach (var window in new[] { TimeSpan.FromMilliseconds(250), TimeSpan.FromSeconds(1) })
|
||||
{
|
||||
var result = Validate(new CommunicationOptions { GrpcStreamBatchWindow = window });
|
||||
Assert.True(result.Failed, $"{window} was accepted");
|
||||
Assert.Contains("GrpcStreamBatchWindow", result.FailureMessage);
|
||||
}
|
||||
|
||||
// Just inside the ceiling is accepted — the bound is exclusive, not a round-down.
|
||||
Assert.True(Validate(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchWindow = CommunicationOptionsValidator.StreamBatchWindowCeiling
|
||||
- TimeSpan.FromMilliseconds(1)
|
||||
}).Succeeded);
|
||||
}
|
||||
|
||||
// ── Aggregated live alarm cache options (plan #10, Task 6) ───────────────────
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -20,6 +20,17 @@ public class ProtoContractTests
|
||||
SiteStreamEvent.EventOneofCase.AlarmChanged
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Oneof variants that are NOT per-event payloads and so are deliberately absent from
|
||||
/// <see cref="HandledCases"/>. <c>Batch</c> (R2) is a framing envelope: it is unpacked
|
||||
/// by <see cref="SiteStreamGrpcClient.ForEachEvent"/> into the per-event cases above
|
||||
/// BEFORE conversion, and never reaches <c>ConvertToDomainEvent</c> as a whole frame.
|
||||
/// </summary>
|
||||
private static readonly SiteStreamEvent.EventOneofCase[] FramingCases =
|
||||
[
|
||||
SiteStreamEvent.EventOneofCase.Batch
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void AllOneofVariants_HaveConversionHandlers()
|
||||
{
|
||||
@@ -27,9 +38,37 @@ public class ProtoContractTests
|
||||
.Where(c => c != SiteStreamEvent.EventOneofCase.None)
|
||||
.ToArray();
|
||||
|
||||
Assert.Equal(allCases.Length, HandledCases.Length);
|
||||
var accountedFor = HandledCases.Concat(FramingCases).ToArray();
|
||||
Assert.Equal(allCases.Length, accountedFor.Length);
|
||||
foreach (var c in allCases)
|
||||
Assert.Contains(c, HandledCases);
|
||||
Assert.Contains(c, accountedFor);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BatchFrame_IsUnpackedIntoPerEventCases_NotConvertedWhole()
|
||||
{
|
||||
// The framing case's contract: ForEachEvent hands the per-event cases to the
|
||||
// handler in order, and ConvertToDomainEvent is never asked to make sense of the
|
||||
// envelope itself (it would return null, silently dropping the whole batch).
|
||||
var inner = new[]
|
||||
{
|
||||
CreateTestEvent(SiteStreamEvent.EventOneofCase.AttributeChanged),
|
||||
CreateTestEvent(SiteStreamEvent.EventOneofCase.AlarmChanged)
|
||||
};
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "test",
|
||||
Batch = new SiteStreamEventBatch { Events = { inner } }
|
||||
};
|
||||
|
||||
Assert.Null(SiteStreamGrpcClient.ConvertToDomainEvent(frame));
|
||||
|
||||
var seen = new List<SiteStreamEvent.EventOneofCase>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, e => seen.Add(e.EventCase));
|
||||
|
||||
Assert.Equal(
|
||||
[SiteStreamEvent.EventOneofCase.AttributeChanged, SiteStreamEvent.EventOneofCase.AlarmChanged],
|
||||
seen);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
using System.Diagnostics;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Unit tests for the per-subscriber coalescing pump behind R2 (gRPC event batching).
|
||||
/// The pump is the only new behaviour on the site→central hot path, so its contract is
|
||||
/// pinned directly rather than only through the server: the size cap, the time cap, the
|
||||
/// flush when the channel writer completes, the "never reorder" guarantee, and the
|
||||
/// un-negotiated (maxBatchEvents == 1) shape that keeps an older central working.
|
||||
/// </summary>
|
||||
public class SiteStreamEventBatcherTests
|
||||
{
|
||||
private const string Corr = "corr-batch";
|
||||
|
||||
private static SiteStreamEvent Event(int seq) => new()
|
||||
{
|
||||
CorrelationId = Corr,
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = "SiteA.Pump01",
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Seq",
|
||||
Value = seq.ToString(),
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UnixEpoch.AddSeconds(seq))
|
||||
}
|
||||
};
|
||||
|
||||
/// <summary>Flattens an emitted frame into the sequence numbers it carried, in order.</summary>
|
||||
private static IEnumerable<int> Seqs(SiteStreamEvent frame)
|
||||
{
|
||||
if (frame.EventCase == SiteStreamEvent.EventOneofCase.Batch)
|
||||
{
|
||||
foreach (var inner in frame.Batch.Events)
|
||||
yield return int.Parse(inner.AttributeChanged.Value);
|
||||
yield break;
|
||||
}
|
||||
|
||||
yield return int.Parse(frame.AttributeChanged.Value);
|
||||
}
|
||||
|
||||
private sealed record PumpRun(List<SiteStreamEvent> Frames, List<int> FrameSizes, Task Completion);
|
||||
|
||||
private static PumpRun StartPump(
|
||||
ChannelReader<SiteStreamEvent> reader,
|
||||
int maxBatchEvents,
|
||||
TimeSpan window,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var frames = new List<SiteStreamEvent>();
|
||||
var sizes = new List<int>();
|
||||
var task = SiteStreamEventBatcher.PumpAsync(
|
||||
reader,
|
||||
Corr,
|
||||
maxBatchEvents,
|
||||
window,
|
||||
(evt, _) =>
|
||||
{
|
||||
lock (frames) { frames.Add(evt); }
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
size => { lock (frames) { sizes.Add(size); } },
|
||||
ct);
|
||||
return new PumpRun(frames, sizes, task);
|
||||
}
|
||||
|
||||
// ── Size cap ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task SizeCap_SplitsABacklogIntoFramesOfAtMostMaxEvents()
|
||||
{
|
||||
// A backlog already sitting in the channel is drained without waiting, but never
|
||||
// beyond the size cap — 250 queued events at a cap of 100 must come out as
|
||||
// 100 + 100 + 50, in order, with nothing lost or duplicated.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 250; i++)
|
||||
Assert.True(channel.Writer.TryWrite(Event(i)));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
Assert.All(run.FrameSizes, s => Assert.True(s <= 100, $"frame carried {s} events (cap 100)"));
|
||||
Assert.Equal([100, 100, 50], run.FrameSizes);
|
||||
Assert.Equal(Enumerable.Range(0, 250), run.Frames.SelectMany(Seqs));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SizeCapOfOne_EmitsPlainPerEventFrames_TheUnnegotiatedShape()
|
||||
{
|
||||
// maxBatchEvents == 1 is what an un-negotiated subscription (an older central)
|
||||
// gets. Every event must ride its own plain frame — never a Batch case, which
|
||||
// that central's generated code cannot parse.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 5; i++)
|
||||
channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 1, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(5, run.Frames.Count);
|
||||
Assert.All(run.Frames, f =>
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, f.EventCase));
|
||||
Assert.All(run.Frames, f => Assert.Equal(Corr, f.CorrelationId));
|
||||
Assert.Equal(Enumerable.Range(0, 5), run.Frames.SelectMany(Seqs));
|
||||
}
|
||||
|
||||
// ── Time cap ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TimeCap_ClosesAnUnderfullBatchWhenTheWindowElapses()
|
||||
{
|
||||
// Two events arrive (a backlog, so the pump lingers), then the source goes quiet
|
||||
// well short of the size cap. The window — not the cap — must close the batch,
|
||||
// and it must do so within a bounded time rather than waiting for a 100th event
|
||||
// that never comes.
|
||||
var window = TimeSpan.FromMilliseconds(120);
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
channel.Writer.TryWrite(Event(0));
|
||||
channel.Writer.TryWrite(Event(1));
|
||||
|
||||
var started = Stopwatch.GetTimestamp();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window);
|
||||
|
||||
SiteStreamEvent frame;
|
||||
while (true)
|
||||
{
|
||||
lock (run.Frames)
|
||||
{
|
||||
if (run.Frames.Count > 0) { frame = run.Frames[0]; break; }
|
||||
}
|
||||
Assert.True(Stopwatch.GetElapsedTime(started) < TimeSpan.FromSeconds(5),
|
||||
"the window never closed the underfull batch");
|
||||
await Task.Delay(5);
|
||||
}
|
||||
|
||||
var elapsed = Stopwatch.GetElapsedTime(started);
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.Batch, frame.EventCase);
|
||||
Assert.Equal([0, 1], Seqs(frame));
|
||||
// The batch waited (it did not close instantly on the two queued events) and it
|
||||
// closed on the window, not on a cap it never reached.
|
||||
Assert.True(elapsed >= window - TimeSpan.FromMilliseconds(20),
|
||||
$"batch closed after {elapsed.TotalMilliseconds:0.0} ms, before the {window.TotalMilliseconds:0} ms window");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LoneEventOnAQuietStream_IsNeverDelayedByTheWindow()
|
||||
{
|
||||
// The latency contract: the window applies only AFTER a backlog has been observed.
|
||||
// A single event on an idle stream must be emitted immediately as a plain frame,
|
||||
// so per-event latency on a quiet site is unchanged by batching.
|
||||
var window = TimeSpan.FromSeconds(5);
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window);
|
||||
|
||||
var started = Stopwatch.GetTimestamp();
|
||||
channel.Writer.TryWrite(Event(7));
|
||||
|
||||
while (true)
|
||||
{
|
||||
lock (run.Frames)
|
||||
{
|
||||
if (run.Frames.Count > 0) break;
|
||||
}
|
||||
Assert.True(Stopwatch.GetElapsedTime(started) < TimeSpan.FromSeconds(3),
|
||||
"a lone event was held by the coalescing window");
|
||||
await Task.Delay(2);
|
||||
}
|
||||
|
||||
var elapsed = Stopwatch.GetElapsedTime(started);
|
||||
channel.Writer.Complete();
|
||||
await run.Completion;
|
||||
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.AttributeChanged, run.Frames[0].EventCase);
|
||||
Assert.True(elapsed < TimeSpan.FromSeconds(1),
|
||||
$"lone event took {elapsed.TotalMilliseconds:0.0} ms against a {window.TotalSeconds:0} s window");
|
||||
}
|
||||
|
||||
// ── Flush on stream close ───────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task WriterCompletion_FlushesTheInFlightBatchBeforeReturning()
|
||||
{
|
||||
// The channel writer completing mid-window (the site stopping the relay actor and
|
||||
// calling channel.Writer.TryComplete()) must flush what is already buffered rather
|
||||
// than silently discarding it while waiting out the window.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
channel.Writer.TryWrite(Event(0));
|
||||
channel.Writer.TryWrite(Event(1));
|
||||
|
||||
// A long window guarantees the pump is lingering, not already past the emit.
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromSeconds(30));
|
||||
|
||||
await Task.Delay(100);
|
||||
lock (run.Frames)
|
||||
{
|
||||
Assert.Empty(run.Frames); // still lingering
|
||||
}
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Single(run.Frames);
|
||||
Assert.Equal([0, 1], Seqs(run.Frames[0]));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WriterCompletionWithNothingBuffered_ReturnsWithoutEmitting()
|
||||
{
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
|
||||
channel.Writer.Complete();
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Empty(run.Frames);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancellation_EndsThePumpWithOperationCanceled()
|
||||
{
|
||||
// Client disconnect / duplicate replacement / site shutdown. The pump must
|
||||
// surface OperationCanceledException exactly as the pre-batching await-foreach
|
||||
// loop did, so SiteStreamGrpcServer's existing catch and finally are unchanged.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
using var cts = new CancellationTokenSource();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25), cts.Token);
|
||||
|
||||
await cts.CancelAsync();
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
() => run.Completion.WaitAsync(TimeSpan.FromSeconds(5)));
|
||||
}
|
||||
|
||||
// ── Ordering ────────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Ordering_IsPreservedAcrossManyBatchesUnderAProducerRace()
|
||||
{
|
||||
// Batching is a framing change and nothing else: with a producer writing
|
||||
// concurrently with the pump, the flattened output must be the exact input
|
||||
// sequence — no reordering, no loss, no duplication, across many frames.
|
||||
const int total = 5_000;
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 32, window: TimeSpan.FromMilliseconds(5));
|
||||
|
||||
var producer = Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < total; i++)
|
||||
{
|
||||
channel.Writer.TryWrite(Event(i));
|
||||
if (i % 250 == 0) await Task.Yield();
|
||||
}
|
||||
channel.Writer.Complete();
|
||||
});
|
||||
|
||||
await producer;
|
||||
await run.Completion.WaitAsync(TimeSpan.FromSeconds(30));
|
||||
|
||||
Assert.Equal(Enumerable.Range(0, total), run.Frames.SelectMany(Seqs));
|
||||
Assert.All(run.FrameSizes, s => Assert.InRange(s, 1, 32));
|
||||
Assert.Equal(total, run.FrameSizes.Sum());
|
||||
}
|
||||
|
||||
// ── Frame shape ─────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task BatchFrame_CarriesTheCorrelationIdOnceAndBlanksItOnInnerEvents()
|
||||
{
|
||||
// The byte saving batching exists for: the correlation id is stamped once on the
|
||||
// enclosing frame, not repeated on every packed event. No consumer reads the
|
||||
// inner value (SiteStreamGrpcClient.ForEachEvent ignores it).
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 4; i++) channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
var frame = Assert.Single(run.Frames);
|
||||
Assert.Equal(SiteStreamEvent.EventOneofCase.Batch, frame.EventCase);
|
||||
Assert.Equal(Corr, frame.CorrelationId);
|
||||
Assert.All(frame.Batch.Events, e => Assert.Equal(string.Empty, e.CorrelationId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PerEventTimestampsSurviveBatching()
|
||||
{
|
||||
// End-to-end latency measurement rides the per-event Timestamp; coalescing must
|
||||
// not rewrite it to a single frame-level stamp.
|
||||
var channel = Channel.CreateUnbounded<SiteStreamEvent>();
|
||||
for (var i = 0; i < 3; i++) channel.Writer.TryWrite(Event(i));
|
||||
channel.Writer.Complete();
|
||||
|
||||
var run = StartPump(channel.Reader, maxBatchEvents: 100, window: TimeSpan.FromMilliseconds(25));
|
||||
await run.Completion;
|
||||
|
||||
var frame = Assert.Single(run.Frames);
|
||||
Assert.Equal(
|
||||
[
|
||||
DateTimeOffset.UnixEpoch,
|
||||
DateTimeOffset.UnixEpoch.AddSeconds(1),
|
||||
DateTimeOffset.UnixEpoch.AddSeconds(2)
|
||||
],
|
||||
frame.Batch.Events.Select(e => e.AttributeChanged.Timestamp.ToDateTimeOffset()));
|
||||
}
|
||||
}
|
||||
@@ -508,6 +508,125 @@ public class SiteStreamGrpcClientTests
|
||||
}
|
||||
}
|
||||
|
||||
// ── R2: batch unpacking on the client ───────────────────────────────────────
|
||||
|
||||
private static SiteStreamEvent Attr(string value, DateTimeOffset ts) => new()
|
||||
{
|
||||
AttributeChanged = new AttributeValueUpdate
|
||||
{
|
||||
InstanceUniqueName = "SiteA.Pump01",
|
||||
AttributePath = "Modules.IO",
|
||||
AttributeName = "Seq",
|
||||
Value = value,
|
||||
Quality = Quality.Good,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(ts)
|
||||
}
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_PlainFrame_IsDeliveredAsIs()
|
||||
{
|
||||
// An OLD SITE (or any un-negotiated stream) sends one event per frame. The new
|
||||
// client's unpack path must pass it straight through — this is the new-central ↔
|
||||
// old-site skew direction.
|
||||
var frame = Attr("1", DateTimeOffset.UnixEpoch);
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Same(frame, Assert.Single(seen));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_BatchFrame_UnpacksInOrderPreservingPerEventTimestamps()
|
||||
{
|
||||
// Order and per-event Timestamp fidelity are the two properties the downstream
|
||||
// consumers (SiteAlarmAggregatorActor, DebugStreamBridgeActor) and the end-to-end
|
||||
// latency measurement depend on.
|
||||
var t0 = new DateTimeOffset(2026, 8, 15, 9, 0, 0, TimeSpan.Zero);
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-batch",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events =
|
||||
{
|
||||
Attr("0", t0),
|
||||
Attr("1", t0.AddMilliseconds(3)),
|
||||
Attr("2", t0.AddMilliseconds(11))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Equal(["0", "1", "2"], seen.Select(e => e.AttributeChanged.Value));
|
||||
Assert.Equal(
|
||||
[t0, t0.AddMilliseconds(3), t0.AddMilliseconds(11)],
|
||||
seen.Select(e => e.AttributeChanged.Timestamp.ToDateTimeOffset()));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ForEachEvent_IgnoresNestedAndUnknownInnerCases()
|
||||
{
|
||||
// The server never nests a batch inside a batch. A nested (or empty) inner frame
|
||||
// from a malformed or hostile peer must be skipped, not followed — unpacking is
|
||||
// deliberately non-recursive so a crafted frame cannot drive unbounded recursion.
|
||||
var frame = new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-nested",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events =
|
||||
{
|
||||
Attr("0", DateTimeOffset.UnixEpoch),
|
||||
new SiteStreamEvent { Batch = new SiteStreamEventBatch { Events = { Attr("hidden", DateTimeOffset.UnixEpoch) } } },
|
||||
new SiteStreamEvent(),
|
||||
Attr("1", DateTimeOffset.UnixEpoch)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var seen = new List<SiteStreamEvent>();
|
||||
SiteStreamGrpcClient.ForEachEvent(frame, seen.Add);
|
||||
|
||||
Assert.Equal(["0", "1"], seen.Select(e => e.AttributeChanged.Value));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConsumeStream_MixedBatchedAndPlainFrames_DeliverEveryEventInOrder()
|
||||
{
|
||||
// A reconnect can straddle a site upgrade, so one stream may legitimately carry
|
||||
// both frame shapes. Driving the real ConsumeStreamAsync with the real unpack
|
||||
// proves the combination is flat and ordered from the consumer's point of view.
|
||||
var client = SiteStreamGrpcClient.CreateForTesting();
|
||||
var cts = new CancellationTokenSource();
|
||||
var delivered = new List<string>();
|
||||
|
||||
void Deliver(SiteStreamEvent e) => delivered.Add(e.AttributeChanged.Value);
|
||||
|
||||
await client.ConsumeStreamAsync(
|
||||
"corr-mixed",
|
||||
cts,
|
||||
() => FakeCall(new StubStreamReader(
|
||||
Attr("0", DateTimeOffset.UnixEpoch),
|
||||
new SiteStreamEvent
|
||||
{
|
||||
CorrelationId = "corr-mixed",
|
||||
Batch = new SiteStreamEventBatch
|
||||
{
|
||||
Events = { Attr("1", DateTimeOffset.UnixEpoch), Attr("2", DateTimeOffset.UnixEpoch) }
|
||||
}
|
||||
},
|
||||
Attr("3", DateTimeOffset.UnixEpoch))),
|
||||
frame => SiteStreamGrpcClient.ForEachEvent(frame, Deliver),
|
||||
_ => { },
|
||||
() => { });
|
||||
|
||||
Assert.Equal(["0", "1", "2", "3"], delivered);
|
||||
}
|
||||
|
||||
private static AsyncServerStreamingCall<SiteStreamEvent> FakeCall(StubStreamReader reader) =>
|
||||
FakeCall(reader, Task.FromResult(new Metadata()));
|
||||
|
||||
|
||||
@@ -580,4 +580,277 @@ public class SiteStreamGrpcServerTests : TestKit
|
||||
var server = CreateServer();
|
||||
Assert.Equal(0, server.DroppedStreamEventCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DuplicateReplacement_CancelsTheReplacedStream_WithoutDisposingItsCts()
|
||||
{
|
||||
// Regression: the duplicate-replacement path used to Cancel AND Dispose the
|
||||
// replaced stream's CancellationTokenSource. That CTS belongs to the replaced
|
||||
// handler's own `using var streamCts`, which is still running and still has to
|
||||
// read `streamCts.Token` — so the Dispose raced that read and escaped the RPC as
|
||||
// an unhandled ObjectDisposedException. It surfaced only under full-suite load
|
||||
// (GrpcStreamIntegrationTests.Pipeline_DuplicateCorrelationId_ReplacesStream) and
|
||||
// predates R2: the same Dispose and the same first-token-read relationship existed
|
||||
// when the handler still used `ReadAllAsync(streamCts.Token)`.
|
||||
//
|
||||
// The race is made DETERMINISTIC here by gating the first stream inside its setup
|
||||
// window (its _activeStreams entry is registered before Subscribe is called), so
|
||||
// the replacement always lands before the first stream reads its token.
|
||||
using var gate = new ManualResetEventSlim(false);
|
||||
var calls = 0;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
var n = Interlocked.Increment(ref calls);
|
||||
if (n == 1)
|
||||
gate.Wait(TimeSpan.FromSeconds(15));
|
||||
return $"sub-dup-race-{n}";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(subscriber, _logger);
|
||||
server.SetReady(Sys);
|
||||
|
||||
using var cts1 = new CancellationTokenSource();
|
||||
var stream1 = Task.Run(() => server.SubscribeInstance(
|
||||
MakeRequest("corr-dup-race"),
|
||||
Substitute.For<IServerStreamWriter<SiteStreamEvent>>(),
|
||||
CreateMockContext(cts1.Token)));
|
||||
|
||||
await WaitForConditionAsync(() => server.ActiveStreamCount == 1);
|
||||
await WaitForConditionAsync(() => Volatile.Read(ref calls) == 1);
|
||||
|
||||
using var cts2 = new CancellationTokenSource();
|
||||
var stream2 = Task.Run(() => server.SubscribeInstance(
|
||||
MakeRequest("corr-dup-race"),
|
||||
Substitute.For<IServerStreamWriter<SiteStreamEvent>>(),
|
||||
CreateMockContext(cts2.Token)));
|
||||
|
||||
// The replacement has taken the slot (and cancelled stream 1's CTS) by the time
|
||||
// its own Subscribe has been called.
|
||||
await WaitForConditionAsync(() => Volatile.Read(ref calls) == 2);
|
||||
|
||||
gate.Set();
|
||||
|
||||
// Pre-fix this threw ObjectDisposedException out of the RPC. Post-fix the replaced
|
||||
// stream observes a plain cancellation and unwinds through its normal finally.
|
||||
await stream1;
|
||||
|
||||
cts2.Cancel();
|
||||
await stream2;
|
||||
|
||||
Assert.Equal(0, server.ActiveStreamCount);
|
||||
}
|
||||
|
||||
// ── R2: gRPC event batching, and its negotiation ────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void BatchOptions_AreBoundFromOptions_AndClampDegenerateValues()
|
||||
{
|
||||
var options = Microsoft.Extensions.Options.Options.Create(new CommunicationOptions());
|
||||
var server = new SiteStreamGrpcServer(_subscriber, _logger, options);
|
||||
|
||||
Assert.Equal(100, server.StreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.FromMilliseconds(25), server.StreamBatchWindow);
|
||||
|
||||
// CommunicationOptionsValidator fails the boot on these, but a host composed
|
||||
// without validation must not blow up deep inside a live RPC.
|
||||
var degenerate = new SiteStreamGrpcServer(_subscriber, _logger,
|
||||
Microsoft.Extensions.Options.Options.Create(new CommunicationOptions
|
||||
{
|
||||
GrpcStreamBatchMaxEvents = 0,
|
||||
GrpcStreamBatchWindow = TimeSpan.FromMilliseconds(-5),
|
||||
}));
|
||||
Assert.Equal(1, degenerate.StreamBatchMaxEvents);
|
||||
Assert.Equal(TimeSpan.Zero, degenerate.StreamBatchWindow);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UnnegotiatedSubscription_NeverEmitsABatchFrame()
|
||||
{
|
||||
// OLD-CENTRAL ↔ NEW-SITE skew. proto3 defaults batching_supported to false, which
|
||||
// is exactly what a central built before R2 sends. The site must then keep to one
|
||||
// event per frame — a Batch frame would arrive at that central as
|
||||
// EventOneofCase.None and be silently dropped by its ConvertToDomainEvent.
|
||||
var (server, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: false);
|
||||
|
||||
for (var i = 0; i < 50; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= 50, 10_000);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
lock (capture)
|
||||
{
|
||||
Assert.All(capture, f => Assert.NotEqual(SiteStreamEvent.EventOneofCase.Batch, f.EventCase));
|
||||
Assert.Equal(50, capture.Count);
|
||||
}
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NegotiatedSubscription_CoalescesABacklogIntoFewerFramesThanEvents()
|
||||
{
|
||||
// NEW-CENTRAL ↔ NEW-SITE. A burst pushed at the relay faster than the pump drains
|
||||
// it must come out in strictly fewer frames than events, with every event
|
||||
// preserved in order.
|
||||
const int burst = 400;
|
||||
var (server, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: true);
|
||||
|
||||
for (var i = 0; i < burst; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= burst, 15_000);
|
||||
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
List<SiteStreamEvent> frames;
|
||||
lock (capture) { frames = [.. capture]; }
|
||||
|
||||
Assert.Equal(burst, frames.Sum(CountFrameEvents));
|
||||
Assert.True(frames.Count < burst,
|
||||
$"batching produced {frames.Count} frames for {burst} events — no coalescing happened");
|
||||
Assert.Contains(frames, f => f.EventCase == SiteStreamEvent.EventOneofCase.Batch);
|
||||
|
||||
// Order is preserved end to end: the values arrive 0..burst-1 exactly once each.
|
||||
var values = frames.SelectMany(FlattenAttributeValues).ToArray();
|
||||
Assert.Equal(Enumerable.Range(0, burst).Select(i => i.ToString()), values);
|
||||
|
||||
GC.KeepAlive(server);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task BatchSizeHistogram_IsRecordedOnlyForNegotiatedStreams()
|
||||
{
|
||||
// scadabridge.site.stream.batch_size rides ScadaBridgeTelemetry.MeterName, which is
|
||||
// already in SiteServiceRegistration.ObservedMeters — an unlisted meter exports
|
||||
// nothing, silently. Assert the instrument actually fires, and that it does NOT
|
||||
// fire on an un-negotiated stream (where it would degenerate into a per-event
|
||||
// instrument on the hottest path in the product).
|
||||
var measurements = new List<int>();
|
||||
using var listener = new MeterListener();
|
||||
listener.InstrumentPublished = (instrument, l) =>
|
||||
{
|
||||
if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName &&
|
||||
instrument.Name == "scadabridge.site.stream.batch_size")
|
||||
{
|
||||
l.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<int>((_, m, _, _) =>
|
||||
{
|
||||
lock (measurements) { measurements.Add(m); }
|
||||
});
|
||||
listener.Start();
|
||||
|
||||
// Un-negotiated: no measurements at all.
|
||||
var (_, plainCapture, plainCts, plainTask, plainRelay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: false, correlationId: "corr-hist-off");
|
||||
plainRelay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", 1, "Good", DateTimeOffset.UtcNow));
|
||||
await WaitForConditionAsync(() => CountEvents(plainCapture) >= 1);
|
||||
plainCts.Cancel();
|
||||
await plainTask;
|
||||
|
||||
lock (measurements) { Assert.Empty(measurements); }
|
||||
|
||||
// Negotiated: one measurement per emitted frame, each within the size cap.
|
||||
var (_, capture, cts, streamTask, relay) =
|
||||
await StartCapturingStreamAsync(batchingSupported: true, correlationId: "corr-hist-on");
|
||||
for (var i = 0; i < 20; i++)
|
||||
{
|
||||
relay.Tell(new Commons.Messages.Streaming.AttributeValueChanged(
|
||||
"Site1.Pump01", "Path", "Attr", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
await WaitForConditionAsync(() => CountEvents(capture) >= 20, 10_000);
|
||||
cts.Cancel();
|
||||
await streamTask;
|
||||
|
||||
lock (measurements)
|
||||
{
|
||||
Assert.NotEmpty(measurements);
|
||||
Assert.Equal(20, measurements.Sum());
|
||||
Assert.All(measurements, m => Assert.InRange(m, 1, SiteStreamGrpcServer.DefaultStreamBatchMaxEvents));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Total events carried across all captured frames (unpacking batch frames).</summary>
|
||||
private static int CountEvents(List<SiteStreamEvent> capture)
|
||||
{
|
||||
lock (capture) { return capture.Sum(CountFrameEvents); }
|
||||
}
|
||||
|
||||
private static int CountFrameEvents(SiteStreamEvent frame) =>
|
||||
frame.EventCase == SiteStreamEvent.EventOneofCase.Batch ? frame.Batch.Events.Count : 1;
|
||||
|
||||
private static IEnumerable<string> FlattenAttributeValues(SiteStreamEvent frame)
|
||||
{
|
||||
if (frame.EventCase == SiteStreamEvent.EventOneofCase.Batch)
|
||||
{
|
||||
foreach (var inner in frame.Batch.Events)
|
||||
yield return inner.AttributeChanged.Value;
|
||||
yield break;
|
||||
}
|
||||
|
||||
yield return frame.AttributeChanged.Value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a SubscribeInstance stream with the given batch negotiation, capturing every
|
||||
/// written frame and handing back the relay actor so the test can drive domain events.
|
||||
/// </summary>
|
||||
private async Task<(SiteStreamGrpcServer Server, List<SiteStreamEvent> Capture,
|
||||
CancellationTokenSource Cts, Task StreamTask, IActorRef Relay)>
|
||||
StartCapturingStreamAsync(bool batchingSupported, string correlationId = "corr-batch")
|
||||
{
|
||||
IActorRef? capturedActor = null;
|
||||
var subscriber = Substitute.For<ISiteStreamSubscriber>();
|
||||
subscriber.Subscribe(Arg.Any<string>(), Arg.Any<IActorRef>())
|
||||
.Returns(ci =>
|
||||
{
|
||||
capturedActor = ci.Arg<IActorRef>();
|
||||
return "sub-batch";
|
||||
});
|
||||
|
||||
var server = new SiteStreamGrpcServer(subscriber, _logger,
|
||||
Microsoft.Extensions.Options.Options.Create(new CommunicationOptions()));
|
||||
server.SetReady(Sys);
|
||||
|
||||
var capture = new List<SiteStreamEvent>();
|
||||
var writer = Substitute.For<IServerStreamWriter<SiteStreamEvent>>();
|
||||
writer.WriteAsync(Arg.Any<SiteStreamEvent>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.CompletedTask)
|
||||
.AndDoes(ci =>
|
||||
{
|
||||
var frame = ci.Arg<SiteStreamEvent>();
|
||||
lock (capture) { capture.Add(frame); }
|
||||
});
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var context = CreateMockContext(cts.Token);
|
||||
|
||||
var request = new InstanceStreamRequest
|
||||
{
|
||||
CorrelationId = correlationId,
|
||||
InstanceUniqueName = "Site1.Pump01",
|
||||
BatchingSupported = batchingSupported
|
||||
};
|
||||
|
||||
var streamTask = Task.Run(() => server.SubscribeInstance(request, writer, context));
|
||||
await WaitForConditionAsync(() => capturedActor != null);
|
||||
|
||||
return (server, capture, cts, streamTask, capturedActor!);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user