feat(comm): batch site→central stream events over gRPC (R2)

Every AttributeValueChanged/AlarmStateChanged rode its own gRPC message on SiteStreamService; at target scale that is ~37.5k messages/s/site of pure framing overhead. Coalesce them, additively, with no new RPC. Wire (sitestream.proto, regenerated via docker/regen-proto.sh sitestream): InstanceStreamRequest.batching_supported = 3 SiteStreamRequest.batching_supported = 2 SiteStreamEvent.batch = 4 (new oneof case) SiteStreamEventBatch { repeated SiteStreamEvent events = 1 } The proto3 default of batching_supported IS the negotiation, and it is load-bearing: a batch frame reaches a pre-R2 central as EventOneofCase.None, whose ConvertToDomainEvent returns null — the whole batch would vanish with no error anywhere. An old central cannot set the flag so it never receives one; an old site ignores the unknown request field and keeps sending per-event frames, which the new client's ForEachEvent handles as the single-event case. Both skew directions are covered by tests that go through a real proto serialize/parse round-trip. Server: SiteStreamEventBatcher, a per-subscriber pump replacing the handler's await-foreach/WriteAsync loop and byte-identical to it at a cap of 1. It never delays a lone event — it drains the already-queued backlog for free and lingers only once a backlog is proven — emits a single-event buffer as a plain frame, preserves order and per-event Timestamps exactly, and flushes what is buffered when the send channel's writer completes. It sits DOWNSTREAM of StreamRelayActor's bounded DropOldest channel, so it changes framing only and does not move the burst ceiling (deferred register row 31, which lives in the shared publish stage upstream of the BroadcastHub). Client: sets the flag on both subscriptions and unpacks in order into the existing per-event pipeline, so SiteAlarmAggregatorActor, DebugStreamBridgeActor, consumer-keepalive/orphan logic, reconnect-on-graceful-completion, generation fencing, the (siteId, endpoint) factory key and IsLive semantics are untouched. Options (validated): GrpcStreamBatchMaxEvents 100 (1 disables), GrpcStreamBatchWindow 25 ms — validated strictly under 250 ms, the load test's end-to-end P99 threshold. Measured worst case (trickle-with-backlog, window-bound rather than cap-bound): P50 13.8 ms, P99 25.4 ms, max 25.8 ms; cap-bound case sent 600 queued events in 6 frames. Telemetry: histogram scadabridge.site.stream.batch_size tagged by stream kind, recorded only on negotiated streams (per-event otherwise). It rides ScadaBridgeTelemetry.MeterName, already in the ObservedMeters allowlist. Docs: Component-Communication.md gains an Event Batching section; CLAUDE.md's gRPC streaming bullet records the wire shape and the negotiation rationale.
2026-08-15 03:39:59 -04:00
parent 2b74851f96
commit 9b5cb3dd9d
16 changed files with 2150 additions and 131 deletions
@@ -29,6 +29,8 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
    private readonly TimeSpan _maxStreamLifetime;
    private readonly int _instanceChannelCapacity;
    private readonly int _siteAlarmChannelCapacity;
+    private readonly int _streamBatchMaxEvents;
+    private readonly TimeSpan _streamBatchWindow;
    private volatile bool _ready;
    // Flipped by CancelAllStreams() when the host enters
    // CoordinatedShutdown so SubscribeInstance refuses new streams with
@@ -75,7 +77,8 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
        ILogger<SiteStreamGrpcServer> logger,
        int maxConcurrentStreams = 100)
        : this(streamSubscriber, logger, maxConcurrentStreams, TimeSpan.FromHours(4),
-            DefaultInstanceChannelCapacity, DefaultSiteAlarmChannelCapacity)
+            DefaultInstanceChannelCapacity, DefaultSiteAlarmChannelCapacity,
+            DefaultStreamBatchMaxEvents, DefaultStreamBatchWindow)
    {
    }

@@ -85,6 +88,12 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
    /// <summary>Fallback site-wide alarm send-channel capacity when no options are bound.</summary>
    internal const int DefaultSiteAlarmChannelCapacity = 20_000;

+    /// <summary>Fallback stream-batch size cap when no options are bound (R2).</summary>
+    internal const int DefaultStreamBatchMaxEvents = 100;
+
+    /// <summary>Fallback stream-batch coalescing window when no options are bound (R2).</summary>
+    internal static readonly TimeSpan DefaultStreamBatchWindow = TimeSpan.FromMilliseconds(25);
+
    /// <summary>
    /// DI constructor — binds <see cref="CommunicationOptions.GrpcMaxConcurrentStreams"/>
    /// and <see cref="CommunicationOptions.GrpcMaxStreamLifetime"/> so the documented
@@ -102,7 +111,9 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
            options.Value.GrpcMaxConcurrentStreams,
            options.Value.GrpcMaxStreamLifetime,
            options.Value.GrpcInstanceStreamChannelCapacity,
-            options.Value.GrpcSiteAlarmStreamChannelCapacity)
+            options.Value.GrpcSiteAlarmStreamChannelCapacity,
+            options.Value.GrpcStreamBatchMaxEvents,
+            options.Value.GrpcStreamBatchWindow)
    {
    }

@@ -112,7 +123,9 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
        int maxConcurrentStreams,
        TimeSpan maxStreamLifetime,
        int instanceChannelCapacity,
-        int siteAlarmChannelCapacity)
+        int siteAlarmChannelCapacity,
+        int streamBatchMaxEvents,
+        TimeSpan streamBatchWindow)
    {
        _streamSubscriber = streamSubscriber;
        _logger = logger;
@@ -120,6 +133,11 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
        _maxStreamLifetime = maxStreamLifetime;
        _instanceChannelCapacity = Math.Max(1, instanceChannelCapacity);
        _siteAlarmChannelCapacity = Math.Max(1, siteAlarmChannelCapacity);
+        // Floored/clamped rather than thrown on: CommunicationOptionsValidator already
+        // fails the boot on a bad value, and a degenerate one must not blow up deep
+        // inside a live RPC on a host composed without validation (tests, embedded use).
+        _streamBatchMaxEvents = Math.Max(1, streamBatchMaxEvents);
+        _streamBatchWindow = streamBatchWindow < TimeSpan.Zero ? TimeSpan.Zero : streamBatchWindow;
    }

    /// <summary>
@@ -230,6 +248,12 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
    /// <summary>Effective site-wide alarm send-channel capacity. Exposed for tests.</summary>
    internal int SiteAlarmChannelCapacity => _siteAlarmChannelCapacity;

+    /// <summary>Effective stream-batch size cap (R2). Exposed for tests.</summary>
+    internal int StreamBatchMaxEvents => _streamBatchMaxEvents;
+
+    /// <summary>Effective stream-batch coalescing window (R2). Exposed for tests.</summary>
+    internal TimeSpan StreamBatchWindow => _streamBatchWindow;
+
    /// <summary>
    /// Total events evicted from stream send channels on this node since start (both stream
    /// kinds). Exported as <c>scadabridge.site.stream.events_dropped</c>; exposed here so a
@@ -251,7 +275,10 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
            relay => _streamSubscriber.Subscribe(request.InstanceUniqueName, relay),
            request.InstanceUniqueName,
            _instanceChannelCapacity,
-            streamKind: "instance");
+            streamKind: "instance",
+            // R2 batch negotiation: proto3 defaults this to false, so a central built
+            // before batching existed keeps getting one frame per event.
+            batchingSupported: request.BatchingSupported);

    /// <inheritdoc />
    public override Task SubscribeSite(
@@ -271,7 +298,8 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
            // DropOldest meant an alarm burst during a WAN stall silently evicted operator-
            // visible transitions to make room for diagnostics traffic.
            _siteAlarmChannelCapacity,
-            streamKind: "site-alarms");
+            streamKind: "site-alarms",
+            batchingSupported: request.BatchingSupported);

    /// <summary>
    /// Shared streaming pipeline behind <see cref="SubscribeInstance"/> and
@@ -289,6 +317,12 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
    /// <param name="description">Human-readable subscription description for logging.</param>
    /// <param name="channelCapacity">Send-channel capacity for this stream kind.</param>
    /// <param name="streamKind">Telemetry tag for this stream kind (<c>instance</c>/<c>site-alarms</c>).</param>
+    /// <param name="batchingSupported">
+    /// Whether the SUBSCRIBING CLIENT declared it understands the <c>SiteStreamEventBatch</c>
+    /// oneof case (R2). False — the proto3 default an older central necessarily sends —
+    /// pins this stream to one event per frame, so a peer that predates batching can never
+    /// receive a frame case its generated code drops on the floor.
+    /// </param>
    private async Task RunSubscriptionStreamAsync(
        string correlationId,
        IServerStreamWriter<SiteStreamEvent> responseStream,
@@ -296,7 +330,8 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
        Func<IActorRef, string> subscribe,
        string description,
        int channelCapacity,
-        string streamKind)
+        string streamKind,
+        bool batchingSupported)
    {
        if (!_ready)
            throw new RpcException(new GrpcStatus(StatusCode.Unavailable, "Server not ready"));
@@ -422,10 +457,24 @@ public class SiteStreamGrpcServer : SiteStreamService.SiteStreamServiceBase
        ScadaBridgeTelemetry.SiteConnectionOpened();
        try
        {
-            await foreach (var evt in channel.Reader.ReadAllAsync(streamCts.Token))
-            {
-                await responseStream.WriteAsync(evt, streamCts.Token);
-            }
+            // R2 — event batching. The pump replaces the old per-event
+            // `await foreach (…) WriteAsync(evt)` loop and is byte-for-byte identical to
+            // it when maxBatchEvents == 1, which is exactly what an un-negotiated
+            // subscription gets. It sits DOWNSTREAM of the bounded DropOldest channel
+            // above, so it changes framing only — never the site's burst ceiling.
+            await SiteStreamEventBatcher.PumpAsync(
+                channel.Reader,
+                correlationId,
+                maxBatchEvents: batchingSupported ? _streamBatchMaxEvents : 1,
+                maxWindow: _streamBatchWindow,
+                (evt, token) => responseStream.WriteAsync(evt, token),
+                // Recorded only on a negotiated stream: on an un-negotiated one every
+                // frame carries exactly one event, so the histogram would degenerate
+                // into a per-event instrument on the product's hottest path.
+                batchingSupported
+                    ? size => ScadaBridgeTelemetry.RecordSiteStreamBatchSize(streamKind, size)
+                    : null,
+                streamCts.Token);
        }
        catch (OperationCanceledException)
        {