From e9e44a9ef0c1b0fe6d544c3a5147d4712823976a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 18 Aug 2026 10:32:46 -0400 Subject: [PATCH] =?UTF-8?q?test(loadharness)+docs:=20wire-path=20stream=20?= =?UTF-8?q?probes=20=E2=80=94=20R2=20batching=20benchmark-proven=20at=20ta?= =?UTF-8?q?rget=20scale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WP-4 runs measured the stream path only up to the bounded send channel; the probe deliberately substituted the socket hop with a controllable reader, which is exactly where the R2 SiteStreamEventBatcher lives — so batching had never been in any benchmark's hot path. The harness now also attaches per-site wire probes: a real Kestrel h2c listener hosting the production SiteStreamGrpcServer, dialed by the production SiteStreamGrpcClient with batching negotiated, batch sizes read from the production batch_size histogram via a MeterListener (--wire-probes-per-site, default 10; 0 restores the old shape). Full-scale run (10x500x75, 20-min window, results doc §8 + raw JSON): 899,807 events in 12,462 frames (mean 72.2 ev/frame, p50=p99=75), wire P99 43.4 ms vs the 250 ms budget — every batch pays the full 25 ms linger since a 75-event burst never hits the 100 cap — zero drops, zero faults, offered load 100.0% of nominal with fan-out doubled. CI smoke Fact now asserts the wire path carries traffic and observes frames. --- CLAUDE.md | 2 +- ...026-08-15-target-scale-load-test-design.md | 2 +- ...26-08-15-target-scale-load-test-results.md | 116 +- .../2026-08-18-loadharness-results-wire.json | 2762 +++++++++++++++++ .../HarnessConfig.cs | 14 + .../HarnessRun.cs | 76 +- .../Metrics/BatchSizeAggregator.cs | 136 + .../Probes/SiteWireStreamHost.cs | 96 + .../Probes/WireStreamSubscriber.cs | 108 + .../Program.cs | 1 + .../ResultsFormatter.cs | 12 + .../SiteRuntimeFixture.cs | 62 + .../ZB.MOM.WW.ScadaBridge.LoadHarness.csproj | 8 + .../TargetScaleHarnessSmokeTests.cs | 13 + 14 files changed, 3400 insertions(+), 8 deletions(-) create mode 100644 docs/plans/2026-08-18-loadharness-results-wire.json create mode 100644 tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Metrics/BatchSizeAggregator.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Probes/SiteWireStreamHost.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Probes/WireStreamSubscriber.cs diff --git a/CLAUDE.md b/CLAUDE.md index 1f3252d4..e20942bc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,7 +99,7 @@ spec for each is `docs/requirements/Component-.md`, and `README.md` carrie - **`ActiveNodeEvaluator.SelfIsOldestUp` is THE single definition of "active node"** (`Communication/ClusterState/ActiveNodeEvaluator.cs`) — the **oldest Up member** in a role scope, and explicitly **never `cluster.State.Leader`**: leadership (lowest address) is an Akka-internal concept that diverges from singleton placement permanently once the original first node restarts and rejoins, and both sides claim it during a partition. The equivalence *oldest-Up == where `ClusterSingletonManager` places singletons* **is** the design. `ClusterActivityEvaluator.SelfIsOldest`, the S&F delivery gate, `/health/active` and the heartbeat `IsActive` stamp all delegate here. - Site nodes carry **two Akka roles**: the base `Site` plus a site-specific `site-{SiteId}` (`AkkaHostedService.BuildRoles`). Singletons scope to the **site-specific** role. - **The gRPC boundary is authenticated (PSK) as of 2026-07-22; Akka remoting still is not, and nothing is encrypted.** Akka remoting sets no `enable-ssl`, no secure cookie, no `trusted-selection-paths` — so intra-cluster Akka remoting remains open to anyone who can reach the remoting port, and that boundary still assumes a trusted network. The gRPC listener stays **h2c**, but `SiteStreamService` is no longer open: `ControlPlaneAuthInterceptor` (`Host/ControlPlaneAuthInterceptor.cs`) gates `/sitestream.SiteStreamService/` — including the `PullAuditEvents`/`PullSiteCalls` RPCs that return audit rows — against a **per-site preshared key**, fail-closed, constant-time compared, alongside the separate `LocalDbSyncAuthInterceptor` on `/localdb_sync.v1.LocalDbSync/` with its own separate key. **Site side:** `ScadaBridge:Communication:GrpcPsk`, in production `${secret:SB-GRPC-PSK-}`, and **`StartupValidator` refuses to boot a site node without it** (an unset key would leave the node healthy-looking but serving nothing). **Central side:** `SitePskProvider` resolves `SB-GRPC-PSK-{siteId}` from the secrets store at channel-build time (sites are added at runtime, so no boot-time expansion is possible), with `ScadaBridge:Communication:SitePsks:{siteId}` as an override for hosts running without a master key — the docker rig uses the latter. One key per site, never fleet-wide. A bearer token over h2c is readable and replayable on-path; TLS is the follow-on hardening and needs no change to this design. Introduced by Phase 0 of the ClusterClient→gRPC migration (`docs/plans/2026-07-22-clusterclient-to-grpc-plan.md`). -- gRPC streaming channel — **note the direction is inverted from the data flow**: data moves site→central, but each **site node hosts the gRPC server** (`SiteStreamGrpcServer`, Kestrel h2c, port 8083, mapped **only in the Site branch** of `Program.cs`) and **central is the client**, dialling in. Central creates per-site `SiteStreamGrpcClient` via `SiteStreamGrpcClientFactory`, keyed **`(siteId, endpoint)`** — the key was widened from site-only to fix an arch-review High where one session's NodeA→NodeB flip disposed a channel another session was still using. Proto evolution is **additive only** and field numbers are never reused (`AlarmStateUpdate` grew 7→23 fields for the native-alarm mirror, then 24 for `AckTime`). Generated C# is **vendored** under `Communication/SiteStreamGrpc/` with the `` include commented out — regeneration is a manual toggle-build-copy-untoggle, automated by `docker/regen-proto.sh [sitestream|centralcontrol|sitecommand|all]` (it always restores the csproj; an active `` item must never be committed — it breaks the Docker build). **Stream events are BATCHED as of R2 (2026-08-15)** — a site emitting ~37.5k events/s used to pay one gRPC message per event. Additive wire shape, no new RPC: `batching_supported` on the request (`InstanceStreamRequest` field 3 / `SiteStreamRequest` field 2) plus a `SiteStreamEvent.batch` oneof case (**field 4**) carrying a new `SiteStreamEventBatch { repeated SiteStreamEvent events = 1 }`. **The proto3 default of that flag IS the negotiation, and it is load-bearing** — a batch frame reaches a pre-R2 central as `EventOneofCase.None`, whose `ConvertToDomainEvent` returns null, so the whole batch would vanish with no error anywhere; an old central cannot set the flag, so it never gets one, and an old site ignores the unknown request field and keeps sending per-event frames the new client's `ForEachEvent` unpack handles as the single-event case. Server side is `SiteStreamEventBatcher`, a per-subscriber pump **downstream of `StreamRelayActor`'s bounded DropOldest channel** — so it changes framing only and **does NOT move the burst ceiling** (deferred register row 31; that ceiling lives in the shared publish stage upstream of the BroadcastHub). It never delays a lone event: it drains the already-queued backlog for free and lingers only once a backlog is proven, then emits a single-event buffer as a plain frame. `GrpcStreamBatchMaxEvents` (100, `1` disables) and `GrpcStreamBatchWindow` (25 ms, validated **strictly under 250 ms** = the load test's end-to-end P99 threshold); measured worst case P99 25.4 ms. Histogram `scadabridge.site.stream.batch_size`, recorded only on negotiated streams (`ScadaBridgeTelemetry.MeterName` is already in the `ObservedMeters` allowlist). **The client reconnects on graceful (OK-status) stream completion, not just on fault (arch-review remediation, commit `34a3f4bb`)** — Kestrel's `MaxStreamLifetime`/`Grpc.AspNetCore.Server` max-connection-age periodically ends a healthy stream with a normal completion, which the client used to treat as terminal (no reconnect attempt), silently killing a site's live feed until the next process restart (observed up to ~4h on the rig); live-probed at a 2-minute forced lifetime, reconnect lands within one reconcile tick and `IsLive` reflects the gap in between. +- gRPC streaming channel — **note the direction is inverted from the data flow**: data moves site→central, but each **site node hosts the gRPC server** (`SiteStreamGrpcServer`, Kestrel h2c, port 8083, mapped **only in the Site branch** of `Program.cs`) and **central is the client**, dialling in. Central creates per-site `SiteStreamGrpcClient` via `SiteStreamGrpcClientFactory`, keyed **`(siteId, endpoint)`** — the key was widened from site-only to fix an arch-review High where one session's NodeA→NodeB flip disposed a channel another session was still using. Proto evolution is **additive only** and field numbers are never reused (`AlarmStateUpdate` grew 7→23 fields for the native-alarm mirror, then 24 for `AckTime`). Generated C# is **vendored** under `Communication/SiteStreamGrpc/` with the `` include commented out — regeneration is a manual toggle-build-copy-untoggle, automated by `docker/regen-proto.sh [sitestream|centralcontrol|sitecommand|all]` (it always restores the csproj; an active `` item must never be committed — it breaks the Docker build). **Stream events are BATCHED as of R2 (2026-08-15)** — a site emitting ~37.5k events/s used to pay one gRPC message per event. Additive wire shape, no new RPC: `batching_supported` on the request (`InstanceStreamRequest` field 3 / `SiteStreamRequest` field 2) plus a `SiteStreamEvent.batch` oneof case (**field 4**) carrying a new `SiteStreamEventBatch { repeated SiteStreamEvent events = 1 }`. **The proto3 default of that flag IS the negotiation, and it is load-bearing** — a batch frame reaches a pre-R2 central as `EventOneofCase.None`, whose `ConvertToDomainEvent` returns null, so the whole batch would vanish with no error anywhere; an old central cannot set the flag, so it never gets one, and an old site ignores the unknown request field and keeps sending per-event frames the new client's `ForEachEvent` unpack handles as the single-event case. Server side is `SiteStreamEventBatcher`, a per-subscriber pump **downstream of `StreamRelayActor`'s bounded DropOldest channel** — so it changes framing only and **does NOT move the burst ceiling** (deferred register row 31; that ceiling lives in the shared publish stage upstream of the BroadcastHub). It never delays a lone event: it drains the already-queued backlog for free and lingers only once a backlog is proven, then emits a single-event buffer as a plain frame. `GrpcStreamBatchMaxEvents` (100, `1` disables) and `GrpcStreamBatchWindow` (25 ms, validated **strictly under 250 ms** = the load test's end-to-end P99 threshold); measured worst case P99 25.4 ms. **Benchmark-proven at target scale 2026-08-18** (the LoadHarness wire path — real Kestrel h2c + real `SiteStreamGrpcClient`, `--wire-probes-per-site`): 899,807 events in 12,462 frames (mean 72.2 ev/frame), wire P99 43.4 ms (5.8× inside the 250 ms budget — a 75-event burst never hits the 100 cap, so every batch pays the full 25 ms window: this workload's worst-case latency posture), zero drops/faults; results doc §8. Histogram `scadabridge.site.stream.batch_size`, recorded only on negotiated streams (`ScadaBridgeTelemetry.MeterName` is already in the `ObservedMeters` allowlist). **The client reconnects on graceful (OK-status) stream completion, not just on fault (arch-review remediation, commit `34a3f4bb`)** — Kestrel's `MaxStreamLifetime`/`Grpc.AspNetCore.Server` max-connection-age periodically ends a healthy stream with a normal completion, which the client used to treat as terminal (no reconnect attempt), silently killing a site's live feed until the next process restart (observed up to ~4h on the rig); live-probed at a 2-minute forced lifetime, reconnect lands within one reconcile tick and `IsLive` reflects the gap in between. - Native alarms are a **read-only** mirror of OPC UA Alarms & Conditions and MxAccess Gateway alarms — **no ack-back, no central tables**; state lives in the site's `native_alarm_state`, survives failover, and is cleared on redeploy/undeploy (mirrors static overrides). Central's per-site live alarm cache (`ISiteAlarmLiveCache`) is **transient in-memory only** — there is deliberately no persisted central alarm store, so the 15s poll remains the NotReporting authority behind the live stream. See `Component-DataConnectionLayer.md` / `Component-CentralUI.md` for the model and the authoring surface. - **`AckTime` mirror enrichment + the `Alarms` script accessor (MES alarm-status API Phase 1, 2026-08-01).** `AlarmStateChanged` carries an additive `AckTime` (`DateTimeOffset?`), mirrored on the vendored `AlarmStateUpdate` proto as **field 24** and persisted inside `native_alarm_state`'s `metadata_json` — deliberately NOT a new column, because that table is `RegisterReplicated` and LocalDb builds its CDC triggers from the column list at registration time. Set only while a condition is active AND acknowledged (so it is null while unacked and cleared on re-raise); the DCL stamps the source's own ack instant for OPC UA (new SelectClause **index 18** = `AckedState/TransitionTime`) and its observation time of the ack transition for MxGateway, which supplies none. Site `Call` scripts read alarms via the new **`Alarms.CurrentAsync()`** accessor (`ScriptRuntimeContext` + `ScriptGlobals`, local Ask on `GetAlarmSnapshotRequest`, returns `Commons.Types.Scripts.ScriptAlarm`), mirrored on `ScriptCompileSurface` AND the Central UI `SandboxScriptHost` editor surface. The trust model needed no change — it is a deny-list over API roots, not an allow-list of context members. Plan: `docs/plans/2026-06-30-mes-alarm-status-api.md` (Phases 2–4 are deployed config, not repo). - OPC UA cert trust is **site-local and not persisted centrally** (follow-up): the verify-endpoint probe captures an untrusted server cert but **NEVER trusts it**, and DeploymentManager broadcasts `TrustServerCertCommand`/`RemoveServerCertCommand` to **BOTH** site nodes — `CertStoreActor` runs on every site node, not as a singleton, so PKI stores stay consistent across failover. diff --git a/docs/plans/2026-08-15-target-scale-load-test-design.md b/docs/plans/2026-08-15-target-scale-load-test-design.md index d90eabd3..9801ea85 100644 --- a/docs/plans/2026-08-15-target-scale-load-test-design.md +++ b/docs/plans/2026-08-15-target-scale-load-test-design.md @@ -191,7 +191,7 @@ Each is a deliberate, recorded trade — not an omission. | D1 | **Sustained window shortened from 1 hour to 20 minutes** at full scale | Practicality on a single shared workstation. Memory growth is reported as a *slope* precisely so a shorter window still answers the leak question: a leak shows as a positive slope in 20 min just as in 60. The 1-hour run is a single flag — `--sustain-minutes 60` — and the exact command is recorded in the results doc. | | D2 | **In-process, non-clustered sites** rather than 10 real two-node Akka clusters | A single box cannot host 20 clustered nodes at this scale. Cluster membership/failover is already measured on a real two-node rig (`FailoverTimingTests`, `docker/failover-drill.sh`); WP-4's question is about the hierarchy under the singleton. | | D3 | **Simulated data source** instead of real OPC UA | No simulated adapter exists in the DCL, and 375k real monitored items would measure the OPC UA SDK. Injected at the documented `RegisterAdapter` seam, at the exact hand-off point the real adapter uses. | -| D4 | **gRPC socket replaced by a reader task** on the stream path | Required by the row-50 measurement itself: a stalled subscriber must be *held* stalled. All of `SiteStreamGrpcServer`'s per-subscription machinery above the socket is real. | +| D4 | **gRPC socket replaced by a reader task** on the stream path | Required by the row-50 measurement itself: a stalled subscriber must be *held* stalled. All of `SiteStreamGrpcServer`'s per-subscription machinery above the socket is real. **Partially retired 2026-08-18:** the harness now ALSO attaches per-site wire probes — real Kestrel h2c + real `SiteStreamGrpcClient` with R2 batching negotiated (`--wire-probes-per-site`, default 10) — so the batcher/serialization/HTTP/2 hop is measured too; the substituted-reader probes remain because the slow-subscriber scenario still needs them. Results doc §8. | | D5 | **Instance Actors are `/user` children**, not children of a `DeploymentManagerActor` | Constructing them directly is what lets the ramp be timed in isolation and keeps the harness independent of the deploy round-trip. Visible only as `InstanceActorInitialized` dead letters, which the harness suppresses. No measured path differs. | | D6 | **Akka logging at WARNING** | At 37,500 updates/s, INFO output would itself become a measured load. | | D7 | `[xc-3]` site event logging volume, `[xc-4]` audit-log degradation, `[xc-5]` template flattening, `[xc-8]` UI responsiveness **not covered** | Out of scope for this harness: the first three need central MS SQL and the Template Engine (a central-cluster fixture, not a site one), and `[xc-8]` needs a browser. Recorded as residual scope in the results doc rather than silently dropped. | diff --git a/docs/plans/2026-08-15-target-scale-load-test-results.md b/docs/plans/2026-08-15-target-scale-load-test-results.md index 941b4975..0da69533 100644 --- a/docs/plans/2026-08-15-target-scale-load-test-results.md +++ b/docs/plans/2026-08-15-target-scale-load-test-results.md @@ -275,7 +275,10 @@ Stated plainly so the evidence is not over-read: - **Not a real-network run.** The gRPC, health-transport and S&F-to-central hops are in-process. Serialization cost, TLS, WAN latency and socket backpressure are out of frame; the *stream* path's backpressure was measured deliberately by substituting a - controllable reader for the socket (D4). + controllable reader for the socket (D4). **Partially superseded by §8 (2026-08-18):** + the site→central stream hop now also runs over a real loopback HTTP/2 socket with + the R2 batcher, protobuf serialization and Kestrel in the measured path. TLS and + WAN latency remain out of frame. - **Not a real OPC UA run.** 375,000 monitored items were simulated at the adapter callback (D3). The DCL above that callback is real and fully exercised. - **Not 1 hour.** 20 minutes (D1). See finding F1 for the one question this leaves @@ -323,6 +326,111 @@ dotnet test tests/ZB.MOM.WW.ScadaBridge.PerformanceTests \ `--help`-style knobs: `--sites`, `--instances-per-site`, `--tags-per-instance`, `--tag-update-period-seconds`, `--settle-minutes`, `--sustain-minutes`, `--sample-seconds`, `--health-interval-seconds`, `--debug-probe-interval-seconds`, -`--subscribe-settle-seconds`, `--stream-probes-per-site`, `--sf-drain-messages`, -`--slow-subscriber-events`, `--data-dir`, `--results`. An unknown key is rejected -rather than ignored, so a typo cannot silently change the measured scale. +`--subscribe-settle-seconds`, `--stream-probes-per-site`, `--wire-probes-per-site`, +`--sf-drain-messages`, `--slow-subscriber-events`, `--data-dir`, `--results`. An +unknown key is rejected rather than ignored, so a typo cannot silently change the +measured scale. + +Since 2026-08-18 the default protocol also attaches 10 **wire** stream subscribers +per site (real Kestrel + real `SiteStreamGrpcClient`, §8); `--wire-probes-per-site 0` +reproduces the original in-process-only shape. + +--- + +## 8. Wire-path run — R2 stream batching in the hot path (2026-08-18) + +**Raw metrics:** `docs/plans/2026-08-18-loadharness-results-wire.json` + +### 8.1 Why this run exists + +Both prior runs (§2 and the §4 F1 1-hour sustain) measured the stream path only up +to the bounded send channel: `StreamSubscriberProbe` deliberately substitutes the +final socket hop with a controllable reader (deviation D4 — required by the row-50 +slow-subscriber measurement). The R2 gRPC event batcher (`SiteStreamEventBatcher`, +merged `c75c79f8` on 2026-08-15, about an hour *after* the §2 run) lives exactly in +that substituted hop — so no benchmark run had ever exercised it, only R2's own +component-level measurement (worst-case P99 25.4 ms) and trickle-rate live traffic +on the rig (register row 48). + +This run adds a **wire path** to the harness without touching the existing probes: +each site hosts the production `SiteStreamGrpcServer` on a real Kestrel HTTP/2 (h2c) +loopback listener (production `CommunicationOptions` defaults — batch cap 100, +window 25 ms, channel capacity 1000), and the production `SiteStreamGrpcClient` +dials in with `BatchingSupported = true`, one channel per site with 10 streams +multiplexed on it — the exact production shape and the exact production code on both +ends. Latency is recorded in the client callback, i.e. after protobuf +deserialization, batch unpacking and domain mapping. The batch-size figures are the +production `scadabridge.site.stream.batch_size` histogram itself, captured by a +`MeterListener`. Deliberately not in the path: `ControlPlaneAuthInterceptor` +(authenticates once per stream lifetime, nothing per-event) and TLS (production is +h2c too). + +The 100 wire subscribers are attached **in addition to** the 100 in-process probes, +watching the same instances (same stride), so one run yields both distributions and +their difference isolates the wire hop's cost. Total hub fan-out is therefore +double that of the §2 run. + +### 8.2 Environment + +Identical posture to §1: same host (M4 Pro, 14 cores, 48 GB), .NET 10.0.5, server +GC, Release; the 8-node `docker/` rig stayed running throughout (D8); 20-minute +window after a 2-minute settle; harness at the state committed with this section. + +### 8.3 Measured + +| | in-process probes (§2 path) | **wire probes (new)** | +|---|---|---| +| Subscribers | 100 | 100 | +| Events delivered (window) | 899,807 | **899,807** | +| Events dropped / faults | 0 / — | **0 / 0** | +| Latency mean | 1.79 ms | **28.00 ms** | +| Latency P50 | 1.36 ms | **26.97 ms** | +| Latency P95 | 4.19 ms | **32.07 ms** | +| Latency P99 | 6.74 ms | **43.44 ms** | +| Latency P99.9 / max | 25.83 / 32.17 ms | **53.94 / 62.42 ms** | + +Batching (production histogram, window deltas): + +``` +events 899,807 frames 12,462 → mean 72.20 events/frame (72x frame reduction) +batch size p50/p95/p99/max = 75/75/75/75 batched frames 11,998 single-event frames 464 +server channel drops 0 stream faults 0 graceful completions 0 +``` + +Offered load held at **37,497/s = 100.0% of nominal** with the doubled fan-out +(driver overrun 0.0 s, skipped 0). Whole-run CPU mean **66% of one core (4.7% of +the box)** vs 41% in §2 — the added ~25% of one core buys 10 Kestrel listeners, 100 +live HTTP/2 streams, protobuf serialization of ~900 k events and the client-side +unpack. Working-set slope **−116 MB/min** (a large mid-window release; peak +3,413 MB), heap sawtooth with 5 gen-2 collects and endpoints 1,876→1,943 MB — +consistent with the §4 F1 resolution (row 29: not a leak). + +### 8.4 Reading the numbers + +- **The batcher engaged exactly as designed, at its natural batch size.** The + driver emits each instance's 75 tags contiguously within a slice, so a subscriber + receives one 75-event burst per 10 s period. 75 < the 100-event cap, so the + size cap never closes a batch early — nearly every burst becomes exactly one + 75-event frame (p50 = p99 = max = 75). The 464 single-event frames are burst + stragglers that arrived after their batch's window closed. +- **The ~26 ms wire P50 is dominated by the deliberate linger, and that is the + documented trade.** Because bursts never hit the cap, every batch pays the full + 25 ms `GrpcStreamBatchWindow` — this workload is the batcher's *worst-case latency + posture and best-case framing win* simultaneously. Wire P50 26.97 ms ≈ in-process + P50 1.36 ms + the 25 ms window + sub-millisecond socket/serialize cost. Against + the 250 ms P99 threshold the wire path passes with a **5.8× margin** (43.44 ms), + and the `CommunicationOptionsValidator` bound (window strictly < 250 ms) is + confirmed to compose: end-to-end latency stayed threshold-compliant with the + window fully paid on every frame. +- **Framing efficiency at scale is now measured, not extrapolated:** 899,807 events + crossed the wire in 12,462 frames. Pre-R2, the same window would have cost + 899,807 frames. +- **Nothing degraded elsewhere.** The in-process path under doubled fan-out reads + P99 6.74 ms vs 4.57 ms in §2 — same order, still a 37× margin — and every §2 + pass criterion that this run re-exercises (offered load, zero drops at healthy + subscribers, health reports, debug snapshots, S&F drain, slow-subscriber + isolation) repeated its pass. + +**Verdict: PASS.** R2's negotiated batching is now benchmark-proven at target scale +over a real socket: 72× fewer frames, zero loss, zero faults, and worst-case-posture +latency 5.8× inside the budget. diff --git a/docs/plans/2026-08-18-loadharness-results-wire.json b/docs/plans/2026-08-18-loadharness-results-wire.json new file mode 100644 index 00000000..f639ed89 --- /dev/null +++ b/docs/plans/2026-08-18-loadharness-results-wire.json @@ -0,0 +1,2762 @@ +{ + "Config": { + "Sites": 10, + "InstancesPerSite": 500, + "TagsPerInstance": 75, + "TagUpdatePeriod": "00:00:10", + "SustainDuration": "00:20:00", + "SettleDuration": "00:02:00", + "SampleInterval": "00:00:10", + "HealthReportInterval": "00:00:30", + "DebugProbeInterval": "00:00:05", + "SubscribeSettleDuration": "00:00:30", + "StreamProbesPerSite": 10, + "WireProbesPerSite": 10, + "StoreAndForwardDrainMessages": 20000, + "SlowSubscriberEvents": 200000, + "DataDirectory": null, + "ResultsPath": "docs/plans/2026-08-18-loadharness-results-wire.json", + "TotalSubscriptions": 375000, + "NominalUpdatesPerSecond": 37500 + }, + "Environment": { + "MachineName": "Josephs-MacBook-Pro", + "OsDescription": "macOS 26.5.2", + "ProcessorCount": 14, + "RuntimeVersion": ".NET 10.0.5", + "ServerGc": true + }, + "StartedUtc": "2026-08-18T14:03:52.298984+00:00", + "TotalSeconds": 1511.5348864, + "SiteRampSeconds": 0.201042, + "InstanceRampSeconds": 2.6594027, + "SlowestSiteInstanceRampSeconds": 2.6361791, + "TagUpdateLatency": { + "Count": 1099807, + "MeanMs": 1.7946889054170414, + "P50Ms": 1.357360692013006, + "P95Ms": 4.186672706411344, + "P99Ms": 6.742595916711287, + "P999Ms": 25.826927881070144, + "MaxMs": 32.168 + }, + "EmittedTagUpdates": 49503375, + "SteadyStateEmittedTagUpdates": 44997000, + "AchievedUpdatesPerSecond": 37497.46801778457, + "NominalUpdatesPerSecond": 37500, + "DriverLagSeconds": 0, + "DriverSkippedNoCallback": 0, + "SteadyStateResources": { + "SampleCount": 121, + "DurationSeconds": 1199.9999851, + "WorkingSetStartMb": 3400.71875, + "WorkingSetEndMb": 1617.625, + "WorkingSetPeakMb": 3412.640625, + "WorkingSetSlopeMbPerMinute": -116.23247972367047, + "ManagedHeapStartMb": 1876.2783966064453, + "ManagedHeapEndMb": 1943.1891632080078, + "ManagedHeapPeakMb": 2459.9215927124023, + "ManagedHeapSlopeMbPerMinute": 25.408545872205078, + "MeanCpuPercentOfOneCore": 65.60327395470735, + "PeakCpuPercentOfOneCore": 79.11544111203865, + "MeanThreadCount": 77.41322314049587, + "Gen2Collections": 5 + }, + "WholeRunResources": { + "SampleCount": 151, + "DurationSeconds": 1499.9973645, + "WorkingSetStartMb": 1125.34375, + "WorkingSetEndMb": 1630.09375, + "WorkingSetPeakMb": 3412.640625, + "WorkingSetSlopeMbPerMinute": -79.63711252602297, + "ManagedHeapStartMb": 1311.9069519042969, + "ManagedHeapEndMb": 1961.5810928344727, + "ManagedHeapPeakMb": 3465.6229248046875, + "ManagedHeapSlopeMbPerMinute": 10.827540888085572, + "MeanCpuPercentOfOneCore": 60.28927415611865, + "PeakCpuPercentOfOneCore": 83.23026138119138, + "MeanThreadCount": 76.31788079470199, + "Gen2Collections": 5 + }, + "ResourceSamples": [ + { + "ElapsedSeconds": 10.0046215, + "WorkingSetMb": 1125.34375, + "ManagedHeapMb": 1311.9069519042969, + "CpuPercent": 79.23970936831543, + "ThreadCount": 73, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 20.0022617, + "WorkingSetMb": 1125.546875, + "ManagedHeapMb": 1311.986473083496, + "CpuPercent": 6.859328664378222, + "ThreadCount": 73, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 30.0020525, + "WorkingSetMb": 1125.765625, + "ManagedHeapMb": 1312.0720977783203, + "CpuPercent": 7.166469922550778, + "ThreadCount": 47, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 40.0028333, + "WorkingSetMb": 1648.765625, + "ManagedHeapMb": 1830.2910995483398, + "CpuPercent": 83.23026138119138, + "ThreadCount": 73, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 50.002089, + "WorkingSetMb": 2359.5, + "ManagedHeapMb": 2539.4089279174805, + "CpuPercent": 76.0481802660572, + "ThreadCount": 73, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 60.002026, + "WorkingSetMb": 2891.25, + "ManagedHeapMb": 1556.3661193847656, + "CpuPercent": 73.19095110299193, + "ThreadCount": 76, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 70.0018274, + "WorkingSetMb": 2891.4375, + "ManagedHeapMb": 2157.843505859375, + "CpuPercent": 62.27261673416837, + "ThreadCount": 76, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 80.001426, + "WorkingSetMb": 2896.09375, + "ManagedHeapMb": 2830.5467376708984, + "CpuPercent": 63.26599949721982, + "ThreadCount": 75, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 90.0012377, + "WorkingSetMb": 3068.78125, + "ManagedHeapMb": 3465.6229248046875, + "CpuPercent": 53.27344313893426, + "ThreadCount": 75, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 100.0012281, + "WorkingSetMb": 3396.515625, + "ManagedHeapMb": 1590.7351989746094, + "CpuPercent": 61.50709904681507, + "ThreadCount": 76, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 110.0013068, + "WorkingSetMb": 3396.671875, + "ManagedHeapMb": 2248.36043548584, + "CpuPercent": 56.293906966952235, + "ThreadCount": 77, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 120.0013745, + "WorkingSetMb": 3396.734375, + "ManagedHeapMb": 2874.7490768432617, + "CpuPercent": 53.13431028071939, + "ThreadCount": 78, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 130.0011233, + "WorkingSetMb": 3400.140625, + "ManagedHeapMb": 1805.1724243164062, + "CpuPercent": 50.94122964368869, + "ThreadCount": 78, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 140.0015607, + "WorkingSetMb": 3400.203125, + "ManagedHeapMb": 2465.302635192871, + "CpuPercent": 53.62669436838827, + "ThreadCount": 78, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 150.001256, + "WorkingSetMb": 3400.71875, + "ManagedHeapMb": 1876.2783966064453, + "CpuPercent": 53.07246711807302, + "ThreadCount": 78, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 160.0014276, + "WorkingSetMb": 3400.953125, + "ManagedHeapMb": 1577.4280624389648, + "CpuPercent": 54.02912286025179, + "ThreadCount": 77, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 170.0011798, + "WorkingSetMb": 3400.953125, + "ManagedHeapMb": 1510.4443054199219, + "CpuPercent": 53.069125052918864, + "ThreadCount": 74, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 180.001003, + "WorkingSetMb": 3401.078125, + "ManagedHeapMb": 1541.088119506836, + "CpuPercent": 54.65403628336144, + "ThreadCount": 75, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 190.0021585, + "WorkingSetMb": 3401.28125, + "ManagedHeapMb": 1632.288558959961, + "CpuPercent": 47.31134317429616, + "ThreadCount": 75, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 200.0013325, + "WorkingSetMb": 3401.171875, + "ManagedHeapMb": 1781.8268814086914, + "CpuPercent": 48.452872207244404, + "ThreadCount": 71, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 210.0022035, + "WorkingSetMb": 3401.140625, + "ManagedHeapMb": 1605.4158401489258, + "CpuPercent": 57.460885156902734, + "ThreadCount": 70, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 220.0009635, + "WorkingSetMb": 3401.203125, + "ManagedHeapMb": 1878.6061477661133, + "CpuPercent": 51.11479823498111, + "ThreadCount": 71, + "Gen2Collections": 1 + }, + { + "ElapsedSeconds": 230.0009058, + "WorkingSetMb": 3410.328125, + "ManagedHeapMb": 1377.405517578125, + "CpuPercent": 58.669308521910246, + "ThreadCount": 86, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 240.0017027, + "WorkingSetMb": 3410.328125, + "ManagedHeapMb": 1509.4178619384766, + "CpuPercent": 62.76183850908914, + "ThreadCount": 87, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 250.0016132, + "WorkingSetMb": 3410.390625, + "ManagedHeapMb": 1426.2226867675781, + "CpuPercent": 66.10047159922082, + "ThreadCount": 87, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 260.0012266, + "WorkingSetMb": 3410.421875, + "ManagedHeapMb": 1453.654441833496, + "CpuPercent": 64.57416643727456, + "ThreadCount": 87, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 270.0022043, + "WorkingSetMb": 3410.328125, + "ManagedHeapMb": 1482.2178421020508, + "CpuPercent": 64.37667589239787, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 280.0012995, + "WorkingSetMb": 3410.375, + "ManagedHeapMb": 1503.0744323730469, + "CpuPercent": 59.82352283234588, + "ThreadCount": 84, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 290.00182, + "WorkingSetMb": 3410.359375, + "ManagedHeapMb": 1628.948471069336, + "CpuPercent": 47.431171207538675, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 300.0037504, + "WorkingSetMb": 3410.34375, + "ManagedHeapMb": 1570.8058319091797, + "CpuPercent": 56.892427485798194, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 310.0038134, + "WorkingSetMb": 3410.28125, + "ManagedHeapMb": 1668.692985534668, + "CpuPercent": 72.04134613951923, + "ThreadCount": 81, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 320.0017698, + "WorkingSetMb": 3410.265625, + "ManagedHeapMb": 1729.3491439819336, + "CpuPercent": 62.24657070919035, + "ThreadCount": 79, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 330.0008899, + "WorkingSetMb": 3410.3125, + "ManagedHeapMb": 1654.1545104980469, + "CpuPercent": 71.88935554439416, + "ThreadCount": 80, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 340.0018668, + "WorkingSetMb": 3412.5625, + "ManagedHeapMb": 1676.8509826660156, + "CpuPercent": 62.60901372544908, + "ThreadCount": 80, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 350.0011261, + "WorkingSetMb": 3412.515625, + "ManagedHeapMb": 1716.8530883789062, + "CpuPercent": 65.83199617595672, + "ThreadCount": 79, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 360.0016549, + "WorkingSetMb": 3412.53125, + "ManagedHeapMb": 1797.4564361572266, + "CpuPercent": 73.12765300970896, + "ThreadCount": 79, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 370.0019215, + "WorkingSetMb": 3412.5625, + "ManagedHeapMb": 1817.5244369506836, + "CpuPercent": 63.990744006764935, + "ThreadCount": 80, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 380.0016015, + "WorkingSetMb": 3412.59375, + "ManagedHeapMb": 1834.446189880371, + "CpuPercent": 67.68645596659086, + "ThreadCount": 82, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 390.0014752, + "WorkingSetMb": 3412.640625, + "ManagedHeapMb": 1864.423683166504, + "CpuPercent": 66.08317463049544, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 400.0012459, + "WorkingSetMb": 3397.46875, + "ManagedHeapMb": 1862.945167541504, + "CpuPercent": 63.63060904986552, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 410.002547, + "WorkingSetMb": 3341.34375, + "ManagedHeapMb": 1881.2931442260742, + "CpuPercent": 65.04317723221045, + "ThreadCount": 86, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 420.0010552, + "WorkingSetMb": 3266.375, + "ManagedHeapMb": 1890.5906295776367, + "CpuPercent": 62.411030477526616, + "ThreadCount": 86, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 430.0018194, + "WorkingSetMb": 3198.234375, + "ManagedHeapMb": 1922.3261337280273, + "CpuPercent": 63.902496571212076, + "ThreadCount": 85, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 440.0013702, + "WorkingSetMb": 3139.390625, + "ManagedHeapMb": 1995.944190979004, + "CpuPercent": 68.13214049575102, + "ThreadCount": 82, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 450.0020454, + "WorkingSetMb": 3082.25, + "ManagedHeapMb": 2020.7418518066406, + "CpuPercent": 66.16033285432576, + "ThreadCount": 82, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 460.0016055, + "WorkingSetMb": 3030.546875, + "ManagedHeapMb": 2009.6987075805664, + "CpuPercent": 63.87690994526851, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 470.0022638, + "WorkingSetMb": 2970.359375, + "ManagedHeapMb": 2006.8369522094727, + "CpuPercent": 60.69951415098345, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 480.0023333, + "WorkingSetMb": 2920.125, + "ManagedHeapMb": 2033.5534286499023, + "CpuPercent": 70.6396690543001, + "ThreadCount": 84, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 490.0013975, + "WorkingSetMb": 2872.390625, + "ManagedHeapMb": 2071.0333709716797, + "CpuPercent": 67.59581561642524, + "ThreadCount": 84, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 500.0015153, + "WorkingSetMb": 2822.5, + "ManagedHeapMb": 2077.0046463012695, + "CpuPercent": 63.38396333691191, + "ThreadCount": 84, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 510.0026452, + "WorkingSetMb": 2780.15625, + "ManagedHeapMb": 2098.1446990966797, + "CpuPercent": 65.231229523376, + "ThreadCount": 82, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 520.0007808, + "WorkingSetMb": 2736.328125, + "ManagedHeapMb": 2121.769515991211, + "CpuPercent": 64.61805739061982, + "ThreadCount": 81, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 530.0034737, + "WorkingSetMb": 2700.53125, + "ManagedHeapMb": 2241.304573059082, + "CpuPercent": 54.346315080812204, + "ThreadCount": 81, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 540.0032402, + "WorkingSetMb": 2662.421875, + "ManagedHeapMb": 2227.9443969726562, + "CpuPercent": 63.2370965862048, + "ThreadCount": 80, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 550.0019432, + "WorkingSetMb": 2618.5625, + "ManagedHeapMb": 2218.3480377197266, + "CpuPercent": 59.85904371797035, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 560.0036122, + "WorkingSetMb": 2584, + "ManagedHeapMb": 2210.653923034668, + "CpuPercent": 58.125458860916154, + "ThreadCount": 83, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 570.0023632, + "WorkingSetMb": 2547.6875, + "ManagedHeapMb": 2235.250503540039, + "CpuPercent": 60.089765211675115, + "ThreadCount": 84, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 580.0010871, + "WorkingSetMb": 2515.046875, + "ManagedHeapMb": 2266.6464233398438, + "CpuPercent": 66.8715834827685, + "ThreadCount": 85, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 590.0026287, + "WorkingSetMb": 2490.625, + "ManagedHeapMb": 2374.9184799194336, + "CpuPercent": 58.68610295036919, + "ThreadCount": 85, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 600.0015718, + "WorkingSetMb": 2455.109375, + "ManagedHeapMb": 2349.8499145507812, + "CpuPercent": 59.11400775947999, + "ThreadCount": 80, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 610.003942, + "WorkingSetMb": 2424.515625, + "ManagedHeapMb": 2346.107650756836, + "CpuPercent": 61.90884636523397, + "ThreadCount": 80, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 620.000774, + "WorkingSetMb": 2403.984375, + "ManagedHeapMb": 2459.9215927124023, + "CpuPercent": 60.214585980839175, + "ThreadCount": 79, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 630.0026415, + "WorkingSetMb": 2377.359375, + "ManagedHeapMb": 2453.065704345703, + "CpuPercent": 61.60641500199836, + "ThreadCount": 79, + "Gen2Collections": 2 + }, + { + "ElapsedSeconds": 640.0020672, + "WorkingSetMb": 2356.03125, + "ManagedHeapMb": 1933.604476928711, + "CpuPercent": 65.3811348385733, + "ThreadCount": 79, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 650.0016431, + "WorkingSetMb": 2329.75, + "ManagedHeapMb": 1947.86669921875, + "CpuPercent": 64.65422198555461, + "ThreadCount": 80, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 660.0023834, + "WorkingSetMb": 2292.390625, + "ManagedHeapMb": 1949.0791320800781, + "CpuPercent": 65.39042914653052, + "ThreadCount": 79, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 670.0039246, + "WorkingSetMb": 2262.53125, + "ManagedHeapMb": 1964.8410186767578, + "CpuPercent": 65.91367138496602, + "ThreadCount": 79, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 680.0013231, + "WorkingSetMb": 2227.609375, + "ManagedHeapMb": 1989.0984802246094, + "CpuPercent": 71.97957548656261, + "ThreadCount": 79, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 690.0027725, + "WorkingSetMb": 2198.734375, + "ManagedHeapMb": 2009.9393692016602, + "CpuPercent": 68.03415912897614, + "ThreadCount": 79, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 700.0010812, + "WorkingSetMb": 2167.96875, + "ManagedHeapMb": 2022.5504989624023, + "CpuPercent": 66.72022439155099, + "ThreadCount": 79, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 710.0016879, + "WorkingSetMb": 2138.703125, + "ManagedHeapMb": 2001.736572265625, + "CpuPercent": 59.373527808068275, + "ThreadCount": 79, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 720.0027373, + "WorkingSetMb": 2108.21875, + "ManagedHeapMb": 2034.7251739501953, + "CpuPercent": 65.20159774433235, + "ThreadCount": 77, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 730.001639, + "WorkingSetMb": 2081.71875, + "ManagedHeapMb": 2040.3736190795898, + "CpuPercent": 67.54661864512633, + "ThreadCount": 77, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 740.0031335, + "WorkingSetMb": 2056.46875, + "ManagedHeapMb": 2089.090087890625, + "CpuPercent": 68.32442891409855, + "ThreadCount": 77, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 750.0012498, + "WorkingSetMb": 2029.9375, + "ManagedHeapMb": 2110.9550247192383, + "CpuPercent": 63.86443014270603, + "ThreadCount": 77, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 760.0014379, + "WorkingSetMb": 2005.015625, + "ManagedHeapMb": 2094.2857208251953, + "CpuPercent": 63.95353703396801, + "ThreadCount": 77, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 770.0022275, + "WorkingSetMb": 1982.328125, + "ManagedHeapMb": 2071.0819396972656, + "CpuPercent": 66.64632760597249, + "ThreadCount": 77, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 780.0129828, + "WorkingSetMb": 1956.828125, + "ManagedHeapMb": 2078.327980041504, + "CpuPercent": 68.08227547026327, + "ThreadCount": 76, + "Gen2Collections": 3 + }, + { + "ElapsedSeconds": 790.001791, + "WorkingSetMb": 1941.9375, + "ManagedHeapMb": 1894.500617980957, + "CpuPercent": 70.04065810373659, + "ThreadCount": 75, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 800.003349, + "WorkingSetMb": 1921.90625, + "ManagedHeapMb": 1936.6790313720703, + "CpuPercent": 75.19865404969957, + "ThreadCount": 74, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 810.0017289, + "WorkingSetMb": 1903.96875, + "ManagedHeapMb": 1973.395362854004, + "CpuPercent": 72.68215523596955, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 820.0018947, + "WorkingSetMb": 1886.78125, + "ManagedHeapMb": 2012.0318908691406, + "CpuPercent": 71.94169720666038, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 830.0018002, + "WorkingSetMb": 1866.1875, + "ManagedHeapMb": 2015.769905090332, + "CpuPercent": 66.92180241103233, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 840.0027828, + "WorkingSetMb": 1847.28125, + "ManagedHeapMb": 2018.2055435180664, + "CpuPercent": 65.4745964661517, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 850.0019701, + "WorkingSetMb": 1831.25, + "ManagedHeapMb": 2057.6731491088867, + "CpuPercent": 67.97646444726551, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 860.002712, + "WorkingSetMb": 1813.421875, + "ManagedHeapMb": 2068.4942016601562, + "CpuPercent": 61.68093389151471, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 870.0028295, + "WorkingSetMb": 1797.078125, + "ManagedHeapMb": 2061.173828125, + "CpuPercent": 66.23623172427732, + "ThreadCount": 74, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 880.0015281, + "WorkingSetMb": 1782.71875, + "ManagedHeapMb": 2094.463935852051, + "CpuPercent": 65.26101306824063, + "ThreadCount": 74, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 890.0024701, + "WorkingSetMb": 1768.53125, + "ManagedHeapMb": 2131.120895385742, + "CpuPercent": 68.97369267814966, + "ThreadCount": 74, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 900.0016959, + "WorkingSetMb": 1752, + "ManagedHeapMb": 2055.368667602539, + "CpuPercent": 72.46194000339523, + "ThreadCount": 74, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 910.0014187, + "WorkingSetMb": 1740.25, + "ManagedHeapMb": 2109.3386993408203, + "CpuPercent": 68.01728543915179, + "ThreadCount": 74, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 920.0019612, + "WorkingSetMb": 1728.5, + "ManagedHeapMb": 2120.814308166504, + "CpuPercent": 62.697198676972164, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 930.0021805, + "WorkingSetMb": 1717.828125, + "ManagedHeapMb": 2151.9377822875977, + "CpuPercent": 62.066578879925004, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 940.0029029, + "WorkingSetMb": 1704.6875, + "ManagedHeapMb": 2072.240463256836, + "CpuPercent": 70.95469423288881, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 950.0008074, + "WorkingSetMb": 1693.765625, + "ManagedHeapMb": 2089.1425857543945, + "CpuPercent": 66.71208951835854, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 960.0036615, + "WorkingSetMb": 1684.15625, + "ManagedHeapMb": 2109.6972732543945, + "CpuPercent": 63.9464990297117, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 970.0018161, + "WorkingSetMb": 1676.203125, + "ManagedHeapMb": 2126.8689880371094, + "CpuPercent": 68.9635365310313, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 980.0015226, + "WorkingSetMb": 1669.453125, + "ManagedHeapMb": 2156.0795669555664, + "CpuPercent": 69.63133367964348, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 990.0008257, + "WorkingSetMb": 1664.015625, + "ManagedHeapMb": 2168.75732421875, + "CpuPercent": 66.34294343972893, + "ThreadCount": 72, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 1000.0032369, + "WorkingSetMb": 1658.984375, + "ManagedHeapMb": 2208.806816101074, + "CpuPercent": 68.46108266374745, + "ThreadCount": 72, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 1010.0029888, + "WorkingSetMb": 1653.890625, + "ManagedHeapMb": 2200.356315612793, + "CpuPercent": 64.28848499731292, + "ThreadCount": 72, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 1020.0023762, + "WorkingSetMb": 1648.5, + "ManagedHeapMb": 2216.3390350341797, + "CpuPercent": 70.44514546961194, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 1030.0016367, + "WorkingSetMb": 1644.03125, + "ManagedHeapMb": 2235.3355560302734, + "CpuPercent": 67.06718961867159, + "ThreadCount": 73, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 1040.0012363, + "WorkingSetMb": 1639.78125, + "ManagedHeapMb": 2260.992774963379, + "CpuPercent": 71.59125651391075, + "ThreadCount": 76, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 1050.0020219, + "WorkingSetMb": 1634.984375, + "ManagedHeapMb": 2289.9648361206055, + "CpuPercent": 69.62832999839551, + "ThreadCount": 76, + "Gen2Collections": 4 + }, + { + "ElapsedSeconds": 1060.0018873, + "WorkingSetMb": 1632.6875, + "ManagedHeapMb": 1969.4179763793945, + "CpuPercent": 74.40257145861285, + "ThreadCount": 76, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1070.002042, + "WorkingSetMb": 1630.625, + "ManagedHeapMb": 1962.0653076171875, + "CpuPercent": 61.12446440453465, + "ThreadCount": 76, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1080.0013753, + "WorkingSetMb": 1629.078125, + "ManagedHeapMb": 1990.356101989746, + "CpuPercent": 68.06754806342946, + "ThreadCount": 75, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1090.0032911, + "WorkingSetMb": 1628.53125, + "ManagedHeapMb": 2023.6001358032227, + "CpuPercent": 68.39598669686858, + "ThreadCount": 78, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1100.0012159, + "WorkingSetMb": 1628.484375, + "ManagedHeapMb": 1996.2192306518555, + "CpuPercent": 63.952971520650195, + "ThreadCount": 78, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1110.002226, + "WorkingSetMb": 1627.953125, + "ManagedHeapMb": 2065.747428894043, + "CpuPercent": 74.87409696746509, + "ThreadCount": 81, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1120.0015833, + "WorkingSetMb": 1627.796875, + "ManagedHeapMb": 1980.4240493774414, + "CpuPercent": 74.9290156878388, + "ThreadCount": 81, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1130.00196, + "WorkingSetMb": 1627.109375, + "ManagedHeapMb": 2015.4603118896484, + "CpuPercent": 70.87812002121836, + "ThreadCount": 81, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1140.0017095, + "WorkingSetMb": 1626.890625, + "ManagedHeapMb": 2019.577049255371, + "CpuPercent": 70.04190454970892, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1150.002018, + "WorkingSetMb": 1626.890625, + "ManagedHeapMb": 2058.9554748535156, + "CpuPercent": 71.03936843548485, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1160.002683, + "WorkingSetMb": 1626.78125, + "ManagedHeapMb": 2094.0753173828125, + "CpuPercent": 67.84620822715272, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1170.003368, + "WorkingSetMb": 1625.0625, + "ManagedHeapMb": 2139.49275970459, + "CpuPercent": 64.03566355704649, + "ThreadCount": 72, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1180.0032086, + "WorkingSetMb": 1623.484375, + "ManagedHeapMb": 2064.0842514038086, + "CpuPercent": 69.64488013938805, + "ThreadCount": 72, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1190.0015183, + "WorkingSetMb": 1622.546875, + "ManagedHeapMb": 2106.517265319824, + "CpuPercent": 64.76047646333701, + "ThreadCount": 72, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1200.003802, + "WorkingSetMb": 1621.765625, + "ManagedHeapMb": 2165.9246215820312, + "CpuPercent": 68.54515634264666, + "ThreadCount": 72, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1210.0017089, + "WorkingSetMb": 1621.703125, + "ManagedHeapMb": 2114.9686279296875, + "CpuPercent": 66.22910241342557, + "ThreadCount": 72, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1220.002161, + "WorkingSetMb": 1621.703125, + "ManagedHeapMb": 2095.1331100463867, + "CpuPercent": 73.87114028574881, + "ThreadCount": 72, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1230.002131, + "WorkingSetMb": 1621.734375, + "ManagedHeapMb": 2206.2094192504883, + "CpuPercent": 72.2832668498, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1240.002618, + "WorkingSetMb": 1621.828125, + "ManagedHeapMb": 2177.2350692749023, + "CpuPercent": 65.46471186853186, + "ThreadCount": 76, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1250.001948, + "WorkingSetMb": 1621.796875, + "ManagedHeapMb": 2173.9827270507812, + "CpuPercent": 75.83069065627322, + "ThreadCount": 76, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1260.001744, + "WorkingSetMb": 1620.96875, + "ManagedHeapMb": 2164.8643188476562, + "CpuPercent": 74.43920855985586, + "ThreadCount": 74, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1270.0020742, + "WorkingSetMb": 1619.953125, + "ManagedHeapMb": 2270.037437438965, + "CpuPercent": 72.24500446995235, + "ThreadCount": 74, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1280.0018522, + "WorkingSetMb": 1618.890625, + "ManagedHeapMb": 2216.3951873779297, + "CpuPercent": 72.6912937467204, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1290.0022248, + "WorkingSetMb": 1618.546875, + "ManagedHeapMb": 2274.146629333496, + "CpuPercent": 69.13959385873288, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1300.0017478, + "WorkingSetMb": 1618.125, + "ManagedHeapMb": 2244.5165634155273, + "CpuPercent": 74.43549057290068, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1310.002328, + "WorkingSetMb": 1617.734375, + "ManagedHeapMb": 2310.938606262207, + "CpuPercent": 68.56158205700865, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1320.0134926, + "WorkingSetMb": 1617.5625, + "ManagedHeapMb": 2331.117691040039, + "CpuPercent": 65.6623905674269, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1330.0016259, + "WorkingSetMb": 1617.5625, + "ManagedHeapMb": 2272.4801330566406, + "CpuPercent": 67.67816164407922, + "ThreadCount": 73, + "Gen2Collections": 5 + }, + { + "ElapsedSeconds": 1340.0020138, + "WorkingSetMb": 1617.625, + "ManagedHeapMb": 1911.3805618286133, + "CpuPercent": 79.11544111203865, + "ThreadCount": 74, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1350.0012411, + "WorkingSetMb": 1617.625, + "ManagedHeapMb": 1943.1891632080078, + "CpuPercent": 66.39628044058945, + "ThreadCount": 74, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1360.0015097, + "WorkingSetMb": 1626.875, + "ManagedHeapMb": 1910.2868041992188, + "CpuPercent": 44.836795683667816, + "ThreadCount": 75, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1370.0035196, + "WorkingSetMb": 1628.46875, + "ManagedHeapMb": 1911.2174835205078, + "CpuPercent": 25.554553790234092, + "ThreadCount": 74, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1380.0026894, + "WorkingSetMb": 1627.9375, + "ManagedHeapMb": 1912.0518035888672, + "CpuPercent": 24.908697920101034, + "ThreadCount": 58, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1390.0033012, + "WorkingSetMb": 1630.328125, + "ManagedHeapMb": 1999.4393844604492, + "CpuPercent": 28.492436832714894, + "ThreadCount": 61, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1400.0019931, + "WorkingSetMb": 1630.6875, + "ManagedHeapMb": 1927.115608215332, + "CpuPercent": 28.221251621924555, + "ThreadCount": 71, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1410.0021712, + "WorkingSetMb": 1630.703125, + "ManagedHeapMb": 1980.130271911621, + "CpuPercent": 24.271127731214904, + "ThreadCount": 72, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1420.0008713, + "WorkingSetMb": 1630.6875, + "ManagedHeapMb": 2032.772346496582, + "CpuPercent": 25.22006835668581, + "ThreadCount": 71, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1430.0012091, + "WorkingSetMb": 1630.546875, + "ManagedHeapMb": 1946.8932723999023, + "CpuPercent": 24.57485986123358, + "ThreadCount": 74, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1440.0020225, + "WorkingSetMb": 1630.484375, + "ManagedHeapMb": 1992.9519882202148, + "CpuPercent": 23.920774284219796, + "ThreadCount": 74, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1450.0024986, + "WorkingSetMb": 1630.421875, + "ManagedHeapMb": 2017.8553848266602, + "CpuPercent": 21.89403762486865, + "ThreadCount": 74, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1460.0014163, + "WorkingSetMb": 1630.34375, + "ManagedHeapMb": 1936.5043869018555, + "CpuPercent": 25.41592076510442, + "ThreadCount": 74, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1470.001871, + "WorkingSetMb": 1630.25, + "ManagedHeapMb": 1981.6953811645508, + "CpuPercent": 24.995803440817884, + "ThreadCount": 72, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1480.0021472, + "WorkingSetMb": 1630.265625, + "ManagedHeapMb": 2026.9382934570312, + "CpuPercent": 24.01971657542784, + "ThreadCount": 72, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1490.0008811, + "WorkingSetMb": 1630.25, + "ManagedHeapMb": 1939.652816772461, + "CpuPercent": 22.86135447609037, + "ThreadCount": 72, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1500.0020759, + "WorkingSetMb": 1630.125, + "ManagedHeapMb": 1961.3844528198242, + "CpuPercent": 9.248444995792001, + "ThreadCount": 68, + "Gen2Collections": 6 + }, + { + "ElapsedSeconds": 1510.001986, + "WorkingSetMb": 1630.09375, + "ManagedHeapMb": 1961.5810928344727, + "CpuPercent": 7.198204711860303, + "ThreadCount": 67, + "Gen2Collections": 6 + } + ], + "HealthReportLatency": { + "Count": 440, + "MeanMs": 0.02325909090909091, + "P50Ms": 0.010154789466897974, + "P95Ms": 0.035668741878751194, + "P99Ms": 0.05500871985409165, + "P999Ms": 4.372032542909791, + "MaxMs": 4.343 + }, + "HealthReportsDelivered": 440, + "SitesTrackedByAggregator": 10, + "DebugSnapshotLatency": { + "Count": 264, + "MeanMs": 0.12096969696969696, + "P50Ms": 0.08483504325081287, + "P95Ms": 0.2200348794163666, + "P99Ms": 0.4595533115881782, + "P999Ms": 4.186672706411344, + "MaxMs": 4.101 + }, + "DebugSnapshotsCompleted": 264, + "DebugSnapshotTimeouts": 0, + "StreamProbeReceived": 899807, + "StreamProbeDropped": 0, + "StoreAndForwardDrain": { + "SiteId": "site-01", + "MessageCount": 20000, + "EnqueueSeconds": 1.170159, + "EnqueuePerSecond": 17091.694376576175, + "TimeToFirstDeliverySeconds": 29.0012197, + "DrainSeconds": 34.9398315, + "DrainPerSecond": 3367.7904321006486, + "ResidualDepth": 0, + "Progress": [ + { + "ElapsedSeconds": 0.1096947, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 0.226133, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 0.3480735, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 0.4748545, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 0.5989505, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 0.7208015, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 0.847525, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 0.9743501, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 1.0994346, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 1.2275051, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 1.3545158, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 1.4791523, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 1.6014682, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 1.7244499, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 1.850791, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 1.9768048, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 2.1011816, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 2.226466, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 2.3501744, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 2.4671824, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 2.585836, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 2.7097355, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 2.8346726, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 2.9628894, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 3.086988, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 3.20999, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 3.3367705, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 3.460584, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 3.5873738, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 3.7120374, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 3.8342123, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 3.9586048, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 4.0867452, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 4.2147997, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 4.3399957, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 4.4650582, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 4.5924552, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 4.7169289, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 4.8433912, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 4.9705741, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 5.0876133, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 5.2023642, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 5.3253996, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 5.4501215, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 5.5729115, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 5.698952, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 5.8192036, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 5.9441058, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 6.0712687, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 6.1979353, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 6.3235371, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 6.4487573, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 6.5736405, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 6.6981717, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 6.8213244, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 6.9455509, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 7.0703392, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 7.1980298, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 7.321838, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 7.4471535, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 7.5732241, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 7.698456, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 7.8267447, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 7.95131, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 8.078348, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 8.1998186, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 8.3251568, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 8.4488091, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 8.5747636, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 8.7013662, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 8.826747, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 8.9504586, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 9.0673325, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 9.1886873, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 9.3141902, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 9.4381652, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 9.5638477, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 9.6912336, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 9.8156615, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 9.9412244, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 10.064209, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 10.1790152, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 10.301836, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 10.4252977, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 10.5505925, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 10.6757274, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 10.8024896, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 10.9287267, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 11.0531436, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 11.1734113, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 11.3007087, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 11.4274334, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 11.5511937, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 11.6755146, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 11.8008965, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 11.9301654, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 12.0558283, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 12.1814477, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 12.3084497, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 12.4355369, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 12.5602009, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 12.6869343, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 12.8081354, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 12.9328172, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 13.057054, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 13.1824196, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 13.30859, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 13.4342607, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 13.5583517, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 13.6827145, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 13.8060124, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 13.9331465, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 14.0598465, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 14.1866256, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 14.313893, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 14.4405506, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 14.5668365, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 14.6889932, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 14.8119519, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 14.9377093, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 15.0595963, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 15.1881695, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 15.3035333, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 15.4200003, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 15.5453812, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 15.6698633, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 15.7953595, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 15.9196012, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 16.0477094, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 16.1722166, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 16.2908067, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 16.4087254, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 16.5332383, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 16.6609383, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 16.7841847, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 16.9113904, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 17.0373537, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 17.1630051, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 17.2885335, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 17.4123515, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 17.5397248, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 17.6648637, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 17.7936111, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 17.919238, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 18.0454113, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 18.1675797, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 18.289695, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 18.412259, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 18.5388729, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 18.6635497, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 18.7897611, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 18.9158502, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 19.0341875, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 19.1490106, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 19.2690384, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 19.396388, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 19.5233622, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 19.6481091, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 19.7732054, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 19.895955, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 20.0201672, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 20.1461514, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 20.2721012, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 20.3903628, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 20.509009, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 20.6321465, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 20.75824, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 20.8799895, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 21.0052414, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 21.1316291, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 21.2583003, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 21.384257, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 21.5078275, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 21.632625, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 21.7572226, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 21.8816319, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 22.0072149, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 22.1302087, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 22.2561388, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 22.3799331, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 22.507535, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 22.6338157, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 22.7616941, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 22.8872222, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 23.0096055, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 23.1350624, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 23.2590936, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 23.3814478, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 23.5089192, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 23.6358776, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 23.758388, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 23.8825856, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 24.0066247, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 24.1294127, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 24.2543117, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 24.3812327, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 24.5063874, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 24.6300947, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 24.7560639, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 24.8802847, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.0056838, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.1284791, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.2553501, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.3815007, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.4966467, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.6159793, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.7401271, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.8671629, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 25.9920755, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 26.1165302, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 26.2392446, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 26.365072, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 26.4905256, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 26.6134102, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 26.7378569, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 26.8630431, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 26.9872455, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 27.1116807, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 27.2347912, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 27.3592967, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 27.4814475, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 27.6002722, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 27.7261782, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 27.8507768, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 27.9755922, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 28.1012532, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 28.2225523, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 28.3458986, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 28.4689818, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 28.5952184, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 28.7180634, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 28.8451627, + "Delivered": 0, + "Depth": 20000 + }, + { + "ElapsedSeconds": 29.0012197, + "Delivered": 155, + "Depth": 19845 + }, + { + "ElapsedSeconds": 29.1581664, + "Delivered": 655, + "Depth": 19345 + }, + { + "ElapsedSeconds": 29.3038994, + "Delivered": 1155, + "Depth": 18845 + }, + { + "ElapsedSeconds": 29.4468802, + "Delivered": 1655, + "Depth": 18345 + }, + { + "ElapsedSeconds": 29.5950572, + "Delivered": 2155, + "Depth": 17845 + }, + { + "ElapsedSeconds": 29.7456235, + "Delivered": 2655, + "Depth": 17345 + }, + { + "ElapsedSeconds": 29.9041925, + "Delivered": 3155, + "Depth": 16845 + }, + { + "ElapsedSeconds": 30.0525223, + "Delivered": 3655, + "Depth": 16345 + }, + { + "ElapsedSeconds": 30.2025815, + "Delivered": 4155, + "Depth": 15845 + }, + { + "ElapsedSeconds": 30.351383, + "Delivered": 4655, + "Depth": 15345 + }, + { + "ElapsedSeconds": 30.4939692, + "Delivered": 5155, + "Depth": 14845 + }, + { + "ElapsedSeconds": 30.6301537, + "Delivered": 5655, + "Depth": 14345 + }, + { + "ElapsedSeconds": 30.7656217, + "Delivered": 6155, + "Depth": 13845 + }, + { + "ElapsedSeconds": 30.9162445, + "Delivered": 6655, + "Depth": 13345 + }, + { + "ElapsedSeconds": 31.0649217, + "Delivered": 7155, + "Depth": 12845 + }, + { + "ElapsedSeconds": 31.2155745, + "Delivered": 7655, + "Depth": 12345 + }, + { + "ElapsedSeconds": 31.3669169, + "Delivered": 8155, + "Depth": 11845 + }, + { + "ElapsedSeconds": 31.517637, + "Delivered": 8655, + "Depth": 11345 + }, + { + "ElapsedSeconds": 31.6695083, + "Delivered": 9155, + "Depth": 10845 + }, + { + "ElapsedSeconds": 31.8079279, + "Delivered": 9655, + "Depth": 10345 + }, + { + "ElapsedSeconds": 31.9532064, + "Delivered": 10155, + "Depth": 9845 + }, + { + "ElapsedSeconds": 32.1022951, + "Delivered": 10655, + "Depth": 9345 + }, + { + "ElapsedSeconds": 32.2500595, + "Delivered": 11155, + "Depth": 8845 + }, + { + "ElapsedSeconds": 32.4021763, + "Delivered": 11655, + "Depth": 8345 + }, + { + "ElapsedSeconds": 32.5505562, + "Delivered": 12155, + "Depth": 7845 + }, + { + "ElapsedSeconds": 32.7033974, + "Delivered": 12655, + "Depth": 7345 + }, + { + "ElapsedSeconds": 32.8507355, + "Delivered": 13155, + "Depth": 6845 + }, + { + "ElapsedSeconds": 32.9994114, + "Delivered": 13655, + "Depth": 6345 + }, + { + "ElapsedSeconds": 33.1449238, + "Delivered": 14155, + "Depth": 5845 + }, + { + "ElapsedSeconds": 33.2935937, + "Delivered": 14655, + "Depth": 5345 + }, + { + "ElapsedSeconds": 33.4485562, + "Delivered": 15155, + "Depth": 4845 + }, + { + "ElapsedSeconds": 33.5994162, + "Delivered": 15655, + "Depth": 4345 + }, + { + "ElapsedSeconds": 33.754945, + "Delivered": 16155, + "Depth": 3845 + }, + { + "ElapsedSeconds": 33.9059512, + "Delivered": 16655, + "Depth": 3345 + }, + { + "ElapsedSeconds": 34.0579469, + "Delivered": 17155, + "Depth": 2845 + }, + { + "ElapsedSeconds": 34.2043016, + "Delivered": 17655, + "Depth": 2345 + }, + { + "ElapsedSeconds": 34.3556321, + "Delivered": 18155, + "Depth": 1845 + }, + { + "ElapsedSeconds": 34.5045718, + "Delivered": 18655, + "Depth": 1345 + }, + { + "ElapsedSeconds": 34.6509902, + "Delivered": 19155, + "Depth": 845 + }, + { + "ElapsedSeconds": 34.8009321, + "Delivered": 19655, + "Depth": 345 + }, + { + "ElapsedSeconds": 34.9398306, + "Delivered": 20000, + "Depth": 0 + } + ] + }, + "SlowSubscriber": { + "PublishedEvents": 200000, + "PublishSeconds": 100.0906053, + "PublishPerSecond": 1998.1895343778083, + "Outcomes": [ + { + "Name": "site-02-slowprobe-00", + "IsSlow": true, + "Received": 2367, + "Dropped": 197028, + "DeliveryRatio": 0.011835 + }, + { + "Name": "site-02-slowprobe-01", + "IsSlow": false, + "Received": 200000, + "Dropped": 0, + "DeliveryRatio": 1 + }, + { + "Name": "site-02-slowprobe-02", + "IsSlow": false, + "Received": 200000, + "Dropped": 0, + "DeliveryRatio": 1 + }, + { + "Name": "site-02-slowprobe-03", + "IsSlow": false, + "Received": 200000, + "Dropped": 0, + "DeliveryRatio": 1 + }, + { + "Name": "site-02-slowprobe-04", + "IsSlow": false, + "Received": 200000, + "Dropped": 0, + "DeliveryRatio": 1 + } + ], + "HealthyMinDeliveryRatio": 1, + "SlowDeliveryRatio": 0.011835 + }, + "WireStream": { + "ProbeCount": 100, + "Latency": { + "Count": 899807, + "MeanMs": 27.996919231568548, + "P50Ms": 26.970383666845148, + "P95Ms": 32.07337215096543, + "P99Ms": 43.43554214441619, + "P999Ms": 53.940767333690296, + "MaxMs": 62.417 + }, + "EventsReceived": 899807, + "ServerDroppedEvents": 0, + "StreamFaults": 0, + "GracefulCompletions": 0, + "Batches": { + "Frames": 12462, + "Events": 899807, + "BatchedFrames": 11998, + "MeanEventsPerFrame": 72.20406034344407, + "P50": 75, + "P95": 75, + "P99": 75, + "Max": 75 + } + } +} \ No newline at end of file diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessConfig.cs b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessConfig.cs index 12c2b27b..c111fd6e 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessConfig.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessConfig.cs @@ -68,6 +68,19 @@ public sealed record HarnessConfig /// public int StreamProbesPerSite { get; init; } = 10; + /// + /// Instances per site additionally watched over a REAL gRPC hop: a per-site + /// Kestrel h2c listener hosts the production SiteStreamGrpcServer and the + /// production SiteStreamGrpcClient dials in with batching negotiated, so + /// the R2 SiteStreamEventBatcher, protobuf serialization and HTTP/2 flow + /// control are inside the measured path (they are NOT for + /// , whose final hop is deliberately an + /// in-process reader). 0 disables the wire path entirely. Like the in-process + /// probes, each wire subscriber's stream graph sees the full site event flow and + /// filters it, so this adds to the fan-out multiplier. + /// + public int WireProbesPerSite { get; init; } = 10; + /// Store-and-forward messages enqueued for the drain-rate measurement (register row 50). public int StoreAndForwardDrainMessages { get; init; } = 20_000; @@ -131,6 +144,7 @@ public sealed record HarnessConfig "health-interval-seconds" => config with { HealthReportInterval = TimeSpan.FromSeconds(double.Parse(value)) }, "debug-probe-interval-seconds" => config with { DebugProbeInterval = TimeSpan.FromSeconds(double.Parse(value)) }, "stream-probes-per-site" => config with { StreamProbesPerSite = int.Parse(value) }, + "wire-probes-per-site" => config with { WireProbesPerSite = int.Parse(value) }, "sf-drain-messages" => config with { StoreAndForwardDrainMessages = int.Parse(value) }, "slow-subscriber-events" => config with { SlowSubscriberEvents = int.Parse(value) }, "data-dir" => config with { DataDirectory = value }, diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessRun.cs b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessRun.cs index 9947a785..51f401e9 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessRun.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/HarnessRun.cs @@ -40,6 +40,10 @@ namespace ZB.MOM.WW.ScadaBridge.LoadHarness; /// Events evicted by live subscribers' bounded channels. /// Store-and-forward drain measurement (register row 50). /// Slow-subscriber isolation measurement (register row 50). +/// +/// Wire-path stream measurement — real Kestrel + real SiteStreamGrpcClient with +/// R2 batching negotiated — or null when is 0. +/// public sealed record HarnessRunResult( HarnessConfig Config, EnvironmentInfo Environment, @@ -67,7 +71,30 @@ public sealed record HarnessRunResult( long StreamProbeReceived, long StreamProbeDropped, StoreAndForwardDrainResult? StoreAndForwardDrain, - SlowSubscriberResult? SlowSubscriber); + SlowSubscriberResult? SlowSubscriber, + WireStreamResult? WireStream); + +/// +/// Steady-state-window measurement of the wire-path stream probes: the production +/// gRPC server (Kestrel h2c) and client with the R2 batcher, protobuf serialization +/// and HTTP/2 flow control INSIDE the measured path. All figures are deltas over the +/// steady-state window only. +/// +/// Wire subscribers attached across the fleet. +/// End-to-end DCL-boundary → post-unpack client callback latency. +/// Domain events delivered by the client in the window. +/// Events evicted by the wire servers' bounded send channels in the window. +/// Stream faults reported by the client (expected 0). +/// Graceful server-side stream completions (expected 0 inside the 4 h lifetime). +/// The frame/batch-size distribution the R2 batcher actually produced. +public sealed record WireStreamResult( + int ProbeCount, + LatencySnapshot Latency, + long EventsReceived, + long ServerDroppedEvents, + long StreamFaults, + long GracefulCompletions, + BatchSizeWindow Batches); /// Host and runtime facts recorded alongside the numbers. /// Host name. @@ -120,6 +147,7 @@ public static class HarnessRun ObservabilityProbes? probes = null; StoreAndForwardDrainResult? drainResult = null; SlowSubscriberResult? slowResult = null; + BatchSizeAggregator? batchAggregator = null; var aggregator = new CentralHealthAggregator( Options.Create(new HealthMonitoringOptions @@ -154,6 +182,18 @@ public static class HarnessRun site.AttachStreamProbes(latency); log($" {sites.Sum(s => s.Probes.Count)} live stream subscribers attached"); + // Wire path (real gRPC): the aggregator must exist BEFORE the first stream + // opens or the MeterListener misses early frames. + var wireLatency = new LatencyHistogram(); + if (config.WireProbesPerSite > 0) + { + batchAggregator = new BatchSizeAggregator(); + foreach (var site in sites) + await site.AttachWireStreamProbesAsync(config.WireProbesPerSite, wireLatency, log); + log($" {sites.Sum(s => s.WireSubscribers.Count)} wire stream subscribers attached " + + "(real Kestrel + SiteStreamGrpcClient, batching negotiated)"); + } + // Let Instance Actors complete their DCL subscribe round-trips before the // driver starts; an emit before SubscribeBatchAsync has captured the // callback would be silently discarded. @@ -184,6 +224,18 @@ public static class HarnessRun probe.RetargetLatency(steadyLatency); } + // Wire path: same ramp/steady histogram separation, plus window-start + // baselines for the client/server counters and the batch distribution. + var steadyWireLatency = new LatencyHistogram(); + var wireReceivedAtWindowStart = sites.Sum(s => s.WireSubscribers.Sum(p => p.ReceivedEvents)); + var wireDroppedAtWindowStart = sites.Sum(s => s.WireHost?.Server.DroppedStreamEventCount ?? 0); + var batchesAtWindowStart = batchAggregator?.Snapshot(); + foreach (var site in sites) + { + foreach (var wire in site.WireSubscribers) + wire.RetargetLatency(steadyWireLatency); + } + var windowWatch = Stopwatch.StartNew(); log($"Sustained measurement window: {config.SustainDuration.TotalMinutes:F1} min..."); await Task.Delay(config.SustainDuration, cancellationToken); @@ -199,6 +251,24 @@ public static class HarnessRun var probeReceived = sites.Sum(s => s.Probes.Sum(p => p.ReceivedEvents)) - receivedAtWindowStart; var probeDropped = sites.Sum(s => s.Probes.Sum(p => p.DroppedEvents)) - droppedAtWindowStart; + // Wire-path window deltas, captured BEFORE the row-50 scenarios below so + // their publishing cannot contaminate the wire figures. + WireStreamResult? wireResult = null; + if (batchAggregator is not null) + { + wireResult = new WireStreamResult( + ProbeCount: sites.Sum(s => s.WireSubscribers.Count), + Latency: steadyWireLatency.Snapshot(), + EventsReceived: sites.Sum(s => s.WireSubscribers.Sum(p => p.ReceivedEvents)) - wireReceivedAtWindowStart, + ServerDroppedEvents: sites.Sum(s => s.WireHost?.Server.DroppedStreamEventCount ?? 0) - wireDroppedAtWindowStart, + StreamFaults: sites.Sum(s => s.WireSubscribers.Sum(p => p.Faults)), + GracefulCompletions: sites.Sum(s => s.WireSubscribers.Sum(p => p.GracefulCompletions)), + Batches: BatchSizeWindow.From(batchesAtWindowStart!, batchAggregator.Snapshot())); + log($" wire path: {wireResult.EventsReceived:N0} events in {wireResult.Batches.Frames:N0} frames " + + $"(mean {wireResult.Batches.MeanEventsPerFrame:F1} ev/frame, max {wireResult.Batches.Max}), " + + $"P99 {wireResult.Latency.P99Ms:F2}ms"); + } + // Register row 50 — measured after the sustained window so the numbers are // not competing with the full tag load for CPU, and reported separately for // the same reason. @@ -250,12 +320,14 @@ public static class HarnessRun StreamProbeReceived: probeReceived, StreamProbeDropped: probeDropped, StoreAndForwardDrain: drainResult, - SlowSubscriber: slowResult); + SlowSubscriber: slowResult, + WireStream: wireResult); } finally { if (driver != null) await driver.DisposeAsync(); if (probes != null) await probes.DisposeAsync(); + batchAggregator?.Dispose(); await sampler.DisposeAsync(); log("Tearing down sites..."); diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Metrics/BatchSizeAggregator.cs b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Metrics/BatchSizeAggregator.cs new file mode 100644 index 00000000..243261a8 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Metrics/BatchSizeAggregator.cs @@ -0,0 +1,136 @@ +using System.Diagnostics.Metrics; +using ZB.MOM.WW.ScadaBridge.Commons.Observability; + +namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics; + +/// +/// Captures the production scadabridge.site.stream.batch_size histogram via a +/// , so a wire-path run can report the actual frame/batch +/// distribution the R2 batcher produced — not a client-side reconstruction. The +/// instrument is recorded once per emitted gRPC frame (negotiated streams only), with +/// the frame's event count as the value, which makes count = frames and +/// sum = events exactly. +/// +/// +/// The harness hosts every site's +/// in-process, so this listener sees every wire frame the run emits and nothing else — +/// no other component in the harness records this instrument. +/// +/// +public sealed class BatchSizeAggregator : IDisposable +{ + /// + /// Sizes above this are clamped into the last bucket. The production cap is + /// GrpcStreamBatchMaxEvents = 100, so 512 leaves generous headroom for a + /// configured-up run without an unbounded array. + /// + public const int MaxTrackedBatchSize = 512; + + private readonly MeterListener _listener; + private readonly long[] _framesBySize = new long[MaxTrackedBatchSize + 1]; + + /// Starts listening. Must be created BEFORE the first stream opens. + public BatchSizeAggregator() + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == ScadaBridgeTelemetry.MeterName && + instrument.Name == "scadabridge.site.stream.batch_size") + { + listener.EnableMeasurementEvents(instrument); + } + }, + }; + _listener.SetMeasurementEventCallback((_, value, _, _) => + { + var size = Math.Clamp(value, 1, MaxTrackedBatchSize); + Interlocked.Increment(ref _framesBySize[size]); + }); + _listener.Start(); + } + + /// Copies the cumulative per-size frame counts. + /// Index = events-per-frame (clamped), value = frames emitted at that size. + public long[] Snapshot() + { + var copy = new long[_framesBySize.Length]; + for (var i = 0; i < copy.Length; i++) + copy[i] = Interlocked.Read(ref _framesBySize[i]); + return copy; + } + + /// + public void Dispose() => _listener.Dispose(); +} + +/// +/// Batch-size distribution over one measurement window, computed from two +/// s. Percentiles are exact (integer sizes, +/// full counts — nothing is bucketed away). +/// +/// gRPC frames written in the window. +/// Events those frames carried. +/// Frames that carried more than one event. +/// Events / Frames. +/// Median events-per-frame. +/// 95th percentile events-per-frame. +/// 99th percentile events-per-frame. +/// Largest frame observed in the window. +public sealed record BatchSizeWindow( + long Frames, + long Events, + long BatchedFrames, + double MeanEventsPerFrame, + int P50, + int P95, + int P99, + int Max) +{ + /// Computes the window distribution from cumulative start/end snapshots. + /// Snapshot at window start. + /// Snapshot at window end. + /// The distribution of frames emitted between the two snapshots. + public static BatchSizeWindow From(long[] start, long[] end) + { + long frames = 0, events = 0, batched = 0; + var max = 0; + var delta = new long[end.Length]; + for (var size = 1; size < end.Length; size++) + { + var n = end[size] - start[size]; + if (n <= 0) continue; + delta[size] = n; + frames += n; + events += n * size; + if (size > 1) batched += n; + max = size; + } + + return new BatchSizeWindow( + frames, events, batched, + frames == 0 ? 0 : events / (double)frames, + Percentile(delta, frames, 50), + Percentile(delta, frames, 95), + Percentile(delta, frames, 99), + max); + } + + private static int Percentile(long[] delta, long frames, double percentile) + { + if (frames == 0) return 0; + var target = (long)Math.Ceiling(frames * percentile / 100.0); + if (target < 1) target = 1; + + long cumulative = 0; + for (var size = 1; size < delta.Length; size++) + { + cumulative += delta[size]; + if (cumulative >= target) + return size; + } + + return delta.Length - 1; + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Probes/SiteWireStreamHost.cs b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Probes/SiteWireStreamHost.cs new file mode 100644 index 00000000..bda5df43 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Probes/SiteWireStreamHost.cs @@ -0,0 +1,96 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.Hosting.Server.Features; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.ScadaBridge.Communication; +using ZB.MOM.WW.ScadaBridge.Communication.Grpc; + +namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Probes; + +/// +/// The site half of the wire-path stream probe: a real Kestrel HTTP/2 (h2c) listener on +/// a loopback ephemeral port, serving the PRODUCTION +/// over the site's real . +/// +/// +/// This is the hop deliberately substitutes with a +/// controllable in-process reader (which the slow-subscriber scenario needs), and +/// therefore the hop the original WP-4 runs never exercised: the +/// SiteStreamEventBatcher pump (R2), protobuf serialization, +/// IServerStreamWriter.WriteAsync, and HTTP/2 flow control. The server is built +/// through its production DI constructor with default , +/// so the batch negotiation (100 events / 25 ms window), channel capacities and the +/// concurrency cap are exactly what ships. +/// +/// +/// +/// Deliberately NOT in the path: ControlPlaneAuthInterceptor (it lives in the +/// Host project and authenticates once per CALL — one header compare per stream +/// lifetime, nothing on the per-event path) and TLS (production is h2c too). +/// +/// +public sealed class SiteWireStreamHost : IAsyncDisposable +{ + private readonly WebApplication _app; + + /// The hosted production gRPC server (drop counters, active stream count). + public SiteStreamGrpcServer Server { get; } + + /// The dialable endpoint, e.g. http://127.0.0.1:54321. + public string Endpoint { get; } + + private SiteWireStreamHost(WebApplication app, SiteStreamGrpcServer server, string endpoint) + { + _app = app; + Server = server; + Endpoint = endpoint; + } + + /// + /// Builds, starts and readies the listener for one site. + /// + /// The site whose stream manager and actor system back the server. + /// The started host. + public static async Task StartAsync(SiteRuntimeFixture site) + { + var server = new SiteStreamGrpcServer( + site.StreamManager, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + Options.Create(new CommunicationOptions())); + + var builder = WebApplication.CreateSlimBuilder(); + builder.Logging.ClearProviders(); + builder.WebHost.ConfigureKestrel(options => + options.Listen(System.Net.IPAddress.Loopback, 0, listenOptions => + listenOptions.Protocols = HttpProtocols.Http2)); + builder.Services.AddGrpc(); + builder.Services.AddSingleton(server); + + var app = builder.Build(); + app.MapGrpcService(); + await app.StartAsync(); + + // Same post-start readiness handoff the production host performs. + server.SetReady(site.System); + + var address = app.Services.GetRequiredService() + .Features.Get()?.Addresses.FirstOrDefault() + ?? throw new InvalidOperationException("Kestrel bound no address."); + + return new SiteWireStreamHost(app, server, address); + } + + /// + public async ValueTask DisposeAsync() + { + // Mirrors the production shutdown ordering: cancel active streams first so + // their pumps observe cancellation, then stop the listener. + Server.CancelAllStreams(); + await _app.StopAsync(); + await _app.DisposeAsync(); + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Probes/WireStreamSubscriber.cs b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Probes/WireStreamSubscriber.cs new file mode 100644 index 00000000..120d0389 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Probes/WireStreamSubscriber.cs @@ -0,0 +1,108 @@ +using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming; +using ZB.MOM.WW.ScadaBridge.Communication.Grpc; +using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics; + +namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Probes; + +/// +/// The central half of the wire-path stream probe: one instance subscription opened +/// through the PRODUCTION — the exact +/// SubscribeAsync path central's DebugStreamBridgeActor uses, including +/// the R2 batch negotiation (BatchingSupported = true) and the client-side +/// batch unpack + proto→domain conversion. +/// +/// +/// Latency is recorded in onEvent, i.e. AFTER wire deserialization, batch +/// unpacking and domain mapping — the same "DCL boundary → subscriber" measurement as +/// , extended across the real socket. The emit +/// instant travels verbatim on the proto Timestamp, so the subtraction stays a +/// genuine end-to-end figure (same process, same clock). +/// +/// +public sealed class WireStreamSubscriber +{ + private Task _streamTask = Task.CompletedTask; + private volatile LatencyHistogram? _latency; + private long _received; + private long _faults; + private long _completions; + + /// Probe name (also the stream's correlation id). + public string Name { get; } + + /// Domain events delivered by the client callback. + public long ReceivedEvents => Interlocked.Read(ref _received); + + /// Stream faults reported via onError. Expected 0 for a healthy run. + public long Faults => Interlocked.Read(ref _faults); + + /// Graceful server-side completions via onCompleted. Expected 0 inside a run shorter than the 4 h stream lifetime. + public long GracefulCompletions => Interlocked.Read(ref _completions); + + private WireStreamSubscriber(string name, LatencyHistogram? latency) + { + Name = name; + _latency = latency; + } + + /// + /// Opens one instance stream on the shared per-site client. The subscription task + /// runs until cancels — the harness deliberately does NOT + /// reconnect on fault (a fault inside a 25-minute loopback run is a finding, not + /// an operational event to paper over), it counts it. + /// + /// The site's shared production gRPC client (one channel per site, streams multiplexed). + /// Instance whose events this probe receives. + /// Probe/correlation name (must be a valid Akka path element). + /// Histogram fed with end-to-end event latency. + /// Progress sink for fault reporting. + /// Stops the subscription. + /// The attached subscriber. + public static WireStreamSubscriber Attach( + SiteStreamGrpcClient client, + string instanceUniqueName, + string name, + LatencyHistogram? latency, + Action log, + CancellationToken ct) + { + var self = new WireStreamSubscriber(name, latency); + + self._streamTask = client.SubscribeAsync( + correlationId: name, + instanceUniqueName: instanceUniqueName, + onEvent: evt => + { + Interlocked.Increment(ref self._received); + if (self._latency is { } histogram && evt is AttributeValueChanged changed) + histogram.Record(DateTimeOffset.UtcNow - changed.Timestamp); + }, + onError: ex => + { + Interlocked.Increment(ref self._faults); + log($" wire probe {name} stream fault: {ex.GetType().Name}: {ex.Message}"); + }, + onCompleted: () => Interlocked.Increment(ref self._completions), + ct); + + return self; + } + + /// Repoints the latency histogram (steady-state window separation) — mirrors . + /// The histogram to record into from now on, or null to stop recording. + public void RetargetLatency(LatencyHistogram? latency) => _latency = latency; + + /// Awaits the subscription task after its token has been cancelled. + /// A task that completes when the stream loop has exited. + public async Task WaitForShutdownAsync() + { + try + { + await _streamTask; + } + catch (OperationCanceledException) + { + // Normal teardown. + } + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Program.cs b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Program.cs index 30fc5e11..2f4b76e7 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Program.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Program.cs @@ -22,6 +22,7 @@ Console.WriteLine($" sites {config.Sites}"); Console.WriteLine($" instances/site {config.InstancesPerSite}"); Console.WriteLine($" tags/instance {config.TagsPerInstance}"); Console.WriteLine($" total subscriptions {config.TotalSubscriptions:N0}"); +Console.WriteLine($" stream probes/site {config.StreamProbesPerSite} in-process + {config.WireProbesPerSite} wire (real gRPC)"); Console.WriteLine($" nominal update rate {config.NominalUpdatesPerSecond:N0}/s"); Console.WriteLine($" settle / sustain {config.SettleDuration.TotalMinutes:F1} / {config.SustainDuration.TotalMinutes:F1} min"); Console.WriteLine(); diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ResultsFormatter.cs b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ResultsFormatter.cs index 5c9e40c3..cbc11bc1 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ResultsFormatter.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ResultsFormatter.cs @@ -47,6 +47,18 @@ public static class ResultsFormatter builder.AppendLine(culture, $"events dropped {result.StreamProbeDropped:N0}"); builder.AppendLine(); + if (result.WireStream is { } wire) + { + builder.AppendLine("-- wire stream subscribers (real gRPC, R2 batching in path) --"); + builder.AppendLine(culture, $"probes {wire.ProbeCount} (Kestrel h2c + SiteStreamGrpcClient, batching negotiated)"); + AppendLatency(builder, culture, wire.Latency); + builder.AppendLine(culture, $"events delivered {wire.EventsReceived:N0} in {wire.Batches.Frames:N0} frames"); + builder.AppendLine(culture, $"batching mean {wire.Batches.MeanEventsPerFrame:F2} ev/frame p50 {wire.Batches.P50} p95 {wire.Batches.P95} p99 {wire.Batches.P99} max {wire.Batches.Max} ({wire.Batches.BatchedFrames:N0} frames >1 event)"); + builder.AppendLine(culture, $"server channel drops {wire.ServerDroppedEvents:N0}"); + builder.AppendLine(culture, $"faults / completions {wire.StreamFaults:N0} / {wire.GracefulCompletions:N0}"); + builder.AppendLine(); + } + builder.AppendLine("-- health report delivery --"); AppendLatency(builder, culture, result.HealthReportLatency); builder.AppendLine(culture, $"reports ingested {result.HealthReportsDelivered:N0}"); diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/SiteRuntimeFixture.cs b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/SiteRuntimeFixture.cs index 649c0c9c..565db7b5 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/SiteRuntimeFixture.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/SiteRuntimeFixture.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection; +using ZB.MOM.WW.ScadaBridge.Communication.Grpc; using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening; using ZB.MOM.WW.ScadaBridge.DataConnectionLayer; using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors; @@ -43,7 +44,11 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable private readonly ILocalDb _localDb; private readonly List _instanceActors = new(); private readonly List _probes = new(); + private readonly List _wireSubscribers = new(); private readonly string _dataDirectory; + private SiteWireStreamHost? _wireHost; + private SiteStreamGrpcClient? _wireClient; + private CancellationTokenSource? _wireCts; /// Site identifier, e.g. site-01. public string SiteId { get; } @@ -72,6 +77,12 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable /// The live stream subscribers attached to this site. public IReadOnlyList Probes => _probes; + /// The wire-path (real gRPC) subscribers attached to this site, if any. + public IReadOnlyList WireSubscribers => _wireSubscribers; + + /// The site's real gRPC listener, when wire probes are attached. + public SiteWireStreamHost? WireHost => _wireHost; + /// Instance Actors created on this site. public IReadOnlyList InstanceActors => _instanceActors; @@ -344,12 +355,63 @@ public sealed class SiteRuntimeFixture : IAsyncDisposable } } + /// + /// Starts the site's real Kestrel/gRPC listener and attaches + /// subscribers through the production + /// — one channel per site with the streams + /// multiplexed on it, exactly the production shape (central keys one client per + /// (siteId, endpoint)). Instance selection uses the same stride as + /// , so the wire probes watch the SAME instances + /// as the in-process probes and the two latency distributions are directly + /// comparable within one run. + /// + /// Wire subscribers to attach. + /// Histogram that receives end-to-end (over-the-wire) latencies. + /// Progress sink for fault reporting. + /// + /// A task that completes when the listener is up and every stream has been + /// initiated. Stream ESTABLISHMENT finishes asynchronously — comfortably inside + /// the subscribe-settle + settle windows that separate attach from measurement. + /// + public async Task AttachWireStreamProbesAsync(int count, LatencyHistogram latency, Action log) + { + _wireHost = await SiteWireStreamHost.StartAsync(this); + _wireClient = new SiteStreamGrpcClient(_wireHost.Endpoint, NullLogger.Instance); + _wireCts = new CancellationTokenSource(); + + var stride = Math.Max(1, _config.InstancesPerSite / Math.Max(1, count)); + for (var p = 0; p < count; p++) + { + var instanceIndex = Math.Min(p * stride, _config.InstancesPerSite - 1); + var subscriber = WireStreamSubscriber.Attach( + _wireClient, InstanceName(instanceIndex), $"wire-{SiteId}-p{p:D2}", + latency, log, _wireCts.Token); + _wireSubscribers.Add(subscriber); + } + } + /// Total tag paths registered across this site's connections. public int TotalTagPaths => TagPathsByConnection.Sum(list => list.Count); /// public async ValueTask DisposeAsync() { + // Wire path first: cancel the client streams, wait for their loops to exit, + // then drop the channel and the listener — the same order production teardown + // takes (client goes away, server cancels, actor system last). + if (_wireCts is not null) + { + await _wireCts.CancelAsync(); + foreach (var subscriber in _wireSubscribers) + await subscriber.WaitForShutdownAsync(); + _wireCts.Dispose(); + } + + if (_wireClient is not null) + await _wireClient.DisposeAsync(); + if (_wireHost is not null) + await _wireHost.DisposeAsync(); + foreach (var probe in _probes) await probe.DisposeAsync(); diff --git a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ZB.MOM.WW.ScadaBridge.LoadHarness.csproj b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ZB.MOM.WW.ScadaBridge.LoadHarness.csproj index 480b3344..6b62d01a 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ZB.MOM.WW.ScadaBridge.LoadHarness.csproj +++ b/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/ZB.MOM.WW.ScadaBridge.LoadHarness.csproj @@ -27,6 +27,14 @@ true + + + + + + diff --git a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/TargetScale/TargetScaleHarnessSmokeTests.cs b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/TargetScale/TargetScaleHarnessSmokeTests.cs index b5dbb5fe..4d47f85f 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/TargetScale/TargetScaleHarnessSmokeTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.PerformanceTests/TargetScale/TargetScaleHarnessSmokeTests.cs @@ -49,6 +49,7 @@ public class TargetScaleHarnessSmokeTests DebugProbeInterval = TimeSpan.FromSeconds(2), SubscribeSettleDuration = TimeSpan.FromSeconds(5), StreamProbesPerSite = 3, + WireProbesPerSite = 2, StoreAndForwardDrainMessages = 200, SlowSubscriberEvents = 2_000, ResultsPath = Path.Combine(Path.GetTempPath(), $"loadharness-smoke-{Guid.NewGuid():N}.json"), @@ -64,6 +65,18 @@ public class TargetScaleHarnessSmokeTests Assert.True(result.StreamProbeReceived > 0, "Live stream subscribers received nothing."); Assert.Equal(0, result.DriverSkippedNoCallback); + // The wire path (real Kestrel + real SiteStreamGrpcClient, R2 batching + // negotiated) carried traffic end to end: events crossed a real HTTP/2 + // socket, the batch-size histogram observed real frames, and nothing faulted. + Assert.NotNull(result.WireStream); + Assert.True(result.WireStream!.EventsReceived > 0, + "Wire subscribers received nothing — the Kestrel/gRPC hop did not carry traffic."); + Assert.True(result.WireStream.Latency.Count > 0, + "No wire latency samples — the over-the-wire measurement path is broken."); + Assert.True(result.WireStream.Batches.Frames > 0, + "The batch-size MeterListener observed no frames — the R2 histogram path is broken."); + Assert.Equal(0, result.WireStream.StreamFaults); + // Observability held up. Assert.Equal(config.Sites, result.SitesTrackedByAggregator); Assert.True(result.DebugSnapshotsCompleted > 0, "No debug snapshot completed.");