using System.Diagnostics; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming; using ZB.MOM.WW.ScadaBridge.LoadHarness.Probes; namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios; /// Per-subscriber outcome from the slow-subscriber isolation probe. /// Probe name. /// Whether this probe's reader was deliberately stalled. /// Events the reader drained. /// Events evicted by this probe's bounded DropOldest channel. /// Received / published, before accounting for the site-stream buffer. public sealed record SubscriberOutcome( string Name, bool IsSlow, long Received, long Dropped, double DeliveryRatio); /// Result of the slow-subscriber isolation measurement. /// Events published to the site stream during the probe. /// Wall time the publisher took. /// Publish throughput observed by the producer. /// Per-subscriber outcomes. /// Worst delivery ratio among the healthy subscribers. /// Delivery ratio of the stalled subscriber. public sealed record SlowSubscriberResult( int PublishedEvents, double PublishSeconds, double PublishPerSecond, IReadOnlyList Outcomes, double HealthyMinDeliveryRatio, double SlowDeliveryRatio); /// /// Register row 50, second half: what does a slow or stalled gRPC subscriber do to /// per-subscriber buffering when several subscribers are attached? /// /// /// Several probes are attached to the SAME instance so every one of them is offered /// exactly the same event sequence — otherwise a difference in delivery could be a /// difference in offered load rather than a backpressure effect. One probe's reader /// is then stalled (a large per-event delay, standing in for a wedged client or a /// dead WAN link) while the rest read as fast as they can. Events are published /// through the real SiteStreamManager. /// /// /// The question the numbers answer: does the stalled subscriber's backlog propagate /// upstream — evicting events for the healthy subscribers or slowing the publisher — /// or is it confined to its own Buffer(DropHead) stage and its own bounded /// DropOldest channel? The design intends the latter; this measures it. /// /// public static class SlowSubscriberScenario { /// Per-event reader delay applied to the stalled subscriber. public const int SlowReaderDelayMicroseconds = 50_000; /// /// Publish rate for the probe. Deliberately paced rather than a tight burst: the /// publish source is a single Source.ActorRef(StreamBufferSize, DropHead) /// SHARED by every attribute subscriber, so an unpaced burst saturates that shared /// stage and every subscriber loses events for a reason that has nothing to do /// with the slow one. Pacing below the shared stage's capacity is what isolates /// the variable under test. /// public const int PublishRatePerSecond = 2_000; /// Runs the isolation probe on a dedicated site. /// Site whose stream manager is used. /// Total subscribers to attach (one of them is stalled). /// Events to publish. /// Cancels the measurement. /// The measured result. public static async Task RunAsync( SiteRuntimeFixture site, int subscriberCount, int eventCount, CancellationToken cancellationToken) { var instanceName = site.InstanceName(0); var probes = new List(subscriberCount); try { for (var i = 0; i < subscriberCount; i++) { var probe = StreamSubscriberProbe.Attach( site.System, site.StreamManager, instanceName, $"{site.SiteId}-slowprobe-{i:D2}", latency: null); // Probe 0 is the pathological one. if (i == 0) probe.ReaderDelayMicroseconds = SlowReaderDelayMicroseconds; probes.Add(probe); } // Let every subscription's stream graph finish materializing before the burst. await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); var watch = Stopwatch.StartNew(); const int sliceMilliseconds = 50; var perSlice = Math.Max(1, PublishRatePerSecond * sliceMilliseconds / 1000); var published = 0; while (published < eventCount && !cancellationToken.IsCancellationRequested) { var sliceStart = Stopwatch.GetTimestamp(); var end = Math.Min(published + perSlice, eventCount); for (var i = published; i < end; i++) { site.StreamManager.PublishAttributeValueChanged(new AttributeValueChanged( instanceName, "Tag000", "Tag000", i, "Good", DateTimeOffset.UtcNow)); } published = end; var elapsedMs = (Stopwatch.GetTimestamp() - sliceStart) * 1000.0 / Stopwatch.Frequency; if (elapsedMs < sliceMilliseconds) await Task.Delay(TimeSpan.FromMilliseconds(sliceMilliseconds - elapsedMs), cancellationToken); } watch.Stop(); // Give the healthy readers time to finish; the stalled one will not. await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken); var outcomes = probes .Select((p, i) => new SubscriberOutcome( p.Name, IsSlow: i == 0, p.ReceivedEvents, p.DroppedEvents, p.ReceivedEvents / (double)eventCount)) .ToList(); var healthy = outcomes.Where(o => !o.IsSlow).ToList(); return new SlowSubscriberResult( PublishedEvents: eventCount, PublishSeconds: watch.Elapsed.TotalSeconds, PublishPerSecond: eventCount / Math.Max(0.001, watch.Elapsed.TotalSeconds), Outcomes: outcomes, HealthyMinDeliveryRatio: healthy.Count == 0 ? 0 : healthy.Min(o => o.DeliveryRatio), SlowDeliveryRatio: outcomes[0].DeliveryRatio); } finally { foreach (var probe in probes) await probe.DisposeAsync(); } } }