using System.Diagnostics;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.LoadHarness.Probes;
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
/// Per-subscriber outcome from the slow-subscriber isolation probe.
/// Probe name.
/// Whether this probe's reader was deliberately stalled.
/// Events the reader drained.
/// Events evicted by this probe's bounded DropOldest channel.
/// Received / published, before accounting for the site-stream buffer.
public sealed record SubscriberOutcome(
string Name,
bool IsSlow,
long Received,
long Dropped,
double DeliveryRatio);
/// Result of the slow-subscriber isolation measurement.
/// Events published to the site stream during the probe.
/// Wall time the publisher took.
/// Publish throughput observed by the producer.
/// Per-subscriber outcomes.
/// Worst delivery ratio among the healthy subscribers.
/// Delivery ratio of the stalled subscriber.
public sealed record SlowSubscriberResult(
int PublishedEvents,
double PublishSeconds,
double PublishPerSecond,
IReadOnlyList Outcomes,
double HealthyMinDeliveryRatio,
double SlowDeliveryRatio);
///
/// Register row 50, second half: what does a slow or stalled gRPC subscriber do to
/// per-subscriber buffering when several subscribers are attached?
///
///
/// Several probes are attached to the SAME instance so every one of them is offered
/// exactly the same event sequence — otherwise a difference in delivery could be a
/// difference in offered load rather than a backpressure effect. One probe's reader
/// is then stalled (a large per-event delay, standing in for a wedged client or a
/// dead WAN link) while the rest read as fast as they can. Events are published
/// through the real SiteStreamManager.
///
///
/// The question the numbers answer: does the stalled subscriber's backlog propagate
/// upstream — evicting events for the healthy subscribers or slowing the publisher —
/// or is it confined to its own Buffer(DropHead) stage and its own bounded
/// DropOldest channel? The design intends the latter; this measures it.
///
///
public static class SlowSubscriberScenario
{
/// Per-event reader delay applied to the stalled subscriber.
public const int SlowReaderDelayMicroseconds = 50_000;
///
/// Publish rate for the probe. Deliberately paced rather than a tight burst: the
/// publish source is a single Source.ActorRef(StreamBufferSize, DropHead)
/// SHARED by every attribute subscriber, so an unpaced burst saturates that shared
/// stage and every subscriber loses events for a reason that has nothing to do
/// with the slow one. Pacing below the shared stage's capacity is what isolates
/// the variable under test.
///
public const int PublishRatePerSecond = 2_000;
/// Runs the isolation probe on a dedicated site.
/// Site whose stream manager is used.
/// Total subscribers to attach (one of them is stalled).
/// Events to publish.
/// Cancels the measurement.
/// The measured result.
public static async Task RunAsync(
SiteRuntimeFixture site,
int subscriberCount,
int eventCount,
CancellationToken cancellationToken)
{
var instanceName = site.InstanceName(0);
var probes = new List(subscriberCount);
try
{
for (var i = 0; i < subscriberCount; i++)
{
var probe = StreamSubscriberProbe.Attach(
site.System, site.StreamManager, instanceName,
$"{site.SiteId}-slowprobe-{i:D2}", latency: null);
// Probe 0 is the pathological one.
if (i == 0)
probe.ReaderDelayMicroseconds = SlowReaderDelayMicroseconds;
probes.Add(probe);
}
// Let every subscription's stream graph finish materializing before the burst.
await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken);
var watch = Stopwatch.StartNew();
const int sliceMilliseconds = 50;
var perSlice = Math.Max(1, PublishRatePerSecond * sliceMilliseconds / 1000);
var published = 0;
while (published < eventCount && !cancellationToken.IsCancellationRequested)
{
var sliceStart = Stopwatch.GetTimestamp();
var end = Math.Min(published + perSlice, eventCount);
for (var i = published; i < end; i++)
{
site.StreamManager.PublishAttributeValueChanged(new AttributeValueChanged(
instanceName, "Tag000", "Tag000", i, "Good", DateTimeOffset.UtcNow));
}
published = end;
var elapsedMs = (Stopwatch.GetTimestamp() - sliceStart) * 1000.0 / Stopwatch.Frequency;
if (elapsedMs < sliceMilliseconds)
await Task.Delay(TimeSpan.FromMilliseconds(sliceMilliseconds - elapsedMs), cancellationToken);
}
watch.Stop();
// Give the healthy readers time to finish; the stalled one will not.
await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken);
var outcomes = probes
.Select((p, i) => new SubscriberOutcome(
p.Name,
IsSlow: i == 0,
p.ReceivedEvents,
p.DroppedEvents,
p.ReceivedEvents / (double)eventCount))
.ToList();
var healthy = outcomes.Where(o => !o.IsSlow).ToList();
return new SlowSubscriberResult(
PublishedEvents: eventCount,
PublishSeconds: watch.Elapsed.TotalSeconds,
PublishPerSecond: eventCount / Math.Max(0.001, watch.Elapsed.TotalSeconds),
Outcomes: outcomes,
HealthyMinDeliveryRatio: healthy.Count == 0 ? 0 : healthy.Min(o => o.DeliveryRatio),
SlowDeliveryRatio: outcomes[0].DeliveryRatio);
}
finally
{
foreach (var probe in probes)
await probe.DisposeAsync();
}
}
}