test(loadharness): target-scale load harness for WP-4 / register #25 + row 50
Standalone console harness under tests/ZB.MOM.WW.ScadaBridge.LoadHarness plus a scaled-down Category=Performance smoke [Fact] in PerformanceTests. Deliberately an Exe rather than an xunit suite: the Performance trait enables a filter but does not exclude by default, so a 20-minute test would run on every 'dotnet test' of the slnx. What is real: per-site ActorSystem + LocalDb SQLite file, the real DCL (DataConnectionManagerActor/DataConnectionActor over a SimulatedDataConnection registered through the documented DataConnectionFactory.RegisterAdapter seam), real InstanceActors fed real TagValueUpdates, the real SiteStreamManager, real StreamRelayActor + production-capacity bounded DropOldest channel, real StoreAndForwardService/Storage, real SiteHealthCollector + CentralHealthAggregator. Only the socket hops are stood in for. Measures: end-to-end tag update latency (the emit instant rides TagValueUpdate.Timestamp verbatim to the subscriber), instance ramp, memory growth/CPU over a steady-state window, health report and debug view latency under load, S&F concurrent buffering + drain throughput, and slow-subscriber isolation.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,160 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Metrics;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
|
||||
|
||||
/// <summary>
|
||||
/// The two "does observability still work at scale?" probes WP-4's test protocol
|
||||
/// names — health report delivery timing and debug view latency — run continuously
|
||||
/// alongside the sustained load rather than after it, so both are measured against a
|
||||
/// site that is actually busy.
|
||||
/// </summary>
|
||||
public sealed class ObservabilityProbes : IAsyncDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private readonly List<Task> _tasks = new();
|
||||
|
||||
/// <summary>
|
||||
/// End-to-end health report latency: <c>SiteHealthCollector.CollectReport</c> plus
|
||||
/// the transport hop plus <c>CentralHealthAggregator.ProcessReport</c>. The
|
||||
/// interesting term at scale is <c>CollectReport</c>, which materializes the
|
||||
/// per-connection dictionaries for a site carrying 37,500 subscriptions.
|
||||
/// </summary>
|
||||
public LatencyHistogram HealthReportLatency { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Debug view snapshot round-trip: an <c>Ask</c> of <c>DebugSnapshotRequest</c> to
|
||||
/// a live Instance Actor. This lands the request in the mailbox of an actor that
|
||||
/// is concurrently ingesting tag updates, so the measured time includes real
|
||||
/// queueing behind production traffic — which is the whole point of measuring it
|
||||
/// under load.
|
||||
/// </summary>
|
||||
public LatencyHistogram DebugSnapshotLatency { get; } = new();
|
||||
|
||||
/// <summary>Health reports successfully ingested by the central aggregator.</summary>
|
||||
public long HealthReportsDelivered => Interlocked.Read(ref _healthReports);
|
||||
|
||||
/// <summary>Debug snapshots that completed within the ask timeout.</summary>
|
||||
public long DebugSnapshotsCompleted => Interlocked.Read(ref _debugSnapshots);
|
||||
|
||||
/// <summary>Debug snapshot asks that timed out.</summary>
|
||||
public long DebugSnapshotTimeouts => Interlocked.Read(ref _debugTimeouts);
|
||||
|
||||
private long _healthReports;
|
||||
private long _debugSnapshots;
|
||||
private long _debugTimeouts;
|
||||
|
||||
/// <summary>
|
||||
/// Starts both probes.
|
||||
/// </summary>
|
||||
/// <param name="sites">Sites to probe.</param>
|
||||
/// <param name="aggregator">The real central aggregator receiving the reports.</param>
|
||||
/// <param name="reportInterval">Health report cadence (production default 30 s).</param>
|
||||
/// <param name="debugProbeInterval">How often to take a debug snapshot.</param>
|
||||
/// <returns>The running probes.</returns>
|
||||
public static ObservabilityProbes Start(
|
||||
IReadOnlyList<SiteRuntimeFixture> sites,
|
||||
CentralHealthAggregator aggregator,
|
||||
TimeSpan reportInterval,
|
||||
TimeSpan debugProbeInterval)
|
||||
{
|
||||
var probes = new ObservabilityProbes();
|
||||
probes._tasks.Add(Task.Run(() => probes.HealthLoopAsync(sites, aggregator, reportInterval, probes._cts.Token)));
|
||||
probes._tasks.Add(Task.Run(() => probes.DebugLoopAsync(sites, debugProbeInterval, probes._cts.Token)));
|
||||
return probes;
|
||||
}
|
||||
|
||||
private async Task HealthLoopAsync(
|
||||
IReadOnlyList<SiteRuntimeFixture> sites,
|
||||
CentralHealthAggregator aggregator,
|
||||
TimeSpan interval,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken))
|
||||
{
|
||||
foreach (var site in sites)
|
||||
{
|
||||
var watch = Stopwatch.StartNew();
|
||||
var report = site.HealthCollector.CollectReport(site.SiteId);
|
||||
aggregator.ProcessReport(report);
|
||||
watch.Stop();
|
||||
|
||||
HealthReportLatency.Record(watch.Elapsed);
|
||||
Interlocked.Increment(ref _healthReports);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Normal teardown.
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DebugLoopAsync(
|
||||
IReadOnlyList<SiteRuntimeFixture> sites,
|
||||
TimeSpan interval,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var random = new Random(20260815);
|
||||
using var timer = new PeriodicTimer(interval);
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken))
|
||||
{
|
||||
var site = sites[random.Next(sites.Count)];
|
||||
if (site.InstanceActors.Count == 0)
|
||||
continue;
|
||||
|
||||
var index = random.Next(site.InstanceActors.Count);
|
||||
var actor = site.InstanceActors[index];
|
||||
var request = new DebugSnapshotRequest(site.InstanceName(index), Guid.NewGuid().ToString("N"));
|
||||
|
||||
var watch = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
await actor.Ask<DebugViewSnapshot>(request, TimeSpan.FromSeconds(10), cancellationToken);
|
||||
watch.Stop();
|
||||
DebugSnapshotLatency.Record(watch.Elapsed);
|
||||
Interlocked.Increment(ref _debugSnapshots);
|
||||
}
|
||||
catch (AskTimeoutException)
|
||||
{
|
||||
Interlocked.Increment(ref _debugTimeouts);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Normal teardown.
|
||||
}
|
||||
}
|
||||
|
||||
private int _disposed;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
// Idempotent: the orchestrator stops the probes early (so the register row 50
|
||||
// scenarios do not compete with them) and again in its finally block.
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
return;
|
||||
|
||||
await _cts.CancelAsync();
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(_tasks);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Expected.
|
||||
}
|
||||
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
using System.Diagnostics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.LoadHarness.Probes;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
|
||||
|
||||
/// <summary>Per-subscriber outcome from the slow-subscriber isolation probe.</summary>
|
||||
/// <param name="Name">Probe name.</param>
|
||||
/// <param name="IsSlow">Whether this probe's reader was deliberately stalled.</param>
|
||||
/// <param name="Received">Events the reader drained.</param>
|
||||
/// <param name="Dropped">Events evicted by this probe's bounded DropOldest channel.</param>
|
||||
/// <param name="DeliveryRatio">Received / published, before accounting for the site-stream buffer.</param>
|
||||
public sealed record SubscriberOutcome(
|
||||
string Name,
|
||||
bool IsSlow,
|
||||
long Received,
|
||||
long Dropped,
|
||||
double DeliveryRatio);
|
||||
|
||||
/// <summary>Result of the slow-subscriber isolation measurement.</summary>
|
||||
/// <param name="PublishedEvents">Events published to the site stream during the probe.</param>
|
||||
/// <param name="PublishSeconds">Wall time the publisher took.</param>
|
||||
/// <param name="PublishPerSecond">Publish throughput observed by the producer.</param>
|
||||
/// <param name="Outcomes">Per-subscriber outcomes.</param>
|
||||
/// <param name="HealthyMinDeliveryRatio">Worst delivery ratio among the healthy subscribers.</param>
|
||||
/// <param name="SlowDeliveryRatio">Delivery ratio of the stalled subscriber.</param>
|
||||
public sealed record SlowSubscriberResult(
|
||||
int PublishedEvents,
|
||||
double PublishSeconds,
|
||||
double PublishPerSecond,
|
||||
IReadOnlyList<SubscriberOutcome> Outcomes,
|
||||
double HealthyMinDeliveryRatio,
|
||||
double SlowDeliveryRatio);
|
||||
|
||||
/// <summary>
|
||||
/// Register row 50, second half: what does a slow or stalled gRPC subscriber do to
|
||||
/// per-subscriber buffering when several subscribers are attached?
|
||||
///
|
||||
/// <para>
|
||||
/// Several probes are attached to the SAME instance so every one of them is offered
|
||||
/// exactly the same event sequence — otherwise a difference in delivery could be a
|
||||
/// difference in offered load rather than a backpressure effect. One probe's reader
|
||||
/// is then stalled (a large per-event delay, standing in for a wedged client or a
|
||||
/// dead WAN link) while the rest read as fast as they can. Events are published
|
||||
/// through the real <c>SiteStreamManager</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The question the numbers answer: does the stalled subscriber's backlog propagate
|
||||
/// upstream — evicting events for the healthy subscribers or slowing the publisher —
|
||||
/// or is it confined to its own <c>Buffer(DropHead)</c> stage and its own bounded
|
||||
/// <c>DropOldest</c> channel? The design intends the latter; this measures it.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class SlowSubscriberScenario
|
||||
{
|
||||
/// <summary>Per-event reader delay applied to the stalled subscriber.</summary>
|
||||
public const int SlowReaderDelayMicroseconds = 50_000;
|
||||
|
||||
/// <summary>
|
||||
/// Publish rate for the probe. Deliberately paced rather than a tight burst: the
|
||||
/// publish source is a single <c>Source.ActorRef(StreamBufferSize, DropHead)</c>
|
||||
/// SHARED by every attribute subscriber, so an unpaced burst saturates that shared
|
||||
/// stage and every subscriber loses events for a reason that has nothing to do
|
||||
/// with the slow one. Pacing below the shared stage's capacity is what isolates
|
||||
/// the variable under test.
|
||||
/// </summary>
|
||||
public const int PublishRatePerSecond = 2_000;
|
||||
|
||||
/// <summary>Runs the isolation probe on a dedicated site.</summary>
|
||||
/// <param name="site">Site whose stream manager is used.</param>
|
||||
/// <param name="subscriberCount">Total subscribers to attach (one of them is stalled).</param>
|
||||
/// <param name="eventCount">Events to publish.</param>
|
||||
/// <param name="cancellationToken">Cancels the measurement.</param>
|
||||
/// <returns>The measured result.</returns>
|
||||
public static async Task<SlowSubscriberResult> RunAsync(
|
||||
SiteRuntimeFixture site,
|
||||
int subscriberCount,
|
||||
int eventCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var instanceName = site.InstanceName(0);
|
||||
var probes = new List<StreamSubscriberProbe>(subscriberCount);
|
||||
|
||||
try
|
||||
{
|
||||
for (var i = 0; i < subscriberCount; i++)
|
||||
{
|
||||
var probe = StreamSubscriberProbe.Attach(
|
||||
site.System, site.StreamManager, instanceName,
|
||||
$"{site.SiteId}-slowprobe-{i:D2}", latency: null);
|
||||
|
||||
// Probe 0 is the pathological one.
|
||||
if (i == 0)
|
||||
probe.ReaderDelayMicroseconds = SlowReaderDelayMicroseconds;
|
||||
|
||||
probes.Add(probe);
|
||||
}
|
||||
|
||||
// Let every subscription's stream graph finish materializing before the burst.
|
||||
await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken);
|
||||
|
||||
var watch = Stopwatch.StartNew();
|
||||
const int sliceMilliseconds = 50;
|
||||
var perSlice = Math.Max(1, PublishRatePerSecond * sliceMilliseconds / 1000);
|
||||
var published = 0;
|
||||
while (published < eventCount && !cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
var sliceStart = Stopwatch.GetTimestamp();
|
||||
var end = Math.Min(published + perSlice, eventCount);
|
||||
for (var i = published; i < end; i++)
|
||||
{
|
||||
site.StreamManager.PublishAttributeValueChanged(new AttributeValueChanged(
|
||||
instanceName, "Tag000", "Tag000", i, "Good", DateTimeOffset.UtcNow));
|
||||
}
|
||||
|
||||
published = end;
|
||||
|
||||
var elapsedMs = (Stopwatch.GetTimestamp() - sliceStart) * 1000.0 / Stopwatch.Frequency;
|
||||
if (elapsedMs < sliceMilliseconds)
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(sliceMilliseconds - elapsedMs), cancellationToken);
|
||||
}
|
||||
|
||||
watch.Stop();
|
||||
|
||||
// Give the healthy readers time to finish; the stalled one will not.
|
||||
await Task.Delay(TimeSpan.FromSeconds(20), cancellationToken);
|
||||
|
||||
var outcomes = probes
|
||||
.Select((p, i) => new SubscriberOutcome(
|
||||
p.Name,
|
||||
IsSlow: i == 0,
|
||||
p.ReceivedEvents,
|
||||
p.DroppedEvents,
|
||||
p.ReceivedEvents / (double)eventCount))
|
||||
.ToList();
|
||||
|
||||
var healthy = outcomes.Where(o => !o.IsSlow).ToList();
|
||||
|
||||
return new SlowSubscriberResult(
|
||||
PublishedEvents: eventCount,
|
||||
PublishSeconds: watch.Elapsed.TotalSeconds,
|
||||
PublishPerSecond: eventCount / Math.Max(0.001, watch.Elapsed.TotalSeconds),
|
||||
Outcomes: outcomes,
|
||||
HealthyMinDeliveryRatio: healthy.Count == 0 ? 0 : healthy.Min(o => o.DeliveryRatio),
|
||||
SlowDeliveryRatio: outcomes[0].DeliveryRatio);
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var probe in probes)
|
||||
await probe.DisposeAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
using System.Diagnostics;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;
|
||||
|
||||
/// <summary>Result of one store-and-forward drain measurement.</summary>
|
||||
/// <param name="SiteId">Site the buffer belonged to.</param>
|
||||
/// <param name="MessageCount">Messages buffered before the drain began.</param>
|
||||
/// <param name="EnqueueSeconds">Wall time to buffer them (concurrent, many origin instances).</param>
|
||||
/// <param name="EnqueuePerSecond">Buffering throughput.</param>
|
||||
/// <param name="TimeToFirstDeliverySeconds">
|
||||
/// Wall time from the first sweep to the first successful delivery. With
|
||||
/// <c>attemptImmediateDelivery: false</c> the engine stamps <c>LastAttemptAt</c>, so the
|
||||
/// row is not due until one <c>DefaultRetryInterval</c> (30 s) has passed — this is the
|
||||
/// configured retry latency, not drain slowness, and is reported separately for that reason.
|
||||
/// </param>
|
||||
/// <param name="DrainSeconds">Wall time from the first sweep to an empty buffer (includes the retry wait).</param>
|
||||
/// <param name="DrainPerSecond">
|
||||
/// Drain throughput measured from the FIRST delivery to an empty buffer — the engine's
|
||||
/// actual capacity, and the headline number for register row 50.
|
||||
/// </param>
|
||||
/// <param name="ResidualDepth">Buffer depth left when the measurement stopped (0 = fully drained).</param>
|
||||
/// <param name="Progress">Delivered-count samples during the drain, so a steady rate can be told from a stall-then-burst.</param>
|
||||
public sealed record StoreAndForwardDrainResult(
|
||||
string SiteId,
|
||||
int MessageCount,
|
||||
double EnqueueSeconds,
|
||||
double EnqueuePerSecond,
|
||||
double TimeToFirstDeliverySeconds,
|
||||
double DrainSeconds,
|
||||
double DrainPerSecond,
|
||||
int ResidualDepth,
|
||||
IReadOnlyList<DrainProgressSample> Progress);
|
||||
|
||||
/// <summary>One observation during the drain.</summary>
|
||||
/// <param name="ElapsedSeconds">Seconds since the drain began.</param>
|
||||
/// <param name="Delivered">Cumulative successful deliveries.</param>
|
||||
/// <param name="Depth">Remaining buffer depth.</param>
|
||||
public sealed record DrainProgressSample(double ElapsedSeconds, long Delivered, int Depth);
|
||||
|
||||
/// <summary>
|
||||
/// Measures store-and-forward buffering and drain throughput (deferred-work register
|
||||
/// row 50, first half) using the real <c>StoreAndForwardService</c>, the real
|
||||
/// <c>StoreAndForwardStorage</c> and the real SQLite file — only the delivery target
|
||||
/// is a counting stub, because what is being measured is the site-local buffer's
|
||||
/// throughput, not a remote endpoint's.
|
||||
///
|
||||
/// <para>
|
||||
/// Phase 1 buffers <paramref name="messageCount"/> messages with
|
||||
/// <c>attemptImmediateDelivery: false</c>, spread across many origin instance names
|
||||
/// and issued from many concurrent tasks — the "concurrent buffering from multiple
|
||||
/// instances" WP-4 asks about (<c>[xc-7]</c>). Phase 2 registers a delivery handler
|
||||
/// that always succeeds and drives sweeps to completion, timing the drain.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The sweep is driven explicitly rather than waiting on the 10 s
|
||||
/// <c>RetryTimerInterval</c> so the number reported is the engine's drain capacity,
|
||||
/// not its polling cadence. The per-sweep batch is <c>SweepBatchLimit</c> (500) with
|
||||
/// <c>SweepTargetParallelism</c> (4) lanes, both at their production defaults.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class StoreAndForwardDrainScenario
|
||||
{
|
||||
/// <summary>Runs the drain measurement against one site's real S&F engine.</summary>
|
||||
/// <param name="site">The site whose store-and-forward engine is exercised.</param>
|
||||
/// <param name="messageCount">Messages to buffer.</param>
|
||||
/// <param name="concurrency">Concurrent enqueue tasks (distinct origin instances).</param>
|
||||
/// <param name="cancellationToken">Cancels the measurement.</param>
|
||||
/// <returns>The measured result.</returns>
|
||||
public static async Task<StoreAndForwardDrainResult> RunAsync(
|
||||
SiteRuntimeFixture site,
|
||||
int messageCount,
|
||||
int concurrency,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var service = site.StoreAndForward;
|
||||
var payload = $"{{\"site\":\"{site.SiteId}\",\"body\":\"{new string('x', 256)}\"}}";
|
||||
|
||||
// Phase 1 — concurrent buffering from many instances, no delivery attempted.
|
||||
var enqueueWatch = Stopwatch.StartNew();
|
||||
var perTask = messageCount / concurrency;
|
||||
var enqueueTasks = new List<Task>(concurrency);
|
||||
for (var t = 0; t < concurrency; t++)
|
||||
{
|
||||
var taskIndex = t;
|
||||
enqueueTasks.Add(Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < perTask; i++)
|
||||
{
|
||||
await service.EnqueueAsync(
|
||||
StoreAndForwardCategory.ExternalSystem,
|
||||
target: $"load-target-{taskIndex % 4}",
|
||||
payloadJson: payload,
|
||||
originInstanceName: site.InstanceName(taskIndex),
|
||||
attemptImmediateDelivery: false);
|
||||
}
|
||||
}, cancellationToken));
|
||||
}
|
||||
|
||||
await Task.WhenAll(enqueueTasks);
|
||||
enqueueWatch.Stop();
|
||||
var buffered = perTask * concurrency;
|
||||
|
||||
// Phase 2 — a delivery target that always succeeds; time the drain to empty.
|
||||
var delivered = 0L;
|
||||
service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ =>
|
||||
{
|
||||
Interlocked.Increment(ref delivered);
|
||||
return Task.FromResult(true);
|
||||
});
|
||||
|
||||
// Sweeps are driven explicitly rather than waiting on the 10 s RetryTimerInterval:
|
||||
// the number wanted is the engine's drain CAPACITY, not its polling cadence. One
|
||||
// sweep moves at most SweepBatchLimit (500) messages, so a large backlog needs
|
||||
// many, and the progress series below is what distinguishes a genuinely slow
|
||||
// drain from an artefact of this polling loop.
|
||||
var drainWatch = Stopwatch.StartNew();
|
||||
var deadline = DateTimeOffset.UtcNow.AddMinutes(10);
|
||||
var progress = new List<DrainProgressSample>();
|
||||
int depth;
|
||||
while (true)
|
||||
{
|
||||
service.TriggerSweep();
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
|
||||
|
||||
var depths = await service.GetBufferDepthAsync();
|
||||
depth = depths.Values.Sum();
|
||||
progress.Add(new DrainProgressSample(
|
||||
drainWatch.Elapsed.TotalSeconds, Interlocked.Read(ref delivered), depth));
|
||||
|
||||
if (depth == 0 || DateTimeOffset.UtcNow > deadline)
|
||||
break;
|
||||
}
|
||||
|
||||
drainWatch.Stop();
|
||||
|
||||
// Split the retry wait from the drain: the first sample with a non-zero delivered
|
||||
// count marks the moment the backlog actually became due.
|
||||
var firstDelivery = progress.FirstOrDefault(s => s.Delivered > 0);
|
||||
var timeToFirstDelivery = firstDelivery?.ElapsedSeconds ?? drainWatch.Elapsed.TotalSeconds;
|
||||
var activeDrainSeconds = Math.Max(0.001, drainWatch.Elapsed.TotalSeconds - timeToFirstDelivery);
|
||||
|
||||
return new StoreAndForwardDrainResult(
|
||||
SiteId: site.SiteId,
|
||||
MessageCount: buffered,
|
||||
EnqueueSeconds: enqueueWatch.Elapsed.TotalSeconds,
|
||||
EnqueuePerSecond: buffered / Math.Max(0.001, enqueueWatch.Elapsed.TotalSeconds),
|
||||
TimeToFirstDeliverySeconds: timeToFirstDelivery,
|
||||
DrainSeconds: drainWatch.Elapsed.TotalSeconds,
|
||||
DrainPerSecond: Interlocked.Read(ref delivered) / activeDrainSeconds,
|
||||
ResidualDepth: depth,
|
||||
Progress: progress);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user