ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.LoadHarness/Scenarios/StoreAndForwardDrainScenario.cs

using System.Diagnostics;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;

namespace ZB.MOM.WW.ScadaBridge.LoadHarness.Scenarios;

/// <summary>Result of one store-and-forward drain measurement.</summary>
/// <param name="SiteId">Site the buffer belonged to.</param>
/// <param name="MessageCount">Messages buffered before the drain began.</param>
/// <param name="EnqueueSeconds">Wall time to buffer them (concurrent, many origin instances).</param>
/// <param name="EnqueuePerSecond">Buffering throughput.</param>
/// <param name="TimeToFirstDeliverySeconds">
/// Wall time from the first sweep to the first successful delivery. With
/// <c>attemptImmediateDelivery: false</c> the engine stamps <c>LastAttemptAt</c>, so the
/// row is not due until one <c>DefaultRetryInterval</c> (30 s) has passed — this is the
/// configured retry latency, not drain slowness, and is reported separately for that reason.
/// </param>
/// <param name="DrainSeconds">Wall time from the first sweep to an empty buffer (includes the retry wait).</param>
/// <param name="DrainPerSecond">
/// Drain throughput measured from the FIRST delivery to an empty buffer — the engine's
/// actual capacity, and the headline number for register row 50.
/// </param>
/// <param name="ResidualDepth">Buffer depth left when the measurement stopped (0 = fully drained).</param>
/// <param name="Progress">Delivered-count samples during the drain, so a steady rate can be told from a stall-then-burst.</param>
public sealed record StoreAndForwardDrainResult(
    string SiteId,
    int MessageCount,
    double EnqueueSeconds,
    double EnqueuePerSecond,
    double TimeToFirstDeliverySeconds,
    double DrainSeconds,
    double DrainPerSecond,
    int ResidualDepth,
    IReadOnlyList<DrainProgressSample> Progress);

/// <summary>One observation during the drain.</summary>
/// <param name="ElapsedSeconds">Seconds since the drain began.</param>
/// <param name="Delivered">Cumulative successful deliveries.</param>
/// <param name="Depth">Remaining buffer depth.</param>
public sealed record DrainProgressSample(double ElapsedSeconds, long Delivered, int Depth);

/// <summary>
/// Measures store-and-forward buffering and drain throughput (deferred-work register
/// row 50, first half) using the real <c>StoreAndForwardService</c>, the real
/// <c>StoreAndForwardStorage</c> and the real SQLite file — only the delivery target
/// is a counting stub, because what is being measured is the site-local buffer's
/// throughput, not a remote endpoint's.
///
/// <para>
/// Phase 1 buffers <paramref name="messageCount"/> messages with
/// <c>attemptImmediateDelivery: false</c>, spread across many origin instance names
/// and issued from many concurrent tasks — the "concurrent buffering from multiple
/// instances" WP-4 asks about (<c>[xc-7]</c>). Phase 2 registers a delivery handler
/// that always succeeds and drives sweeps to completion, timing the drain.
/// </para>
/// <para>
/// The sweep is driven explicitly rather than waiting on the 10 s
/// <c>RetryTimerInterval</c> so the number reported is the engine's drain capacity,
/// not its polling cadence. The per-sweep batch is <c>SweepBatchLimit</c> (500) with
/// <c>SweepTargetParallelism</c> (4) lanes, both at their production defaults.
/// </para>
/// </summary>
public static class StoreAndForwardDrainScenario
{
    /// <summary>Runs the drain measurement against one site's real S&amp;F engine.</summary>
    /// <param name="site">The site whose store-and-forward engine is exercised.</param>
    /// <param name="messageCount">Messages to buffer.</param>
    /// <param name="concurrency">Concurrent enqueue tasks (distinct origin instances).</param>
    /// <param name="cancellationToken">Cancels the measurement.</param>
    /// <returns>The measured result.</returns>
    public static async Task<StoreAndForwardDrainResult> RunAsync(
        SiteRuntimeFixture site,
        int messageCount,
        int concurrency,
        CancellationToken cancellationToken)
    {
        var service = site.StoreAndForward;
        var payload = $"{{\"site\":\"{site.SiteId}\",\"body\":\"{new string('x', 256)}\"}}";

        // Phase 1 — concurrent buffering from many instances, no delivery attempted.
        var enqueueWatch = Stopwatch.StartNew();
        var perTask = messageCount / concurrency;
        var enqueueTasks = new List<Task>(concurrency);
        for (var t = 0; t < concurrency; t++)
        {
            var taskIndex = t;
            enqueueTasks.Add(Task.Run(async () =>
            {
                for (var i = 0; i < perTask; i++)
                {
                    await service.EnqueueAsync(
                        StoreAndForwardCategory.ExternalSystem,
                        target: $"load-target-{taskIndex % 4}",
                        payloadJson: payload,
                        originInstanceName: site.InstanceName(taskIndex),
                        attemptImmediateDelivery: false);
                }
            }, cancellationToken));
        }

        await Task.WhenAll(enqueueTasks);
        enqueueWatch.Stop();
        var buffered = perTask * concurrency;

        // Phase 2 — a delivery target that always succeeds; time the drain to empty.
        var delivered = 0L;
        service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem, _ =>
        {
            Interlocked.Increment(ref delivered);
            return Task.FromResult(true);
        });

        // Sweeps are driven explicitly rather than waiting on the 10 s RetryTimerInterval:
        // the number wanted is the engine's drain CAPACITY, not its polling cadence. One
        // sweep moves at most SweepBatchLimit (500) messages, so a large backlog needs
        // many, and the progress series below is what distinguishes a genuinely slow
        // drain from an artefact of this polling loop.
        var drainWatch = Stopwatch.StartNew();
        var deadline = DateTimeOffset.UtcNow.AddMinutes(10);
        var progress = new List<DrainProgressSample>();
        int depth;
        while (true)
        {
            service.TriggerSweep();
            await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);

            var depths = await service.GetBufferDepthAsync();
            depth = depths.Values.Sum();
            progress.Add(new DrainProgressSample(
                drainWatch.Elapsed.TotalSeconds, Interlocked.Read(ref delivered), depth));

            if (depth == 0 || DateTimeOffset.UtcNow > deadline)
                break;
        }

        drainWatch.Stop();

        // Split the retry wait from the drain: the first sample with a non-zero delivered
        // count marks the moment the backlog actually became due.
        var firstDelivery = progress.FirstOrDefault(s => s.Delivered > 0);
        var timeToFirstDelivery = firstDelivery?.ElapsedSeconds ?? drainWatch.Elapsed.TotalSeconds;
        var activeDrainSeconds = Math.Max(0.001, drainWatch.Elapsed.TotalSeconds - timeToFirstDelivery);

        return new StoreAndForwardDrainResult(
            SiteId: site.SiteId,
            MessageCount: buffered,
            EnqueueSeconds: enqueueWatch.Elapsed.TotalSeconds,
            EnqueuePerSecond: buffered / Math.Max(0.001, enqueueWatch.Elapsed.TotalSeconds),
            TimeToFirstDeliverySeconds: timeToFirstDelivery,
            DrainSeconds: drainWatch.Elapsed.TotalSeconds,
            DrainPerSecond: Interlocked.Read(ref delivered) / activeDrainSeconds,
            ResidualDepth: depth,
            Progress: progress);
    }
}