using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Observability;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;

namespace ZB.MOM.WW.ScadaBridge.StoreAndForward;

/// <summary>
/// WP-9/10: Core store-and-forward service.
///
/// Lifecycle:
///   1. Caller attempts immediate delivery via IDeliveryHandler
///   2. On transient failure → buffer in SQLite → retry loop
///   3. On success → remove from buffer
///   4. On reaching MaxRetries → park (a MaxRetries of 0 means "no limit" — the
///      message is retried until delivered and is never parked for retry exhaustion)
///   5. Permanent failures are returned to caller immediately (never buffered)
///
/// WP-10: Fixed retry interval (not exponential). Per-source-entity retry settings.
///        Background timer-based retry sweep.
///
/// WP-12: Parked messages queryable, retryable, and discardable.
///
/// WP-14: Buffer depth reported as health metric. Activity logged to site event log.
///
/// WP-15: CachedCall idempotency is the caller's responsibility.
///        This service does not deduplicate — if the same message is enqueued twice,
///        it will be delivered twice. Callers using ExternalSystem.CachedCall() must
///        design their payloads to be idempotent (e.g., include unique request IDs
///        and handle duplicate detection on the remote end).
/// </summary>
public class StoreAndForwardService
{
    private readonly StoreAndForwardStorage _storage;
    private readonly StoreAndForwardOptions _options;
    private readonly ReplicationService? _replication;
    private readonly ILogger<StoreAndForwardService> _logger;
    /// <summary>
    /// Audit Log #23 (M3 Bundle E — Task E4): site-side observer notified
    /// after every cached-call delivery attempt. Optional — when null no
    /// telemetry is emitted; the legacy pre-M3 retry loop behaviour is
    /// preserved exactly.
    /// </summary>
    private readonly ICachedCallLifecycleObserver? _cachedCallObserver;
    /// <summary>
    /// Audit Log #23 (M3 Bundle E — Task E4): site id stamped onto the
    /// cached-call attempt context so the audit bridge can build the
    /// <see cref="SiteCallOperational"/> half of the telemetry packet.
    /// <para>
    /// <b>StoreAndForward-023:</b> an empty-string site id must never reach
    /// downstream consumers — the central audit pipeline keys
    /// <c>(SourceSite, TrackedOperationId)</c> off this value, so an empty
    /// string degrades correlation to a per-id-only index and breaks the
    /// per-site routing of <c>RetryParkedOperation</c>/<c>DiscardParkedOperation</c>
    /// commands. The constructor normalises a null/empty/whitespace
    /// <paramref name="siteId"/> argument to <see cref="UnknownSiteSentinel"/>
    /// so a misconfigured host (no <c>IStoreAndForwardSiteContext</c>
    /// registered) produces a distinctive marker in the central audit log
    /// rather than silently merging multiple sites into the empty bucket.
    /// </para>
    /// </summary>
    private readonly string _siteId;

    /// <summary>
    /// StoreAndForward-023: distinctive marker stamped onto cached-call audit
    /// telemetry when the host has not registered an
    /// <see cref="IStoreAndForwardSiteContext"/>. Chosen with a leading <c>$</c>
    /// so it cannot collide with a real site id (which is a configuration
    /// identifier and never starts with <c>$</c>). Surfacing this in the
    /// central audit log makes a missing site-context binding immediately
    /// recognisable instead of an unattributable empty string.
    /// </summary>
    public const string UnknownSiteSentinel = "$unknown-site";
    private Timer? _retryTimer;
    private int _retryInProgress;

    /// <summary>
    /// StoreAndForward-024: the in-flight retry sweep <see cref="Task"/>, or
    /// <c>null</c> when no sweep is currently running. Captured when the timer
    /// callback starts a sweep so <see cref="StopAsync"/> can wait for it to
    /// finish before the host disposes downstream dependencies
    /// (<see cref="_storage"/>, <see cref="_replication"/>) that the sweep is
    /// still touching. Written from the timer thread and from
    /// <see cref="StopAsync"/>, so reads are synchronised via the
    /// <see cref="Volatile"/> APIs.
    /// </summary>
    private Task? _sweepTask;

    /// <summary>
    /// StoreAndForward-024: how long <see cref="StopAsync"/> waits for an
    /// in-flight retry sweep to finish before returning. The default — 10 s —
    /// is generous enough to let a typical sweep over the buffered queue drain,
    /// but bounded so a hung downstream call (a stuck SQLite write, a
    /// long-running delivery handler) cannot block host shutdown indefinitely.
    /// On timeout the wait is abandoned and the timer is still disposed; the
    /// sweep keeps running but will throw on the next call into a disposed
    /// dependency — preferred to blocking shutdown forever.
    /// </summary>
    private static readonly TimeSpan SweepShutdownWaitTimeout = TimeSpan.FromSeconds(10);

    /// <summary>
    /// WP-14 (telemetry): cached count of messages currently buffered for
    /// forwarding — i.e. rows in <see cref="StoreAndForwardMessageStatus.Pending"/>,
    /// the live store-and-forward queue waiting to be delivered. This backs the
    /// <c>scadabridge.store_and_forward.queue.depth</c> observable gauge.
    /// <para>
    /// The gauge's collection callback is synchronous and is invoked frequently by
    /// the OpenTelemetry/Prometheus collector, so it must never run an async SQLite
    /// <c>COUNT(*)</c>. Instead this <see cref="long"/> is seeded once from storage
    /// in <see cref="StartAsync"/> and then adjusted in-process on the existing
    /// paths that change the Pending population: <see cref="BufferAsync"/> (+1),
    /// successful-retry removal and Pending→Parked transitions in
    /// <see cref="RetryMessageAsync"/> (-1), and operator requeue in
    /// <see cref="RetryParkedMessageAsync"/> (+1). The provider registered with
    /// <see cref="ScadaBridgeTelemetry.SetQueueDepthProvider"/> reads it via
    /// <see cref="Interlocked.Read"/> — non-blocking and sync-safe. It is an
    /// approximate, eventually-consistent gauge (concurrent failover replication
    /// applies to the standby's own store, not this counter), which is exactly
    /// what a queue-depth metric needs.
    /// </para>
    /// </summary>
    private long _bufferedCount;

    /// <summary>
    /// Test seam (WP-14 telemetry): simulates a concurrent pre-seed
    /// <see cref="BufferAsync"/> increment landing on <see cref="_bufferedCount"/>
    /// before <see cref="StartAsync"/> seeds it, so a test can prove the seed uses
    /// <see cref="Interlocked.Add"/> (additive) rather than Exchange (clobbering).
    /// </summary>
    internal void TestOnly_IncrementBufferedCount() =>
        Interlocked.Increment(ref _bufferedCount);

    /// <summary>
    /// WP-14 (telemetry): an instance field that guards against a single instance
    /// registering the queue-depth provider (and re-seeding the counter) more than
    /// once — e.g. a second <see cref="StartAsync"/> on the same instance. It does NOT
    /// coordinate across instances: the gauge slot in <see cref="ScadaBridgeTelemetry"/>
    /// is process-global, so in a multi-instance process the last <see cref="StartAsync"/>
    /// wins the global slot. 0 = not yet registered, 1 = done.
    /// </summary>
    private int _queueDepthProviderRegistered;

    /// <summary>
    /// WP-10: Delivery handler delegate. The return value / exception is interpreted
    /// the same way on both the immediate-delivery path (<see cref="EnqueueAsync"/>)
    /// and the background retry path (<c>RetryMessageAsync</c>):
    /// <list type="bullet">
    ///   <item><description><c>true</c> — delivered successfully. The message is
    ///   removed from the buffer (or, on the immediate path, never buffered).</description></item>
    ///   <item><description><c>false</c> — permanent failure. On the immediate path
    ///   the message is NOT buffered; on a retry the message is already buffered and
    ///   is parked immediately (no further retries).</description></item>
    ///   <item><description>throws — transient failure. On the immediate path the
    ///   message is buffered for retry; on a retry the retry count is incremented and
    ///   the message is parked once <see cref="StoreAndForwardMessage.MaxRetries"/> is
    ///   reached.</description></item>
    /// </list>
    /// </summary>
    private readonly Dictionary<StoreAndForwardCategory, Func<StoreAndForwardMessage, Task<bool>>> _deliveryHandlers = new();

    /// <summary>
    /// WP-14: Event callback for logging S&amp;F activity to site event log.
    /// </summary>
    public event Action<string, StoreAndForwardCategory, string>? OnActivity;

    /// <summary>
    /// Initializes a new instance of the StoreAndForwardService.
    /// </summary>
    /// <param name="storage">The storage backend for buffered messages.</param>
    /// <param name="options">Configuration options.</param>
    /// <param name="logger">Logger instance.</param>
    /// <param name="replication">Optional replication service for standby synchronization.</param>
    /// <param name="cachedCallObserver">Optional observer for cached call lifecycle events.</param>
    /// <param name="siteId">The site identifier this service belongs to.</param>
    public StoreAndForwardService(
        StoreAndForwardStorage storage,
        StoreAndForwardOptions options,
        ILogger<StoreAndForwardService> logger,
        ReplicationService? replication = null,
        ICachedCallLifecycleObserver? cachedCallObserver = null,
        string siteId = "")
    {
        _storage = storage;
        _options = options;
        _logger = logger;
        _replication = replication;
        _cachedCallObserver = cachedCallObserver;
        // StoreAndForward-023: normalise an empty / whitespace site id to the
        // distinctive UnknownSiteSentinel so downstream consumers (the central
        // audit pipeline keying off SourceSite) never see an empty string and
        // a misconfigured host is recognisable in the central log.
        _siteId = string.IsNullOrWhiteSpace(siteId) ? UnknownSiteSentinel : siteId;
    }

    /// <summary>
    /// Registers a delivery handler for a given message category. See the
    /// <c>_deliveryHandlers</c> field documentation for the true/false/throws contract,
    /// which applies identically on the immediate and retry paths.
    /// </summary>
    /// <param name="category">The message category to handle.</param>
    /// <param name="handler">The delivery handler function.</param>
    public void RegisterDeliveryHandler(
        StoreAndForwardCategory category,
        Func<StoreAndForwardMessage, Task<bool>> handler)
    {
        _deliveryHandlers[category] = handler;
    }

    /// <summary>
    /// Initializes storage and starts the background retry timer.
    /// </summary>
    /// <returns>A task representing the asynchronous start operation.</returns>
    public async Task StartAsync()
    {
        await _storage.InitializeAsync();

        // WP-14 (telemetry): seed the cached buffered-message count from the
        // store exactly once (the gauge callback cannot run an async COUNT), then
        // register the sync, non-blocking provider with the process-global
        // ScadaBridgeTelemetry gauge. Both steps are inside the one-time guard so a
        // second StartAsync on the same instance cannot double-seed.
        //
        // The seed is an Interlocked.Add — NOT an Exchange — to avoid a startup race:
        // between the await above returning and this point, a concurrent BufferAsync
        // could already have Interlocked.Increment'd _bufferedCount. Exchange would
        // clobber that increment (losing a +1); Add preserves it. _bufferedCount
        // starts at 0 and only BufferAsync increments it before the seed, so
        // 0 + pending + (any concurrent increments) is the correct live count.
        var pending = await _storage.GetMessageCountByStatusAsync(
            StoreAndForwardMessageStatus.Pending);
        if (Interlocked.CompareExchange(ref _queueDepthProviderRegistered, 1, 0) == 0)
        {
            Interlocked.Add(ref _bufferedCount, pending);
            ScadaBridgeTelemetry.SetQueueDepthProvider(() => Interlocked.Read(ref _bufferedCount));
        }

        _retryTimer = new Timer(
            // StoreAndForward-024: capture the sweep Task on each tick so
            // StopAsync can await any in-flight invocation before the host
            // disposes _storage/_replication underneath it. The RetryPending
            // path is self-guarded against overlapping sweeps via the
            // _retryInProgress Interlocked flag, so unconditionally re-assigning
            // the field here cannot lose a still-running task (the new tick
            // will short-circuit if one is already running).
            _ => Volatile.Write(ref _sweepTask, RetryPendingMessagesAsync()),
            null,
            _options.RetryTimerInterval,
            _options.RetryTimerInterval);

        _logger.LogInformation(
            "Store-and-forward service started. Retry interval: {Interval}s",
            _options.DefaultRetryInterval.TotalSeconds);
    }

    /// <summary>
    /// Stops the background retry timer and waits (bounded) for any in-flight
    /// retry sweep to finish before returning.
    ///
    /// StoreAndForward-024: prior to this fix, <see cref="StopAsync"/> only
    /// disposed the timer — a sweep already inside
    /// <see cref="RetryPendingMessagesAsync"/> continued running against
    /// <see cref="_storage"/> and <see cref="_replication"/> after this method
    /// returned, and could then NRE / throw on a disposed dependency once the
    /// DI container ran its own shutdown. We now await the captured sweep task
    /// (with a bounded <see cref="SweepShutdownWaitTimeout"/> so a hung
    /// dependency cannot block host shutdown indefinitely) before returning.
    /// </summary>
    /// <returns>A task representing the asynchronous stop operation.</returns>
    public async Task StopAsync()
    {
        if (_retryTimer != null)
        {
            // Stop the periodic callback first so no new sweep starts while we
            // are waiting for the in-flight one to drain.
            await _retryTimer.DisposeAsync();
            _retryTimer = null;
        }

        var inflight = Volatile.Read(ref _sweepTask);
        if (inflight is null || inflight.IsCompleted)
        {
            return;
        }

        try
        {
            // WaitAsync with a finite timeout: a hung delivery handler /
            // storage call cannot block host shutdown indefinitely. On timeout
            // the sweep keeps running but the host is free to proceed with
            // disposal — preferred to never returning.
            await inflight.WaitAsync(SweepShutdownWaitTimeout).ConfigureAwait(false);
        }
        catch (TimeoutException)
        {
            _logger.LogWarning(
                "Store-and-forward retry sweep did not finish within {Timeout}; " +
                "shutdown is proceeding while the sweep is still in-flight",
                SweepShutdownWaitTimeout);
        }
        catch (Exception ex)
        {
            // The sweep itself already logs at Error on failure (see
            // RetryPendingMessagesAsync's catch); we only log here so a
            // surprise fault during shutdown is still visible. Swallow so the
            // host's shutdown sequence can continue regardless.
            _logger.LogWarning(ex,
                "Store-and-forward retry sweep faulted during shutdown wait");
        }
    }

    /// <summary>
    /// WP-10: Enqueues a message for store-and-forward delivery.
    /// Attempts immediate delivery first. On transient failure, buffers for retry.
    /// On permanent failure (handler returns false), returns false immediately.
    ///
    /// WP-10: Retry-count lifecycle — the immediate (or caller-made) delivery attempt
    /// is attempt 0 and is not counted; the background retry sweep increments
    /// <see cref="StoreAndForwardMessage.RetryCount"/> on each retry. A buffered
    /// message is parked once <c>RetryCount</c> reaches <paramref name="maxRetries"/>
    /// — <b>but only when <paramref name="maxRetries"/> is greater than 0</b>. A
    /// <paramref name="maxRetries"/> of <c>0</c> means <b>no limit</b>: the message is
    /// retried on every sweep until it is delivered and is <b>never parked</b> on a
    /// retry-count basis. It is therefore <i>not</i> a "do not retry" value — callers
    /// that want delivery abandoned after a bounded number of attempts must pass a
    /// positive <paramref name="maxRetries"/>.
    ///
    /// WP-15: CachedCall idempotency note — this method does not deduplicate.
    /// The caller (e.g., ExternalSystem.CachedCall()) is responsible for ensuring
    /// that the remote system can handle duplicate deliveries safely.
    /// </summary>
    /// <param name="category">Message category (selects the delivery handler).</param>
    /// <param name="target">Target system name (external system / notification list / DB connection).</param>
    /// <param name="payloadJson">JSON-serialized call payload, treated opaquely.</param>
    /// <param name="originInstanceName">Instance that originated the message (WP-13: survives instance deletion).</param>
    /// <param name="maxRetries">
    /// Maximum background retry-sweep attempts before the message is parked.
    /// <b><c>0</c> = no limit</b> — the message is retried on every sweep until
    /// delivered and is never parked for exhausting retries; it is <b>not</b> a
    /// "never retry" value. <c>null</c> uses <see cref="StoreAndForwardOptions.DefaultMaxRetries"/>.
    /// Must be positive to bound delivery attempts. Mirrors the
    /// <see cref="StoreAndForwardMessage.MaxRetries"/> contract.
    /// </param>
    /// <param name="retryInterval">Fixed interval between retry sweeps for this message; <c>null</c> uses the configured default.</param>
    /// <param name="attemptImmediateDelivery">
    /// When <c>false</c>, the caller has already made its own delivery attempt and the
    /// message is buffered directly for the retry sweep (the handler is not invoked here).
    /// </param>
    /// <param name="messageId">
    /// An explicit, caller-supplied message id. <c>null</c> (the default) makes the
    /// service mint a fresh GUID. The Notification Outbox enqueue path supplies its own
    /// id so the script-generated <c>NotificationId</c> is the single idempotency key —
    /// it is the buffered row's <see cref="StoreAndForwardMessage.Id"/>, it is carried
    /// inside the payload, and it is the id the forwarder submits to central.
    /// </param>
    /// <param name="executionId">
    /// Audit Log #23 (ExecutionId Task 4): the originating script execution's
    /// per-run correlation id. Threaded onto the buffered row so the retry-loop
    /// cached-call audit rows carry it. <c>null</c> for callers (notifications,
    /// pre-Task-4 callers) that do not supply one.
    /// </param>
    /// <param name="sourceScript">
    /// Audit Log #23 (ExecutionId Task 4): the originating script identifier,
    /// threaded onto the buffered row alongside <paramref name="executionId"/>
    /// so the retry-loop audit rows carry the same provenance the script-side
    /// cached rows do. <c>null</c> when not known.
    /// </param>
    /// <param name="parentExecutionId">
    /// Audit Log #23 (ParentExecutionId Task 6): the <c>ExecutionId</c> of the
    /// inbound-API request that spawned the originating script execution.
    /// Threaded onto the buffered row alongside <paramref name="executionId"/>
    /// so the retry-loop cached-call audit rows carry it. <c>null</c> for a
    /// non-routed run and for callers (notifications, pre-Task-6 callers) that
    /// do not supply one.
    /// </param>
    /// <returns>A task that resolves to a result indicating whether the message was delivered or buffered.</returns>
    public async Task<StoreAndForwardResult> EnqueueAsync(
        StoreAndForwardCategory category,
        string target,
        string payloadJson,
        string? originInstanceName = null,
        int? maxRetries = null,
        TimeSpan? retryInterval = null,
        bool attemptImmediateDelivery = true,
        string? messageId = null,
        Guid? executionId = null,
        string? sourceScript = null,
        Guid? parentExecutionId = null)
    {
        var message = new StoreAndForwardMessage
        {
            Id = messageId ?? Guid.NewGuid().ToString("N"),
            Category = category,
            Target = target,
            PayloadJson = payloadJson,
            RetryCount = 0,
            MaxRetries = maxRetries ?? _options.DefaultMaxRetries,
            RetryIntervalMs = (long)(retryInterval ?? _options.DefaultRetryInterval).TotalMilliseconds,
            CreatedAt = DateTimeOffset.UtcNow,
            Status = StoreAndForwardMessageStatus.Pending,
            OriginInstanceName = originInstanceName,
            ExecutionId = executionId,
            SourceScript = sourceScript,
            ParentExecutionId = parentExecutionId
        };

        // Attempt immediate delivery — unless the caller has already made a
        // delivery attempt of its own (attemptImmediateDelivery: false). In that
        // case re-invoking the handler here would dispatch the request twice.
        if (attemptImmediateDelivery && _deliveryHandlers.TryGetValue(category, out var handler))
        {
            try
            {
                var success = await handler(message);
                if (success)
                {
                    RaiseActivity("Delivered", category, $"Immediate delivery to {target}");
                    return new StoreAndForwardResult(true, message.Id, false);
                }

                // Permanent failure — do not buffer
                return new StoreAndForwardResult(false, message.Id, false);
            }
            catch (Exception ex)
            {
                // Transient failure — buffer for retry. The immediate attempt is
                // attempt 0; RetryCount tracks only sweep retries, so it stays 0
                // here (StoreAndForward-003).
                _logger.LogWarning(ex,
                    "Immediate delivery to {Target} failed (transient), buffering for retry",
                    target);

                message.LastAttemptAt = DateTimeOffset.UtcNow;
                message.LastError = ex.Message;
                await BufferAsync(message);

                RaiseActivity("Queued", category, $"Buffered for retry: {target} ({ex.Message})");
                return new StoreAndForwardResult(true, message.Id, true);
            }
        }

        // Either no handler is registered yet, or the caller already attempted
        // delivery itself — buffer for the background retry sweep to deliver.
        // The initial attempt (caller-made, or skipped because no handler is
        // registered) is attempt 0; RetryCount tracks only sweep retries and
        // therefore stays 0 here (StoreAndForward-003).
        if (!attemptImmediateDelivery)
        {
            message.LastAttemptAt = DateTimeOffset.UtcNow;
        }
        await BufferAsync(message);
        RaiseActivity("Queued", category, attemptImmediateDelivery
            ? $"No handler registered, buffered: {target}"
            : $"Buffered for retry: {target}");
        return new StoreAndForwardResult(true, message.Id, true);
    }

    /// <summary>
    /// Persists a message to the local SQLite buffer and (WP-11) replicates the
    /// add to the standby node so a failover does not lose the buffered message.
    /// </summary>
    private async Task BufferAsync(StoreAndForwardMessage message)
    {
        await _storage.EnqueueAsync(message);
        _replication?.ReplicateEnqueue(message);
        // WP-14 (telemetry): a freshly buffered row is Pending → grows the live
        // queue depth. Bumped after the durable write so the gauge never leads the
        // store.
        Interlocked.Increment(ref _bufferedCount);
    }

    /// <summary>
    /// WP-10: Background retry sweep. Processes all pending messages that are due for retry.
    /// </summary>
    /// <returns>A task representing the asynchronous retry sweep.</returns>
    internal async Task RetryPendingMessagesAsync()
    {
        // Prevent overlapping retry sweeps
        if (Interlocked.CompareExchange(ref _retryInProgress, 1, 0) != 0)
            return;

        try
        {
            var messages = await _storage.GetMessagesForRetryAsync();
            if (messages.Count == 0) return;

            _logger.LogDebug("Retry sweep: {Count} messages due for retry", messages.Count);

            foreach (var message in messages)
            {
                await RetryMessageAsync(message);
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Error during retry sweep");
        }
        finally
        {
            Interlocked.Exchange(ref _retryInProgress, 0);
        }
    }

    private async Task RetryMessageAsync(StoreAndForwardMessage message)
    {
        if (!_deliveryHandlers.TryGetValue(message.Category, out var handler))
        {
            _logger.LogWarning("No delivery handler for category {Category}", message.Category);
            return;
        }

        // Audit Log #23 (M3 Bundle E — Tasks E4/E5): measure per-attempt
        // duration so the audit row carries a meaningful DurationMs. Captured
        // around the handler invocation only — storage / replication overhead
        // is excluded.
        var attemptStartUtc = DateTime.UtcNow;
        var attemptStopwatch = System.Diagnostics.Stopwatch.StartNew();

        try
        {
            var success = await handler(message);
            attemptStopwatch.Stop();
            if (success)
            {
                await _storage.RemoveMessageAsync(message.Id);
                _replication?.ReplicateRemove(message.Id);
                // WP-14 (telemetry): a delivered row leaves the Pending queue.
                Interlocked.Decrement(ref _bufferedCount);
                RaiseActivity("Delivered", message.Category,
                    $"Delivered to {message.Target} after {message.RetryCount} retries");

                // M3: terminal Delivered observer notification — the audit
                // bridge maps this to Attempted + CachedResolve(Delivered).
                await NotifyCachedCallObserverAsync(
                    message,
                    CachedCallAttemptOutcome.Delivered,
                    lastError: null,
                    httpStatus: null,
                    occurredAtUtc: attemptStartUtc,
                    durationMs: (int)attemptStopwatch.ElapsedMilliseconds);
                return;
            }

            // Permanent failure on retry — park immediately.
            // StoreAndForward-005: the sweep observed this row as Pending; only commit
            // the park if it is still Pending so a concurrent operator action that
            // moved it (retry/discard) is not silently overwritten.
            message.Status = StoreAndForwardMessageStatus.Parked;
            message.LastAttemptAt = DateTimeOffset.UtcNow;
            message.LastError = "Permanent failure (handler returned false)";
            var parked = await _storage.UpdateMessageIfStatusAsync(
                message, StoreAndForwardMessageStatus.Pending);
            if (!parked)
            {
                _logger.LogDebug(
                    "Message {MessageId} changed status during delivery; sweep park skipped",
                    message.Id);
                return;
            }
            // WP-14 (telemetry): the row committed Pending→Parked, leaving the live
            // forward queue. Only counted when the conditional update actually won.
            Interlocked.Decrement(ref _bufferedCount);
            _replication?.ReplicatePark(message);
            RaiseActivity("Parked", message.Category,
                $"Permanent failure for {message.Target}: handler returned false");

            // M3: terminal PermanentFailure observer notification — the
            // audit bridge maps this to Attempted(Failed) + CachedResolve(Parked).
            await NotifyCachedCallObserverAsync(
                message,
                CachedCallAttemptOutcome.PermanentFailure,
                lastError: message.LastError,
                httpStatus: null,
                occurredAtUtc: attemptStartUtc,
                durationMs: (int)attemptStopwatch.ElapsedMilliseconds);
        }
        catch (Exception ex)
        {
            attemptStopwatch.Stop();
            // Transient failure — increment retry, check max
            message.RetryCount++;
            message.LastAttemptAt = DateTimeOffset.UtcNow;
            message.LastError = ex.Message;

            if (message.MaxRetries > 0 && message.RetryCount >= message.MaxRetries)
            {
                // StoreAndForward-005: conditional park — see the permanent-failure
                // branch above for rationale.
                message.Status = StoreAndForwardMessageStatus.Parked;
                var parked = await _storage.UpdateMessageIfStatusAsync(
                    message, StoreAndForwardMessageStatus.Pending);
                if (!parked)
                {
                    _logger.LogDebug(
                        "Message {MessageId} changed status during delivery; sweep park skipped",
                        message.Id);
                    return;
                }
                // WP-14 (telemetry): the row committed Pending→Parked, leaving the
                // live forward queue. Only counted when the conditional update won.
                Interlocked.Decrement(ref _bufferedCount);
                _replication?.ReplicatePark(message);
                RaiseActivity("Parked", message.Category,
                    $"Max retries ({message.MaxRetries}) reached for {message.Target}");
                _logger.LogWarning(
                    "Message {MessageId} parked after {MaxRetries} retries to {Target}",
                    message.Id, message.MaxRetries, message.Target);

                // M3: terminal ParkedMaxRetries observer notification — the
                // audit bridge maps this to Attempted(Failed) + CachedResolve(Parked).
                await NotifyCachedCallObserverAsync(
                    message,
                    CachedCallAttemptOutcome.ParkedMaxRetries,
                    lastError: ex.Message,
                    httpStatus: null,
                    occurredAtUtc: attemptStartUtc,
                    durationMs: (int)attemptStopwatch.ElapsedMilliseconds);
            }
            else
            {
                // StoreAndForward-005: the retry-count increment is also conditional
                // on the row still being Pending so it cannot clobber an operator
                // action that ran during the failed delivery.
                if (!await _storage.UpdateMessageIfStatusAsync(
                        message, StoreAndForwardMessageStatus.Pending))
                {
                    _logger.LogDebug(
                        "Message {MessageId} changed status during delivery; sweep retry-count update skipped",
                        message.Id);
                    return;
                }
                RaiseActivity("Retried", message.Category,
                    $"Retry {message.RetryCount}/{message.MaxRetries} for {message.Target}: {ex.Message}");

                // M3: per-attempt TransientFailure observer notification —
                // the audit bridge maps this to Attempted(Failed).
                await NotifyCachedCallObserverAsync(
                    message,
                    CachedCallAttemptOutcome.TransientFailure,
                    lastError: ex.Message,
                    httpStatus: null,
                    occurredAtUtc: attemptStartUtc,
                    durationMs: (int)attemptStopwatch.ElapsedMilliseconds);
            }
        }
    }

    /// <summary>
    /// Audit Log #23 (M3 Bundle E — Tasks E4/E5): notify the registered
    /// <see cref="ICachedCallLifecycleObserver"/> of the just-completed
    /// attempt. Only fires for cached-call categories
    /// (<see cref="StoreAndForwardCategory.ExternalSystem"/> and
    /// <see cref="StoreAndForwardCategory.CachedDbWrite"/>); the
    /// <see cref="StoreAndForwardCategory.Notification"/> category has its
    /// own central-side audit pipeline (Notification Outbox / #21) and must
    /// not surface on this hook.
    /// </summary>
    /// <remarks>
    /// Best-effort: an observer that throws is logged and swallowed so a
    /// failing audit pipeline cannot corrupt S&amp;F retry bookkeeping
    /// (alog.md §7 contract). Messages whose ids are not valid GUIDs (pre-M3
    /// callers that didn't thread a TrackedOperationId in) are silently
    /// skipped — the observer requires a parseable id by contract.
    /// </remarks>
    private async Task NotifyCachedCallObserverAsync(
        StoreAndForwardMessage message,
        CachedCallAttemptOutcome outcome,
        string? lastError,
        int? httpStatus,
        DateTime occurredAtUtc,
        int? durationMs)
    {
        if (_cachedCallObserver == null)
        {
            return;
        }

        // Only cached-call categories generate audit telemetry on this hook —
        // notifications have their own outbox-side audit pipeline.
        var channel = message.Category switch
        {
            StoreAndForwardCategory.ExternalSystem => "ApiOutbound",
            StoreAndForwardCategory.CachedDbWrite => "DbOutbound",
            _ => null,
        };
        if (channel is null)
        {
            return;
        }

        if (!TrackedOperationId.TryParse(message.Id, out var trackedId))
        {
            // StoreAndForward-022: previously a silent skip — but a non-GUID
            // message id means a caller bypassed the audit hot path with zero
            // feedback. The drop is still best-effort (S&F retry bookkeeping
            // must never depend on the audit pipeline) but it is now observable
            // via a Warning so a misconfigured caller can be diagnosed.
            // Engine-minted ids (Guid.NewGuid().ToString("N")) and the current
            // caller set (NotificationOutbox enqueue with NotificationId,
            // cached-call enqueue with TrackedOperationId.ToString()) all
            // parse — this log line fires only when a future caller supplies a
            // non-GUID id, which is exactly when the silent-drop was hardest
            // to diagnose.
            _logger.LogWarning(
                "Cached-call audit observer skipped: message id {MessageId} is not a parseable TrackedOperationId (category {Category}, outcome {Outcome}). " +
                "Audit lifecycle for this operation will have no rows.",
                message.Id, message.Category, outcome);
            return;
        }

        CachedCallAttemptContext context;
        try
        {
            context = new CachedCallAttemptContext(
                TrackedOperationId: trackedId,
                Channel: channel,
                Target: message.Target,
                SourceSite: _siteId,
                Outcome: outcome,
                RetryCount: message.RetryCount,
                LastError: lastError,
                HttpStatus: httpStatus,
                CreatedAtUtc: message.CreatedAt.UtcDateTime,
                OccurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc),
                DurationMs: durationMs,
                SourceInstanceId: message.OriginInstanceName,
                // Audit Log #23 (ExecutionId Task 4): the buffered message
                // carries the originating script execution's ExecutionId +
                // SourceScript; surface them on the context so the bridge can
                // stamp the retry-loop cached audit rows. Null on rows buffered
                // before Task 4 (back-compat).
                ExecutionId: message.ExecutionId,
                SourceScript: message.SourceScript,
                // Audit Log #23 (ParentExecutionId Task 6): the buffered
                // message also carries the spawning inbound-API request's
                // ExecutionId; surface it so the bridge stamps it onto the
                // retry-loop cached rows. Null for a non-routed run and on
                // rows buffered before Task 6 (back-compat).
                ParentExecutionId: message.ParentExecutionId);
        }
        catch (Exception buildEx)
        {
            // Defensive — record construction shouldn't throw, but the alog.md
            // §7 contract requires this path be exception-safe regardless.
            _logger.LogWarning(buildEx,
                "Failed to build cached-call attempt context for {MessageId}; observer skipped",
                message.Id);
            return;
        }

        try
        {
            await _cachedCallObserver.OnAttemptCompletedAsync(context, CancellationToken.None)
                .ConfigureAwait(false);
        }
        catch (Exception ex)
        {
            // alog.md §7 best-effort: an audit observer outage must NEVER be
            // misclassified as a transient delivery failure or corrupt the
            // S&F retry bookkeeping.
            _logger.LogWarning(ex,
                "ICachedCallLifecycleObserver threw for {MessageId} (Outcome {Outcome}); ignored",
                message.Id, outcome);
        }
    }

    /// <summary>
    /// WP-12: Gets parked messages for central query (Pattern 8).
    /// </summary>
    /// <param name="category">Optional category filter, or null for all categories.</param>
    /// <param name="pageNumber">The page number (1-based).</param>
    /// <param name="pageSize">The page size.</param>
    /// <returns>A tuple of parked messages and the total count.</returns>
    public async Task<(List<StoreAndForwardMessage> Messages, int TotalCount)> GetParkedMessagesAsync(
        StoreAndForwardCategory? category = null,
        int pageNumber = 1,
        int pageSize = 50)
    {
        return await _storage.GetParkedMessagesAsync(category, pageNumber, pageSize);
    }

    /// <summary>
    /// WP-12: Retries a parked message (moves back to pending queue).
    ///
    /// StoreAndForward-016: an operator requeue is a buffer state change and is
    /// replicated to the standby (as a <see cref="ReplicationOperationType.Requeue"/>)
    /// so a failover preserves the operator's retry intent.
    /// StoreAndForward-017: the activity-log entry carries the message's true
    /// category rather than a hard-coded one.
    /// StoreAndForward-020: the parked row is captured <i>before</i> the local
    /// requeue write rather than re-read after it, so a concurrent
    /// <c>RemoveMessageAsync</c> or <c>DiscardParkedMessageAsync</c> running
    /// between the two storage calls cannot leave the standby in <c>Parked</c>
    /// while the active node has already requeued — we always have the row in
    /// hand for the <c>Requeue</c> replication.
    /// </summary>
    /// <param name="messageId">The identifier of the message to retry.</param>
    /// <returns>True if successfully retried, false otherwise.</returns>
    public async Task<bool> RetryParkedMessageAsync(string messageId)
    {
        // StoreAndForward-020: capture the parked row up front so the standby
        // gets a Requeue even if a concurrent writer (a sweep delete after a
        // successful delivery, or an operator discard) removes the row between
        // the local update and the re-load. The storage call below is
        // conditional on status = Parked, so if the row has already moved we
        // return false here without replicating — the standby's matching row
        // will be reconciled by whichever other operator path won the race.
        var captured = await _storage.GetMessageByIdAsync(messageId);
        if (captured is null || captured.Status != StoreAndForwardMessageStatus.Parked)
        {
            return false;
        }

        var success = await _storage.RetryParkedMessageAsync(messageId);
        if (!success)
        {
            return false;
        }

        // WP-14 (telemetry): an operator requeue moves Parked→Pending, re-adding the
        // row to the live forward queue. Counted only when the conditional storage
        // update actually flipped the row.
        Interlocked.Increment(ref _bufferedCount);

        // The active node just rewrote this row to Pending with retry_count = 0
        // and cleared last_error / last_attempt_at (see
        // StoreAndForwardStorage.RetryParkedMessageAsync). Reconstruct the
        // post-requeue state on the captured POCO so the standby applies the
        // same mutations even if a concurrent writer has already deleted the
        // row underneath us.
        captured.Status = StoreAndForwardMessageStatus.Pending;
        captured.RetryCount = 0;
        captured.LastError = null;
        captured.LastAttemptAt = null;
        _replication?.ReplicateRequeue(captured);

        RaiseActivity("Retry", captured.Category,
            $"Parked message {messageId} moved back to queue");
        return true;
    }

    /// <summary>
    /// WP-12: Permanently discards a parked message.
    ///
    /// StoreAndForward-016: an operator discard is a buffer removal and is replicated
    /// to the standby (as a <see cref="ReplicationOperationType.Remove"/>) so the
    /// discarded message does not reappear after a failover.
    /// StoreAndForward-017: the activity-log entry carries the message's true
    /// category rather than a hard-coded one.
    /// </summary>
    /// <param name="messageId">The identifier of the message to discard.</param>
    /// <returns>True if successfully discarded, false otherwise.</returns>
    public async Task<bool> DiscardParkedMessageAsync(string messageId)
    {
        // Capture the category before the row is deleted so the activity log is
        // labelled correctly.
        // WP-14 (telemetry): Parked rows are not in _bufferedCount; discarding a Parked row needs no counter adjustment.
        var message = await _storage.GetMessageByIdAsync(messageId);
        var success = await _storage.DiscardParkedMessageAsync(messageId);
        if (success)
        {
            _replication?.ReplicateRemove(messageId);
            RaiseActivity("Discard", message?.Category ?? StoreAndForwardCategory.ExternalSystem,
                $"Parked message {messageId} discarded");
        }
        return success;
    }

    /// <summary>
    /// WP-14: Gets buffer depth by category for health reporting.
    /// </summary>
    /// <returns>A dictionary of buffer depths by category.</returns>
    public async Task<Dictionary<StoreAndForwardCategory, int>> GetBufferDepthAsync()
    {
        return await _storage.GetBufferDepthByCategoryAsync();
    }

    /// <summary>
    /// WP-13: Gets count of S&amp;F messages for a given instance (for verifying survival on deletion).
    /// </summary>
    /// <param name="instanceName">The instance name to query.</param>
    /// <returns>The number of messages originating from the instance.</returns>
    public async Task<int> GetMessageCountForInstanceAsync(string instanceName)
    {
        return await _storage.GetMessageCountByOriginInstanceAsync(instanceName);
    }

    /// <summary>
    /// Notification Outbox: looks up a buffered message by its id, or <c>null</c> if it
    /// is not (or no longer) in the buffer. <c>Notify.Status</c> uses this to detect a
    /// notification still in transit at the site — central reports it not-found while
    /// the S&amp;F buffer still holds it, which is the site-local <c>Forwarding</c> state.
    /// </summary>
    /// <param name="messageId">The message identifier.</param>
    /// <returns>The message, or null if not found.</returns>
    public async Task<StoreAndForwardMessage?> GetMessageByIdAsync(string messageId)
    {
        return await _storage.GetMessageByIdAsync(messageId);
    }

    /// <summary>
    /// WP-14: Raises the S&amp;F activity notification. StoreAndForward-009: the
    /// delegate is snapshotted (so a concurrent unsubscribe cannot NRE) and every
    /// subscriber invocation is wrapped so a slow/throwing subscriber (e.g. the site
    /// event log) cannot abort the caller. Crucially, a subscriber exception raised
    /// from <see cref="EnqueueAsync"/> or <c>RetryMessageAsync</c> must NOT be
    /// misclassified as a transient delivery failure — pre-fix it escaped into the
    /// delivery try/catch and caused a successfully delivered message to be buffered
    /// (or its retry count to be bumped). Activity logging is best-effort.
    /// </summary>
    private void RaiseActivity(string action, StoreAndForwardCategory category, string detail)
    {
        var handlers = OnActivity;
        if (handlers == null) return;

        foreach (var handler in handlers.GetInvocationList().Cast<Action<string, StoreAndForwardCategory, string>>())
        {
            try
            {
                handler(action, category, detail);
            }
            catch (Exception ex)
            {
                _logger.LogWarning(ex,
                    "Store-and-forward activity subscriber threw for action {Action}; ignored",
                    action);
            }
        }
    }
}

/// <summary>
/// Result of an enqueue operation.
/// </summary>
public record StoreAndForwardResult(
    /// <summary>True if the message was accepted (either delivered immediately or buffered).</summary>
    bool Accepted,
    /// <summary>Unique message ID for tracking.</summary>
    string MessageId,
    /// <summary>True if the message was buffered (not delivered immediately).</summary>
    bool WasBuffered);
