lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorAsyncCacheWriteTests.cs

using System.Text.Json;
using Akka.Actor;
using Akka.Event;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;

namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;

/// <summary>
///     Surgical-deploys Task 7 Part A — the artifact-cache write is I/O (hundreds of chunk inserts
///     into LocalDb) and must not run on the actor thread.
/// </summary>
/// <remarks>
///     <para>
///         The ACK was never the thing at risk: <c>HandleArtifactForApply</c> sends it BEFORE it caches, so a
///         blocking store still acked promptly. What it blocked was <b>everything after</b> — the host
///         could not dequeue a driver value burst, a diagnostics request or the next dispatch until
///         SQLite was done. So the load-bearing assertion here is that the actor answers an unrelated
///         message while the store is still in flight, not that the ACK arrived.
///     </para>
///     <para>
///         Runs at <c>INFO</c> so the completion self-message's log line is observable — the pipe-back
///         is private, and its visible effect is that line.
///     </para>
/// </remarks>
public sealed class DriverHostActorAsyncCacheWriteTests()
    : RuntimeActorTestBase(AkkaTestHoconAtLevel("INFO"))
{
    private static readonly NodeId TestNode = NodeId.Parse("driver-test");
    private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));

    [Fact]
    public void AStillRunningCacheStore_DoesNotBlockTheHostMailbox()
    {
        var db = NewInMemoryDbFactory();
        var cache = new GatedArtifactCache();
        var deploymentId = SeedDeployment(db, RevA);

        var coordinator = CreateTestProbe();
        var actor = Sys.ActorOf(DriverHostActor.Props(
            db, TestNode, coordinator.Ref,
            localRoles: new HashSet<string> { "driver" },
            deploymentArtifactCache: cache));

        try
        {
            actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
            coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);

            // The store was entered and has NOT completed — the gate is still closed.
            AwaitAssert(() => cache.StoresEntered.ShouldBe(1), duration: PresenceBudget);
            cache.StoresCompleted.ShouldBe(0);

            // THE assertion: with the store still in flight the host is still processing its mailbox.
            // A blocking `.GetAwaiter().GetResult()` parks the actor thread here and this times out.
            actor.Tell(new GetDiagnostics(CorrelationId.NewId()), coordinator.Ref);
            coordinator.ExpectMsg<NodeDiagnosticsSnapshot>(TimeSpan.FromSeconds(3));

            // Completion is observed rather than fire-and-forgotten: the store pipes back to Self and
            // the handler logs it, so an operator can tell a cached apply from an uncached one.
            EventFilter.Info(contains: "cached applied deployment")
                .ExpectOne(TimeSpan.FromSeconds(5), () => cache.Release());
        }
        finally
        {
            // Never leave the fake gated: a parked store would hang ActorSystem shutdown.
            cache.Release();
        }
    }

    [Fact]
    public void AFailingCacheStore_WarnsAndLeavesTheApplyApplied()
    {
        // The cache is a boot fallback, not part of the deployment protocol. A failed store must
        // surface as a Warning and nothing else — emphatically NOT as a second, contradictory Failed
        // ACK for a deployment the fleet already believes is live.
        var db = NewInMemoryDbFactory();
        var deploymentId = SeedDeployment(db, RevA);

        var coordinator = CreateTestProbe();
        var actor = Sys.ActorOf(DriverHostActor.Props(
            db, TestNode, coordinator.Ref,
            localRoles: new HashSet<string> { "driver" },
            deploymentArtifactCache: new AsyncFailingArtifactCache()));

        // Matched on the EXCEPTION, not just the prose: the pipe-back carries the Exception itself and
        // the handler logs it as the event's Cause. A SQLite failure is routinely a wrapped inner
        // exception whose outer Message names neither the result code nor the failing frame, so a
        // Reason-string-only carrier would have thrown that away before the log line was written.
        // (Custom rather than EventFilter.Exception<T>, which only ever matches ERROR-level events —
        // a failed cache write is deliberately a Warning, since the apply itself succeeded.)
        EventFilter.Custom<Warning>(e =>
                e.Cause is InvalidOperationException &&
                e.Message?.ToString()?.Contains("could not be cached", StringComparison.Ordinal) == true)
            .ExpectOne(TimeSpan.FromSeconds(5), () =>
            {
                actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
                coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
            });

        coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
    }

    /// <summary>Seeds a sealed, parseable deployment and returns its id.</summary>
    private static DeploymentId SeedDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev)
    {
        var artifact = JsonSerializer.SerializeToUtf8Bytes(new { DriverInstances = Array.Empty<object>() });
        var id = DeploymentId.NewId();
        using var ctx = db.CreateDbContext();
        ctx.Deployments.Add(new Deployment
        {
            DeploymentId = id.Value,
            RevisionHash = rev.Value,
            Status = DeploymentStatus.Sealed,
            CreatedBy = "test",
            SealedAtUtc = DateTime.UtcNow,
            ArtifactBlob = artifact,
        });
        ctx.SaveChanges();
        return id;
    }

    /// <summary>
    ///     A store that never completes until the test releases it — the fake shape of a slow LocalDb
    ///     write. It must not complete synchronously, or it would prove nothing about the mailbox.
    /// </summary>
    private sealed class GatedArtifactCache : IDeploymentArtifactCache
    {
        private readonly TaskCompletionSource _gate =
            new(TaskCreationOptions.RunContinuationsAsynchronously);
        private int _entered;
        private int _completed;

        public int StoresEntered => Volatile.Read(ref _entered);
        public int StoresCompleted => Volatile.Read(ref _completed);

        public void Release() => _gate.TrySetResult();

        public async Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
                                     byte[] artifact, CancellationToken ct = default)
        {
            Interlocked.Increment(ref _entered);
            await _gate.Task.ConfigureAwait(false);
            Interlocked.Increment(ref _completed);
        }

        public Task<CachedDeploymentArtifact?> GetCurrentAsync(string clusterId, CancellationToken ct = default)
            => Task.FromResult<CachedDeploymentArtifact?>(null);

        public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
            => Task.FromResult<CachedDeploymentArtifact?>(null);
    }

    /// <summary>
    ///     Fails the store ASYNCHRONOUSLY (a faulted task), which is how a real LocalDb write fails —
    ///     as distinct from <c>DriverHostActorArtifactCacheTests</c>'s synchronously-throwing fake,
    ///     which exercises the other half of the guard.
    /// </summary>
    private sealed class AsyncFailingArtifactCache : IDeploymentArtifactCache
    {
        public async Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
                                     byte[] artifact, CancellationToken ct = default)
        {
            await Task.Yield();
            throw new InvalidOperationException("disk full");
        }

        public Task<CachedDeploymentArtifact?> GetCurrentAsync(string clusterId, CancellationToken ct = default)
            => Task.FromResult<CachedDeploymentArtifact?>(null);

        public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
            => Task.FromResult<CachedDeploymentArtifact?>(null);
    }
}