using System.Text.Json; using Akka.Actor; using Akka.Event; using Microsoft.EntityFrameworkCore; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; /// /// Surgical-deploys Task 7 Part A — the artifact-cache write is I/O (hundreds of chunk inserts /// into LocalDb) and must not run on the actor thread. /// /// /// /// The ACK was never the thing at risk: HandleArtifactForApply sends it BEFORE it caches, so a /// blocking store still acked promptly. What it blocked was everything after — the host /// could not dequeue a driver value burst, a diagnostics request or the next dispatch until /// SQLite was done. So the load-bearing assertion here is that the actor answers an unrelated /// message while the store is still in flight, not that the ACK arrived. /// /// /// Runs at INFO so the completion self-message's log line is observable — the pipe-back /// is private, and its visible effect is that line. /// /// public sealed class DriverHostActorAsyncCacheWriteTests() : RuntimeActorTestBase(AkkaTestHoconAtLevel("INFO")) { private static readonly NodeId TestNode = NodeId.Parse("driver-test"); private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64)); [Fact] public void AStillRunningCacheStore_DoesNotBlockTheHostMailbox() { var db = NewInMemoryDbFactory(); var cache = new GatedArtifactCache(); var deploymentId = SeedDeployment(db, RevA); var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, deploymentArtifactCache: cache)); try { actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); coordinator.ExpectMsg(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied); // The store was entered and has NOT completed — the gate is still closed. AwaitAssert(() => cache.StoresEntered.ShouldBe(1), duration: PresenceBudget); cache.StoresCompleted.ShouldBe(0); // THE assertion: with the store still in flight the host is still processing its mailbox. // A blocking `.GetAwaiter().GetResult()` parks the actor thread here and this times out. actor.Tell(new GetDiagnostics(CorrelationId.NewId()), coordinator.Ref); coordinator.ExpectMsg(TimeSpan.FromSeconds(3)); // Completion is observed rather than fire-and-forgotten: the store pipes back to Self and // the handler logs it, so an operator can tell a cached apply from an uncached one. EventFilter.Info(contains: "cached applied deployment") .ExpectOne(TimeSpan.FromSeconds(5), () => cache.Release()); } finally { // Never leave the fake gated: a parked store would hang ActorSystem shutdown. cache.Release(); } } [Fact] public void AFailingCacheStore_WarnsAndLeavesTheApplyApplied() { // The cache is a boot fallback, not part of the deployment protocol. A failed store must // surface as a Warning and nothing else — emphatically NOT as a second, contradictory Failed // ACK for a deployment the fleet already believes is live. var db = NewInMemoryDbFactory(); var deploymentId = SeedDeployment(db, RevA); var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, deploymentArtifactCache: new AsyncFailingArtifactCache())); // Matched on the EXCEPTION, not just the prose: the pipe-back carries the Exception itself and // the handler logs it as the event's Cause. A SQLite failure is routinely a wrapped inner // exception whose outer Message names neither the result code nor the failing frame, so a // Reason-string-only carrier would have thrown that away before the log line was written. // (Custom rather than EventFilter.Exception, which only ever matches ERROR-level events — // a failed cache write is deliberately a Warning, since the apply itself succeeded.) EventFilter.Custom(e => e.Cause is InvalidOperationException && e.Message?.ToString()?.Contains("could not be cached", StringComparison.Ordinal) == true) .ExpectOne(TimeSpan.FromSeconds(5), () => { actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); coordinator.ExpectMsg(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied); }); coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); } /// Seeds a sealed, parseable deployment and returns its id. private static DeploymentId SeedDeployment(IDbContextFactory db, RevisionHash rev) { var artifact = JsonSerializer.SerializeToUtf8Bytes(new { DriverInstances = Array.Empty() }); var id = DeploymentId.NewId(); using var ctx = db.CreateDbContext(); ctx.Deployments.Add(new Deployment { DeploymentId = id.Value, RevisionHash = rev.Value, Status = DeploymentStatus.Sealed, CreatedBy = "test", SealedAtUtc = DateTime.UtcNow, ArtifactBlob = artifact, }); ctx.SaveChanges(); return id; } /// /// A store that never completes until the test releases it — the fake shape of a slow LocalDb /// write. It must not complete synchronously, or it would prove nothing about the mailbox. /// private sealed class GatedArtifactCache : IDeploymentArtifactCache { private readonly TaskCompletionSource _gate = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _entered; private int _completed; public int StoresEntered => Volatile.Read(ref _entered); public int StoresCompleted => Volatile.Read(ref _completed); public void Release() => _gate.TrySetResult(); public async Task StoreAsync(string clusterId, string deploymentId, string revisionHash, byte[] artifact, CancellationToken ct = default) { Interlocked.Increment(ref _entered); await _gate.Task.ConfigureAwait(false); Interlocked.Increment(ref _completed); } public Task GetCurrentAsync(string clusterId, CancellationToken ct = default) => Task.FromResult(null); public Task GetCurrentUnkeyedAsync(CancellationToken ct = default) => Task.FromResult(null); } /// /// Fails the store ASYNCHRONOUSLY (a faulted task), which is how a real LocalDb write fails — /// as distinct from DriverHostActorArtifactCacheTests's synchronously-throwing fake, /// which exercises the other half of the guard. /// private sealed class AsyncFailingArtifactCache : IDeploymentArtifactCache { public async Task StoreAsync(string clusterId, string deploymentId, string revisionHash, byte[] artifact, CancellationToken ct = default) { await Task.Yield(); throw new InvalidOperationException("disk full"); } public Task GetCurrentAsync(string clusterId, CancellationToken ct = default) => Task.FromResult(null); public Task GetCurrentUnkeyedAsync(CancellationToken ct = default) => Task.FromResult(null); } }