using System.Text.Json; using Akka.Actor; using Akka.Event; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; /// /// Surgical-deploys Task 7 Part B — a dispatch that arrives while an apply is in flight is /// STASHED, not re-forwarded to Self. /// /// /// /// Self.Forward put the message back at the tail of the actor's OWN mailbox, so with /// nothing else to do the host spun dequeue → log → enqueue for the entire fetch window — /// thousands of iterations and thousands of log lines for one deferred deployment, burning a /// core and starving nothing but itself of useful work. The stash holds it outside the mailbox /// until the apply finishes. /// /// /// Runs at DEBUG because the assertion IS a log line: a spin is only distinguishable /// from a single deferral by how many times it is announced. A log event below /// akka.loglevel never reaches the EventStream at all, so at the default WARNING this /// test would count zero either way and pass vacuously. /// /// public sealed class DriverHostActorApplyingStashTests() : RuntimeActorTestBase(AkkaTestHoconAtLevel("DEBUG")) { private static readonly NodeId TestNode = NodeId.Parse("driver-test"); private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64)); private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64)); private static readonly RevisionHash RevC = RevisionHash.Parse(new string('c', 64)); /// How long a forward-spin gets to announce itself thousands of times if it is back. private static readonly TimeSpan SpinWindow = TimeSpan.FromSeconds(1.5); /// Minimal parseable artifact — no drivers. private static byte[] ArtifactBytes() => JsonSerializer.SerializeToUtf8Bytes(new { DriverInstances = Array.Empty() }); [Fact] public void ADispatchArrivingMidApply_IsDeferredOnce_AndAppliedAfterTheApplyCompletes() { var db = NewInMemoryDbFactory(); var fetcher = new GatedFetcher(ArtifactBytes()); var deploymentA = DeploymentId.NewId(); var deploymentB = DeploymentId.NewId(); var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, deploymentArtifactCache: new NullArtifactCache(), fetchAndCacheMode: true, artifactFetcher: fetcher)); try { // A is in flight: the fetch has started and its task will not complete until released. actor.Tell(new DispatchDeployment(deploymentA, RevA, CorrelationId.NewId())); AwaitAssert(() => fetcher.Calls.ShouldBe(1), duration: PresenceBudget); var logs = SubscribeToLogs("deferring"); actor.Tell(new DispatchDeployment(deploymentB, RevB, CorrelationId.NewId())); // PRESENCE first, on the shared budget: the deferral must actually be ANNOUNCED. Counting // only over a fixed window would let "the dispatch was silently dropped" score zero and // then pass the ≤1 half below, which is the wrong bug to be blind to. logs.DrainUntilAllSeen(PresenceBudget); logs["deferring"].ShouldBeGreaterThanOrEqualTo(1); // ABSENCE second: over a window in which a forward-spin would announce itself thousands of // times, no further deferral appears. logs.Drain(SpinWindow); logs["deferring"].ShouldBe(1); // Nothing was lost: B re-enters exactly once, after A's apply transitions back to Steady. fetcher.Release(); coordinator.ExpectMsg(TimeSpan.FromSeconds(10)).Outcome.ShouldBe(ApplyAckOutcome.Applied); coordinator.ExpectMsg(TimeSpan.FromSeconds(10)).Outcome.ShouldBe(ApplyAckOutcome.Applied); fetcher.Calls.ShouldBe(2); } finally { // Never leave the fetch parked: a blocked apply would hang ActorSystem shutdown. fetcher.Release(); } } /// /// The SAME dispatch arriving twice mid-apply is stashed ONCE — the stash is bounded by /// distinct deployments, not by how often the coordinator re-dispatches. /// /// /// /// Verified RED against the pinned Akka (1.5.62): with the dedup disabled, both copies are /// stashed and the deferral is announced twice. That is the failure this pins — a /// coordinator that re-dispatches on a timer would otherwise grow the stash with retries /// rather than with work, and every extra copy is re-delivered on the way back to Steady /// only to short-circuit as a duplicate. /// /// /// It is NOT a crash today, and the review's suggestion that it might be was checked rather /// than assumed. AbstractStash.Stash() is documented to throw /// IllegalActorStateException when an envelope equals the last stashed one, and that /// message string is in the shipped assembly — but a probe actor on this package stashed the /// same instance twice, and two value-equal instances, with and without an explicit sender, /// and never threw. The RED above is a count, not an exception, for exactly that reason. /// /// [Fact] public void TheSameDispatchArrivingTwiceMidApply_IsStashedOnce_AndAppliedOnce() { var db = NewInMemoryDbFactory(); var fetcher = new GatedFetcher(ArtifactBytes()); var deploymentA = DeploymentId.NewId(); var deploymentB = DeploymentId.NewId(); var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, deploymentArtifactCache: new NullArtifactCache(), fetchAndCacheMode: true, artifactFetcher: fetcher)); try { actor.Tell(new DispatchDeployment(deploymentA, RevA, CorrelationId.NewId())); AwaitAssert(() => fetcher.Calls.ShouldBe(1), duration: PresenceBudget); var logs = SubscribeToLogs("deferring", "already deferred"); // The IDENTICAL message object, twice — a record, so the two envelopes are value-equal. var dispatchB = new DispatchDeployment(deploymentB, RevB, CorrelationId.NewId()); actor.Tell(dispatchB); actor.Tell(dispatchB); logs.DrainUntilAllSeen(PresenceBudget); logs["deferring"].ShouldBe(1); logs["already deferred"].ShouldBe(1); // No second stash, therefore no second re-delivery: exactly one Applied ack for B. fetcher.Release(); coordinator.ExpectMsg(TimeSpan.FromSeconds(10)).Outcome.ShouldBe(ApplyAckOutcome.Applied); coordinator.ExpectMsg(TimeSpan.FromSeconds(10)).Outcome.ShouldBe(ApplyAckOutcome.Applied); coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(500)); fetcher.Calls.ShouldBe(2); } finally { fetcher.Release(); } } /// /// Two successive deferrals apply in arrival order, and the second one proves the dedup set is /// CLEARED on the way back to Steady rather than accumulating across applies. /// /// /// C is stashed by B's apply, i.e. after A's apply already unstashed B and cleared the set. A /// dedup set that was never cleared would still hold nothing for C, so what this really pins is /// the re-stash CYCLE working at all — plus the ordering the coordinator depends on: the node /// must end on C's revision, not on B's. /// [Fact] public void ASecondDeferralAppliesAfterTheFirst_AndTheNodeEndsOnTheLastRevision() { var db = NewInMemoryDbFactory(); // Gate the first TWO fetches so C can arrive while B is applying. var fetcher = new GatedFetcher(ArtifactBytes(), gatedCalls: 2); var deploymentA = DeploymentId.NewId(); var deploymentB = DeploymentId.NewId(); var deploymentC = DeploymentId.NewId(); var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, deploymentArtifactCache: new NullArtifactCache(), fetchAndCacheMode: true, artifactFetcher: fetcher)); try { actor.Tell(new DispatchDeployment(deploymentA, RevA, CorrelationId.NewId())); AwaitAssert(() => fetcher.Calls.ShouldBe(1), duration: PresenceBudget); actor.Tell(new DispatchDeployment(deploymentB, RevB, CorrelationId.NewId())); // A completes; B unstashes and starts its own (still gated) fetch. fetcher.Release(1); coordinator.ExpectMsg(TimeSpan.FromSeconds(10)).DeploymentId.ShouldBe(deploymentA); AwaitAssert(() => fetcher.Calls.ShouldBe(2), duration: PresenceBudget); // C arrives mid-B — a fresh stash cycle, on a set that was cleared when B was released. actor.Tell(new DispatchDeployment(deploymentC, RevC, CorrelationId.NewId())); fetcher.Release(2); coordinator.ExpectMsg(TimeSpan.FromSeconds(10)).DeploymentId.ShouldBe(deploymentB); coordinator.ExpectMsg(TimeSpan.FromSeconds(10)).DeploymentId.ShouldBe(deploymentC); actor.Tell(new GetDiagnostics(CorrelationId.NewId()), coordinator.Ref); coordinator.ExpectMsg(TimeSpan.FromSeconds(5)) .CurrentRevision.ShouldBe(RevC); } finally { fetcher.Release(); } } /// /// A host wedged in Applying says so. The pre-stash forward-spin was accidentally the /// only alarm for this — a flood of log lines nobody could miss — and removing it removed the /// signal with it. The watchdog is the deliberate replacement. /// /// /// The wedge delay is a constructor parameter (defaulting to /// ), shortened here so the fire is /// observable in a test's lifetime — the same seam every other interval on this actor uses. /// Nothing is cancelled or failed by the fire, so the assertion is purely the Warning, and it /// must NAME the wedged deployment or it is useless during an incident. /// [Fact] public void AnApplyThatNeverCompletes_IsReportedAsWedged() { var db = NewInMemoryDbFactory(); var fetcher = new GatedFetcher(ArtifactBytes()); var deploymentA = DeploymentId.NewId(); var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, deploymentArtifactCache: new NullArtifactCache(), fetchAndCacheMode: true, artifactFetcher: fetcher, applyWedgeTimeout: TimeSpan.FromMilliseconds(300))); try { EventFilter.Warning(contains: deploymentA.ToString()).ExpectOne(TimeSpan.FromSeconds(10), () => actor.Tell(new DispatchDeployment(deploymentA, RevA, CorrelationId.NewId()))); } finally { fetcher.Release(); } } /// /// A completed apply does NOT report a wedge, however short the delay — the timer is cancelled /// on the way back to Steady, and a fire already in the mailbox drops at Debug. /// /// /// The falsification control for the test above: a watchdog that fires unconditionally would /// pass it and would then cry wedge after every ordinary deployment. /// [Fact] public void ACompletedApply_NeverReportsAWedge() { var db = NewInMemoryDbFactory(); var deploymentA = DeploymentId.NewId(); var coordinator = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, deploymentArtifactCache: new NullArtifactCache(), fetchAndCacheMode: true, // Ungated: the fetch answers immediately. artifactFetcher: new GatedFetcher(ArtifactBytes(), gatedCalls: 0), applyWedgeTimeout: TimeSpan.FromMilliseconds(200))); var logs = SubscribeToLogs("appears WEDGED"); actor.Tell(new DispatchDeployment(deploymentA, RevA, CorrelationId.NewId())); coordinator.ExpectMsg(TimeSpan.FromSeconds(10)).Outcome.ShouldBe(ApplyAckOutcome.Applied); // Well past the (200 ms) delay — a live timer would have fired several times over. logs.Drain(SpinWindow); logs["appears WEDGED"].ShouldBe(0); } /// Subscribes a fresh counter to the EventStream's log events. /// /// Every fragment of interest must be declared up front, because draining is destructive: a /// probe read for one fragment discards every other log event it passes over, so counting two /// fragments with two independent drains would always score zero on the second. /// private LogCounter SubscribeToLogs(params string[] fragments) { var probe = CreateTestProbe(); Sys.EventStream.Subscribe(probe.Ref, typeof(LogEvent)); return new LogCounter(probe, fragments); } /// Counts occurrences of a fixed set of log-message fragments off one EventStream probe. private sealed class LogCounter(Akka.TestKit.TestProbe logs, string[] fragments) { private readonly Dictionary _counts = fragments.ToDictionary(f => f, _ => 0, StringComparer.Ordinal); public int this[string fragment] => _counts[fragment]; /// Drains the probe for a fixed window, tallying every tracked fragment. public void Drain(TimeSpan window) { var deadline = DateTime.UtcNow + window; while (DateTime.UtcNow < deadline) { if (logs.ReceiveOne(TimeSpan.FromMilliseconds(100)) is LogEvent e && e.Message?.ToString() is { } text) { foreach (var fragment in fragments) if (text.Contains(fragment, StringComparison.Ordinal)) _counts[fragment]++; } } } /// Drains until every tracked fragment has been seen at least once, or the budget expires. public void DrainUntilAllSeen(TimeSpan budget) { var deadline = DateTime.UtcNow + budget; while (DateTime.UtcNow < deadline && _counts.Values.Any(n => n == 0)) Drain(TimeSpan.FromMilliseconds(200)); } } /// /// Gates the first gatedCalls fetches on the test; later fetches complete immediately. /// private sealed class GatedFetcher(byte[] bytes, int gatedCalls = 1) : IDeploymentArtifactFetcher { private readonly TaskCompletionSource[] _gates = Enumerable.Range(0, gatedCalls) .Select(_ => new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)) .ToArray(); private int _calls; public int Calls => Volatile.Read(ref _calls); /// Releases every gate — also the test-teardown safety net. public void Release() { foreach (var gate in _gates) gate.TrySetResult(); } /// Releases the gate on the -th fetch (1-based). public void Release(int call) => _gates[call - 1].TrySetResult(); public async Task FetchAsync(string deploymentId, string expectedRevisionHash, CancellationToken ct) { var call = Interlocked.Increment(ref _calls); if (call <= _gates.Length) await _gates[call - 1].Task.ConfigureAwait(false); return bytes; } } /// Accepts every store; this class asserts on dispatch handling, not on caching. private sealed class NullArtifactCache : IDeploymentArtifactCache { public Task StoreAsync(string clusterId, string deploymentId, string revisionHash, byte[] artifact, CancellationToken ct = default) => Task.CompletedTask; public Task GetCurrentAsync(string clusterId, CancellationToken ct = default) => Task.FromResult(null); public Task GetCurrentUnkeyedAsync(CancellationToken ct = default) => Task.FromResult(null); } }