perf(runtime): apply-path cache write off-mailbox; stash deferred dispatches
This commit is contained in:
Joseph Doherty
+175
@@ -0,0 +1,175 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.DeploymentCache;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// Surgical-deploys Task 7 Part A — the artifact-cache write is I/O (hundreds of chunk inserts
|
||||
/// into LocalDb) and must not run on the actor thread.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The ACK was never the thing at risk: <c>ApplyAndAck</c> sends it BEFORE it caches, so a
|
||||
/// blocking store still acked promptly. What it blocked was <b>everything after</b> — the host
|
||||
/// could not dequeue a driver value burst, a diagnostics request or the next dispatch until
|
||||
/// SQLite was done. So the load-bearing assertion here is that the actor answers an unrelated
|
||||
/// message while the store is still in flight, not that the ACK arrived.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Runs at <c>INFO</c> so the completion self-message's log line is observable — the pipe-back
|
||||
/// is private, and its visible effect is that line.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DriverHostActorAsyncCacheWriteTests()
|
||||
: RuntimeActorTestBase(AkkaTestHoconAtLevel("INFO"))
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
|
||||
[Fact]
|
||||
public void AStillRunningCacheStore_DoesNotBlockTheHostMailbox()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var cache = new GatedArtifactCache();
|
||||
var deploymentId = SeedDeployment(db, RevA);
|
||||
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: cache));
|
||||
|
||||
try
|
||||
{
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
|
||||
// The store was entered and has NOT completed — the gate is still closed.
|
||||
AwaitAssert(() => cache.StoresEntered.ShouldBe(1), duration: PresenceBudget);
|
||||
cache.StoresCompleted.ShouldBe(0);
|
||||
|
||||
// THE assertion: with the store still in flight the host is still processing its mailbox.
|
||||
// A blocking `.GetAwaiter().GetResult()` parks the actor thread here and this times out.
|
||||
actor.Tell(new GetDiagnostics(CorrelationId.NewId()), coordinator.Ref);
|
||||
coordinator.ExpectMsg<NodeDiagnosticsSnapshot>(TimeSpan.FromSeconds(3));
|
||||
|
||||
// Completion is observed rather than fire-and-forgotten: the store pipes back to Self and
|
||||
// the handler logs it, so an operator can tell a cached apply from an uncached one.
|
||||
EventFilter.Info(contains: "cached applied deployment")
|
||||
.ExpectOne(TimeSpan.FromSeconds(5), () => cache.Release());
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Never leave the fake gated: a parked store would hang ActorSystem shutdown.
|
||||
cache.Release();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AFailingCacheStore_WarnsAndLeavesTheApplyApplied()
|
||||
{
|
||||
// The cache is a boot fallback, not part of the deployment protocol. A failed store must
|
||||
// surface as a Warning and nothing else — emphatically NOT as a second, contradictory Failed
|
||||
// ACK for a deployment the fleet already believes is live.
|
||||
var db = NewInMemoryDbFactory();
|
||||
var deploymentId = SeedDeployment(db, RevA);
|
||||
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
deploymentArtifactCache: new AsyncFailingArtifactCache()));
|
||||
|
||||
EventFilter.Warning(contains: "could not be cached").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
||||
{
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
});
|
||||
|
||||
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
/// <summary>Seeds a sealed, parseable deployment and returns its id.</summary>
|
||||
private static DeploymentId SeedDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev)
|
||||
{
|
||||
var artifact = JsonSerializer.SerializeToUtf8Bytes(new { DriverInstances = Array.Empty<object>() });
|
||||
var id = DeploymentId.NewId();
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = id.Value,
|
||||
RevisionHash = rev.Value,
|
||||
Status = DeploymentStatus.Sealed,
|
||||
CreatedBy = "test",
|
||||
SealedAtUtc = DateTime.UtcNow,
|
||||
ArtifactBlob = artifact,
|
||||
});
|
||||
ctx.SaveChanges();
|
||||
return id;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A store that never completes until the test releases it — the fake shape of a slow LocalDb
|
||||
/// write. It must not complete synchronously, or it would prove nothing about the mailbox.
|
||||
/// </summary>
|
||||
private sealed class GatedArtifactCache : IDeploymentArtifactCache
|
||||
{
|
||||
private readonly TaskCompletionSource _gate =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private int _entered;
|
||||
private int _completed;
|
||||
|
||||
public int StoresEntered => Volatile.Read(ref _entered);
|
||||
public int StoresCompleted => Volatile.Read(ref _completed);
|
||||
|
||||
public void Release() => _gate.TrySetResult();
|
||||
|
||||
public async Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
|
||||
byte[] artifact, CancellationToken ct = default)
|
||||
{
|
||||
Interlocked.Increment(ref _entered);
|
||||
await _gate.Task.ConfigureAwait(false);
|
||||
Interlocked.Increment(ref _completed);
|
||||
}
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentAsync(string clusterId, CancellationToken ct = default)
|
||||
=> Task.FromResult<CachedDeploymentArtifact?>(null);
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
|
||||
=> Task.FromResult<CachedDeploymentArtifact?>(null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fails the store ASYNCHRONOUSLY (a faulted task), which is how a real LocalDb write fails —
|
||||
/// as distinct from <c>DriverHostActorArtifactCacheTests</c>'s synchronously-throwing fake,
|
||||
/// which exercises the other half of the guard.
|
||||
/// </summary>
|
||||
private sealed class AsyncFailingArtifactCache : IDeploymentArtifactCache
|
||||
{
|
||||
public async Task StoreAsync(string clusterId, string deploymentId, string revisionHash,
|
||||
byte[] artifact, CancellationToken ct = default)
|
||||
{
|
||||
await Task.Yield();
|
||||
throw new InvalidOperationException("disk full");
|
||||
}
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentAsync(string clusterId, CancellationToken ct = default)
|
||||
=> Task.FromResult<CachedDeploymentArtifact?>(null);
|
||||
|
||||
public Task<CachedDeploymentArtifact?> GetCurrentUnkeyedAsync(CancellationToken ct = default)
|
||||
=> Task.FromResult<CachedDeploymentArtifact?>(null);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user