feat(runtime): DriverHostActor state machine with PreStart recovery + DispatchDeployment + stale fallback
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,143 @@
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
public sealed class DriverHostActorTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
||||
|
||||
[Fact]
|
||||
public void Bootstrap_with_no_prior_state_enters_Steady()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
||||
|
||||
// No-rev Steady: an incoming dispatch should be processed as a fresh apply, not a no-op.
|
||||
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.Sealed);
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
|
||||
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
||||
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
ack.NodeId.ShouldBe(TestNode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Same_revision_dispatch_is_acked_immediately_with_no_apply_work()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.Sealed);
|
||||
|
||||
// Seed an Applied NodeDeploymentState for self at RevA so PreStart recovers Steady@RevA.
|
||||
using (var ctx = db.CreateDbContext())
|
||||
{
|
||||
ctx.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
||||
{
|
||||
NodeId = TestNode.Value,
|
||||
DeploymentId = deploymentId.Value,
|
||||
Status = NodeDeploymentStatus.Applied,
|
||||
AppliedAtUtc = DateTime.UtcNow.AddMinutes(-1),
|
||||
});
|
||||
ctx.SaveChanges();
|
||||
}
|
||||
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
||||
|
||||
// Dispatch the SAME deployment again.
|
||||
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
||||
|
||||
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
||||
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
|
||||
// No new NodeDeploymentState row got added — the rev matched, so nothing changed.
|
||||
using var verify = db.CreateDbContext();
|
||||
verify.NodeDeploymentStates.Count(s => s.NodeId == TestNode.Value).ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void New_revision_dispatch_writes_Applied_NodeDeploymentState()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var deploymentB = SeedDeployment(db, RevB, DeploymentStatus.Dispatching);
|
||||
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
||||
|
||||
actor.Tell(new DispatchDeployment(deploymentB, RevB, CorrelationId.NewId()));
|
||||
|
||||
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
using var verify = db.CreateDbContext();
|
||||
var row = verify.NodeDeploymentStates.Single(s =>
|
||||
s.NodeId == TestNode.Value && s.DeploymentId == deploymentB.Value);
|
||||
row.Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
row.AppliedAtUtc.ShouldNotBeNull();
|
||||
}, duration: TimeSpan.FromSeconds(3));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Orphan_Applying_row_on_bootstrap_replays_apply()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.AwaitingApplyAcks);
|
||||
|
||||
// Crash-orphan: a prior actor was mid-apply and never finished.
|
||||
using (var ctx = db.CreateDbContext())
|
||||
{
|
||||
ctx.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
||||
{
|
||||
NodeId = TestNode.Value,
|
||||
DeploymentId = deploymentId.Value,
|
||||
Status = NodeDeploymentStatus.Applying,
|
||||
StartedAtUtc = DateTime.UtcNow.AddMinutes(-2),
|
||||
});
|
||||
ctx.SaveChanges();
|
||||
}
|
||||
|
||||
var coordinator = CreateTestProbe();
|
||||
Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
||||
|
||||
// PreStart should replay → ApplyAck back to coordinator with the new correlation id.
|
||||
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
||||
ack.DeploymentId.ShouldBe(deploymentId);
|
||||
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
|
||||
using var verify = db.CreateDbContext();
|
||||
verify.NodeDeploymentStates.Single(s =>
|
||||
s.NodeId == TestNode.Value && s.DeploymentId == deploymentId.Value)
|
||||
.Status.ShouldBe(NodeDeploymentStatus.Applied);
|
||||
}
|
||||
|
||||
private static DeploymentId SeedDeployment(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db,
|
||||
RevisionHash rev,
|
||||
DeploymentStatus status)
|
||||
{
|
||||
var id = DeploymentId.NewId();
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Add(new Configuration.Entities.Deployment
|
||||
{
|
||||
DeploymentId = id.Value,
|
||||
RevisionHash = rev.Value,
|
||||
Status = status,
|
||||
CreatedBy = "test",
|
||||
SealedAtUtc = status == DeploymentStatus.Sealed ? DateTime.UtcNow : null,
|
||||
});
|
||||
ctx.SaveChanges();
|
||||
return id;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
using Akka.Cluster;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
/// <summary>
|
||||
/// Mirrors the ControlPlane test harness: single-node Akka cluster (self-join) + in-memory
|
||||
/// EF Core <see cref="OtOpcUaConfigDbContext"/> via an <see cref="IDbContextFactory{TContext}"/>.
|
||||
/// </summary>
|
||||
public abstract class RuntimeActorTestBase : TestKit
|
||||
{
|
||||
protected static string AkkaTestHocon => @"
|
||||
akka {
|
||||
loglevel = ""WARNING""
|
||||
extensions = [
|
||||
""Akka.Cluster.Tools.PublishSubscribe.DistributedPubSubExtensionProvider, Akka.Cluster.Tools""
|
||||
]
|
||||
actor {
|
||||
provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster""
|
||||
}
|
||||
remote.dot-netty.tcp {
|
||||
hostname = ""127.0.0.1""
|
||||
port = 0
|
||||
}
|
||||
cluster {
|
||||
seed-nodes = []
|
||||
roles = [""driver""]
|
||||
min-nr-of-members = 1
|
||||
run-coordinated-shutdown-when-down = off
|
||||
}
|
||||
}";
|
||||
|
||||
protected RuntimeActorTestBase() : base(AkkaTestHocon)
|
||||
{
|
||||
var cluster = Akka.Cluster.Cluster.Get(Sys);
|
||||
cluster.Join(cluster.SelfAddress);
|
||||
AwaitCondition(() => cluster.State.Members.Any(m => m.Status == MemberStatus.Up),
|
||||
TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
protected static IDbContextFactory<OtOpcUaConfigDbContext> NewInMemoryDbFactory(string? dbName = null)
|
||||
{
|
||||
dbName ??= Guid.NewGuid().ToString("N");
|
||||
return new InMemoryConfigDbFactory(dbName);
|
||||
}
|
||||
|
||||
private sealed class InMemoryConfigDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
|
||||
{
|
||||
public OtOpcUaConfigDbContext CreateDbContext()
|
||||
{
|
||||
var opts = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
|
||||
.UseInMemoryDatabase(dbName)
|
||||
.Options;
|
||||
return new OtOpcUaConfigDbContext(opts);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
<RootNamespace>ZB.MOM.WW.OtOpcUa.Runtime.Tests</RootNamespace>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- xunit v2 because Akka.TestKit.Xunit2 pins to it (same as ControlPlane.Tests). -->
|
||||
<PackageReference Include="xunit"/>
|
||||
<PackageReference Include="Shouldly"/>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk"/>
|
||||
<PackageReference Include="Akka.TestKit.Xunit2"/>
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory"/>
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\Server\ZB.MOM.WW.OtOpcUa.Runtime\ZB.MOM.WW.OtOpcUa.Runtime.csproj"/>
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Configuration\ZB.MOM.WW.OtOpcUa.Configuration.csproj"/>
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Commons\ZB.MOM.WW.OtOpcUa.Commons.csproj"/>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<NuGetAuditSuppress Include="https://github.com/advisories/GHSA-g94r-2vxg-569j"/>
|
||||
<NuGetAuditSuppress Include="https://github.com/advisories/GHSA-h958-fxgg-g7w3"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
Reference in New Issue
Block a user