ef41a43102
ApplyAndAck advanced _currentRevision and recorded NodeDeploymentState=Applied immediately after ReconcileDrivers, which returns null when the artifact could not be read. So a node that applied NOTHING reported success, claimed a revision whose configuration it never applied, and — because HandleDispatchFromSteady short-circuits on a revision match — could never be healed by re-dispatching that revision. Only a later, different revision recovered it. Now a null blob fails the apply: the revision is left where it was, NodeDeploymentState records Failed with the reason, and the coordinator gets a Failed ACK. Leaving the revision alone is what makes the retry actually land instead of being waved through. This is about telling the truth, not about tearing anything down — the node keeps serving its last-known-good address space, drivers and subscriptions throughout (#485). Tests, RED-first (both verified failing with "should be Failed but was Applied"): the ACK/revision assertion, plus the one that matters — after a failed apply, re-dispatching the SAME revision now genuinely applies. Its recovered artifact adds a second driver precisely so a short-circuited ACK (which has no side effects) cannot satisfy it. Four existing tests changed, and both changes are deliberate: - DriverHostActorTests.SeedDeployment never set ArtifactBlob at all. Its three consumers are about the apply/ACK/state machine, not the artifact, and now need a genuine apply — so the helper seeds a well-formed artifact declaring no drivers. That is what the composer emits for an empty configuration, and is precisely what "bytes we could not read" is NOT. - EmptyArtifact_IsNotCached asserted "the apply still succeeds — an empty artifact is a legitimate no-op deployment". That premise is the bug. Flipped to Failed; the test's actual subject (nothing is cached) is untouched and now holds for two independent reasons. Closes #486 Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
287 lines
12 KiB
C#
287 lines
12 KiB
C#
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
public sealed class DriverHostActorTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
|
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
|
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
|
|
|
/// <summary>Verifies that bootstrap with no prior state enters the Steady state.</summary>
|
|
[Fact]
|
|
public void Bootstrap_with_no_prior_state_enters_Steady()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
|
|
|
// No-rev Steady: an incoming dispatch should be processed as a fresh apply, not a no-op.
|
|
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.Sealed);
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
ack.NodeId.ShouldBe(TestNode);
|
|
}
|
|
|
|
/// <summary>Verifies that dispatching the same revision is acked immediately without apply work.</summary>
|
|
[Fact]
|
|
public void Same_revision_dispatch_is_acked_immediately_with_no_apply_work()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.Sealed);
|
|
|
|
// Seed an Applied NodeDeploymentState for self at RevA so PreStart recovers Steady@RevA.
|
|
using (var ctx = db.CreateDbContext())
|
|
{
|
|
ctx.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = TestNode.Value,
|
|
DeploymentId = deploymentId.Value,
|
|
Status = NodeDeploymentStatus.Applied,
|
|
AppliedAtUtc = DateTime.UtcNow.AddMinutes(-1),
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
|
|
|
// Dispatch the SAME deployment again.
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
// No new NodeDeploymentState row got added — the rev matched, so nothing changed.
|
|
using var verify = db.CreateDbContext();
|
|
verify.NodeDeploymentStates.Count(s => s.NodeId == TestNode.Value).ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>Verifies that a new revision dispatch writes an Applied NodeDeploymentState.</summary>
|
|
[Fact]
|
|
public void New_revision_dispatch_writes_Applied_NodeDeploymentState()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentB = SeedDeployment(db, RevB, DeploymentStatus.Dispatching);
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentB, RevB, CorrelationId.NewId()));
|
|
|
|
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
using var verify = db.CreateDbContext();
|
|
var row = verify.NodeDeploymentStates.Single(s =>
|
|
s.NodeId == TestNode.Value && s.DeploymentId == deploymentB.Value);
|
|
row.Status.ShouldBe(NodeDeploymentStatus.Applied);
|
|
row.AppliedAtUtc.ShouldNotBeNull();
|
|
}, duration: TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
/// <summary>Verifies that an orphaned Applying row on bootstrap is replayed.</summary>
|
|
[Fact]
|
|
public void Orphan_Applying_row_on_bootstrap_replays_apply()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.AwaitingApplyAcks);
|
|
|
|
// Crash-orphan: a prior actor was mid-apply and never finished.
|
|
using (var ctx = db.CreateDbContext())
|
|
{
|
|
ctx.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = TestNode.Value,
|
|
DeploymentId = deploymentId.Value,
|
|
Status = NodeDeploymentStatus.Applying,
|
|
StartedAtUtc = DateTime.UtcNow.AddMinutes(-2),
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
|
|
var coordinator = CreateTestProbe();
|
|
Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
|
|
|
// PreStart should replay → ApplyAck back to coordinator with the new correlation id.
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
|
ack.DeploymentId.ShouldBe(deploymentId);
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
using var verify = db.CreateDbContext();
|
|
verify.NodeDeploymentStates.Single(s =>
|
|
s.NodeId == TestNode.Value && s.DeploymentId == deploymentId.Value)
|
|
.Status.ShouldBe(NodeDeploymentStatus.Applied);
|
|
}
|
|
|
|
/// <summary>Fresh apply: dispatching a deployment whose artifact carries one Equipment
|
|
/// ScriptedAlarm forwards a <see cref="ScriptedAlarmHostActor.ApplyScriptedAlarms"/> carrying that
|
|
/// plan to the injected ScriptedAlarm host (via the <c>scriptedAlarmHostOverride</c> seam, mirroring
|
|
/// the VirtualTag-host wiring).</summary>
|
|
[Fact]
|
|
public void Apply_forwards_EquipmentScriptedAlarms_to_scripted_alarm_host()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeploymentWithScriptedAlarm(db, RevA);
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var alarmHost = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
scriptedAlarmHostOverride: alarmHost.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
var apply = alarmHost.ExpectMsg<ScriptedAlarmHostActor.ApplyScriptedAlarms>(TimeSpan.FromSeconds(5));
|
|
var plan = apply.Plans.ShouldHaveSingleItem();
|
|
plan.ScriptedAlarmId.ShouldBe("al-1");
|
|
plan.EquipmentId.ShouldBe("eq-1");
|
|
plan.Name.ShouldBe("Overheat");
|
|
plan.PredicateSource.ShouldContain("ctx.GetTag");
|
|
}
|
|
|
|
/// <summary>Bootstrap-restore: a node that already has an <c>Applied</c> NodeDeploymentState
|
|
/// row for a ScriptedAlarm-carrying deployment re-forwards the
|
|
/// <see cref="ScriptedAlarmHostActor.ApplyScriptedAlarms"/> on PreStart (no dispatch needed), so a
|
|
/// restarted node restores its live ScriptedAlarm children.</summary>
|
|
[Fact]
|
|
public void Restore_on_bootstrap_forwards_EquipmentScriptedAlarms_to_scripted_alarm_host()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeploymentWithScriptedAlarm(db, RevA);
|
|
|
|
// Seed an Applied NodeDeploymentState row so Bootstrap() detects the Applied branch and
|
|
// calls RestoreApplied — no DispatchDeployment needed.
|
|
using (var ctx = db.CreateDbContext())
|
|
{
|
|
ctx.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = TestNode.Value,
|
|
DeploymentId = deploymentId.Value,
|
|
Status = NodeDeploymentStatus.Applied,
|
|
StartedAtUtc = DateTime.UtcNow,
|
|
AppliedAtUtc = DateTime.UtcNow,
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var alarmHost = CreateTestProbe();
|
|
// No DispatchDeployment — Bootstrap() should detect the Applied row and run RestoreApplied,
|
|
// which routes through PushDesiredSubscriptions and forwards ApplyScriptedAlarms.
|
|
Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
scriptedAlarmHostOverride: alarmHost.Ref));
|
|
|
|
var apply = alarmHost.ExpectMsg<ScriptedAlarmHostActor.ApplyScriptedAlarms>(TimeSpan.FromSeconds(5));
|
|
var plan = apply.Plans.ShouldHaveSingleItem();
|
|
plan.ScriptedAlarmId.ShouldBe("al-1");
|
|
plan.EquipmentId.ShouldBe("eq-1");
|
|
plan.Name.ShouldBe("Overheat");
|
|
plan.PredicateSource.ShouldContain("ctx.GetTag");
|
|
}
|
|
|
|
private static DeploymentId SeedDeploymentWithScriptedAlarm(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev)
|
|
{
|
|
// Artifact carries a ScriptedAlarm joined (by PredicateScriptId) to its predicate Script —
|
|
// the same shape ConfigComposer emits and DeploymentArtifact.ParseComposition decodes into
|
|
// composition.EquipmentScriptedAlarms.
|
|
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
Scripts = new[]
|
|
{
|
|
new
|
|
{
|
|
ScriptId = "scr-1",
|
|
SourceCode = "return System.Convert.ToDouble(ctx.GetTag(\"Mach1.Temp\").Value) > 80;",
|
|
},
|
|
},
|
|
ScriptedAlarms = new[]
|
|
{
|
|
new
|
|
{
|
|
ScriptedAlarmId = "al-1",
|
|
EquipmentId = "eq-1",
|
|
Name = "Overheat",
|
|
AlarmType = "LimitAlarm",
|
|
Severity = 700,
|
|
MessageTemplate = "Machine 1 hot",
|
|
PredicateScriptId = "scr-1",
|
|
HistorizeToAveva = true,
|
|
Retain = true,
|
|
Enabled = true,
|
|
},
|
|
},
|
|
});
|
|
|
|
var id = DeploymentId.NewId();
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.Deployments.Add(new Configuration.Entities.Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = rev.Value,
|
|
Status = DeploymentStatus.Sealed,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = DateTime.UtcNow,
|
|
ArtifactBlob = artifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
return id;
|
|
}
|
|
|
|
private static DeploymentId SeedDeployment(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db,
|
|
RevisionHash rev,
|
|
DeploymentStatus status)
|
|
{
|
|
var id = DeploymentId.NewId();
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.Deployments.Add(new Configuration.Entities.Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = rev.Value,
|
|
Status = status,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = status == DeploymentStatus.Sealed ? DateTime.UtcNow : null,
|
|
// A REAL (if empty-of-content) artifact. These tests are about the apply/ACK/state machine,
|
|
// not about the artifact, and used to omit the blob entirely — but since #486 an unreadable
|
|
// artifact (which a missing blob is indistinguishable from) correctly FAILS the apply, so an
|
|
// artifact-less row would no longer exercise the success path they are asserting on. A
|
|
// configuration that declares no drivers is the genuine "nothing to run" deployment.
|
|
ArtifactBlob = EmptyButRealArtifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
return id;
|
|
}
|
|
|
|
/// <summary>A well-formed artifact declaring no drivers — what the composer emits for a configuration
|
|
/// with nothing in it, as distinct from bytes that could not be read.</summary>
|
|
private static readonly byte[] EmptyButRealArtifact = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = Array.Empty<object>(),
|
|
DriverInstances = Array.Empty<object>(),
|
|
Devices = Array.Empty<object>(),
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = Array.Empty<object>(),
|
|
});
|
|
}
|