90d7f6dfca
DriverHostActor no longer acks Applied the moment the revision advances. It asks OpcUaPublishActor for the outcome (RebuildAddressSpace.ReportOutcome) and acks Applied+Degraded(reason) when the apply was degraded; a missing outcome times out (60 s, inside the coordinator's 2 min deadline) to a degraded ack, never a clean one. Re-dispatching a revision whose apply was degraded or is still unconfirmed re-applies instead of taking the clean idempotent short-circuit - that re-apply is also what repairs the degraded nodes. Tests with a bare publish probe use a clean-outcome auto-pilot probe. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
329 lines
15 KiB
C#
329 lines
15 KiB
C#
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
public sealed class DriverHostActorTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
|
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
|
private static readonly RevisionHash RevB = RevisionHash.Parse(new string('b', 64));
|
|
|
|
/// <summary>Verifies that bootstrap with no prior state enters the Steady state.</summary>
|
|
[Fact]
|
|
public void Bootstrap_with_no_prior_state_enters_Steady()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
|
|
|
// No-rev Steady: an incoming dispatch should be processed as a fresh apply, not a no-op.
|
|
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.Sealed);
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
ack.NodeId.ShouldBe(TestNode);
|
|
}
|
|
|
|
/// <summary>Verifies that dispatching the same revision is acked immediately without apply work.</summary>
|
|
[Fact]
|
|
public void Same_revision_dispatch_is_acked_immediately_with_no_apply_work()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.Sealed);
|
|
|
|
// Seed an Applied NodeDeploymentState for self at RevA so PreStart recovers Steady@RevA.
|
|
using (var ctx = db.CreateDbContext())
|
|
{
|
|
ctx.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = TestNode.Value,
|
|
DeploymentId = deploymentId.Value,
|
|
Status = NodeDeploymentStatus.Applied,
|
|
AppliedAtUtc = DateTime.UtcNow.AddMinutes(-1),
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
|
|
|
// Dispatch the SAME deployment again.
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
// No new NodeDeploymentState row got added — the rev matched, so nothing changed.
|
|
using var verify = db.CreateDbContext();
|
|
verify.NodeDeploymentStates.Count(s => s.NodeId == TestNode.Value).ShouldBe(1);
|
|
}
|
|
|
|
/// <summary>Verifies that a new revision dispatch writes an Applied NodeDeploymentState.</summary>
|
|
[Fact]
|
|
public void New_revision_dispatch_writes_Applied_NodeDeploymentState()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentB = SeedDeployment(db, RevB, DeploymentStatus.Dispatching);
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentB, RevB, CorrelationId.NewId()));
|
|
|
|
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
using var verify = db.CreateDbContext();
|
|
var row = verify.NodeDeploymentStates.Single(s =>
|
|
s.NodeId == TestNode.Value && s.DeploymentId == deploymentB.Value);
|
|
row.Status.ShouldBe(NodeDeploymentStatus.Applied);
|
|
row.AppliedAtUtc.ShouldNotBeNull();
|
|
}, duration: TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
/// <summary>Verifies that an orphaned Applying row on bootstrap is replayed.</summary>
|
|
[Fact]
|
|
public void Orphan_Applying_row_on_bootstrap_replays_apply()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeployment(db, RevA, DeploymentStatus.AwaitingApplyAcks);
|
|
|
|
// Crash-orphan: a prior actor was mid-apply and never finished.
|
|
using (var ctx = db.CreateDbContext())
|
|
{
|
|
ctx.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = TestNode.Value,
|
|
DeploymentId = deploymentId.Value,
|
|
Status = NodeDeploymentStatus.Applying,
|
|
StartedAtUtc = DateTime.UtcNow.AddMinutes(-2),
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
|
|
var coordinator = CreateTestProbe();
|
|
Sys.ActorOf(DriverHostActor.Props(db, TestNode, coordinator.Ref));
|
|
|
|
// PreStart should replay → ApplyAck back to coordinator with the new correlation id.
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5));
|
|
ack.DeploymentId.ShouldBe(deploymentId);
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
using var verify = db.CreateDbContext();
|
|
verify.NodeDeploymentStates.Single(s =>
|
|
s.NodeId == TestNode.Value && s.DeploymentId == deploymentId.Value)
|
|
.Status.ShouldBe(NodeDeploymentStatus.Applied);
|
|
}
|
|
|
|
/// <summary>Fresh apply: dispatching a deployment whose artifact carries one Equipment
|
|
/// ScriptedAlarm forwards a <see cref="ScriptedAlarmHostActor.ApplyScriptedAlarms"/> carrying that
|
|
/// plan to the injected ScriptedAlarm host (via the <c>scriptedAlarmHostOverride</c> seam, mirroring
|
|
/// the VirtualTag-host wiring).</summary>
|
|
[Fact]
|
|
public void Apply_forwards_EquipmentScriptedAlarms_to_scripted_alarm_host()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeploymentWithScriptedAlarm(db, RevA);
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var alarmHost = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
scriptedAlarmHostOverride: alarmHost.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
var apply = alarmHost.ExpectMsg<ScriptedAlarmHostActor.ApplyScriptedAlarms>(TimeSpan.FromSeconds(5));
|
|
var plan = apply.Plans.ShouldHaveSingleItem();
|
|
plan.ScriptedAlarmId.ShouldBe("al-1");
|
|
plan.EquipmentId.ShouldBe("eq-1");
|
|
plan.Name.ShouldBe("Overheat");
|
|
plan.PredicateSource.ShouldContain("ctx.GetTag");
|
|
}
|
|
|
|
/// <summary>Bootstrap-restore: a node that already has an <c>Applied</c> NodeDeploymentState
|
|
/// row for a ScriptedAlarm-carrying deployment re-forwards the
|
|
/// <see cref="ScriptedAlarmHostActor.ApplyScriptedAlarms"/> on PreStart (no dispatch needed), so a
|
|
/// restarted node restores its live ScriptedAlarm children.</summary>
|
|
[Fact]
|
|
public void Restore_on_bootstrap_forwards_EquipmentScriptedAlarms_to_scripted_alarm_host()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = SeedDeploymentWithScriptedAlarm(db, RevA);
|
|
|
|
// Seed an Applied NodeDeploymentState row so Bootstrap() detects the Applied branch and
|
|
// calls RestoreApplied — no DispatchDeployment needed.
|
|
using (var ctx = db.CreateDbContext())
|
|
{
|
|
ctx.NodeDeploymentStates.Add(new Configuration.Entities.NodeDeploymentState
|
|
{
|
|
NodeId = TestNode.Value,
|
|
DeploymentId = deploymentId.Value,
|
|
Status = NodeDeploymentStatus.Applied,
|
|
StartedAtUtc = DateTime.UtcNow,
|
|
AppliedAtUtc = DateTime.UtcNow,
|
|
});
|
|
ctx.SaveChanges();
|
|
}
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var alarmHost = CreateTestProbe();
|
|
// No DispatchDeployment — Bootstrap() should detect the Applied row and run RestoreApplied,
|
|
// which routes through PushDesiredSubscriptions and forwards ApplyScriptedAlarms.
|
|
Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
scriptedAlarmHostOverride: alarmHost.Ref));
|
|
|
|
var apply = alarmHost.ExpectMsg<ScriptedAlarmHostActor.ApplyScriptedAlarms>(TimeSpan.FromSeconds(5));
|
|
var plan = apply.Plans.ShouldHaveSingleItem();
|
|
plan.ScriptedAlarmId.ShouldBe("al-1");
|
|
plan.EquipmentId.ShouldBe("eq-1");
|
|
plan.Name.ShouldBe("Overheat");
|
|
plan.PredicateSource.ShouldContain("ctx.GetTag");
|
|
}
|
|
|
|
/// <summary>Verifies that a Direct-mode apply hands the artifact bytes it has already read to the
|
|
/// address-space rebuild instead of leaving the publish actor to re-read the same row from central SQL.</summary>
|
|
/// <remarks>
|
|
/// The Direct apply reads <c>Deployment.ArtifactBlob</c> once (<c>ReconcileDrivers</c> returns it as
|
|
/// <c>appliedBlob</c>) and then used to Tell <c>RebuildAddressSpace</c> with NO bytes — so
|
|
/// <c>OpcUaPublishActor.HandleRebuild</c> re-read the row, and <c>PushDesiredSubscriptions</c> read it
|
|
/// a third time: three central-SQL reads of one immutable blob per apply. Carrying the in-hand bytes
|
|
/// collapses that to one. The FetchAndCache path has always been shaped this way, because it has no
|
|
/// ConfigDb to re-read from — this asserts Direct mode now matches it. Byte-equality (rather than
|
|
/// merely non-null) is the assertion that matters: a rebuild fed the WRONG artifact would materialise
|
|
/// a configuration nobody deployed.
|
|
/// </remarks>
|
|
[Fact]
|
|
public void Direct_mode_apply_hands_the_already_read_artifact_to_the_rebuild()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var deploymentId = V3ArtifactSeeder.SeedV3Deployment(db, RevA,
|
|
rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) },
|
|
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
|
|
|
|
byte[] seeded;
|
|
using (var seedCtx = db.CreateDbContext())
|
|
seeded = seedCtx.Deployments.Single(d => d.DeploymentId == deploymentId.Value).ArtifactBlob!;
|
|
seeded.Length.ShouldBeGreaterThan(0);
|
|
|
|
var coordinator = CreateTestProbe();
|
|
var publish = this.CreateCleanOutcomePublishProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref,
|
|
localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
coordinator.ExpectMsg<ApplyAck>(PresenceBudget).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
|
|
var rebuild = publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(PresenceBudget);
|
|
rebuild.DeploymentId.ShouldBe(deploymentId);
|
|
rebuild.Artifact.ShouldNotBeNull();
|
|
rebuild.Artifact.ShouldBe(seeded);
|
|
}
|
|
|
|
private static DeploymentId SeedDeploymentWithScriptedAlarm(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev)
|
|
{
|
|
// Artifact carries a ScriptedAlarm joined (by PredicateScriptId) to its predicate Script —
|
|
// the same shape ConfigComposer emits and DeploymentArtifact.ParseComposition decodes into
|
|
// composition.EquipmentScriptedAlarms.
|
|
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
Scripts = new[]
|
|
{
|
|
new
|
|
{
|
|
ScriptId = "scr-1",
|
|
SourceCode = "return System.Convert.ToDouble(ctx.GetTag(\"Mach1.Temp\").Value) > 80;",
|
|
},
|
|
},
|
|
ScriptedAlarms = new[]
|
|
{
|
|
new
|
|
{
|
|
ScriptedAlarmId = "al-1",
|
|
EquipmentId = "eq-1",
|
|
Name = "Overheat",
|
|
AlarmType = "LimitAlarm",
|
|
Severity = 700,
|
|
MessageTemplate = "Machine 1 hot",
|
|
PredicateScriptId = "scr-1",
|
|
HistorizeToAveva = true,
|
|
Retain = true,
|
|
Enabled = true,
|
|
},
|
|
},
|
|
});
|
|
|
|
var id = DeploymentId.NewId();
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.Deployments.Add(new Configuration.Entities.Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = rev.Value,
|
|
Status = DeploymentStatus.Sealed,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = DateTime.UtcNow,
|
|
ArtifactBlob = artifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
return id;
|
|
}
|
|
|
|
private static DeploymentId SeedDeployment(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> db,
|
|
RevisionHash rev,
|
|
DeploymentStatus status)
|
|
{
|
|
var id = DeploymentId.NewId();
|
|
using var ctx = db.CreateDbContext();
|
|
ctx.Deployments.Add(new Configuration.Entities.Deployment
|
|
{
|
|
DeploymentId = id.Value,
|
|
RevisionHash = rev.Value,
|
|
Status = status,
|
|
CreatedBy = "test",
|
|
SealedAtUtc = status == DeploymentStatus.Sealed ? DateTime.UtcNow : null,
|
|
// A REAL (if empty-of-content) artifact. These tests are about the apply/ACK/state machine,
|
|
// not about the artifact, and used to omit the blob entirely — but since #486 an unreadable
|
|
// artifact (which a missing blob is indistinguishable from) correctly FAILS the apply, so an
|
|
// artifact-less row would no longer exercise the success path they are asserting on. A
|
|
// configuration that declares no drivers is the genuine "nothing to run" deployment.
|
|
ArtifactBlob = EmptyButRealArtifact,
|
|
});
|
|
ctx.SaveChanges();
|
|
return id;
|
|
}
|
|
|
|
/// <summary>A well-formed artifact declaring no drivers — what the composer emits for a configuration
|
|
/// with nothing in it, as distinct from bytes that could not be read.</summary>
|
|
private static readonly byte[] EmptyButRealArtifact = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
RawFolders = Array.Empty<object>(),
|
|
DriverInstances = Array.Empty<object>(),
|
|
Devices = Array.Empty<object>(),
|
|
TagGroups = Array.Empty<object>(),
|
|
Tags = Array.Empty<object>(),
|
|
});
|
|
}
|