Joseph Doherty
0a821417ea
60 s labelled a slow-but-clean large rebuild degraded. The deadline moves to a shared Commons DeployTimings constant that ConfigPublishCoordinator reads, and the node's timeout is deadline - 15 s margin, so the held ack still lands in time. A timed-out ack's reason now says the outcome was not reported in time, distinct from a real sink failure. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
245 lines
14 KiB
C#
245 lines
14 KiB
C#
using Akka.Actor;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
|
|
|
/// <summary>
|
|
/// Gitea #601 (archreview 01/S-9) — the deploy ack must be honest about the address-space apply. It used
|
|
/// to be sent the moment the revision advanced, before <c>OpcUaPublishActor</c> had even dequeued the
|
|
/// rebuild, so a degraded apply (swallowed sink faults; <c>_lastApplied</c> deliberately not advanced)
|
|
/// still acked a clean <see cref="ApplyAckOutcome.Applied"/> and the deployment sealed green fleet-wide.
|
|
/// Now the Applied ack waits for the publish actor's <see cref="OpcUaPublishActor.RebuildCompleted"/> and
|
|
/// carries its <see cref="ApplyAck.Degraded"/> verdict; a missing verdict times out to degraded, never clean.
|
|
/// </summary>
|
|
[Collection("opcua-publish-meters")]
|
|
public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
|
|
{
|
|
private static readonly NodeId TestNode = NodeId.Parse("driver-test");
|
|
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
|
|
|
[Fact]
|
|
public void The_Applied_ack_waits_for_the_address_space_outcome_and_a_clean_one_acks_clean()
|
|
{
|
|
var (db, deploymentId) = SeedOneTagDeployment();
|
|
var coordinator = CreateTestProbe();
|
|
var publish = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
|
|
rebuild.ReportOutcome.ShouldBeTrue();
|
|
// The ack is HELD until the outcome lands — the pre-#601 code had already sent it by now.
|
|
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
|
|
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
ack.Degraded.ShouldBeFalse();
|
|
ack.DegradedReason.ShouldBeNull();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_degraded_address_space_outcome_acks_Applied_but_degraded_with_the_reason()
|
|
{
|
|
var (db, deploymentId) = SeedOneTagDeployment();
|
|
var coordinator = CreateTestProbe();
|
|
var publish = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
|
|
publish.Reply(new OpcUaPublishActor.RebuildCompleted(
|
|
rebuild.Correlation, rebuild.DeploymentId, Degraded: true, DegradedReason: "failedNodes=2"));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
ack.Degraded.ShouldBeTrue();
|
|
ack.DegradedReason.ShouldBe("failedNodes=2");
|
|
ack.DeploymentId.ShouldBe(deploymentId);
|
|
}
|
|
|
|
[Fact]
|
|
public void A_real_publish_actor_whose_sink_faults_produces_a_degraded_ack_end_to_end()
|
|
{
|
|
// No hand-written verdict: the real OpcUaPublishActor + AddressSpaceApplier over a sink that faults on
|
|
// every node. This is the production shape of the defect — a degraded apply that sealed green.
|
|
var (db, deploymentId) = SeedOneTagDeployment();
|
|
var coordinator = CreateTestProbe();
|
|
var sink = new FaultingSink();
|
|
var publish = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
|
sink: sink, applier: new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance)));
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
ack.Degraded.ShouldBeTrue();
|
|
ack.DegradedReason.ShouldNotBeNull();
|
|
ack.DegradedReason.ShouldContain("failedNodes=");
|
|
}
|
|
|
|
[Fact]
|
|
public void A_real_publish_actor_with_a_healthy_sink_produces_a_clean_ack_end_to_end()
|
|
{
|
|
// The control for the test above: same wiring, a sink that does not fault ⇒ clean.
|
|
var (db, deploymentId) = SeedOneTagDeployment();
|
|
var coordinator = CreateTestProbe();
|
|
var sink = new FaultingSink(fault: false);
|
|
var publish = Sys.ActorOf(OpcUaPublishActor.PropsForTests(
|
|
sink: sink, applier: new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance)));
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
ack.Degraded.ShouldBeFalse();
|
|
}
|
|
|
|
[Fact]
|
|
public void No_outcome_within_the_timeout_acks_degraded_never_clean()
|
|
{
|
|
var (db, deploymentId) = SeedOneTagDeployment();
|
|
var coordinator = CreateTestProbe();
|
|
var publish = CreateTestProbe(); // never replies — a hung / lost apply
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish.Ref, applyOutcomeTimeout: TimeSpan.FromMilliseconds(300)));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
|
|
var ack = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
|
|
ack.Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
|
ack.Degraded.ShouldBeTrue();
|
|
ack.DegradedReason.ShouldNotBeNull();
|
|
// Distinct from a real sink failure: it says the outcome was not reported in time.
|
|
ack.DegradedReason.ShouldStartWith(DriverHostActor.OutcomeNotReportedReasonPrefix);
|
|
ack.DegradedReason.ShouldNotContain("failedNodes=");
|
|
|
|
// A late outcome after the timeout must not produce a second ack.
|
|
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
|
|
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null));
|
|
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
}
|
|
|
|
[Fact]
|
|
public void Redispatching_a_degraded_revision_reapplies_instead_of_acking_clean()
|
|
{
|
|
var (db, deploymentId) = SeedOneTagDeployment();
|
|
var coordinator = CreateTestProbe();
|
|
var publish = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
var first = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
|
|
publish.Reply(new OpcUaPublishActor.RebuildCompleted(first.Correlation, first.DeploymentId, true, "failedNodes=1"));
|
|
coordinator.ExpectMsg<ApplyAck>(PresenceBudget).Degraded.ShouldBeTrue();
|
|
|
|
// Same revision again: the old idempotent short-circuit would ack a CLEAN Applied right here. It must
|
|
// re-apply instead (a second rebuild request), and ack whatever THAT apply reports.
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
var second = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
|
|
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
|
publish.Reply(new OpcUaPublishActor.RebuildCompleted(second.Correlation, second.DeploymentId, false, null));
|
|
coordinator.ExpectMsg<ApplyAck>(PresenceBudget).Degraded.ShouldBeFalse();
|
|
|
|
// Now clean: a third dispatch of the same revision takes the short-circuit again (no rebuild).
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
var third = coordinator.ExpectMsg<ApplyAck>(PresenceBudget);
|
|
third.Degraded.ShouldBeFalse();
|
|
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
|
}
|
|
|
|
[Fact]
|
|
public void A_degraded_apply_records_its_reason_on_the_local_NodeDeploymentState_row()
|
|
{
|
|
// #601 review — a node WITH a ConfigDb writes its own row Applied/null before the outcome is known.
|
|
// Central later adds the reason from the ack, but only when it accepts that ack (a stale/timed-out
|
|
// deployment's ack is discarded), so the node must not leave its own row reading clean.
|
|
var (db, deploymentId) = SeedOneTagDeployment();
|
|
var coordinator = CreateTestProbe();
|
|
var publish = CreateTestProbe();
|
|
var actor = Sys.ActorOf(DriverHostActor.Props(
|
|
db, TestNode, coordinator.Ref, localRoles: new HashSet<string> { "driver" },
|
|
opcUaPublishActor: publish.Ref));
|
|
|
|
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
|
|
var rebuild = publish.FishForMessage<OpcUaPublishActor.RebuildAddressSpace>(_ => true, PresenceBudget);
|
|
publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, true, "failedNodes=3"));
|
|
coordinator.ExpectMsg<ApplyAck>(PresenceBudget).Degraded.ShouldBeTrue();
|
|
|
|
AwaitAssert(() =>
|
|
{
|
|
using var ctx = db.CreateDbContext();
|
|
var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value);
|
|
row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied);
|
|
row.FailureReason.ShouldBe(ApplyAck.DegradedReasonPrefix + "failedNodes=3");
|
|
}, PresenceBudget);
|
|
}
|
|
|
|
[Fact]
|
|
public void The_default_outcome_timeout_is_derived_from_centrals_apply_deadline_minus_a_margin()
|
|
{
|
|
// A slow-but-clean rebuild must not be labelled degraded at 60 s, yet the held ack must still
|
|
// reach central before its apply deadline. Both sides read ONE shared constant.
|
|
DeployTimings.ApplyDeadline.ShouldBe(TimeSpan.FromMinutes(2));
|
|
DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBe(DeployTimings.ApplyDeadline - DeployTimings.ApplyOutcomeSafetyMargin);
|
|
DeployTimings.ApplyOutcomeSafetyMargin.ShouldBe(TimeSpan.FromSeconds(15));
|
|
DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBeGreaterThan(TimeSpan.FromSeconds(60));
|
|
DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBeLessThan(DeployTimings.ApplyDeadline);
|
|
}
|
|
|
|
private (IDbContextFactory<OtOpcUaConfigDbContext> Db, DeploymentId Id) SeedOneTagDeployment()
|
|
{
|
|
var db = NewInMemoryDbFactory();
|
|
var id = V3ArtifactSeeder.SeedV3Deployment(db, RevA,
|
|
rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) },
|
|
unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
|
|
return (db, id);
|
|
}
|
|
|
|
/// <summary>A sink that throws from every node-creating call when <c>fault</c> is set.</summary>
|
|
private sealed class FaultingSink(bool fault = true) : IOpcUaAddressSpaceSink
|
|
{
|
|
private void MaybeFault()
|
|
{
|
|
if (fault) throw new InvalidOperationException("simulated sink fault");
|
|
}
|
|
|
|
public void WriteValue(string nodeId, object? value, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
|
|
public void WriteAlarmQuality(string alarmNodeId, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm) { }
|
|
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime occurredUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
|
|
public void MaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, AddressSpaceRealm realm, bool isNative = false) => MaybeFault();
|
|
public void EnsureFolder(string folderNodeId, string? parentNodeId, string displayName, AddressSpaceRealm realm = AddressSpaceRealm.Uns) => MaybeFault();
|
|
public void EnsureVariable(string variableNodeId, string? parentFolderNodeId, string displayName, string dataType, bool writable, AddressSpaceRealm realm, string? historianTagname = null, bool isArray = false, uint? arrayLength = null, bool refuseArrayOnScalar = false) => MaybeFault();
|
|
public void RebuildAddressSpace() { }
|
|
public void RaiseNodesAddedModelChange(string affectedNodeId, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { }
|
|
public void WireAlarmNotifiers(string alarmNodeId, AddressSpaceRealm alarmRealm, IReadOnlyList<string> notifierFolderNodeIds, AddressSpaceRealm notifierFolderRealm) { }
|
|
public void AddReference(string sourceNodeId, AddressSpaceRealm sourceRealm, string targetNodeId, AddressSpaceRealm targetRealm, string referenceType = "Organizes") { }
|
|
}
|
|
}
|