using Akka.Actor; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.OpcUaServer; using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers; /// /// Gitea #601 (archreview 01/S-9) — the deploy ack must be honest about the address-space apply. It used /// to be sent the moment the revision advanced, before OpcUaPublishActor had even dequeued the /// rebuild, so a degraded apply (swallowed sink faults; _lastApplied deliberately not advanced) /// still acked a clean and the deployment sealed green fleet-wide. /// Now the Applied ack waits for the publish actor's and /// carries its verdict; a missing verdict times out to degraded, never clean. /// [Collection("opcua-publish-meters")] public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase { private static readonly NodeId TestNode = NodeId.Parse("driver-test"); private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64)); [Fact] public void The_Applied_ack_waits_for_the_address_space_outcome_and_a_clean_one_acks_clean() { var (db, deploymentId) = SeedOneTagDeployment(); var coordinator = CreateTestProbe(); var publish = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, opcUaPublishActor: publish.Ref)); actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var rebuild = publish.FishForMessage(_ => true, PresenceBudget); rebuild.ReportOutcome.ShouldBeTrue(); // The ack is HELD until the outcome lands — the pre-#601 code had already sent it by now. coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null)); var ack = coordinator.ExpectMsg(PresenceBudget); ack.Outcome.ShouldBe(ApplyAckOutcome.Applied); ack.Degraded.ShouldBeFalse(); ack.DegradedReason.ShouldBeNull(); } [Fact] public void A_degraded_address_space_outcome_acks_Applied_but_degraded_with_the_reason() { var (db, deploymentId) = SeedOneTagDeployment(); var coordinator = CreateTestProbe(); var publish = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, opcUaPublishActor: publish.Ref)); actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var rebuild = publish.FishForMessage(_ => true, PresenceBudget); publish.Reply(new OpcUaPublishActor.RebuildCompleted( rebuild.Correlation, rebuild.DeploymentId, Degraded: true, DegradedReason: "failedNodes=2")); var ack = coordinator.ExpectMsg(PresenceBudget); ack.Outcome.ShouldBe(ApplyAckOutcome.Applied); ack.Degraded.ShouldBeTrue(); ack.DegradedReason.ShouldBe("failedNodes=2"); ack.DeploymentId.ShouldBe(deploymentId); } [Fact] public void A_real_publish_actor_whose_sink_faults_produces_a_degraded_ack_end_to_end() { // No hand-written verdict: the real OpcUaPublishActor + AddressSpaceApplier over a sink that faults on // every node. This is the production shape of the defect — a degraded apply that sealed green. var (db, deploymentId) = SeedOneTagDeployment(); var coordinator = CreateTestProbe(); var sink = new FaultingSink(); var publish = Sys.ActorOf(OpcUaPublishActor.PropsForTests( sink: sink, applier: new AddressSpaceApplier(sink, NullLogger.Instance))); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, opcUaPublishActor: publish)); actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var ack = coordinator.ExpectMsg(PresenceBudget); ack.Outcome.ShouldBe(ApplyAckOutcome.Applied); ack.Degraded.ShouldBeTrue(); ack.DegradedReason.ShouldNotBeNull(); ack.DegradedReason.ShouldContain("failedNodes="); } [Fact] public void A_real_publish_actor_with_a_healthy_sink_produces_a_clean_ack_end_to_end() { // The control for the test above: same wiring, a sink that does not fault ⇒ clean. var (db, deploymentId) = SeedOneTagDeployment(); var coordinator = CreateTestProbe(); var sink = new FaultingSink(fault: false); var publish = Sys.ActorOf(OpcUaPublishActor.PropsForTests( sink: sink, applier: new AddressSpaceApplier(sink, NullLogger.Instance))); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, opcUaPublishActor: publish)); actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var ack = coordinator.ExpectMsg(PresenceBudget); ack.Outcome.ShouldBe(ApplyAckOutcome.Applied); ack.Degraded.ShouldBeFalse(); } [Fact] public void No_outcome_within_the_timeout_acks_degraded_never_clean() { var (db, deploymentId) = SeedOneTagDeployment(); var coordinator = CreateTestProbe(); var publish = CreateTestProbe(); // never replies — a hung / lost apply var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, opcUaPublishActor: publish.Ref, applyOutcomeTimeout: TimeSpan.FromMilliseconds(300))); actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var ack = coordinator.ExpectMsg(PresenceBudget); ack.Outcome.ShouldBe(ApplyAckOutcome.Applied); ack.Degraded.ShouldBeTrue(); ack.DegradedReason.ShouldNotBeNull(); // Distinct from a real sink failure: it says the outcome was not reported in time. ack.DegradedReason.ShouldStartWith(DriverHostActor.OutcomeNotReportedReasonPrefix); ack.DegradedReason.ShouldNotContain("failedNodes="); // A late outcome after the timeout must not produce a second ack. var rebuild = publish.FishForMessage(_ => true, PresenceBudget); publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null)); coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); } [Fact] public void Redispatching_a_degraded_revision_reapplies_instead_of_acking_clean() { var (db, deploymentId) = SeedOneTagDeployment(); var coordinator = CreateTestProbe(); var publish = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, opcUaPublishActor: publish.Ref)); actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var first = publish.FishForMessage(_ => true, PresenceBudget); publish.Reply(new OpcUaPublishActor.RebuildCompleted(first.Correlation, first.DeploymentId, true, "failedNodes=1")); coordinator.ExpectMsg(PresenceBudget).Degraded.ShouldBeTrue(); // Same revision again: the old idempotent short-circuit would ack a CLEAN Applied right here. It must // re-apply instead (a second rebuild request), and ack whatever THAT apply reports. actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var second = publish.FishForMessage(_ => true, PresenceBudget); coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); publish.Reply(new OpcUaPublishActor.RebuildCompleted(second.Correlation, second.DeploymentId, false, null)); coordinator.ExpectMsg(PresenceBudget).Degraded.ShouldBeFalse(); // Now clean: a third dispatch of the same revision takes the short-circuit again (no rebuild). actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var third = coordinator.ExpectMsg(PresenceBudget); third.Degraded.ShouldBeFalse(); publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); } [Fact] public void A_degraded_apply_records_its_reason_on_the_local_NodeDeploymentState_row() { // #601 review — a node WITH a ConfigDb writes its own row Applied/null before the outcome is known. // Central later adds the reason from the ack, but only when it accepts that ack (a stale/timed-out // deployment's ack is discarded), so the node must not leave its own row reading clean. var (db, deploymentId) = SeedOneTagDeployment(); var coordinator = CreateTestProbe(); var publish = CreateTestProbe(); var actor = Sys.ActorOf(DriverHostActor.Props( db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" }, opcUaPublishActor: publish.Ref)); actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId())); var rebuild = publish.FishForMessage(_ => true, PresenceBudget); publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, true, "failedNodes=3")); coordinator.ExpectMsg(PresenceBudget).Degraded.ShouldBeTrue(); AwaitAssert(() => { using var ctx = db.CreateDbContext(); var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value); row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied); row.FailureReason.ShouldBe(ApplyAck.DegradedReasonPrefix + "failedNodes=3"); }, PresenceBudget); } [Fact] public void The_default_outcome_timeout_is_derived_from_centrals_apply_deadline_minus_a_margin() { // A slow-but-clean rebuild must not be labelled degraded at 60 s, yet the held ack must still // reach central before its apply deadline. Both sides read ONE shared constant. DeployTimings.ApplyDeadline.ShouldBe(TimeSpan.FromMinutes(2)); DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBe(DeployTimings.ApplyDeadline - DeployTimings.ApplyOutcomeSafetyMargin); DeployTimings.ApplyOutcomeSafetyMargin.ShouldBe(TimeSpan.FromSeconds(15)); DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBeGreaterThan(TimeSpan.FromSeconds(60)); DriverHostActor.DefaultApplyOutcomeTimeout.ShouldBeLessThan(DeployTimings.ApplyDeadline); } private (IDbContextFactory Db, DeploymentId Id) SeedOneTagDeployment() { var db = NewInMemoryDbFactory(); var id = V3ArtifactSeeder.SeedV3Deployment(db, RevA, rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) }, unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) }); return (db, id); } /// A sink that throws from every node-creating call when fault is set. private sealed class FaultingSink(bool fault = true) : IOpcUaAddressSpaceSink { private void MaybeFault() { if (fault) throw new InvalidOperationException("simulated sink fault"); } public void WriteValue(string nodeId, object? value, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { } public void WriteAlarmQuality(string alarmNodeId, uint statusCode, DateTime sourceTimestampUtc, AddressSpaceRealm realm) { } public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime occurredUtc, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { } public void MaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, AddressSpaceRealm realm, bool isNative = false) => MaybeFault(); public void EnsureFolder(string folderNodeId, string? parentNodeId, string displayName, AddressSpaceRealm realm = AddressSpaceRealm.Uns) => MaybeFault(); public void EnsureVariable(string variableNodeId, string? parentFolderNodeId, string displayName, string dataType, bool writable, AddressSpaceRealm realm, string? historianTagname = null, bool isArray = false, uint? arrayLength = null, bool refuseArrayOnScalar = false) => MaybeFault(); public void RebuildAddressSpace() { } public void RaiseNodesAddedModelChange(string affectedNodeId, AddressSpaceRealm realm = AddressSpaceRealm.Uns) { } public void WireAlarmNotifiers(string alarmNodeId, AddressSpaceRealm alarmRealm, IReadOnlyList notifierFolderNodeIds, AddressSpaceRealm notifierFolderRealm) { } public void AddReference(string sourceNodeId, AddressSpaceRealm sourceRealm, string targetNodeId, AddressSpaceRealm targetRealm, string referenceType = "Organizes") { } } }