", security + 1, StringComparison.Ordinal);
+ security.ShouldBeGreaterThan(-1);
+ var panel = razor[security..next];
+ panel.ShouldContain("_form.OpcUa.ValidateServerHostName");
+ panel.ShouldContain("Opt-in");
+ panel.ShouldContain("NAT");
+ panel.ShouldContain("IPv6 zone ids");
+ panel.ShouldContain("Loose match");
+ razor.Split("_form.OpcUa.ValidateServerHostName").Length.ShouldBe(2, "the toggle must be rendered once");
+ }
+
///
/// #648: stored DriverConfig JSON from before targetNamespaceKind/unsMappingTable were removed
/// must still load into the form (the form's options use UnmappedMemberHandling.Skip) and the legacy
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs
new file mode 100644
index 000000000..202969bc7
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorDbOutageTests.cs
@@ -0,0 +1,620 @@
+using Akka.Actor;
+using Microsoft.EntityFrameworkCore;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
+using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests;
+
+///
+/// lmxopcua#868: the coordinator's recovery read used to run synchronously in PreStart. With the
+/// ConfigDb unreachable when the singleton started, the actor died on initialization, was never re-created,
+/// and every deployment hung. It now loads asynchronously with backoff, queues what arrives meanwhile and
+/// reports until the load succeeds.
+///
+public sealed class ConfigPublishCoordinatorDbOutageTests : ControlPlaneActorTestBase
+{
+ private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('d', 64));
+ private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5);
+ private static readonly TimeSpan RetryMin = TimeSpan.FromMilliseconds(50);
+ private static readonly TimeSpan RetryMax = TimeSpan.FromMilliseconds(200);
+ private const string Node = "site-a-1:4053";
+
+ [Fact]
+ public void Db_unreachable_at_start_does_not_kill_the_coordinator_and_a_deploy_sent_during_the_outage_completes_once_it_returns()
+ {
+ var dbName = Guid.NewGuid().ToString("N");
+ var direct = NewInMemoryDbFactory(dbName);
+ var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
+ SeedNodes(direct, Node);
+ var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
+ var readiness = new ConfigPublishCoordinatorReadiness();
+
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+ Watch(coordinator);
+
+ // Several failed loads: the actor keeps retrying (it is alive) and reports Loading with the error.
+ AwaitAssert(() =>
+ {
+ outage.Attempts.ShouldBeGreaterThanOrEqualTo(3);
+ readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Loading);
+ readiness.Current.FailedLoadAttempts.ShouldBeGreaterThanOrEqualTo(2);
+ readiness.Current.LastError.ShouldNotBeNull();
+ readiness.Current.LastError.ShouldContain("unreachable");
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ // A deployment dispatched during the outage is queued, not processed and not dropped.
+ coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
+ ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
+ using (var db = direct.CreateDbContext())
+ {
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching);
+ db.NodeDeploymentStates.ShouldBeEmpty();
+ }
+
+ // The DB comes back: the load succeeds, the queued dispatch runs, and the node's ack seals it.
+ outage.Down = false;
+ AwaitAssert(() =>
+ {
+ readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+ db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { Node });
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ ExpectNoMsg(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); // never Terminated
+ }
+
+ [Fact]
+ public void In_flight_deployment_is_recovered_after_the_outage_and_an_ack_queued_during_it_is_not_lost()
+ {
+ var dbName = Guid.NewGuid().ToString("N");
+ var direct = NewInMemoryDbFactory(dbName);
+ var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
+ SeedNodes(direct, Node, "site-a-2:4053");
+ var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow);
+ using (var db = direct.CreateDbContext())
+ {
+ db.NodeDeploymentStates.Add(new NodeDeploymentState
+ { NodeId = Node, DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applying });
+ db.NodeDeploymentStates.Add(new NodeDeploymentState
+ { NodeId = "site-a-2:4053", DeploymentId = deploymentId.Value, Status = NodeDeploymentStatus.Applied, AppliedAtUtc = DateTime.UtcNow });
+ db.SaveChanges();
+ }
+
+ var readiness = new ConfigPublishCoordinatorReadiness();
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+ AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2),
+ duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ // The last outstanding ack arrives while the coordinator is still loading: it is queued, then applied
+ // against the recovered expected set (site-a-2 already Applied in the DB) and seals the deployment.
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ outage.Down = false;
+
+ AwaitAssert(() =>
+ {
+ readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Review finding 1: a recovered deployment whose every expected ack is already in the DB as a FINAL outcome
+ /// (a seal whose SaveChanges failed, or a fused node's own row — which DriverHostActor writes Applied only
+ /// once the address-space outcome is known; a pre-outcome row stays Applying, second review M1) must be completed — inside the deadline it used to wait for
+ /// acks that had already landed, past it the deadline's "race" guard returned without sealing; either way it
+ /// sat in AwaitingApplyAcks forever and AdminOps refused every later deploy.
+ ///
+ [Theory]
+ [InlineData(false, false, DeploymentStatus.Sealed)]
+ [InlineData(true, false, DeploymentStatus.Sealed)]
+ [InlineData(false, true, DeploymentStatus.PartiallyFailed)]
+ [InlineData(true, true, DeploymentStatus.PartiallyFailed)]
+ public void Recovered_deployment_with_every_ack_already_persisted_is_completed(bool pastDeadline, bool oneFailed, DeploymentStatus expected)
+ {
+ var dbFactory = NewInMemoryDbFactory();
+ SeedNodes(dbFactory, Node, "site-a-2:4053");
+ var deadline = TimeSpan.FromMinutes(1);
+ var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks,
+ pastDeadline ? DateTime.UtcNow.AddMinutes(-10) : DateTime.UtcNow);
+ SeedNodeState(dbFactory, deploymentId, Node, NodeDeploymentStatus.Applied);
+ SeedNodeState(dbFactory, deploymentId, "site-a-2:4053", oneFailed ? NodeDeploymentStatus.Failed : NodeDeploymentStatus.Applied);
+
+ Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, deadline));
+
+ AwaitAssert(() =>
+ {
+ using var db = dbFactory.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(expected);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Review finding 2: a recovered deployment that is already past its deadline must first apply the acks that
+ /// were queued while the load was failing (they arrived in time); the deadline used to run inline before the
+ /// replay, so they were discarded as stale and the deployment was timed out.
+ ///
+ [Fact]
+ public void Ack_queued_during_the_outage_is_applied_before_a_past_deadline_recovery_times_out()
+ {
+ var dbName = Guid.NewGuid().ToString("N");
+ var direct = NewInMemoryDbFactory(dbName);
+ var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
+ SeedNodes(direct, Node);
+ var deploymentId = SeedDeployment(direct, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow.AddMinutes(-10));
+ SeedNodeState(direct, deploymentId, Node, NodeDeploymentStatus.Applying);
+
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ outage, TimeSpan.FromMinutes(1), loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+ AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(2),
+ duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ outage.Down = false;
+
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Review finding 3: an ack whose persist fails (ConfigDb blip while Ready) is retried, not dropped. It used to
+ /// throw, restart the actor and lose the ack — the node never resends and never writes its own row.
+ ///
+ [Fact]
+ public void Ack_whose_persist_fails_is_retried_not_dropped()
+ {
+ var dbName = Guid.NewGuid().ToString("N");
+ var direct = NewInMemoryDbFactory(dbName);
+ var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName));
+ SeedNodes(direct, Node);
+ var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
+ var readiness = new ConfigPublishCoordinatorReadiness();
+
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ outage, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+ coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ outage.Down = true;
+ var before = outage.Attempts;
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ AwaitAssert(() => outage.Attempts.ShouldBeGreaterThanOrEqualTo(before + 2),
+ duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ outage.Down = false;
+
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Second review M2: an ack that arrived but whose persist is still failing at the deadline must not be timed
+ /// out as "no ack from" its node. The deadline is deferred until the backlog persists, then the deployment seals.
+ ///
+ [Fact]
+ public void Deadline_does_not_time_out_a_received_but_unpersisted_ack()
+ {
+ var (direct, outage, deploymentId, coordinator, _) = DispatchedSingleNode(TimeSpan.FromSeconds(2));
+
+ outage.Down = true;
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ // Hold the DB down past the 2 s apply deadline.
+ Thread.Sleep(TimeSpan.FromSeconds(3.5));
+ outage.Down = false;
+
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ db.NodeDeploymentStates.Single().Status.ShouldBe(NodeDeploymentStatus.Applied);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Second review M2: an unpersisted ack must survive an actor restart (its retry timer does not). Here a second
+ /// dispatch's guard read throws while the DB is down, restarting the coordinator.
+ ///
+ [Fact]
+ public void Unpersisted_ack_survives_a_coordinator_restart()
+ {
+ var (direct, outage, deploymentId, coordinator, _) = DispatchedSingleNode(NoTimeout);
+
+ outage.Down = true;
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ var before = outage.Attempts;
+ AwaitAssert(() => outage.Attempts.ShouldBeGreaterThan(before),
+ duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ // Its guard read throws (DB down) => the coordinator restarts.
+ coordinator.Tell(new DispatchDeployment(DeploymentId.NewId(), TestRevision, CorrelationId.NewId()));
+ Thread.Sleep(TimeSpan.FromMilliseconds(500));
+ outage.Down = false;
+
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single(d => d.DeploymentId == deploymentId.Value).Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Second review L3: a retry of a node's OLDER unpersisted ack must not overwrite the newer ack that persisted
+ /// after it (it used to flip the node to Failed and turn a clean deploy PartiallyFailed).
+ ///
+ [Fact]
+ public void A_stale_ack_retry_does_not_overwrite_a_newer_persisted_ack()
+ {
+ var dbName = Guid.NewGuid().ToString("N");
+ var direct = NewInMemoryDbFactory(dbName);
+ var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName));
+ SeedNodes(direct, Node, "site-a-2:4053");
+ var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ outage, NoTimeout, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+ coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ // Node 1's first answer (Failed) cannot be persisted; its corrected answer (Applied) can.
+ outage.FailNext(1);
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Failed, "first try", CorrelationId.NewId()));
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ Thread.Sleep(TimeSpan.FromMilliseconds(600)); // several retry periods
+ using (var db = direct.CreateDbContext())
+ db.NodeDeploymentStates.Single(s => s.NodeId == Node).Status.ShouldBe(NodeDeploymentStatus.Applied);
+
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-2:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Second review L4: a dispatch QUEUED while the load failed for longer than the apply deadline was never sent;
+ /// it must be dispatched (fresh deadline) once the load succeeds, not closed "Dispatch was lost".
+ ///
+ [Fact]
+ public void Dispatch_queued_through_an_outage_longer_than_the_deadline_is_sent_not_closed_as_lost()
+ {
+ var dbName = Guid.NewGuid().ToString("N");
+ var direct = NewInMemoryDbFactory(dbName);
+ var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName)) { Down = true };
+ SeedNodes(direct, Node);
+ var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
+ var deadline = TimeSpan.FromSeconds(3);
+
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ outage, deadline, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+ coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
+ Thread.Sleep(deadline + TimeSpan.FromSeconds(1));
+ outage.Down = false;
+
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+ db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { Node });
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Second review L5: an in-flight row recovery cannot interpret (blank RevisionHash on a row it must re-dispatch)
+ /// used to throw on every load and keep the coordinator Loading forever. It is closed with a reason, and the
+ /// coordinator serves the next deployment.
+ ///
+ [Fact]
+ public void Malformed_in_flight_row_is_closed_with_a_reason_and_the_coordinator_carries_on()
+ {
+ var dbFactory = NewInMemoryDbFactory();
+ SeedNodes(dbFactory, Node);
+ var bad = DeploymentId.NewId();
+ using (var db = dbFactory.CreateDbContext())
+ {
+ db.Deployments.Add(new Deployment
+ {
+ DeploymentId = bad.Value, RevisionHash = " ", Status = DeploymentStatus.Dispatching,
+ CreatedBy = "test", CreatedAtUtc = DateTime.UtcNow,
+ });
+ db.SaveChanges();
+ }
+ var readiness = new ConfigPublishCoordinatorReadiness();
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ dbFactory, NoTimeout, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+
+ AwaitAssert(() =>
+ {
+ readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
+ using var db = dbFactory.CreateDbContext();
+ var row = db.Deployments.Single(d => d.DeploymentId == bad.Value);
+ row.Status.ShouldBe(DeploymentStatus.PartiallyFailed);
+ row.FailureReason.ShouldNotBeNull();
+ row.FailureReason.ShouldContain("RevisionHash");
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ var next = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(1));
+ coordinator.Tell(new DispatchDeployment(next, TestRevision, CorrelationId.NewId()));
+ coordinator.Tell(new ApplyAck(next, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var db = dbFactory.CreateDbContext();
+ db.Deployments.Single(d => d.DeploymentId == next.Value).Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ /// Second review L6: an ack-persist backlog is visible on readiness while Ready, and clears.
+ [Fact]
+ public void Readiness_reports_the_ack_persist_backlog()
+ {
+ var (_, outage, deploymentId, coordinator, readiness) = DispatchedSingleNode(NoTimeout);
+
+ outage.Down = true;
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready);
+ readiness.Current.PendingAckPersists.ShouldBe(1);
+ readiness.Current.LastPersistError.ShouldNotBeNull();
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ outage.Down = false;
+ AwaitAssert(() =>
+ {
+ readiness.Current.PendingAckPersists.ShouldBe(0);
+ readiness.Current.LastPersistError.ShouldBeNull();
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Third review #1: a Failed ack's reason (node free text) was written unbounded; on SQL Server the overflow
+ /// refuses the SaveChanges forever and the deployment wedged. It is truncated to the column.
+ ///
+ [Fact]
+ public void An_oversized_failure_reason_is_truncated_not_retried_forever()
+ {
+ var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout);
+
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Failed,
+ new string('x', 5000), CorrelationId.NewId()));
+
+ AwaitAssert(() =>
+ {
+ using var ctx = db.CreateDbContext();
+ ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.PartiallyFailed);
+ var row = ctx.NodeDeploymentStates.Single();
+ row.Status.ShouldBe(NodeDeploymentStatus.Failed);
+ row.FailureReason!.Length.ShouldBe(FailureReasons.MaxLength);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Third review #1: a write the DB REJECTS (non-transient DbUpdateException) is not retried forever — the ack is
+ /// counted from memory and the deployment completes.
+ ///
+ [Fact]
+ public void A_rejected_ack_write_is_counted_from_memory_not_retried_forever()
+ {
+ var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(NoTimeout);
+ db.PoisonNodeStateUpdates = true;
+
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+
+ AwaitAssert(() =>
+ {
+ using var ctx = db.CreateDbContext();
+ ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Third review #1: a write that keeps failing transiently defers the deadline only up to a cap
+ /// (10 × the retry ceiling); past it the outcome is decided from the acks held in memory.
+ ///
+ [Fact]
+ public void Deadline_deferral_on_unpersisted_acks_is_bounded()
+ {
+ var (db, deploymentId, coordinator) = DispatchedOnConstraintDb(TimeSpan.FromSeconds(1));
+ db.FailNodeStateUpdatesTransiently = true;
+
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+
+ // Deadline 1 s + cap 10 × 200 ms: decided within a few seconds, never left in flight.
+ AwaitAssert(() =>
+ {
+ using var ctx = db.CreateDbContext();
+ ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ ///
+ /// Third review #3: a lost dispatch recovered with only a sliver of its deadline left used to be re-driven with
+ /// that sliver — the nodes applied and the deployment timed out moments later. It gets a full deadline.
+ ///
+ [Fact]
+ public void A_lost_dispatch_redriven_near_its_deadline_gets_a_full_deadline()
+ {
+ var dbFactory = NewInMemoryDbFactory();
+ SeedNodes(dbFactory, Node);
+ var deadline = TimeSpan.FromSeconds(3);
+ var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow - TimeSpan.FromSeconds(2.5));
+
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, deadline));
+ AwaitAssert(() =>
+ {
+ using var db = dbFactory.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ // Well past the 0.5 s that was left, still inside a fresh 3 s deadline.
+ Thread.Sleep(TimeSpan.FromSeconds(1.5));
+ using (var db = dbFactory.CreateDbContext())
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+
+ coordinator.Tell(new ApplyAck(deploymentId, NodeId.Parse(Node), ApplyAckOutcome.Applied, null, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var db = dbFactory.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ private (ConstraintEnforcingDbFactory Db, DeploymentId Id, IActorRef Coordinator) DispatchedOnConstraintDb(TimeSpan applyDeadline)
+ {
+ var db = new ConstraintEnforcingDbFactory(Guid.NewGuid().ToString("N"));
+ SeedNodes(db, Node);
+ var deploymentId = SeedDeployment(db, DeploymentStatus.Dispatching, DateTime.UtcNow);
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ db, applyDeadline, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+ coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var ctx = db.CreateDbContext();
+ ctx.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ return (db, deploymentId, coordinator);
+ }
+
+ /// A single-node deployment dispatched and AwaitingApplyAcks through an outage-capable factory.
+ private (IDbContextFactory Direct, OutageDbFactory Outage, DeploymentId Id, IActorRef Coordinator,
+ ConfigPublishCoordinatorReadiness Readiness) DispatchedSingleNode(TimeSpan applyDeadline)
+ {
+ var dbName = Guid.NewGuid().ToString("N");
+ var direct = NewInMemoryDbFactory(dbName);
+ var outage = new OutageDbFactory(NewInMemoryDbFactory(dbName));
+ SeedNodes(direct, Node);
+ var deploymentId = SeedDeployment(direct, DeploymentStatus.Dispatching, DateTime.UtcNow);
+ var readiness = new ConfigPublishCoordinatorReadiness();
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(
+ outage, applyDeadline, readiness: readiness, loadRetryMin: RetryMin, loadRetryMax: RetryMax));
+ coordinator.Tell(new DispatchDeployment(deploymentId, TestRevision, CorrelationId.NewId()));
+ AwaitAssert(() =>
+ {
+ using var db = direct.CreateDbContext();
+ db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks);
+ }, duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ return (direct, outage, deploymentId, coordinator, readiness);
+ }
+
+ [Fact]
+ public void Readiness_is_NotHosted_after_the_coordinator_stops()
+ {
+ var readiness = new ConfigPublishCoordinatorReadiness();
+ var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(NewInMemoryDbFactory(), NoTimeout, readiness: readiness));
+ AwaitAssert(() => readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.Ready),
+ duration: PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+
+ Watch(coordinator);
+ Sys.Stop(coordinator);
+ ExpectTerminated(coordinator, cancellationToken: TestContext.Current.CancellationToken);
+ readiness.Current.State.ShouldBe(ConfigPublishCoordinatorState.NotHosted);
+ }
+
+ private static DeploymentId SeedDeployment(
+ IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc)
+ {
+ var id = DeploymentId.NewId();
+ using var db = dbFactory.CreateDbContext();
+ db.Deployments.Add(new Deployment
+ {
+ DeploymentId = id.Value,
+ RevisionHash = TestRevision.Value,
+ Status = status,
+ CreatedBy = "test",
+ CreatedAtUtc = createdAtUtc,
+ });
+ db.SaveChanges();
+ return id;
+ }
+
+ private static void SeedNodeState(
+ IDbContextFactory dbFactory, DeploymentId deploymentId, string nodeId, NodeDeploymentStatus status)
+ {
+ using var db = dbFactory.CreateDbContext();
+ db.NodeDeploymentStates.Add(new NodeDeploymentState
+ {
+ NodeId = nodeId, DeploymentId = deploymentId.Value, Status = status,
+ AppliedAtUtc = status == NodeDeploymentStatus.Applying ? null : DateTime.UtcNow,
+ });
+ db.SaveChanges();
+ }
+
+ private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds)
+ {
+ using var db = dbFactory.CreateDbContext();
+ db.ServerClusters.Add(new ServerCluster
+ {
+ ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a",
+ NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test",
+ });
+ foreach (var nodeId in nodeIds)
+ {
+ db.ClusterNodes.Add(new ClusterNode
+ {
+ NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0],
+ ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test",
+ });
+ }
+ db.SaveChanges();
+ }
+
+ /// A factory that throws while — a ConfigDb that is unreachable.
+ private sealed class OutageDbFactory(IDbContextFactory inner) : IDbContextFactory
+ {
+ private volatile bool _down;
+ private int _attempts;
+ private int _failNext;
+
+ public bool Down { get => _down; set => _down = value; }
+
+ public int Attempts => Volatile.Read(ref _attempts);
+
+ /// The next context creations throw, then the DB is reachable again.
+ public void FailNext(int count) => Volatile.Write(ref _failNext, count);
+
+ public OtOpcUaConfigDbContext CreateDbContext()
+ {
+ Interlocked.Increment(ref _attempts);
+ if (_down) throw new InvalidOperationException("ConfigDb unreachable (simulated outage)");
+ if (Volatile.Read(ref _failNext) > 0 && Interlocked.Decrement(ref _failNext) >= 0)
+ throw new InvalidOperationException("ConfigDb unreachable (simulated blip)");
+ return inner.CreateDbContext();
+ }
+ }
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs
index bd73313ad..91a947c8d 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs
@@ -136,9 +136,9 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes
var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout));
var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty()));
- // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and
- // the actor would be stopped, not restarted). Identify is processed after creation, so a
- // resolved selection proves PreStart completed.
+ // Arm the fault AFTER PreStart has run. Since lmxopcua#868 the recovery read is asynchronous and
+ // retried, so a fault that lands on it instead of on HandleDispatch is also recovered (the dispatch is
+ // queued until the retried load succeeds) — either way the deployment must complete.
await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget, TestContext.Current.CancellationToken);
faulting.FailNext();
admin.Tell(new StartDeployment("joe", CorrelationId.NewId()));
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/ConstraintEnforcingDbFactory.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/ConstraintEnforcingDbFactory.cs
new file mode 100644
index 000000000..a3d29603b
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/ConstraintEnforcingDbFactory.cs
@@ -0,0 +1,69 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Diagnostics;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+
+namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
+
+///
+/// An in-memory ConfigDb that behaves like SQL Server where the InMemory provider does not (lmxopcua#868 third
+/// review): a FailureReason longer than the column refuses the whole SaveChanges with a
+/// . It can also reject (, a non-transient
+/// ) or fail transiently () every
+/// update of an existing NodeDeploymentState row.
+///
+public sealed class ConstraintEnforcingDbFactory(string dbName) : IDbContextFactory
+{
+ private readonly Interceptor _interceptor = new();
+
+ /// Every update of an existing node row is rejected as a constraint violation.
+ public bool PoisonNodeStateUpdates { get => _interceptor.Poison; set => _interceptor.Poison = value; }
+
+ /// Every update of an existing node row fails as if the connection dropped.
+ public bool FailNodeStateUpdatesTransiently { get => _interceptor.Transient; set => _interceptor.Transient = value; }
+
+ ///
+ public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder()
+ .UseInMemoryDatabase(dbName)
+ .AddInterceptors(_interceptor)
+ .Options);
+
+ private sealed class Interceptor : SaveChangesInterceptor
+ {
+ public volatile bool Poison;
+ public volatile bool Transient;
+
+ public override InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result)
+ {
+ Check(eventData.Context!);
+ return result;
+ }
+
+ public override ValueTask> SavingChangesAsync(
+ DbContextEventData eventData, InterceptionResult result, CancellationToken cancellationToken = default)
+ {
+ Check(eventData.Context!);
+ return ValueTask.FromResult(result);
+ }
+
+ private void Check(DbContext ctx)
+ {
+ foreach (var e in ctx.ChangeTracker.Entries())
+ {
+ var reason = e.Entity switch
+ {
+ NodeDeploymentState n => n.FailureReason,
+ Deployment d => d.FailureReason,
+ _ => null,
+ };
+ if (reason is { Length: > FailureReasons.MaxLength })
+ throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'.");
+ if (e.Entity is NodeDeploymentState && e.State == EntityState.Modified)
+ {
+ if (Poison) throw new DbUpdateException("The UPDATE statement conflicted with a CHECK constraint.");
+ if (Transient) throw new InvalidOperationException("A network-related error occurred (simulated).");
+ }
+ }
+ }
+ }
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs
new file mode 100644
index 000000000..560028524
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ConfigPublishCoordinatorHealthCheckTests.cs
@@ -0,0 +1,80 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Options;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.Health;
+using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators;
+using ZB.MOM.WW.OtOpcUa.Host.Health;
+
+namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Health;
+
+/// lmxopcua#868: the deploy coordinator reports Degraded (never Unhealthy) while it cannot load from the ConfigDb.
+public sealed class ConfigPublishCoordinatorHealthCheckTests
+{
+ private static async Task Check(ConfigPublishCoordinatorReadiness? readiness) =>
+ await new ConfigPublishCoordinatorHealthCheck(readiness).CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
+
+ [Fact]
+ public async Task Loading_after_failed_attempts_is_degraded_and_names_the_error()
+ {
+ var r = new ConfigPublishCoordinatorReadiness();
+ r.MarkLoading(3, "connection refused");
+ var result = await Check(r);
+ result.Status.ShouldBe(HealthStatus.Degraded);
+ result.Description.ShouldNotBeNull();
+ result.Description.ShouldContain("connection refused");
+ result.Description.ShouldContain("queued");
+ }
+
+ [Fact]
+ public async Task Ready_is_healthy()
+ {
+ var r = new ConfigPublishCoordinatorReadiness();
+ r.MarkLoading(2, "x");
+ r.MarkReady();
+ (await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
+ }
+
+ [Fact]
+ public async Task Ready_with_an_ack_persist_backlog_is_degraded_and_clears()
+ {
+ var r = new ConfigPublishCoordinatorReadiness();
+ r.MarkReady();
+ r.MarkAckPersistBacklog(2, "timeout");
+ var result = await Check(r);
+ result.Status.ShouldBe(HealthStatus.Degraded);
+ result.Description.ShouldNotBeNull();
+ result.Description.ShouldContain("timeout");
+
+ r.MarkAckPersistBacklog(0, null);
+ (await Check(r)).Status.ShouldBe(HealthStatus.Healthy);
+ }
+
+ [Fact]
+ public async Task Not_hosted_and_untracked_are_healthy()
+ {
+ (await Check(new ConfigPublishCoordinatorReadiness())).Status.ShouldBe(HealthStatus.Healthy);
+ (await Check(null)).Status.ShouldBe(HealthStatus.Healthy);
+ }
+
+ [Theory]
+ [InlineData(true, true)]
+ [InlineData(false, false)]
+ public void It_is_registered_on_the_ready_tier_only_on_admin_nodes(bool hasAdmin, bool registered)
+ {
+ var registrations = new ServiceCollection()
+ .AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: true)
+ .BuildServiceProvider()
+ .GetRequiredService>()
+ .Value.Registrations
+ .Where(x => x.Name == ConfigPublishCoordinatorHealthCheck.Name)
+ .ToList();
+ if (!registered)
+ {
+ registrations.ShouldBeEmpty();
+ return;
+ }
+ registrations.ShouldHaveSingleItem().Tags.ShouldBe([ZbHealthTags.Ready]);
+ }
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs
index 18cc6d003..bba4a23eb 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ConditionRefreshDeferralWireTests.cs
@@ -24,16 +24,18 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// every other subscriber still receives them, and the refresh never waits for the stalled drain.
///
///
-/// The overload is produced deterministically: the folder item's server-side MonitoredItem lock is held, so
-/// the SDK's per-node delivery task parks in QueueEvent, its 4096-slot channel fills, and the condition's
-/// later events are deferred — without holding the node manager's Lock, which the refresh needs.
+/// The overload is produced deterministically: delivery is stalled at the EqA folder (), so the
+/// condition's walk suspends there and its later events are deferred — without holding the node manager's Lock,
+/// which the refresh needs. (Before Gitea #855 this held the folder item's SDK lock so the folder's 4096-slot channel
+/// filled; folder items are now queued per item by the reporter, and no notifier of ours defers on its own.) The
+/// folder-item twin of these tests is .
///
public sealed class ConditionRefreshDeferralWireTests
{
private const string EqA = ServerEventRig.EqA;
- // > 4096 (channel) + 1 (parked in the delivery task) + 1 (the in-flight deferred write), < 1024 pending: nothing coalesces.
- private const int FloodSize = 5000;
+ // Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces.
+ private const int FloodSize = 500;
///
Per-folder / Server-object ACL on or off (off is the production default).
///
@@ -84,7 +86,7 @@ public sealed class ConditionRefreshDeferralWireTests
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
await folderSubscription.SetPublishingModeAsync(false, ct);
- using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
@@ -200,7 +202,7 @@ public sealed class ConditionRefreshDeferralWireTests
serverEvents.Has(onServer, "seed").ShouldBeFalse("the key holds no grant yet");
await folderSubscription.SetPublishingModeAsync(false, ct);
- using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Project(native: true, Active("final"));
@@ -271,7 +273,7 @@ public sealed class ConditionRefreshDeferralWireTests
rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
};
- using (var stall = ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
@@ -345,7 +347,7 @@ public sealed class ConditionRefreshDeferralWireTests
proceed.Wait(TimeSpan.FromSeconds(30));
};
- using (ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)))
+ using (NotifierStall.Hold(rig.Nm, EqA))
{
for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
rig.Project(native: true, Active("final"));
@@ -416,7 +418,6 @@ public sealed class ConditionRefreshDeferralWireTests
rig.Project(native: true, Active("s0"));
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "s0") && serverEvents.Has(onServer, "s0"));
await folderSubscription.SetPublishingModeAsync(false, ct);
- var item = FindServerItem(rig.Nm, onA.Status.Id);
using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
var counter = 0;
@@ -440,7 +441,7 @@ public sealed class ConditionRefreshDeferralWireTests
{
while (!stop.IsCancellationRequested)
{
- using (ItemLockStall.Hold(item))
+ using (NotifierStall.Hold(rig.Nm, EqA))
{
Thread.Sleep(400);
if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
@@ -568,7 +569,7 @@ public sealed class ConditionRefreshDeferralWireTests
await RecordedEvents.WaitUntilAsync(() => folderEvents.Has(onA, "seed") && serverEvents.Has(onServer, "seed"));
await folderSubscription.SetPublishingModeAsync(false, ct);
- var stall = deferring ? ItemLockStall.Hold(FindServerItem(rig.Nm, onA.Status.Id)) : null;
+ var stall = deferring ? NotifierStall.Hold(rig.Nm, EqA) : null;
try
{
if (deferring)
@@ -622,7 +623,7 @@ public sealed class ConditionRefreshDeferralWireTests
}
///
The server-side Server-object (all-events) item behind a client item.
- private static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
+ internal static IEventMonitoredItem FindAllEventsItem(OtOpcUaNodeManager nm, uint id)
{
var map = (IDictionary)typeof(OtOpcUaNodeManager)
.GetField("_allEventsItems", BindingFlags.Instance | BindingFlags.NonPublic)!
@@ -636,7 +637,7 @@ public sealed class ConditionRefreshDeferralWireTests
}
///
The server-side event item behind a client item, from the node manager's direct-item map.
- private static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
+ internal static IEventMonitoredItem FindServerItem(OtOpcUaNodeManager nm, uint id)
{
var map = (IDictionary)typeof(OtOpcUaNodeManager)
.GetField("_eventItems", BindingFlags.Instance | BindingFlags.NonPublic)!
@@ -664,45 +665,4 @@ public sealed class ConditionRefreshDeferralWireTests
ScopeId = $"eq-{equipment}",
PermissionFlags = NodePermissions.ReadOnly,
};
-
- ///
- /// Holds a server-side monitored item's private lock on a dedicated thread, so the SDK's per-node delivery task
- /// parks in QueueEvent for that item (a stalled subscriber) without the node manager's Lock held.
- ///
- private sealed class ItemLockStall : IDisposable
- {
- private readonly ManualResetEventSlim _release = new();
- private readonly Thread _thread;
-
- private ItemLockStall(IEventMonitoredItem item)
- {
- var gate = (Lock)typeof(Opc.Ua.Server.MonitoredItem)
- .GetField("m_lock", BindingFlags.Instance | BindingFlags.NonPublic)!
- .GetValue(item)!;
- using var held = new ManualResetEventSlim();
- _thread = new Thread(() =>
- {
- gate.Enter();
- try
- {
- held.Set();
- _release.Wait();
- }
- finally { gate.Exit(); }
- }) { IsBackground = true };
- _thread.Start();
- held.Wait();
- }
-
- public static ItemLockStall Hold(IEventMonitoredItem item) => new(item);
-
- public void Release() => _release.Set();
-
- public void Dispose()
- {
- _release.Set();
- _thread.Join();
- _release.Dispose();
- }
- }
}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventQueueOverflowAclWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventQueueOverflowAclWireTests.cs
index 1ca3b158a..81039208d 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventQueueOverflowAclWireTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/EventQueueOverflowAclWireTests.cs
@@ -27,13 +27,16 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
///
- no ACL-denied event reaches an item during the overflow, including deferred events whose grant was
/// revoked while they waited.
///
+/// Since Gitea #855 the folder items are served per item by the reporter (as the Server-object items always were),
+/// so the flood no longer defers at all — the first test pins that — and the revocation test stalls the folder's
+/// delivery (
) to get a deferred backlog.
///
public sealed class EventQueueOverflowAclWireTests
{
private const string EqA = ServerEventRig.EqA;
private const string EqB = ServerEventRig.EqB;
- // Native events per flood: past the SDK's 4096-slot per-notifier channel, so the native condition defers.
+ // Native events per flood: past the SDK's 4096-slot per-notifier channel (which deferred them before #855).
private const int PerCondition = 6000;
///
@@ -150,13 +153,12 @@ public sealed class EventQueueOverflowAclWireTests
readLatencies.Max().ShouldBeLessThan(TimeSpan.FromSeconds(5), "a Read must not queue behind the whole flood");
(await TimedReadAsync(anonymous, eqA, ct)).ShouldBeLessThan(TimeSpan.FromSeconds(2), "the node manager is free afterwards");
- // Memory: the node manager's own overflow store stayed within its per-condition budget (queued + in flight)…
- Volatile.Read(ref peakDeferred).ShouldBeLessThanOrEqualTo(
- 2 * (OtOpcUaNodeManager.MaxDeferredEventsPerCondition + 1), "deferral is bounded per condition");
- if (singleHold) rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the native tail overflowed and coalesced");
+ // Memory: the node manager held nothing of its own for the overflow (#855: no notifier of ours defers, so the
+ // item queues are the only store, bounded below)…
+ Volatile.Read(ref peakDeferred).ShouldBe(0, "folder items are served per item by the reporter: nothing defers");
+ rig.Nm.DiscardedEventReports.ShouldBe(0);
- // …let the SDK channel finish draining into the (still unpublished) item queues, then publish.
- await Task.Delay(TimeSpan.FromSeconds(2), ct);
+ // …then publish the (still unpublished) item queues.
await keySub.SetPublishingModeAsync(true, ct);
var lastNative = $"n{PerCondition - 1}";
await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(onB, lastNative) && keyEvents.Has(keyServer, lastNative));
@@ -165,10 +167,9 @@ public sealed class EventQueueOverflowAclWireTests
foreach (var item in new[] { onB, keyServer })
{
var received = keyEvents.Of(item);
- // ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (Under a single hold the
- // deferred native tail drains last and evicts every synchronously-queued event from the 1000-slot queue, so
- // there this check cannot see a leak; the "after-s" leg below is the falsifier for that mode — verified by
- // running with enforcement off: chunked fails here, single-hold fails at "after-s".)
+ // ACL: not one event of the denied (EqA-owned) scripted alarm, overflow or not. (The 1000-slot queue keeps
+ // only the last events of the flood, which interleave n/s, so a leak shows here in both modes; the
+ // "after-s" leg below is a second falsifier.)
received.Where(f => RecordedEvents.Message(f)?.StartsWith('s') == true)
.ShouldBeEmpty($"{item.DisplayName}: an ACL-denied event leaked during the overflow");
@@ -211,7 +212,7 @@ public sealed class EventQueueOverflowAclWireTests
}
///
- /// A grant revoked while a deferred backlog waits behind a full notifier channel: the deferred events are decided
+ /// A grant revoked while a deferred backlog waits behind a stalled notifier: the deferred events are decided
/// at delivery against the CURRENT generation, so none of them — nor any event reported after the revocation —
/// reaches the revoked session's folder or Server-object item, while a still-granted session sharing the same
/// folder channel receives the whole tail (positive control).
@@ -249,6 +250,9 @@ public sealed class EventQueueOverflowAclWireTests
await anonSub.ApplyChangesAsync(ct);
anonA.Status.Error.ShouldBeNull();
+ // The EqA folder's delivery stalls (#855: nothing of ours defers on its own any more), so the native condition's
+ // walk suspends there and the rest of the flood is deferred behind it.
+ using var stall = NotifierStall.Hold(rig.Nm, EqA);
var flood = Task.Factory.StartNew(() =>
{
lock (rig.Nm.Lock)
@@ -276,11 +280,12 @@ public sealed class EventQueueOverflowAclWireTests
throw new ShouldAssertException("revoking a grant under Lock while events were deferred stalled (deadlock)");
}
+ stall.Release();
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
await Task.Delay(TimeSpan.FromSeconds(2), ct);
await keySub.SetPublishingModeAsync(true, ct);
- // Positive control: the still-granted session on the SAME folder channel gets the whole deferred tail.
+ // Positive control: the still-granted session on the SAME folder gets the whole deferred tail.
var lastNative = $"n{PerCondition - 1}";
await RecordedEvents.WaitUntilAsync(() =>
anonEvents.Has(anonA, lastNative) && anonEvents.Has(anonA, "after-revoke")
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs
new file mode 100644
index 000000000..b6deba5e2
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderConditionRefreshWireTests.cs
@@ -0,0 +1,841 @@
+using System.Diagnostics;
+using System.Globalization;
+using Opc.Ua;
+using Opc.Ua.Client;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+using MonitoredItem = Opc.Ua.Client.MonitoredItem;
+
+namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
+
+///
+/// Gitea #855: a ConditionRefresh of an event item on an equipment / raw device FOLDER gets the guarantee the
+/// Server object got in #854 — no condition event older than what the refresh replayed reaches the item after
+/// RefreshEnd, per-condition order is kept, the per-folder ACL (#826) decides every event and every replayed condition,
+/// and neither a reporter nor the refresh ever waits on a stalled delivery.
+///
+///
+///
+/// Folder items are now served per item by the node manager (no SDK channel), so in production no notifier defers
+/// at all; pins that under a
+/// flood with concurrent refreshes. Every other test produces the deferral the cut exists for by stalling delivery
+/// at a notifier (), the way a full SDK channel used to.
+///
+///
+/// The rig's native condition is wired into EqA and EqB and hangs under the raw device folder, so its walk is
+/// condition → EqA → … → device folder; a stall at EqA suspends it before EqA's items and every later hop.
+///
+///
+public sealed class FolderConditionRefreshWireTests
+{
+ private const string EqA = ServerEventRig.EqA;
+
+ // Every event after the stalled one is deferred; < 1024 pending, so nothing coalesces.
+ private const int FloodSize = 500;
+
+ /// Per-folder ACL on or off (off is the production default).
+ /// Whether the condition is still Retain at refresh time (re-sent) or cleared (omitted).
+ /// ConditionRefresh2 on the folder item instead of ConditionRefresh on its subscription.
+ ///
+ /// The folder item is modified (ModifyMonitoredItems re-subscribes it to its notifier) between the refresh and the
+ /// drain: its cut must survive, as the Server object's does across an unsubscribe + resubscribe.
+ ///
+ [Theory]
+ [InlineData(false, true, false, false)]
+ [InlineData(true, true, false, false)]
+ [InlineData(false, false, false, false)]
+ [InlineData(true, false, false, false)]
+ [InlineData(false, true, true, false)]
+ [InlineData(true, false, true, false)]
+ [InlineData(false, true, false, true)]
+ [InlineData(true, false, false, true)]
+ public async Task Deferred_events_older_than_a_refresh_never_reach_the_refreshed_folder_item_after_RefreshEnd(
+ bool enforced, bool finalRetained, bool refresh2, bool modify)
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+
+ // A second, non-refreshing subscriber on the SAME folder: it must keep the whole deferred tail.
+ using var otherSession = await rig.ConnectAsync(anonymous: false, ct);
+ var otherEvents = new RecordedEvents();
+ var otherSubscription = await RecordedEvents.SubscribeAsync(otherSession, otherEvents, ct);
+ var otherOnA = RecordedEvents.AddEventItem(otherSubscription, ServerEventRig.UnsNodeId(otherSession, EqA));
+ await otherSubscription.ApplyChangesAsync(ct);
+ otherOnA.Status.Error.ShouldBeNull();
+
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
+ await subscription.ApplyChangesAsync(ct);
+ onA.Status.Error.ShouldBeNull();
+
+ rig.Project(native: true, Active("seed"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed") && otherEvents.Has(otherOnA, "seed"));
+
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
+ {
+ for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
+ rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the stall must actually have deferred events");
+ rig.Project(native: true, finalRetained ? Active("final") : Cleared("final"));
+
+ var refresh = Stopwatch.StartNew();
+ await RefreshAsync(session, subscription, onA, refresh2, ct);
+ refresh.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(10), "ConditionRefresh must not wait for the stalled delivery");
+ await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
+ rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the deferred events were still waiting at RefreshEnd");
+ rig.Nm.ActiveRefreshCuts.ShouldBe(1, "only the refreshed folder item holds a cut");
+
+ if (modify)
+ {
+ onA.QueueSize = 1001;
+ await subscription.ApplyChangesAsync(ct);
+ onA.Status.Error.ShouldBeNull();
+ }
+
+ stall.Release();
+ }
+
+ await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
+ await RecordedEvents.WaitUntilAsync(() => rig.Nm.ActiveRefreshCuts == 0);
+ rig.Project(native: true, Active("after"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after") && otherEvents.Has(otherOnA, "after"));
+ await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
+
+ var (replayed, afterEnd) = Window(events.Of(onA));
+ if (finalRetained)
+ {
+ replayed.ShouldBe(["final"], "the refresh re-sends the condition's current state");
+ afterEnd.ShouldBe(["after"], "nothing older than the refreshed state may follow RefreshEnd");
+ }
+ else
+ {
+ replayed.ShouldBeEmpty("a cleared (Retain=false) condition is not part of a refresh");
+ afterEnd.ShouldBe(["final", "after"], "stale Retain=true events would re-add the alarm; the Retain=false one agrees");
+ events.Of(onA).Last(f => RecordedEvents.Message(f) == "final")[RecordedEvents.RetainField].GetBoolean().ShouldBeFalse();
+ }
+
+ rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0);
+
+ // The other subscriber on the same folder did not refresh: it keeps every deferred event, in order.
+ var flood = Indices(otherEvents.Of(otherOnA), 'f');
+ flood.ShouldBe(Enumerable.Range(0, FloodSize).ToList(), "the non-refreshing folder subscriber keeps the whole tail, in order");
+ otherEvents.Has(otherOnA, "final").ShouldBeTrue();
+ }
+
+ ///
+ /// The race the item gate closes, for a folder item: a deferred event is between its supersession check and its
+ /// queueing to the folder item when the refresh arrives. The refresh must wait for that one delivery, so it lands
+ /// before the replay rather than after RefreshEnd.
+ ///
+ [Fact]
+ public async Task A_folder_refresh_waits_for_a_deferred_delivery_already_past_its_check()
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855Gate", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
+ await subscription.ApplyChangesAsync(ct);
+
+ rig.Project(native: true, Active("seed"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "seed"));
+
+ using var entered = new ManualResetEventSlim();
+ using var proceed = new ManualResetEventSlim();
+ var probed = 0;
+ rig.Nm.DeferredFolderDeliveryProbe = () =>
+ {
+ if (Interlocked.Increment(ref probed) != 1) return;
+ entered.Set();
+ proceed.Wait(TimeSpan.FromSeconds(30));
+ };
+
+ using (NotifierStall.Hold(rig.Nm, EqA))
+ {
+ for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
+ rig.Project(native: true, Active("final"));
+ }
+
+ // The released walk parks on its first deferred delivery to the folder item, past the check, inside the gate.
+ entered.Wait(TimeSpan.FromSeconds(10), ct).ShouldBeTrue("the drain must reach the folder item");
+ var refresh = session.CallAsync(
+ ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
+ await Task.Delay(TimeSpan.FromSeconds(1), ct);
+ events.HasType(onA, ObjectTypeIds.RefreshEndEventType)
+ .ShouldBeFalse("the refresh must wait for the delivery already inside the item's gate");
+ proceed.Set();
+ await refresh.WaitAsync(TimeSpan.FromSeconds(10), ct);
+ await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
+ rig.Nm.DeferredFolderDeliveryProbe = null;
+
+ await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
+ rig.Project(native: true, Active("after"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "after"));
+ await Task.Delay(TimeSpan.FromMilliseconds(300), ct);
+
+ var (window, afterEnd) = Window(events.Of(onA));
+ window.Count.ShouldBe(2, "the parked delivery, then the replay");
+ window[0]!.ShouldStartWith("f");
+ window[1].ShouldBe("final");
+ afterEnd.ShouldBe(["after"]);
+ }
+
+ ///
+ /// The cut path queues its frozen replay OFF Lock. An event newer than the cut that reaches the folder item
+ /// in that window (a condition delivered synchronously, reported from the probe between the cut and the queueing)
+ /// must not be followed by its condition's older frozen replay entry.
+ ///
+ [Fact]
+ public async Task An_event_newer_than_the_folder_cut_is_never_followed_by_its_older_frozen_replay()
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855Pending", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+ // A second condition under the device folder only: its walk never reaches the stalled EqA, so it stays synchronous.
+ const string Free = ServerEventRig.RawDeviceFolder + "/free";
+ rig.Sink.MaterialiseAlarmCondition(Free, ServerEventRig.RawDeviceFolder, "free", "OffNormalAlarm", 700, AddressSpaceRealm.Raw, isNative: true);
+ rig.Sink.WriteAlarmCondition(Free, Active("x1"), DateTime.UtcNow, AddressSpaceRealm.Raw);
+
+ // EqA needs an event item of its own to be a notifier fan-out the stall can apply to.
+ using var eqASession = await rig.ConnectAsync(anonymous: false, ct);
+ var eqAEvents = new RecordedEvents();
+ var eqASubscription = await RecordedEvents.SubscribeAsync(eqASession, eqAEvents, ct);
+ RecordedEvents.AddEventItem(eqASubscription, ServerEventRig.UnsNodeId(eqASession, EqA));
+ await eqASubscription.ApplyChangesAsync(ct);
+
+ // The refreshed item is on the raw device folder, which both conditions report through.
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var onDevice = RecordedEvents.AddEventItem(subscription, RawNodeId(session, ServerEventRig.RawDeviceFolder));
+ await subscription.ApplyChangesAsync(ct);
+ onDevice.Status.Error.ShouldBeNull();
+
+ rig.Project(native: true, Active("seed"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onDevice, "seed"));
+
+ var injected = 0;
+ rig.Nm.DirectRefreshProbe = (cut, _) =>
+ {
+ if (cut && Interlocked.Exchange(ref injected, 1) == 0)
+ rig.Sink.WriteAlarmCondition(Free, Active("x2"), DateTime.UtcNow, AddressSpaceRealm.Raw);
+ };
+
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
+ {
+ for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
+ rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the cut path runs only while something is deferred");
+ await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
+ await RecordedEvents.WaitUntilAsync(() => events.HasType(onDevice, ObjectTypeIds.RefreshEndEventType));
+ stall.Release();
+ }
+
+ rig.Nm.DirectRefreshProbe = null;
+ Volatile.Read(ref injected).ShouldBe(1, "the folder refresh took the cut path");
+ var stream = events.Of(onDevice).ToList();
+ var end = stream.FindIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
+ var free = stream.Take(end).Select(RecordedEvents.Message).Where(m => m is "x1" or "x2").ToList();
+ free.ShouldContain("x2");
+ free[^1].ShouldBe("x2", "the older frozen replay of the condition must not follow its newer event");
+ }
+
+ ///
+ /// ACL on: a condition the folder refresh omitted because the item may not see it there (its reference into EqA was
+ /// dropped by the generation) is not withheld afterwards — the reference returns before the drain, and the item must
+ /// end on the condition's newest state. And while omitted, the replay itself carried nothing of it.
+ ///
+ [Fact]
+ public async Task Deferred_events_of_a_condition_the_folder_refresh_omitted_by_the_acl_are_not_withheld()
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855Acl", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ NodeAcl[] grants = [KeyEquipmentGrant(EqA)];
+ var state = rig.Enforce(true, [ServerEventRig.Key()], grants);
+ Deploy(state, "rev-2", grants, ServerEventRig.EqB); // the native alarm is no longer projected into EqA
+
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
+ await subscription.ApplyChangesAsync(ct);
+ onA.Status.Error.ShouldBeNull();
+
+ rig.Project(native: true, Active("seed"));
+ await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
+ events.Has(onA, "seed").ShouldBeFalse("the generation does not project the alarm into EqA");
+
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
+ {
+ for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
+ rig.Project(native: true, Active("final"));
+ rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
+
+ await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
+ await RecordedEvents.WaitUntilAsync(() => events.HasType(onA, ObjectTypeIds.RefreshEndEventType));
+
+ // The reference returns while the condition's events still wait.
+ Deploy(state, "rev-3", grants, EqA, ServerEventRig.EqB);
+ stall.Release();
+ }
+
+ await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "final"));
+ var (replayed, after) = Window(events.Of(onA));
+ replayed.ShouldBeEmpty("the refresh omitted the condition: not visible through EqA at the time");
+ after.ShouldNotBeEmpty();
+ after[^1].ShouldBe("final", "the item ends on the newest state of a condition it was never refreshed");
+ after.Count(m => m!.StartsWith('f')).ShouldBeGreaterThan(0, "its deferred events were delivered, not withheld");
+ }
+
+ ///
+ /// ACL on: a grant revoked while a refreshed folder item's deferred events wait. Neither the replay (taken while
+ /// granted) nor anything after it leaks once revoked; a still-granted subscriber on the same folder gets the tail.
+ ///
+ [Fact]
+ public async Task A_grant_revoked_after_a_folder_refresh_withholds_every_deferred_event()
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855Revoke", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly);
+ var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant);
+
+ using var keyed = await rig.ConnectAsync(anonymous: false, ct);
+ var keyEvents = new RecordedEvents();
+ var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct);
+ var keyA = RecordedEvents.AddEventItem(keySub, ServerEventRig.UnsNodeId(keyed, EqA));
+ await keySub.ApplyChangesAsync(ct);
+
+ using var anonymous = await rig.ConnectAsync(anonymous: true, ct);
+ var anonEvents = new RecordedEvents();
+ var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct);
+ var anonA = RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA));
+ await anonSub.ApplyChangesAsync(ct);
+
+ rig.Project(native: true, Active("seed"));
+ await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(keyA, "seed") && anonEvents.Has(anonA, "seed"));
+
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
+ {
+ for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
+ await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id);
+ await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(keyA, ObjectTypeIds.RefreshEndEventType));
+
+ state.Apply(
+ ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(),
+ ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
+ ServerEventRig.TagReferences());
+ rig.Project(native: true, Active("after-revoke"));
+ stall.Release();
+ }
+
+ await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
+ await RecordedEvents.WaitUntilAsync(() =>
+ anonEvents.Has(anonA, $"f{FloodSize - 1}") && anonEvents.Has(anonA, "after-revoke"));
+ await Task.Delay(TimeSpan.FromSeconds(1), ct);
+
+ var (_, after) = Window(keyEvents.Of(keyA));
+ after.ShouldBeEmpty("after RefreshEnd the revoked session gets nothing: older events are superseded, later ones denied");
+ keyEvents.Has(keyA, "after-revoke").ShouldBeFalse();
+ }
+
+ ///
+ /// #855 review: the cut path decides its replay before taking Lock; a grant revoked between that decision
+ /// and the queueing of the frozen replay must still withhold the replay (it is re-decided right before each entry).
+ /// The revocation lands from the refresh probe, which runs after the cut is installed and before the replay is queued.
+ ///
+ /// Refresh a Server-object item instead of a folder item.
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public async Task A_grant_revoked_between_the_cut_and_its_replay_withholds_the_replay(bool serverObject)
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855RevokeCut", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ var anonGrant = ServerEventRig.EquipmentGrant(EqA, NodePermissions.ReadOnly);
+ var state = rig.Enforce(true, [ServerEventRig.Key()], KeyEquipmentGrant(EqA), anonGrant);
+
+ // An anonymous subscriber keeps EqA a notifier fan-out the stall applies to (and stays granted throughout).
+ using var anonymous = await rig.ConnectAsync(anonymous: true, ct);
+ var anonEvents = new RecordedEvents();
+ var anonSub = await RecordedEvents.SubscribeAsync(anonymous, anonEvents, ct);
+ RecordedEvents.AddEventItem(anonSub, ServerEventRig.UnsNodeId(anonymous, EqA));
+ await anonSub.ApplyChangesAsync(ct);
+
+ using var keyed = await rig.ConnectAsync(anonymous: false, ct);
+ var keyEvents = new RecordedEvents();
+ var keySub = await RecordedEvents.SubscribeAsync(keyed, keyEvents, ct);
+ var item = RecordedEvents.AddEventItem(keySub, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(keyed, EqA));
+ await keySub.ApplyChangesAsync(ct);
+ item.Status.Error.ShouldBeNull();
+
+ rig.Project(native: true, Active("seed"));
+ await RecordedEvents.WaitUntilAsync(() => keyEvents.Has(item, "seed"));
+
+ var revoked = 0;
+ void Revoke(bool cut, bool _)
+ {
+ if (!cut || Interlocked.Exchange(ref revoked, 1) != 0) return;
+ state.Apply(
+ ServerEventRig.Cluster, "rev-2", [ServerEventRig.Key()], [anonGrant], ServerEventRig.ScopePaths(),
+ ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
+ ServerEventRig.TagReferences());
+ }
+
+ if (serverObject) rig.Nm.ServerObjectRefreshProbe = Revoke;
+ else rig.Nm.DirectRefreshProbe = Revoke;
+
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
+ {
+ for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
+ rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
+ await keyed.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, keySub.Id);
+ await RecordedEvents.WaitUntilAsync(() => keyEvents.HasType(item, ObjectTypeIds.RefreshEndEventType));
+ stall.Release();
+ }
+
+ rig.Nm.ServerObjectRefreshProbe = null;
+ rig.Nm.DirectRefreshProbe = null;
+ Volatile.Read(ref revoked).ShouldBe(1, "the refresh took the cut path");
+ await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
+ await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
+
+ var (replayed, after) = Window(keyEvents.Of(item));
+ replayed.ShouldBeEmpty("the grant was revoked before the replay was queued");
+ after.ShouldBeEmpty("and nothing reaches the item after RefreshEnd either");
+ }
+
+ ///
+ /// #855 review 2: a condition removed by a deploy between the cut and the queueing of its frozen replay must not be
+ /// replayed — its frozen Retain=true state would be a ghost no Retain=false ever follows. ACL off, so nothing but the
+ /// liveness check stands in the way.
+ ///
+ /// Refresh a Server-object item instead of a folder item.
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public async Task A_condition_removed_between_the_cut_and_its_replay_is_not_replayed(bool serverObject)
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855RemoveCut", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+
+ // A second subscriber keeps EqA a notifier fan-out the stall applies to.
+ using var other = await rig.ConnectAsync(anonymous: true, ct);
+ var otherEvents = new RecordedEvents();
+ var otherSub = await RecordedEvents.SubscribeAsync(other, otherEvents, ct);
+ RecordedEvents.AddEventItem(otherSub, ServerEventRig.UnsNodeId(other, EqA));
+ await otherSub.ApplyChangesAsync(ct);
+
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var item = RecordedEvents.AddEventItem(subscription, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(session, EqA));
+ await subscription.ApplyChangesAsync(ct);
+
+ rig.Project(native: true, Active("seed"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(item, "seed"));
+
+ var removed = 0;
+ void Remove(bool cut, bool _)
+ {
+ if (cut && Interlocked.Exchange(ref removed, 1) == 0)
+ rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
+ }
+
+ if (serverObject) rig.Nm.ServerObjectRefreshProbe = Remove;
+ else rig.Nm.DirectRefreshProbe = Remove;
+
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
+ {
+ for (var i = 0; i < FloodSize; i++) rig.Project(native: true, Active($"f{i}"));
+ rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0);
+ await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
+ await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
+ stall.Release();
+ }
+
+ rig.Nm.ServerObjectRefreshProbe = null;
+ rig.Nm.DirectRefreshProbe = null;
+ Volatile.Read(ref removed).ShouldBe(1, "the refresh took the cut path");
+ await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
+ await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
+
+ var (replayed, after) = Window(events.Of(item));
+ replayed.ShouldBeEmpty("the condition was removed before its frozen replay was queued");
+ after.ShouldBeEmpty("and none of its deferred events follows");
+ }
+
+ ///
+ /// #855 review 3: the plain refresh path (nothing deferred — the only path production takes) collects live
+ /// conditions off Lock and queues them afterwards; a condition removed in between must not be replayed.
+ ///
+ /// Refresh a Server-object item instead of a folder item.
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public async Task A_condition_removed_during_a_plain_refresh_is_not_replayed(bool serverObject)
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855RemovePlain", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var item = RecordedEvents.AddEventItem(subscription, serverObject ? ObjectIds.Server : ServerEventRig.UnsNodeId(session, EqA));
+ await subscription.ApplyChangesAsync(ct);
+
+ rig.Project(native: true, Active("seed"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(item, "seed"));
+
+ var removed = 0;
+ void Remove(bool cut, bool _)
+ {
+ if (!cut && Interlocked.Exchange(ref removed, 1) == 0)
+ rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
+ }
+
+ if (serverObject) rig.Nm.ServerObjectRefreshProbe = Remove;
+ else rig.Nm.DirectRefreshProbe = Remove;
+ await session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
+ await RecordedEvents.WaitUntilAsync(() => events.HasType(item, ObjectTypeIds.RefreshEndEventType));
+ rig.Nm.ServerObjectRefreshProbe = null;
+ rig.Nm.DirectRefreshProbe = null;
+
+ Volatile.Read(ref removed).ShouldBe(1, "the plain path ran");
+ Window(events.Of(item)).Replayed.ShouldBeEmpty("the condition was removed before the replay was queued");
+ }
+
+ ///
+ /// #855 review: a delivery that fails inline (here: a hop that throws) is over, not suspended — it must not be booked
+ /// as a deferral episode (which moved the episode a concurrent plain refresh checks, and cleared cuts under the
+ /// event-report gate while Lock was held). The Server-object item, served before the failing hop, still gets
+ /// the event, and the next event is delivered normally.
+ ///
+ [Fact]
+ public async Task A_delivery_that_throws_is_not_booked_as_a_deferral()
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855Fault", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(false, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
+ var onServer = RecordedEvents.AddEventItem(subscription, ObjectIds.Server);
+ await subscription.ApplyChangesAsync(ct);
+
+ var episodes = rig.Nm.DeferralEpisodes;
+ rig.Nm.NotifierDeliveryStall = node =>
+ node.NodeId.IdentifierAsString == EqA ? throw new InvalidOperationException("hop fault (test)") : null;
+ rig.Project(native: true, Active("faulted"));
+ rig.Nm.NotifierDeliveryStall = null;
+
+ rig.Nm.DeferredEventReports.ShouldBe(0);
+ rig.Nm.DeferralEpisodes.ShouldBe(episodes, "a failed delivery is not a deferral episode");
+ rig.Project(native: true, Active("next"));
+ await RecordedEvents.WaitUntilAsync(() =>
+ events.Has(onServer, "faulted") && events.Has(onServer, "next") && events.Has(onA, "next"));
+ events.Has(onA, "faulted").ShouldBeFalse("the faulted hop delivered nothing");
+ }
+
+ ///
+ /// Production shape, no stall: bursts reported under one Lock hold (an apply pass) and single reports, against
+ /// folder items that refresh repeatedly while their client is slow to publish. Folder items are served per item, so
+ /// nothing is ever deferred; each refresh's replay is never older than an event the item already received, nothing
+ /// after RefreshEnd goes backwards, the stream converges on the newest state, and no replay is queued under Lock.
+ ///
+ /// Per-folder ACL on or off.
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public async Task Production_overload_never_defers_and_never_overtakes_a_folder_refresh(bool enforced)
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855Load", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
+ onA.QueueSize = 100_000; // keep every event so the ordering check sees the whole stream
+ await subscription.ApplyChangesAsync(ct);
+
+ rig.Project(native: true, Active("s0"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0"));
+
+ var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
+ rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
+
+ using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
+ var counter = 0;
+ var peakDeferred = 0;
+ var reporter = Task.Factory.StartNew(() =>
+ {
+ while (!stop.IsCancellationRequested)
+ {
+ lock (rig.Nm.Lock)
+ {
+ for (var i = 0; i < 1000; i++)
+ {
+ rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
+ // Sampled inside Lock, after each report: a deferral cannot open and drain unseen (#855 review).
+ peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
+ }
+ }
+
+ rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
+ Thread.Sleep(2);
+ }
+ }, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
+
+ var refusedAsInProgress = 0;
+ for (var r = 0; r < 30; r++)
+ {
+ try
+ {
+ await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct);
+ }
+ catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
+ {
+ refusedAsInProgress++;
+ }
+
+ await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
+ }
+
+ await stop.CancelAsync();
+ await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
+ rig.Nm.DirectRefreshProbe = null;
+
+ peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item by the reporter");
+ rig.Nm.DiscardedEventReports.ShouldBe(0);
+ var newest = $"s{Volatile.Read(ref counter)}";
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest));
+ await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
+
+ CheckStream(events.Of(onA)).ShouldBeEmpty();
+ refusedAsInProgress.ShouldBeLessThan(30, "most refreshes must actually have run");
+ probes.ShouldNotBeEmpty();
+ probes.ShouldAllBe(p => !p.Cut && !p.LockHeld);
+ RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
+ .ShouldBe(newest, "the stream converges on the newest state");
+ }
+
+ ///
+ /// Race / stress with deferral: reports (bursts under Lock and single), repeated refreshes of the folder item
+ /// and an intermittent stall of its folder's delivery all run at once. Outside a refresh the folder item's stream
+ /// never goes backwards, each refresh ends on the newest state the item has seen, and it converges. Nothing
+ /// deadlocks, and neither refresh path queues its replay under Lock.
+ ///
+ /// Per-folder ACL on or off.
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public async Task Concurrent_reports_folder_refreshes_and_stalls_never_deliver_a_pre_refresh_state_after_RefreshEnd(bool enforced)
+ {
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync("Refresh855Stress", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(enforced, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+
+ using var session = await rig.ConnectAsync(anonymous: false, ct);
+ var events = new RecordedEvents();
+ var subscription = await RecordedEvents.SubscribeAsync(session, events, ct);
+ var onA = RecordedEvents.AddEventItem(subscription, ServerEventRig.UnsNodeId(session, EqA));
+ onA.QueueSize = 100_000;
+ await subscription.ApplyChangesAsync(ct);
+
+ rig.Project(native: true, Active("s0"));
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s0"));
+
+ var probes = new System.Collections.Concurrent.ConcurrentQueue<(bool Cut, bool LockHeld)>();
+ rig.Nm.DirectRefreshProbe = (cut, held) => probes.Enqueue((cut, held));
+
+ using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct);
+ var counter = 0;
+ var reporter = Task.Factory.StartNew(() =>
+ {
+ while (!stop.IsCancellationRequested)
+ {
+ lock (rig.Nm.Lock)
+ {
+ for (var i = 0; i < 300; i++) rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
+ }
+
+ rig.Project(native: true, Active($"s{Interlocked.Increment(ref counter)}"));
+ Thread.Sleep(2);
+ }
+ }, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
+
+ var deferredSeen = 0;
+ var staller = Task.Factory.StartNew(() =>
+ {
+ while (!stop.IsCancellationRequested)
+ {
+ using (NotifierStall.Hold(rig.Nm, EqA))
+ {
+ Thread.Sleep(300);
+ if (rig.Nm.DeferredEventReports > 0) Interlocked.Increment(ref deferredSeen);
+ }
+
+ Thread.Sleep(100);
+ }
+ }, stop.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
+
+ var refusedAsInProgress = 0;
+ var stressDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(90);
+ for (var r = 0; r < 40 || (Volatile.Read(ref deferredSeen) <= 1 && DateTime.UtcNow < stressDeadline); r++)
+ {
+ try
+ {
+ await RefreshAsync(session, subscription, onA, refresh2: r % 3 == 2, ct).WaitAsync(TimeSpan.FromSeconds(20), ct);
+ }
+ catch (ServiceResultException ex) when (ex.StatusCode == StatusCodes.BadRefreshInProgress)
+ {
+ refusedAsInProgress++;
+ }
+
+ await Task.Delay(TimeSpan.FromMilliseconds(80), ct);
+ }
+
+ await stop.CancelAsync();
+ await reporter.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
+ await staller.WaitAsync(TimeSpan.FromSeconds(30), CancellationToken.None);
+ rig.Nm.DirectRefreshProbe = null;
+ Volatile.Read(ref deferredSeen).ShouldBeGreaterThan(1, "the stress must actually have run with deferred events");
+
+ await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
+ var newest = $"s{Volatile.Read(ref counter)}";
+ await RecordedEvents.WaitUntilAsync(() => events.Has(onA, newest));
+ await Task.Delay(TimeSpan.FromMilliseconds(500), ct);
+
+ CheckStream(events.Of(onA)).ShouldBeEmpty();
+ refusedAsInProgress.ShouldBeLessThan(40, "most refreshes must actually have run");
+ rig.Nm.RefreshSupersededDeliveries.ShouldBeGreaterThan(0, "some folder refresh must actually have overtaken deferred events");
+ probes.ShouldContain(p => p.Cut, "some folder refresh must have taken the cut path");
+ probes.ShouldAllBe(p => !p.LockHeld);
+ RecordedEvents.Message(events.Of(onA).Last(f => f[RecordedEvents.AckedField].TryGetValue(out bool _)))
+ .ShouldBe(newest, "the stream converges on the newest state");
+ }
+
+ ///
+ /// Walks one condition's stream: outside a refresh its states never go backwards, and each refresh ends on the newest
+ /// state the item has seen (its replay is never older than an event already delivered, and nothing older follows).
+ ///
+ private static List CheckStream(IReadOnlyList> stream)
+ {
+ var max = -1;
+ var last = -1;
+ var replaying = false;
+ var violations = new List();
+ foreach (var f in stream)
+ {
+ var type = f[RecordedEvents.EventTypeField].GetNodeId();
+ if (type == ObjectTypeIds.RefreshStartEventType)
+ {
+ replaying = true;
+ continue;
+ }
+
+ if (type == ObjectTypeIds.RefreshEndEventType)
+ {
+ replaying = false;
+ if (last < max) violations.Add($"a refresh ended on s{last} after s{max} had been delivered");
+ continue;
+ }
+
+ if (!f[RecordedEvents.AckedField].TryGetValue(out bool _)) continue;
+ var n = int.Parse(RecordedEvents.Message(f)![1..], CultureInfo.InvariantCulture);
+ if (!replaying && n < max) violations.Add($"s{n} delivered after s{max}");
+ max = Math.Max(max, n);
+ last = n;
+ }
+
+ return violations;
+ }
+
+ private static Task RefreshAsync(ISession session, Subscription subscription, MonitoredItem item, bool refresh2, CancellationToken ct) =>
+ refresh2
+ ? session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh2, ct, subscription.Id, item.Status.Id)
+ : session.CallAsync(ObjectTypeIds.ConditionType, MethodIds.ConditionType_ConditionRefresh, ct, subscription.Id);
+
+ /// The condition-event messages inside the item's LAST RefreshStart..RefreshEnd, and those after it.
+ private static (List Replayed, List AfterEnd) Window(IReadOnlyList> stream)
+ {
+ var list = stream.ToList();
+ var start = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshStartEventType);
+ var end = list.FindLastIndex(f => f[RecordedEvents.EventTypeField].GetNodeId() == ObjectTypeIds.RefreshEndEventType);
+ start.ShouldBeGreaterThanOrEqualTo(0);
+ end.ShouldBeGreaterThan(start);
+ static bool IsCondition(ArrayOf f) => f[RecordedEvents.AckedField].TryGetValue(out bool _);
+ return (
+ list.Skip(start + 1).Take(end - start - 1).Where(IsCondition).Select(RecordedEvents.Message).ToList(),
+ list.Skip(end + 1).Where(IsCondition).Select(RecordedEvents.Message).ToList());
+ }
+
+ private static List Indices(IReadOnlyList> received, char prefix) =>
+ received
+ .Select(RecordedEvents.Message)
+ .Where(m => m is { Length: > 1 } && m[0] == prefix && char.IsAsciiDigit(m[1]))
+ .Select(m => int.Parse(m![1..], CultureInfo.InvariantCulture))
+ .ToList();
+
+ private static NodeId RawNodeId(ISession session, string id) =>
+ new(id, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));
+
+ private static void Deploy(DataPlaneAuthorityState state, string revision, NodeAcl[] grants, params string[] referencing) =>
+ state.Apply(
+ ServerEventRig.Cluster, revision, [ServerEventRig.Key()], grants, ServerEventRig.ScopePaths(),
+ ServerEventRig.EquipmentOwnerPaths(), ServerEventRig.ScriptedAlarmOwners(), ServerEventRig.FolderScopes(),
+ new Dictionary> { [ServerEventRig.RawAlarmPath] = referencing.ToHashSet() });
+
+ private static AlarmConditionSnapshot Active(string message) =>
+ new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
+
+ private static AlarmConditionSnapshot Cleared(string message) =>
+ new(false, true, true, true, AlarmShelvingKind.Unshelved, 700, message);
+
+ private static NodeAcl KeyEquipmentGrant(string equipment) => new()
+ {
+ NodeAclId = $"acl-key-{equipment}",
+ ClusterId = ServerEventRig.Cluster,
+ Principal = ServerEventRig.KeyPrincipal,
+ ScopeKind = NodeAclScopeKind.Equipment,
+ ScopeId = $"eq-{equipment}",
+ PermissionFlags = NodePermissions.ReadOnly,
+ };
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventDeliveryPerfTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventDeliveryPerfTests.cs
new file mode 100644
index 000000000..25b14788e
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventDeliveryPerfTests.cs
@@ -0,0 +1,350 @@
+using System.Diagnostics;
+using System.Globalization;
+using System.Text;
+using Opc.Ua;
+using Opc.Ua.Client;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
+using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Core.Authorization;
+using MonitoredItem = Opc.Ua.Client.MonitoredItem;
+
+namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
+
+///
+/// Gitea #855 cost measurement: since #855 a condition event is queued to every event item on an equipment folder by
+/// the reporter itself, under the node manager's Lock (per-item verdict, QueueEvent with the SDK's
+/// filter evaluation, duplicate scan over the last 1000 queued events and a List.RemoveAt(0) at a full queue).
+/// Before #855 that work ran on the SDK's per-node channel consumer, off Lock. This harness reports, per alarm
+/// report, the Lock hold time and the end-to-end delivery latency, for one equipment folder carrying N event
+/// items, ACL on / off, with the folder's subscribers publishing or stalled (publishing disabled, item queues full and
+/// discarding oldest) — and the latency of a Server-object (i=2253) subscriber while the folder's subscribers
+/// are stalled (the ScadaBridge shape).
+///
+///
+/// Measurement only: nothing is asserted beyond the harness's own sanity checks. Opt-in and never part of a
+/// default run: every case skips unless OTOPCUA_EVENT_PERF=1.
+/// Run: OTOPCUA_EVENT_PERF=1 dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests -c Release
+/// --filter "FullyQualifiedName~FolderEventDeliveryPerfTests" --logger "console;verbosity=detailed".
+/// OTOPCUA_EVENT_PERF_REPORTS sets the measured reports per case (default 1000) and
+/// OTOPCUA_EVENT_PERF_OUT appends each result row to a file.
+/// The rig is the same one uses and existed before #855, so this file
+/// compiles against the pre-#855 tree for a before/after comparison.
+///
+[Trait("Category", "Perf")]
+public sealed class FolderEventDeliveryPerfTests
+{
+ private const string OptInVariable = "OTOPCUA_EVENT_PERF";
+ private const string EqA = ServerEventRig.EqA;
+ private const uint DefaultQueueSize = 1000;
+ private const double PublishingIntervalMs = 50;
+
+ private static int Reports =>
+ int.TryParse(Environment.GetEnvironmentVariable("OTOPCUA_EVENT_PERF_REPORTS"), out var r) && r > 0 ? r : 1000;
+
+ private static void RequireOptIn()
+ {
+ if (Environment.GetEnvironmentVariable(OptInVariable) != "1")
+ Assert.Skip($"perf harness: set {OptInVariable}=1 to run (not part of a default run)");
+ }
+
+ ///
+ /// Lock hold per report and (publishing only) the time until the LAST of the N folder items received the event.
+ ///
+ [Theory]
+ [InlineData(1, false, false)]
+ [InlineData(10, false, false)]
+ [InlineData(50, false, false)]
+ [InlineData(200, false, false)]
+ [InlineData(1, true, false)]
+ [InlineData(10, true, false)]
+ [InlineData(50, true, false)]
+ [InlineData(200, true, false)]
+ [InlineData(1, false, true)]
+ [InlineData(10, false, true)]
+ [InlineData(50, false, true)]
+ [InlineData(200, false, true)]
+ [InlineData(1, true, true)]
+ [InlineData(10, true, true)]
+ [InlineData(50, true, true)]
+ [InlineData(200, true, true)]
+ public Task Folder_items(int n, bool acl, bool stalled) => RunAsync(n, acl, stalled, serverObjectProbe: false);
+
+ ///
+ /// A publishing Server-object (i=2253) subscriber on its own session, while N folder items on another
+ /// session are stalled with full queues. N = 0 is the baseline.
+ ///
+ [Theory]
+ [InlineData(0, false)]
+ [InlineData(50, false)]
+ [InlineData(200, false)]
+ [InlineData(0, true)]
+ [InlineData(50, true)]
+ [InlineData(200, true)]
+ public Task Server_object_subscriber_while_folder_items_stall(int n, bool acl) =>
+ RunAsync(n, acl, stalled: true, serverObjectProbe: true);
+
+ ///
+ /// Full-queue sensitivity: N = 50 stalled items at different QueueSizes. At a full queue every
+ /// QueueEvent scans the last min(QueueSize, 1000) queued events for a duplicate and shifts the whole queue
+ /// (List.RemoveAt(0)), so the per-item cost grows with the queue, not with N.
+ ///
+ [Theory]
+ [InlineData(10u)]
+ [InlineData(100u)]
+ [InlineData(1000u)]
+ [InlineData(10000u)]
+ public Task Stalled_items_queue_size_sensitivity(uint queueSize) =>
+ RunAsync(50, acl: false, stalled: true, serverObjectProbe: false, queueSize);
+
+ private static async Task RunAsync(
+ int n, bool acl, bool stalled, bool serverObjectProbe, uint queueSize = DefaultQueueSize)
+ {
+ RequireOptIn();
+ var ct = TestContext.Current.CancellationToken;
+ await using var rig = await ServerEventRig.StartAsync($"Perf855-{n}", encrypted: false, ct);
+ rig.KeyRoles = [];
+ rig.MaterialiseNative();
+ rig.Enforce(acl, [ServerEventRig.Key()], KeyEquipmentGrant(EqA));
+
+ var reports = Reports;
+ var reportedAt = new long[reports];
+ var receipts = new Receipts(reports);
+
+ ISession? folderSession = null;
+ ISession? serverSession = null;
+ try
+ {
+ Subscription? folderSubscription = null;
+ var folderHandles = new HashSet();
+ if (n > 0)
+ {
+ folderSession = await rig.ConnectAsync(anonymous: false, ct);
+ folderSubscription = await SubscribeAsync(folderSession, receipts, isServer: false, ct);
+ for (var i = 0; i < n; i++)
+ {
+ var item = RecordedEvents.AddEventItem(folderSubscription, ServerEventRig.UnsNodeId(folderSession, EqA));
+ item.QueueSize = queueSize;
+ folderHandles.Add(item.ClientHandle);
+ }
+
+ await folderSubscription.ApplyChangesAsync(ct);
+ folderSubscription.MonitoredItems.ShouldAllBe(m => m.Status.Error == null);
+ // The server's revision (MaxEventQueueSize) — what the measured queues actually hold.
+ queueSize = folderSubscription.MonitoredItems.First().Status.QueueSize;
+ }
+
+ receipts.FolderHandles = folderHandles;
+
+ MonitoredItem? onServer = null;
+ if (serverObjectProbe)
+ {
+ serverSession = await rig.ConnectAsync(anonymous: false, ct);
+ var serverSubscription = await SubscribeAsync(serverSession, receipts, isServer: true, ct);
+ onServer = RecordedEvents.AddEventItem(serverSubscription, ObjectIds.Server);
+ await serverSubscription.ApplyChangesAsync(ct);
+ onServer.Status.Error.ShouldBeNull();
+ receipts.ServerHandle = onServer.ClientHandle;
+ }
+
+ if (stalled && folderSubscription is not null)
+ {
+ await folderSubscription.SetPublishingModeAsync(false, ct);
+
+ // Fill every folder item's queue to QueueSize (and past it, so the measured reports run at a full queue,
+ // discarding oldest), then let any off-Lock consumer (pre-#855: the SDK's channel task) catch up.
+ for (var i = 0; i < queueSize + 50; i++) rig.Project(native: true, State($"w{i}"));
+ await SettleAsync(rig, n, ct);
+ }
+
+ // Warm-up (JIT, first-verdict caches) — not measured.
+ for (var i = 0; i < 50; i++) rig.Project(native: true, State($"x{i}"));
+ await SettleAsync(rig, n, ct);
+
+ var holds = new double[reports];
+ for (var i = 0; i < reports; i++)
+ {
+ var state = State($"p{i}");
+ long t0, t1;
+ lock (rig.Nm.Lock)
+ {
+ t0 = Stopwatch.GetTimestamp();
+ reportedAt[i] = t0;
+ rig.Project(native: true, state);
+ t1 = Stopwatch.GetTimestamp();
+ }
+
+ holds[i] = Stopwatch.GetElapsedTime(t0, t1).TotalMicroseconds;
+ // ~1 kHz report rate: an alarm storm, not a single burst, so a consumer that keeps up is not penalised.
+ Thread.Sleep(1);
+ }
+
+ // Wait for the tail of every probe that is expected to receive.
+ var expectFolder = n > 0 && !stalled;
+ var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+ while (DateTime.UtcNow < deadline
+ && ((expectFolder && !receipts.FolderComplete(reports - 1, n))
+ || (serverObjectProbe && !receipts.ServerHas(reports - 1))))
+ {
+ await Task.Delay(50, ct);
+ }
+
+ var deferredPeak = rig.Nm.DeferredEventReports;
+ var row = new StringBuilder();
+ row.Append(CultureInfo.InvariantCulture,
+ $"{(serverObjectProbe ? "i=2253" : "folder"),-6} N={n,-3} acl={(acl ? "on " : "off")} {(stalled ? "stalled " : "publishing")} q={queueSize,-5} | ");
+ row.Append(CultureInfo.InvariantCulture, $"lock hold us p50 {Pct(holds, 50),8:F1} p90 {Pct(holds, 90),8:F1} p99 {Pct(holds, 99),8:F1} max {holds.Max(),9:F1} mean {holds.Average(),8:F1}");
+ row.Append(CultureInfo.InvariantCulture, $" | per-item mean {(n > 0 ? holds.Average() / n : 0),6:F2} us");
+ if (expectFolder)
+ {
+ var lat = receipts.FolderLatenciesMs(reportedAt, n);
+ row.Append(CultureInfo.InvariantCulture,
+ $" | folder all-N latency ms p50 {Pct(lat, 50),6:F1} p99 {Pct(lat, 99),6:F1} max {Max(lat),7:F1} (got {lat.Length}/{reports})");
+ }
+
+ if (serverObjectProbe)
+ {
+ var lat = receipts.ServerLatenciesMs(reportedAt);
+ row.Append(CultureInfo.InvariantCulture,
+ $" | i=2253 latency ms p50 {Pct(lat, 50),6:F1} p99 {Pct(lat, 99),6:F1} max {Max(lat),7:F1} (got {lat.Length}/{reports})");
+ }
+
+ row.Append(CultureInfo.InvariantCulture, $" | deferred-now {deferredPeak}");
+ Report(row.ToString());
+ }
+ finally
+ {
+ folderSession?.Dispose();
+ serverSession?.Dispose();
+ }
+ }
+
+ ///
+ /// Gives an off-Lock consumer (pre-#855) time to drain what was just reported; post-#855 nothing is pending.
+ ///
+ private static async Task SettleAsync(ServerEventRig rig, int n, CancellationToken ct)
+ {
+ var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(60);
+ while (rig.Nm.DeferredEventReports > 0 && DateTime.UtcNow < deadline) await Task.Delay(50, ct);
+ await Task.Delay(TimeSpan.FromMilliseconds(500 + (10 * n)), ct);
+ }
+
+ private static async Task SubscribeAsync(ISession session, Receipts receipts, bool isServer, CancellationToken ct)
+ {
+ var subscription = new Subscription(session.DefaultSubscription)
+ {
+ PublishingInterval = (int)PublishingIntervalMs,
+ MaxNotificationsPerPublish = 0,
+ };
+ subscription.FastEventCallback = receipts.OnEvents;
+ session.AddSubscription(subscription);
+ await subscription.CreateAsync(ct);
+ return subscription;
+ }
+
+ private static double Pct(double[] values, double p)
+ {
+ if (values.Length == 0) return double.NaN;
+ var sorted = values.Order().ToArray();
+ var idx = (int)Math.Ceiling(p / 100.0 * sorted.Length) - 1;
+ return sorted[Math.Clamp(idx, 0, sorted.Length - 1)];
+ }
+
+ private static double Max(double[] values) => values.Length == 0 ? double.NaN : values.Max();
+
+ private static void Report(string text)
+ {
+ TestContext.Current.TestOutputHelper?.WriteLine(text);
+ if (Environment.GetEnvironmentVariable("OTOPCUA_EVENT_PERF_OUT") is { Length: > 0 } path)
+ File.AppendAllText(path, text + Environment.NewLine);
+ }
+
+ private static AlarmConditionSnapshot State(string message) =>
+ new(true, false, false, true, AlarmShelvingKind.Unshelved, 700, message);
+
+ private static NodeAcl KeyEquipmentGrant(string equipment) => new()
+ {
+ NodeAclId = $"acl-key-{equipment}",
+ ClusterId = ServerEventRig.Cluster,
+ Principal = ServerEventRig.KeyPrincipal,
+ ScopeKind = NodeAclScopeKind.Equipment,
+ ScopeId = $"eq-{equipment}",
+ PermissionFlags = NodePermissions.ReadOnly,
+ };
+
+ /// Client-side receipt times of the measured (p{i}) events, per probe.
+ private sealed class Receipts(int reports)
+ {
+ private readonly object _gate = new();
+ private readonly int[] _folderCount = new int[reports];
+ private readonly long[] _folderLast = new long[reports];
+ private readonly long[] _serverFirst = new long[reports];
+
+ public IReadOnlySet FolderHandles { get; set; } = new HashSet();
+
+ public uint? ServerHandle { get; set; }
+
+ public void OnEvents(Subscription subscription, EventNotificationList notification, ArrayOf stringTable)
+ {
+ var now = Stopwatch.GetTimestamp();
+ lock (_gate)
+ {
+ foreach (var e in notification.Events)
+ {
+ var message = RecordedEvents.Message(e.EventFields);
+ if (message is not { Length: > 1 } || message[0] != 'p') continue;
+ if (!int.TryParse(message.AsSpan(1), NumberStyles.Integer, CultureInfo.InvariantCulture, out var i)
+ || i < 0 || i >= _folderCount.Length)
+ {
+ continue;
+ }
+
+ if (ServerHandle == e.ClientHandle)
+ {
+ if (_serverFirst[i] == 0) _serverFirst[i] = now;
+ }
+ else if (FolderHandles.Contains(e.ClientHandle))
+ {
+ _folderCount[i]++;
+ _folderLast[i] = now;
+ }
+ }
+ }
+ }
+
+ public bool FolderComplete(int i, int n)
+ {
+ lock (_gate) return _folderCount[i] >= n;
+ }
+
+ public bool ServerHas(int i)
+ {
+ lock (_gate) return _serverFirst[i] != 0;
+ }
+
+ public double[] FolderLatenciesMs(long[] reportedAt, int n)
+ {
+ lock (_gate)
+ {
+ return Enumerable.Range(0, reportedAt.Length)
+ .Where(i => _folderCount[i] >= n)
+ .Select(i => Stopwatch.GetElapsedTime(reportedAt[i], _folderLast[i]).TotalMilliseconds)
+ .ToArray();
+ }
+ }
+
+ public double[] ServerLatenciesMs(long[] reportedAt)
+ {
+ lock (_gate)
+ {
+ return Enumerable.Range(0, reportedAt.Length)
+ .Where(i => _serverFirst[i] != 0)
+ .Select(i => Stopwatch.GetElapsedTime(reportedAt[i], _serverFirst[i]).TotalMilliseconds)
+ .ToArray();
+ }
+ }
+ }
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs
index d99ac83e6..266637e48 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventFloodWireTests.cs
@@ -19,6 +19,13 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// consumer — a deadlock that also freezes every other operation needing Lock. Publishing is switched off
/// on the client so nothing downstream drains either (the slow / absent consumer case).
///
+///
+/// Since Gitea #855 condition events no longer go through that channel at all: the node manager queues them to a
+/// folder's items itself, per item, on the reporter's thread (as it always did for Server-object items), so the same
+/// flood now completes without deferring anything. The deadlock test therefore also pins that per-item delivery is in
+/// force (it would defer again if the bypass stopped recognising the SDK's fan-out); the ghost test produces its
+/// deferral with .
+///
public sealed class FolderEventFloodWireTests
{
private const string EqA = ServerEventRig.EqA;
@@ -56,8 +63,12 @@ public sealed class FolderEventFloodWireTests
// No publishing: the server-side item queue (QueueSize 1000) is the only sink, as with a stalled client.
await subscription.SetPublishingModeAsync(false, ct);
+ // #855: the folder's SDK fan-out is the one the node manager recognises and serves per item itself.
+ rig.Nm.TryGetFolder(EqA)!.OnReportEventAsync?.Target.ShouldBeOfType
();
+
// The whole burst under ONE hold of the node manager's Lock — the shape of any multi-event apply/projection
// pass — starting from an uncached verdict (the item is new), so the delivery task needs Lock for event 1.
+ var peakDeferred = 0;
var flood = Task.Factory.StartNew(() =>
{
lock (rig.Nm.Lock)
@@ -65,6 +76,7 @@ public sealed class FolderEventFloodWireTests
for (var i = 0; i < FloodSize; i++)
{
rig.Project(native: true, State($"f{i}"));
+ peakDeferred = Math.Max(peakDeferred, rig.Nm.DeferredEventReports);
Volatile.Write(ref reported, i + 1);
}
}
@@ -83,11 +95,9 @@ public sealed class FolderEventFloodWireTests
+ "on Lock (deadlock)");
}
- // The overflow was deferred rather than blocking, and drains once Lock is free.
- await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
- // …and the SDK's channel (up to 4096 accepted events) finishes draining into the item queue, still with
- // publishing off, so the count below measures the item queue and not events in transit.
- await Task.Delay(TimeSpan.FromSeconds(2), ct);
+ // #855: nothing was deferred at all — every event was queued to the folder item by the reporter itself.
+ peakDeferred.ShouldBe(0, "no notifier of ours defers: folder items are served per item, not through the SDK channel");
+ rig.Nm.DeferredEventReports.ShouldBe(0);
// Delivery is bounded by the item's queue (QueueSize 1000, oldest discarded), not by the flood, and the
// events that survive arrive in report order, ending with the last one reported.
@@ -104,8 +114,7 @@ public sealed class FolderEventFloodWireTests
flooded.Count.ShouldBeLessThanOrEqualTo((int)onA.Status.QueueSize, "delivery is bounded by the item's queue, not the flood");
flooded.ShouldBe(flooded.Order().ToList(), "events leave in the order they were reported");
flooded[^1].ShouldBe(FloodSize - 1);
- rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0,
- "past the per-condition budget the oldest waiting events give way to the newest (coalesced)");
+ rig.Nm.DiscardedEventReports.ShouldBe(0, "nothing was deferred, so nothing was coalesced (the item queue is the bound)");
// And the node manager is live: a later event is decided by the per-folder ACL and delivered.
rig.Project(native: true, State("after"));
@@ -127,7 +136,8 @@ public sealed class FolderEventFloodWireTests
/// #838 review — events deferred behind a full channel must not outlive their condition. A deploy that removes
/// the condition while its events wait used to let them reach Server-object (i=2253) subscribers after the
/// NodeDeleted — a ghost Retain alarm for ScadaBridge — and the folder subscribers too. Every hop of a deferred
- /// report now re-checks that the condition is still the live instance.
+ /// report now re-checks that the condition is still the live instance (#855: also after a stall at a hop, before
+ /// the folder's items are served).
///
/// Per-folder / Server-object ACL on or off (off is the production default).
[Theory]
@@ -151,16 +161,20 @@ public sealed class FolderEventFloodWireTests
onServer.Status.Error.ShouldBeNull();
await subscription.SetPublishingModeAsync(false, ct);
- // One Lock hold: flood past the channel (the tail is deferred), then the deploy removes the condition.
- await Task.Factory.StartNew(() =>
+ // One Lock hold, delivery stalled at the folder: flood (the tail is deferred), then the deploy removes the condition.
+ using (var stall = NotifierStall.Hold(rig.Nm, EqA))
{
- lock (rig.Nm.Lock)
+ await Task.Factory.StartNew(() =>
{
- for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
- rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
- rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
- }
- }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
+ lock (rig.Nm.Lock)
+ {
+ for (var i = 0; i < FloodSize; i++) rig.Project(native: true, State($"f{i}"));
+ rig.Nm.DeferredEventReports.ShouldBeGreaterThan(0, "the flood must actually have deferred events");
+ rig.Nm.RemoveAlarmConditionNode(ServerEventRig.RawAlarmPath, AddressSpaceRealm.Raw).ShouldBeTrue();
+ }
+ }, ct, TaskCreationOptions.LongRunning, TaskScheduler.Default).WaitAsync(TimeSpan.FromSeconds(60), ct);
+ stall.Release();
+ }
await RecordedEvents.WaitUntilAsync(() => rig.Nm.DeferredEventReports == 0);
rig.Nm.DiscardedEventReports.ShouldBeGreaterThan(0, "the waiting events were dropped, not delivered");
@@ -174,12 +188,15 @@ public sealed class FolderEventFloodWireTests
.Select(m => int.Parse(m![1..], System.Globalization.CultureInfo.InvariantCulture))
.ToArray();
- // Positive control: the events reported before the channel filled did arrive (Server object: synchronously at
- // the condition's own hop).
+ // Positive control: the event reported before the stall did arrive at the Server object (synchronously, at the
+ // condition's own hop, before the walk reached the stalled folder).
Flooded(onServer).ShouldNotBeEmpty();
// None of the deferred tail — which was still waiting when the condition was removed — reached anyone.
Flooded(onServer).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the Server-object subscriber");
- Flooded(onA).DefaultIfEmpty(-1).Max().ShouldBeLessThan(FloodSize - 1, "no ghost reached the folder subscriber");
+ // Only the in-flight f0 ever reached the stalled EqA hop, and the condition was gone when the stall released: the
+ // post-stall liveness re-check must have withheld it (#855 review 4 — "< FloodSize - 1" passed without that check).
+ Flooded(onA).ShouldBeEmpty("no ghost reached the folder subscriber");
+ Flooded(onServer).ShouldBe([0], "only the event reported before the stall reached the Server object");
}
private static AlarmConditionSnapshot State(string message) =>
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventVerdictInvalidationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventVerdictInvalidationTests.cs
index 37fe86526..f20a6ea7a 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventVerdictInvalidationTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/FolderEventVerdictInvalidationTests.cs
@@ -97,10 +97,10 @@ public sealed class FolderEventVerdictInvalidationTests
rig.Project(native: false, State("s1"));
await RecordedEvents.WaitUntilAsync(() => events.Has(onA, "s1"));
- // Holding the node-manager lock parks EqA's SDK delivery task: the native event "n1" is queued first and its
- // verdict (a cache miss after the first invalidation) needs the lock in the inherited role check — its node is
- // never removed, so that check always takes it. The OLD scripted instance's "s2", queued behind it on the same
- // single-reader queue, is therefore decided only after the replace and the apply's invalidation.
+ // Before Gitea #855, holding the node-manager lock parked EqA's SDK delivery task, so the OLD scripted instance's
+ // "s2" was decided (and its verdict cached) only after the replace and the apply's invalidation. Condition events
+ // are now decided per item at report time, uncached — "s2" is delivered while its instance is live — and this
+ // pins that the replace still leaves the new instance's events flowing either way.
lock (rig.Nm.Lock)
{
rig.Nm.CompleteAddressSpaceApply(); // forget the cached allows, so both verdicts must be computed
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs
index 6dffd4484..5857950ab 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/LiveAggregateLateStartFactsTests.cs
@@ -6,9 +6,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
///
/// #821 / #809 facts, through the BOOTED server's runtime AggregateManager — the calculators a live
-/// AggregateFilter monitored item gets, under the server's own default configuration. This pins current behaviour for
-/// the owner decision; it does not choose. The HistoryRead (Part 13) answers and SDK 2.0.1's stock answers for the
-/// same input are pinned beside it in SdkAggregateParityTests.
+/// AggregateFilter monitored item gets, under the server's own default configuration. Both are decided (owner,
+/// 2026-11-11): live PercentGood / DurationGood count Good time only (#809), and live Total / TimeAverage integrate only
+/// the covered part of an interval the data starts inside, even with an Uncertain first sample (#821) — equal to
+/// HistoryRead. The HistoryRead (Part 13) answers, the untouched baseline's and SDK 2.0.1's stock answers for the same
+/// input are pinned beside it in SdkAggregateParityTests.
///
public sealed class LiveAggregateLateStartFactsTests
{
@@ -40,14 +42,16 @@ public sealed class LiveAggregateLateStartFactsTests
return d;
}
- // Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s.
- // The live calculators integrate the uncovered [0,5) as a ramp from 0 (+25) and divide by the whole 20 s.
+ // #821: Part 13 (HistoryRead on SqlServer/QuestDB): Total = 73.5 + 88 + 32/13 = 163.96…, TimeAverage = Total ÷ 15 s
+ // — live as in HistoryRead (before #821 the live calculator integrated the uncovered [0,5) as a ramp from 0 (+25)
+ // and divided by the whole 20 s: 188.96… / 9.448…).
const double part13Total = 73.5 + 88 + 32d / 13;
- One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total + 25, 1e-9); // 188.96…
- One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe((part13Total + 25) / 20, 1e-9); // 9.448… (Part 13: 10.93…)
- // #809: the Uncertain [5,12) counts as Good live (HistoryRead: 8 000 ms / 40 %).
- One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(15_000d, 1e-9);
- One(ObjectIds.AggregateFunction_PercentGood).ShouldBe(75d, 1e-9);
+ One(ObjectIds.AggregateFunction_Total).ShouldBe(part13Total, 1e-9); // 163.96…
+ One(ObjectIds.AggregateFunction_TimeAverage).ShouldBe(part13Total / 15, 1e-9); // 10.93…
+ // #809: the Uncertain [5,12) is not Good — only [12,20) is, live as in HistoryRead (8 000 ms / 40 %; before #809
+ // the live calculator counted the Uncertain time too: 15 000 ms / 75 %).
+ One(ObjectIds.AggregateFunction_DurationGood).ShouldBe(8_000d, 1e-9);
+ One(ObjectIds.AggregateFunction_PercentGood).ShouldBe(40d, 1e-9);
// DurationBad / PercentBad agree with HistoryRead: the 5 s before the data is Bad.
One(ObjectIds.AggregateFunction_DurationBad).ShouldBe(5_000d, 1e-9);
One(ObjectIds.AggregateFunction_PercentBad).ShouldBe(25d, 1e-9);
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/NotifierStall.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/NotifierStall.cs
new file mode 100644
index 000000000..755f26a09
--- /dev/null
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/NotifierStall.cs
@@ -0,0 +1,42 @@
+using Opc.Ua;
+
+namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
+
+///
+/// Stalls condition-event delivery at one of the node manager's notifiers (Gitea #855): every condition event whose walk
+/// reaches that notifier suspends there until , without blocking the reporter or holding the node
+/// manager's Lock — so the condition's later events are deferred, as they were behind a full SDK channel before
+/// folder items were delivered per item.
+///
+///
+/// The pre-#855 tests produced deferral by holding a folder item's server-side MonitoredItem lock, which parked
+/// the SDK's per-node delivery task. Folder items are now queued synchronously by the reporter (as Server-object items
+/// always were), so holding an item lock would block the reporter instead; this goes through the node manager's test
+/// probe (). The probe is consulted only at a notifier that has
+/// event items (the hop the per-item delivery serves), so the stalled notifier must carry at least one.
+///
+internal sealed class NotifierStall : IDisposable
+{
+ private readonly OtOpcUaNodeManager _nm;
+ private readonly TaskCompletionSource _released = new(TaskCreationOptions.RunContinuationsAsynchronously);
+ private readonly Func _probe;
+
+ private NotifierStall(OtOpcUaNodeManager nm, string notifierId)
+ {
+ _nm = nm;
+ _probe = node => !_released.Task.IsCompleted && node.NodeId.IdentifierAsString == notifierId ? _released.Task : null;
+ nm.NotifierDeliveryStall = _probe;
+ }
+
+ /// Starts stalling delivery at the notifier whose NodeId identifier is .
+ public static NotifierStall Hold(OtOpcUaNodeManager nm, string notifierId) => new(nm, notifierId);
+
+ /// Lets every stalled walk continue (on pool threads, off Lock).
+ public void Release() => _released.TrySetResult();
+
+ public void Dispose()
+ {
+ Release();
+ if (ReferenceEquals(_nm.NotifierDeliveryStall, _probe)) _nm.NotifierDeliveryStall = null;
+ }
+}
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs
index 87d1fd57e..f87e13b85 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/Sdk2AggregateCompatibilityTests.cs
@@ -9,8 +9,33 @@ using ZB.MOM.WW.OtOpcUa.OpcUaServer.Compatibility.Aggregates;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
/// Compares all preserved aggregate factories with independently generated SDK 1.5.378.182 vectors.
+///
+/// #809 (owner-approved 2026-11-11): PercentGood / DurationGood no longer count Uncertain time as Good. The baseline
+/// vectors that changes are exempted EXACTLY — a PercentGood / DurationGood vector whose output differs from the same
+/// case computed by the baseline SDK with TreatUncertainAsBad = true (its "twin": there 1.5.378.182 already
+/// counted Good regions only) — and each is pinned to that twin instead. The twin is baseline evidence, not output of
+/// the application's calculator, so the pin never regenerates expectations from the implementation.
+///
+/// #821 (owner-approved 2026-11-11): TimeAverage / Total no longer integrate the span before an Uncertain first value
+/// when there is no start bound. No fixture vector changes (every data set has a usable value at −10 s, before every
+/// window, so no fixture interval lacks a start bound) — all 384 TimeAverage / Total vectors stay held to their own
+/// output. The change is pinned instead over late-start variants of every data set against a baseline-derived oracle
+/// () and by hand.
+///
+///
public sealed class Sdk2AggregateCompatibilityTests
{
+ private const string PercentGood = "AggregateFunction_PercentGood";
+ private const string DurationGood = "AggregateFunction_DurationGood";
+ private const int Issue809PercentGoodExemptions = 38;
+ private const int Issue809DurationGoodExemptions = 26;
+ private const string TimeAverage = "AggregateFunction_TimeAverage";
+ private const string Total = "AggregateFunction_Total";
+ /// #821: fixture vectors changed — none (every data set has a usable value before the window; class remarks).
+ private const int Issue821FixtureExemptions = 0;
+ private const int Issue821Cut5Changed = 150;
+ private const int Issue821Cut25Changed = 142;
+
[Theory]
[InlineData(false)]
[InlineData(true)]
@@ -19,57 +44,248 @@ public sealed class Sdk2AggregateCompatibilityTests
await using var rig = throughRuntimeManager
? await ServerEventRig.StartAsync("AggregateBaseline", encrypted: false, TestContext.Current.CancellationToken)
: null;
- var baseline = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "opcua-1.5.378.182-aggregates.json")), new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
- var telemetry = DefaultTelemetry.Create(_ => { });
- var epoch = new DateTime(baseline.EpochTicks, DateTimeKind.Utc);
+ var baseline = LoadBaseline();
+ var exemptions = Issue809Exemptions(baseline);
var failures = new List();
foreach (var vector in baseline.Vectors)
{
- var id = (NodeId)typeof(ObjectIds).GetProperty(vector.Aggregate)!.GetValue(null)!;
- var start = epoch.AddSeconds(vector.Reverse ? 70 : 0); var end = epoch.AddSeconds(vector.Reverse ? 0 : 70);
- var profile = vector.Configuration;
- var config = new AggregateConfiguration
- {
- TreatUncertainAsBad = profile?.TreatUncertainAsBad ?? vector.UncertainAsBad,
- PercentDataBad = profile?.PercentDataBad ?? 100, PercentDataGood = profile?.PercentDataGood ?? 100,
- UseSlopedExtrapolation = profile?.UseSlopedExtrapolation ?? false,
- UseServerCapabilitiesDefaults = profile?.UseServerCapabilitiesDefaults ?? false,
- };
- var calculator = throughRuntimeManager
- ? rig!.Server.CurrentInstance.AggregateManager.CreateCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config)
- : CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry);
- calculator.ShouldNotBeNull($"Runtime registration must serve {vector.Aggregate}.");
- var output = new List(); var accepted = new List(); string? error = null;
- try
- {
- var samples = baseline.Data[vector.DataSet].OrderBy(x => x.Seconds);
- foreach (var sample in vector.Reverse ? samples.Reverse() : samples)
- {
- var time = epoch.AddSeconds(sample.Seconds);
- accepted.Add(calculator.QueueRawValue(new DataValue(SdkValueBoundary.FromClr(Payload(sample)), sample.Status, time, time.AddTicks(sample.ServerOffsetTicks), sample.SourcePicoseconds, sample.ServerPicoseconds)));
- if (vector.Incremental) Drain(false);
- }
- Drain(true);
- }
- catch (Exception ex) { error = ex.GetType().FullName; }
- if (error != vector.Exception || !output.SequenceEqual(vector.Output) || (vector.QueueAccepted is { } expectedAccepted && !accepted.SequenceEqual(expectedAccepted)))
- {
- failures.Add($"{vector.Aggregate}/{vector.DataSet}/rev={vector.Reverse}/step={vector.Stepped}/uncertainBad={vector.UncertainAsBad}/incremental={vector.Incremental}: error {vector.Exception} vs {error}; expected {JsonSerializer.Serialize(vector.Output)} actual {JsonSerializer.Serialize(output)}");
- }
- void Drain(bool partial)
- {
- for (var guard = 0; guard < 100; guard++)
- {
- if (!calculator.TryGetProcessedValue(partial, out var value)) return;
- var payload = SdkValueBoundary.ToClr(value.WrappedValue);
- output.Add(new(partial, payload?.GetType().Name, Format(payload), value.StatusCode.Code, LegacyTimestampTicks(value.SourceTimestamp), LegacyTimestampTicks(value.ServerTimestamp), value.SourcePicoseconds, value.ServerPicoseconds));
- }
- throw new InvalidOperationException("Baseline calculator did not terminate.");
- }
+ // #809: an exempted case is held to its baseline uncertain-as-bad twin; every other case to its own vector.
+ var expected = exemptions.TryGetValue(vector, out var twin) ? twin.Output : vector.Output;
+ if (Check(baseline, vector, expected, rig) is { } failure) failures.Add(failure);
}
baseline.Vectors.Length.ShouldBe(6144);
baseline.Vectors.Select(x => x.Aggregate).Distinct().Count().ShouldBe(37);
- failures.ShouldBeEmpty("The application must retain baseline values, CLR types, complete status bits, timestamps and incremental processing.");
+ exemptions.Keys.Count(x => x.Aggregate == PercentGood).ShouldBe(Issue809PercentGoodExemptions);
+ exemptions.Keys.Count(x => x.Aggregate == DurationGood).ShouldBe(Issue809DurationGoodExemptions);
+ // #821: no TimeAverage / Total fixture vector is exempt — all 384 are held to their own baseline output above.
+ exemptions.Keys.Count(x => x.Aggregate is TimeAverage or Total).ShouldBe(Issue821FixtureExemptions);
+ failures.ShouldBeEmpty("The application must retain baseline values, CLR types, complete status bits, timestamps and incremental processing (#809 exemptions: the uncertain-as-bad twin).");
+ }
+
+ ///
+ /// #809 pin: the exemptions are exactly the PercentGood / DurationGood cases that counted Uncertain as Good, and
+ /// every PercentGood / DurationGood case — exempt or not, whatever its configuration — now answers its baseline
+ /// uncertain-as-bad twin (Good regions only).
+ ///
+ [Fact]
+ public void Issue809_percent_and_duration_good_count_only_good_time()
+ {
+ var baseline = LoadBaseline();
+ var exemptions = Issue809Exemptions(baseline);
+ var failures = new List();
+ var checkedCount = 0;
+ var twinned = 0;
+ foreach (var vector in baseline.Vectors.Where(v => v.Aggregate is PercentGood or DurationGood))
+ {
+ // A data set with no uncertain-as-bad twin (the numeric-width edge sets) has no Uncertain region to move.
+ var twin = UncertainAsBadTwin(baseline, vector);
+ if (exemptions.ContainsKey(vector))
+ EffectiveTreatUncertainAsBad(vector).ShouldBeFalse($"{Describe(vector)}: only an Uncertain-as-Good case may change.");
+ if (Check(baseline, vector, twin?.Output ?? vector.Output, rig: null) is { } failure) failures.Add(failure);
+ checkedCount++;
+ if (twin is not null) twinned++;
+ }
+ checkedCount.ShouldBe(352);
+ twinned.ShouldBeGreaterThan(exemptions.Count);
+ exemptions.Count.ShouldBe(Issue809PercentGoodExemptions + Issue809DurationGoodExemptions);
+ failures.ShouldBeEmpty("PercentGood / DurationGood must count Good regions only (Part 13 §5.4.3.31/§5.4.3.33, #809).");
+ }
+
+ ///
+ /// #809 hand-computed pin, independent of the expected outputs in any fixture: the "uncertain" data set (0 s Good,
+ /// 5 s Uncertain, 15 s Good, 25 s Bad, 35 s Good, 50 s Good) over 20 s intervals from 0 to 70 s, sloped, forward.
+ /// [0,20): Good 0–5 and 15–20 = 10 s (the baseline said 20 s); [20,40): the bound at 20 slopes into the Bad sample
+ /// (Uncertain), 25–35 Bad, 35–40 Good = 5 s (baseline 10 s); [40,60): 20 s; [60,70): all 10 s of the partial
+ /// interval = 100 %.
+ ///
+ [Theory]
+ [InlineData(DurationGood, new[] { 10000d, 5000d, 20000d, 10000d })]
+ [InlineData(PercentGood, new[] { 50d, 25d, 100d, 100d })]
+ public void Issue809_uncertain_time_is_not_good_hand_computed(string aggregate, double[] expected)
+ {
+ var baseline = LoadBaseline();
+ var vector = baseline.Vectors.Single(v => v.Aggregate == aggregate && v.DataSet == "uncertain" && !v.Reverse && !v.Stepped
+ && !v.UncertainAsBad && !v.Incremental && v.Configuration is null);
+ var output = Run(baseline, vector, rig: null, out var error);
+ error.ShouldBeNull();
+ output.ShouldAllBe(x => x.Type == "Double");
+ output.Select(x => double.Parse(x.Payload!, CultureInfo.InvariantCulture)).ShouldBe(expected);
+ }
+
+ ///
+ /// #821 pin over late-start variants of every fixture data set (the fixture itself has none — see the class remarks):
+ /// each TimeAverage / Total vector's data set cut at (no usable value before the data
+ /// starts, so no start bound) with its first remaining non-Bad sample made Uncertain, run through the application's
+ /// calculator under the vector's own direction, stepping, configuration and incremental draining.
+ ///
+ /// The oracle is the untouched BASELINE, never the application's calculator: under TreatUncertainAsBad = false
+ /// the baseline treats an Uncertain value exactly like a Good one except for statuses, and with a Good first
+ /// value it already leaves the uncovered span out (the #821 control). So the expected value of each interval is
+ /// the baseline's on the same variant with that first sample Good, and the expected status, timestamps and bits
+ /// the baseline's on the variant itself (the region before the data is non-Good either way) — except an
+ /// interval the Good twin answers Bad (nothing covered: only the uncovered span ever fed it), which is the twin
+ /// whole. Stepped variables and TreatUncertainAsBad = true never reached the baseline's defect and must be the
+ /// baseline byte for byte.
+ ///
+ ///
+ [Theory]
+ [InlineData(5, Issue821Cut5Changed)]
+ [InlineData(25, Issue821Cut25Changed)]
+ public void Issue821_late_start_with_an_uncertain_first_sample_integrates_only_the_covered_part(double cutSeconds, int expectedChanged)
+ {
+ var baseline = LoadBaseline();
+ var failures = new List();
+ var changed = 0;
+ // Why the fixture needs no #821 exemption: every data set with a usable value has one before the window (−10 s), so
+ // no fixture interval lacks a start bound; the variants below remove it.
+ baseline.Data.Values.Where(d => d.Any(s => !StatusCode.IsBad(s.Status)))
+ .ShouldAllBe(d => d.Any(s => s.Seconds < 0 && !StatusCode.IsBad(s.Status)));
+ var vectors = baseline.Vectors.Where(v => v.Aggregate is TimeAverage or Total).ToList();
+ foreach (var vector in vectors)
+ {
+ var (variant, goodTwin) = LateStartVariant(baseline.Data[vector.DataSet], cutSeconds);
+ var asIs = Run(baseline, vector, rig: null, out var asIsError, variant, baselineAsIs: true);
+ var twin = Run(baseline, vector, rig: null, out var twinError, goodTwin, baselineAsIs: true);
+ var actual = Run(baseline, vector, rig: null, out var error, variant);
+ asIsError.ShouldBeNull(Describe(vector));
+ twinError.ShouldBeNull(Describe(vector));
+ twin.Count.ShouldBe(asIs.Count, Describe(vector));
+
+ Value[] expected = EffectiveTreatUncertainAsBad(vector) || vector.Stepped
+ ? [.. asIs]
+ : [.. asIs.Zip(twin, (a, g) =>
+ {
+ g.SourceTicks.ShouldBe(a.SourceTicks, Describe(vector));
+ return StatusCode.IsBad(g.Status) ? g : a with { Type = g.Type, Payload = g.Payload };
+ })];
+ if (!expected.SequenceEqual(asIs)) changed++;
+ if (error is not null || !actual.SequenceEqual(expected))
+ failures.Add($"{Describe(vector)} cut {cutSeconds}: error {error}; expected {JsonSerializer.Serialize(expected)} actual {JsonSerializer.Serialize(actual.ToArray())}");
+ }
+ vectors.Count.ShouldBe(384);
+ failures.ShouldBeEmpty("Live TimeAverage / Total must leave the span before an Uncertain first value out (Part 13 'No Start Bound: Calculate Partial Interval', #821).");
+ changed.ShouldBe(expectedChanged, "the late-start variants the #821 override changes from the untouched baseline");
+ }
+
+ ///
+ /// #821 hand-computed pin, independent of every calculator: the "uncertain" data set from +5 (Uncertain 100 at 5,
+ /// Good 18 at 15, Bad at 25, Good 19 at 35, …), forward, sloped, [0,20). Covered [5,20): 100 → 18 over 10 s = 590,
+ /// then 18 → the bound at 20 (sloped 18 → 19 across the Bad sample: 18.25) over 5 s = 90.625 ⇒ Total 680.625,
+ /// TimeAverage 680.625 ÷ 15 = 45.375. The baseline added the ramp 0 → 100 over [0,5) (+250): 930.625 and 46.53125.
+ /// Uncertain either way (an Uncertain value is used, a Bad one skipped, the interval not covered).
+ ///
+ [Theory]
+ [InlineData(Total, 680.625, 930.625)]
+ [InlineData(TimeAverage, 45.375, 46.53125)]
+ public void Issue821_late_start_hand_computed(string aggregate, double part13, double baselineValue)
+ {
+ var baseline = LoadBaseline();
+ var vector = baseline.Vectors.Single(v => v.Aggregate == aggregate && v.DataSet == "uncertain" && !v.Reverse && !v.Stepped
+ && !v.UncertainAsBad && !v.Incremental && v.Configuration is null);
+ var (variant, _) = LateStartVariant(baseline.Data["uncertain"], 5);
+ variant[0].Status.ShouldBe(0x40A40000u, "the data set's own Uncertain 100 at +5");
+ var output = Run(baseline, vector, rig: null, out var error, variant);
+ var asIs = Run(baseline, vector, rig: null, out _, variant, baselineAsIs: true);
+ error.ShouldBeNull();
+ double.Parse(output[0].Payload!, CultureInfo.InvariantCulture).ShouldBe(part13, 1e-9);
+ double.Parse(asIs[0].Payload!, CultureInfo.InvariantCulture).ShouldBe(baselineValue, 1e-9);
+ (output[0].Status & 0xFFFF0000u).ShouldBe(StatusCodes.UncertainDataSubNormal.Code);
+ output[0].Status.ShouldBe(asIs[0].Status);
+ output.Skip(1).ShouldBe(asIs.Skip(1), "only the late-start interval changes");
+ }
+
+ ///
+ /// A data set cut at with its first non-Bad sample Uncertain (kept Uncertain if it already
+ /// is), and the same with that sample Good.
+ ///
+ private static (Sample[] Variant, Sample[] GoodTwin) LateStartVariant(Sample[] data, double cutSeconds)
+ {
+ var cut = data.Where(s => s.Seconds >= cutSeconds).OrderBy(s => s.Seconds).ToArray();
+ var first = Array.FindIndex(cut, s => !StatusCode.IsBad(s.Status));
+ if (first < 0) return (cut, cut);
+ var variant = cut.ToArray();
+ var good = cut.ToArray();
+ variant[first] = cut[first] with { Status = StatusCode.IsUncertain(cut[first].Status) ? cut[first].Status : 0x40A40000u };
+ good[first] = cut[first] with { Status = 0u };
+ return (variant, good);
+ }
+
+ private static Baseline LoadBaseline() =>
+ JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "Fixtures", "opcua-1.5.378.182-aggregates.json")), new JsonSerializerOptions { PropertyNameCaseInsensitive = true })!;
+
+ private static bool EffectiveTreatUncertainAsBad(Vector vector) => vector.Configuration?.TreatUncertainAsBad ?? vector.UncertainAsBad;
+
+ /// The same case computed by the baseline SDK with TreatUncertainAsBad = true and no explicit profile.
+ private static Vector? UncertainAsBadTwin(Baseline baseline, Vector vector) =>
+ baseline.Vectors.SingleOrDefault(v => v.Aggregate == vector.Aggregate && v.DataSet == vector.DataSet && v.Reverse == vector.Reverse
+ && v.Stepped == vector.Stepped && v.Incremental == vector.Incremental && v.UncertainAsBad && v.Configuration is null);
+
+ /// #809: the PercentGood / DurationGood vectors whose baseline output differs from their uncertain-as-bad twin.
+ private static Dictionary Issue809Exemptions(Baseline baseline) =>
+ baseline.Vectors
+ .Where(v => v.Aggregate is PercentGood or DurationGood)
+ .Select(v => (Vector: v, Twin: UncertainAsBadTwin(baseline, v)))
+ .Where(x => x.Twin is not null && !x.Vector.Output.SequenceEqual(x.Twin.Output))
+ .ToDictionary<(Vector Vector, Vector? Twin), Vector, Vector>(x => x.Vector, x => x.Twin!, System.Collections.Generic.ReferenceEqualityComparer.Instance);
+
+ private static string Describe(Vector vector) =>
+ $"{vector.Aggregate}/{vector.DataSet}/rev={vector.Reverse}/step={vector.Stepped}/uncertainBad={vector.UncertainAsBad}/incremental={vector.Incremental}/config={JsonSerializer.Serialize(vector.Configuration)}";
+
+ private static string? Check(Baseline baseline, Vector vector, Value[] expected, ServerEventRig? rig)
+ {
+ var output = Run(baseline, vector, rig, out var error);
+ return error != vector.Exception || !output.SequenceEqual(expected) || (vector.QueueAccepted is { } expectedAccepted && !output.Accepted.SequenceEqual(expectedAccepted))
+ ? $"{Describe(vector)}: error {vector.Exception} vs {error}; expected {JsonSerializer.Serialize(expected)} actual {JsonSerializer.Serialize(output.ToArray())}"
+ : null;
+ }
+
+ private static RunOutput Run(Baseline baseline, Vector vector, ServerEventRig? rig, out string? error,
+ Sample[]? data = null, bool baselineAsIs = false)
+ {
+ var telemetry = DefaultTelemetry.Create(_ => { });
+ var epoch = new DateTime(baseline.EpochTicks, DateTimeKind.Utc);
+ var id = (NodeId)typeof(ObjectIds).GetProperty(vector.Aggregate)!.GetValue(null)!;
+ var start = epoch.AddSeconds(vector.Reverse ? 70 : 0); var end = epoch.AddSeconds(vector.Reverse ? 0 : 70);
+ var profile = vector.Configuration;
+ var config = new AggregateConfiguration
+ {
+ TreatUncertainAsBad = profile?.TreatUncertainAsBad ?? vector.UncertainAsBad,
+ PercentDataBad = profile?.PercentDataBad ?? 100, PercentDataGood = profile?.PercentDataGood ?? 100,
+ UseSlopedExtrapolation = profile?.UseSlopedExtrapolation ?? false,
+ UseServerCapabilitiesDefaults = profile?.UseServerCapabilitiesDefaults ?? false,
+ };
+ var calculator = rig is not null
+ ? rig.Server.CurrentInstance.AggregateManager.CreateCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config)
+ : baselineAsIs
+ ? new BaselineAdapter(Aggregators.CreateStandardCalculator(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry)!)
+ : CompatibilityAggregateFactory.Create(id, start, end, baseline.IntervalMilliseconds, vector.Stepped, config, telemetry);
+ calculator.ShouldNotBeNull($"Runtime registration must serve {vector.Aggregate}.");
+ var output = new RunOutput();
+ error = null;
+ try
+ {
+ var samples = (data ?? baseline.Data[vector.DataSet]).OrderBy(x => x.Seconds);
+ foreach (var sample in vector.Reverse ? samples.Reverse() : samples)
+ {
+ var time = epoch.AddSeconds(sample.Seconds);
+ output.Accepted.Add(calculator.QueueRawValue(new DataValue(SdkValueBoundary.FromClr(Payload(sample)), sample.Status, time, time.AddTicks(sample.ServerOffsetTicks), sample.SourcePicoseconds, sample.ServerPicoseconds)));
+ if (vector.Incremental) Drain(false);
+ }
+ Drain(true);
+ }
+ catch (Exception ex) { error = ex.GetType().FullName; }
+ return output;
+
+ void Drain(bool partial)
+ {
+ for (var guard = 0; guard < 100; guard++)
+ {
+ if (!calculator.TryGetProcessedValue(partial, out var value)) return;
+ var payload = SdkValueBoundary.ToClr(value.WrappedValue);
+ output.Add(new(partial, payload?.GetType().Name, Format(payload), value.StatusCode.Code, LegacyTimestampTicks(value.SourceTimestamp), LegacyTimestampTicks(value.ServerTimestamp), value.SourcePicoseconds, value.ServerPicoseconds));
+ }
+ throw new InvalidOperationException("Baseline calculator did not terminate.");
+ }
}
// UA FILETIME zero is the old CLR unset sentinel, not a different historical instant.
@@ -84,6 +300,21 @@ public sealed class Sdk2AggregateCompatibilityTests
"Single" => float.Parse(sample.Payload!, CultureInfo.InvariantCulture),
"Double" => double.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Int64" => long.Parse(sample.Payload!, CultureInfo.InvariantCulture), "Boolean" => bool.Parse(sample.Payload!), _ => null };
static string? Format(object? value) => value switch { null => null, double number => number.ToString("R", CultureInfo.InvariantCulture), float number => number.ToString("R", CultureInfo.InvariantCulture), IFormattable f => f.ToString(null, CultureInfo.InvariantCulture), _ => value.ToString() };
+ private sealed class RunOutput : List { public List Accepted { get; } = []; }
+
+ /// The untouched baseline calculator behind the SDK 2 protocol (what live served before #809 / #821).
+ private sealed class BaselineAdapter(AggregateCalculator calculator) : Opc.Ua.Server.IAggregateCalculator
+ {
+ public NodeId AggregateId => calculator.AggregateId;
+ public bool QueueRawValue(DataValue value) => calculator.QueueRawValue(LegacyDataValue.FromSdk(value));
+ public bool HasEndTimePassed(DateTimeUtc currentTime) => calculator.HasEndTimePassed((DateTime)currentTime);
+ public bool TryGetProcessedValue(bool returnPartial, out DataValue value)
+ {
+ var processed = calculator.GetProcessedValue(returnPartial);
+ value = processed is null ? default : processed.ToSdk();
+ return processed is not null;
+ }
+ }
record Baseline(long EpochTicks, int IntervalMilliseconds, Dictionary Data, Vector[] Vectors);
record Sample(double Seconds,string Type,string? Payload,uint Status,ushort SourcePicoseconds = 0,ushort ServerPicoseconds = 0,long ServerOffsetTicks = 0);
record Value(bool PartialRequested,string? Type,string? Payload,uint Status,long SourceTicks,long ServerTicks,ushort SourcePicoseconds,ushort ServerPicoseconds);
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs
index 23c2d4195..033c22934 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorBootFromCacheTests.cs
@@ -56,6 +56,31 @@ public sealed class DriverHostActorBootFromCacheTests : RuntimeActorTestBase
cache.UnkeyedReads.ShouldBe(1);
}
+ ///
+ /// lmxopcua#868 third review #2: the cache is written before the address-space outcome, so a cache-booted
+ /// revision may never have been acked. A re-dispatch of it must RE-APPLY (here: fail honestly, since the
+ /// ConfigDb is unreachable) — never short-circuit a clean Applied for an outcome nobody confirmed.
+ ///
+ [Fact]
+ public void A_cache_booted_revision_is_unconfirmed_so_a_redispatch_reapplies_instead_of_acking_clean()
+ {
+ var cached = new CachedDeploymentArtifact(
+ DeploymentId.NewId().ToString(), CachedRev.Value, EmptyArtifact(), DateTimeOffset.UtcNow.AddMinutes(-1));
+ var coordinator = CreateTestProbe();
+ var actor = Sys.ActorOf(DriverHostActor.Props(
+ new ThrowingDbFactory(), TestNode, ackRouter: coordinator.Ref,
+ localRoles: new HashSet { "driver" },
+ deploymentArtifactCache: new StubArtifactCache(cached)));
+ AskDiagnostics(actor).CurrentRevision.ShouldBe(CachedRev);
+
+ actor.Tell(new ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.DispatchDeployment(
+ DeploymentId.NewId(), CachedRev, CorrelationId.NewId()));
+
+ var ack = coordinator.ExpectMsg(
+ PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ ack.Outcome.ShouldNotBe(ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy.ApplyAckOutcome.Applied);
+ }
+
[Fact]
public void BootFromCache_HandsTheCachedArtifactToTheAddressSpaceRebuild()
{
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs
index 7af4f16e1..0875d1755 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDegradedAckTests.cs
@@ -174,12 +174,49 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
publish.ExpectNoMsg(TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken);
}
+ [Fact]
+ public void The_local_row_stays_Applying_until_the_address_space_outcome_is_known()
+ {
+ // lmxopcua#868 second review M1: the node used to write its own row Applied/null the moment the revision
+ // advanced, up to ~105 s before the outcome. ConfigPublishCoordinator's recovery counts a non-Applying row
+ // as an ack, so a coordinator restart in that window sealed the deployment CLEAN while the node was about
+ // to report degraded. The row must carry only the final outcome.
+ var (db, deploymentId) = SeedOneTagDeployment();
+ var coordinator = CreateTestProbe();
+ var publish = CreateTestProbe();
+ var actor = Sys.ActorOf(DriverHostActor.Props(
+ db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" },
+ opcUaPublishActor: publish.Ref));
+
+ actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
+ var rebuild = publish.FishForMessage(_ => true, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300), TestContext.Current.CancellationToken);
+
+ // The ack is held: the row must still read Applying (the apply's synchronous writes are done by now — the
+ // rebuild Tell above is issued after them on the actor thread).
+ using (var ctx = db.CreateDbContext())
+ ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value)
+ .Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applying);
+
+ publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, false, null));
+ coordinator.ExpectMsg(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken).Degraded.ShouldBeFalse();
+
+ AwaitAssert(() =>
+ {
+ using var ctx = db.CreateDbContext();
+ var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value);
+ row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied);
+ row.FailureReason.ShouldBeNull();
+ row.AppliedAtUtc.ShouldNotBeNull();
+ }, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
[Fact]
public void A_degraded_apply_records_its_reason_on_the_local_NodeDeploymentState_row()
{
- // #601 review — a node WITH a ConfigDb writes its own row Applied/null before the outcome is known.
- // Central later adds the reason from the ack, but only when it accepts that ack (a stale/timed-out
- // deployment's ack is discarded), so the node must not leave its own row reading clean.
+ // #601 review — a node WITH a ConfigDb records its own row. Central later adds the reason from the ack,
+ // but only when it accepts that ack (a stale/timed-out deployment's ack is discarded), so the node must
+ // not leave its own row reading clean.
var (db, deploymentId) = SeedOneTagDeployment();
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
@@ -201,6 +238,58 @@ public sealed class DriverHostActorDegradedAckTests : RuntimeActorTestBase
}, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
}
+ [Fact]
+ public void An_oversized_degraded_reason_is_truncated_so_the_local_row_still_records_the_outcome()
+ {
+ // lmxopcua#868 third review #1: the row's FailureReason is nvarchar(2048). An unbounded reason made SQL Server
+ // refuse the write (logged and swallowed), so the row stayed Applying forever. A DB that enforces the column.
+ var db = new LengthEnforcingDbFactory(Guid.NewGuid().ToString("N"));
+ var deploymentId = V3ArtifactSeeder.SeedV3Deployment(db, RevA,
+ rawTags: new[] { (Driver: "drv-1", DriverName: "Modbus", Device: "dev1", Tag: "speed", DataType: "Double", Writable: false) },
+ unsRefs: new[] { (Area: "filling", Line: "line1", Equip: "station1", BackingTag: "speed", Effective: (string?)null) });
+ var coordinator = CreateTestProbe();
+ var publish = CreateTestProbe();
+ var actor = Sys.ActorOf(DriverHostActor.Props(
+ db, TestNode, coordinator.Ref, localRoles: new HashSet { "driver" },
+ opcUaPublishActor: publish.Ref));
+
+ actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
+ var rebuild = publish.FishForMessage(_ => true, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ publish.Reply(new OpcUaPublishActor.RebuildCompleted(rebuild.Correlation, rebuild.DeploymentId, true, new string('f', 5000)));
+ coordinator.ExpectMsg(PresenceBudget, cancellationToken: TestContext.Current.CancellationToken).Degraded.ShouldBeTrue();
+
+ AwaitAssert(() =>
+ {
+ using var ctx = db.CreateDbContext();
+ var row = ctx.NodeDeploymentStates.Single(r => r.NodeId == TestNode.Value && r.DeploymentId == deploymentId.Value);
+ row.Status.ShouldBe(ZB.MOM.WW.OtOpcUa.Configuration.Enums.NodeDeploymentStatus.Applied);
+ row.FailureReason!.Length.ShouldBe(FailureReasons.MaxLength);
+ row.FailureReason.ShouldStartWith(ApplyAck.DegradedReasonPrefix);
+ }, PresenceBudget, cancellationToken: TestContext.Current.CancellationToken);
+ }
+
+ /// In-memory ConfigDb that refuses an over-length FailureReason, as SQL Server does.
+ private sealed class LengthEnforcingDbFactory(string dbName) : IDbContextFactory
+ {
+ public OtOpcUaConfigDbContext CreateDbContext() => new(new DbContextOptionsBuilder()
+ .UseInMemoryDatabase(dbName)
+ .AddInterceptors(new Enforcer())
+ .Options);
+
+ private sealed class Enforcer : Microsoft.EntityFrameworkCore.Diagnostics.SaveChangesInterceptor
+ {
+ public override Microsoft.EntityFrameworkCore.Diagnostics.InterceptionResult SavingChanges(
+ Microsoft.EntityFrameworkCore.Diagnostics.DbContextEventData eventData,
+ Microsoft.EntityFrameworkCore.Diagnostics.InterceptionResult result)
+ {
+ foreach (var e in eventData.Context!.ChangeTracker.Entries())
+ if (e.Entity.FailureReason is { Length: > FailureReasons.MaxLength })
+ throw new DbUpdateException("String or binary data would be truncated in column 'FailureReason'.");
+ return result;
+ }
+ }
+ }
+
[Fact]
public void The_default_outcome_timeout_is_derived_from_centrals_apply_deadline_minus_a_margin()
{
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs
index 86ca424b0..f1b6294ea 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorDiagnosticsTests.cs
@@ -4,6 +4,8 @@ using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
@@ -43,6 +45,7 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
driver.State.ShouldBe("Healthy");
driver.ConnectedDevices.ShouldBe(1);
driver.FaultedDevices.ShouldBe(0);
+ driver.StoppedDevices.ShouldBe(0);
driver.LastChangeUtc.ShouldBeGreaterThan(before);
driver.LastChangeUtc.ShouldBeLessThanOrEqualTo(DateTime.UtcNow);
}
@@ -66,6 +69,46 @@ public sealed class DriverHostActorDiagnosticsTests : RuntimeActorTestBase
driver.FaultedDevices.ShouldBe(0);
}
+ [Fact]
+ public void Faulted_counts_only_Faulted_hosts_and_Stopped_is_reported_separately()
+ {
+ // #786: a Stopped host used to be counted as faulted.
+ var now = DateTime.UtcNow;
+ var health = new ChildHealthRecord(
+ DriverState.Degraded, now, null,
+ new[]
+ {
+ new HostConnectivityStatus("plc-a:502", HostState.Running, now),
+ new HostConnectivityStatus("plc-b:502", HostState.Faulted, now),
+ new HostConnectivityStatus("plc-c:502", HostState.Stopped, now),
+ new HostConnectivityStatus("plc-d:502", HostState.Stopped, now),
+ new HostConnectivityStatus("plc-e:502", HostState.Unknown, now),
+ },
+ now);
+
+ var row = DriverHostActor.DiagnosticsFromHealth(Guid.NewGuid(), DriverId, health);
+
+ row.ConnectedDevices.ShouldBe(1);
+ row.FaultedDevices.ShouldBe(1);
+ row.StoppedDevices.ShouldBe(2);
+ row.FaultedHostNames.ShouldBe(new[] { "plc-b:502" });
+ }
+
+ [Theory]
+ [InlineData(DriverState.Faulted, 0, 1)]
+ [InlineData(DriverState.Healthy, 1, 0)]
+ [InlineData(DriverState.Degraded, 1, 0)]
+ public void A_single_host_driver_is_one_device_with_no_host_name(DriverState state, int connected, int faulted)
+ {
+ var row = DriverHostActor.DiagnosticsFromHealth(
+ Guid.NewGuid(), DriverId, new ChildHealthRecord(state, null, null, null, DateTime.UtcNow));
+
+ row.ConnectedDevices.ShouldBe(connected);
+ row.FaultedDevices.ShouldBe(faulted);
+ row.StoppedDevices.ShouldBe(0);
+ row.FaultedHostNames.ShouldNotBeNull().ShouldBeEmpty();
+ }
+
private async Task AwaitDiagnosticsAsync(IActorRef host, Func ready)
{
NodeDiagnosticsSnapshot? snapshot = null;
diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs
index 2235faa5b..f78053f6b 100644
--- a/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs
+++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ScriptWorker.Tests/ScriptRunnerPerfTests.cs
@@ -32,6 +32,12 @@ public sealed class ScriptRunnerPerfTests
{
private const string OptInVariable = "OTOPCUA_SCRIPT_PERF";
private const int AcceptedGateEvalsPerSecond = 5_000;
+
+ ///
+ /// §8.2 "Worker idle RSS ≤ 200 MB + scripts", per shard (revised by the owner 2026-11-11, lmxopcua#789: compiles
+ /// run in the worker, so Roslyn stays resident; the original 80 MB [A-3] is only met by a warm-PE-cache worker).
+ ///
+ private const int WorkerIdleRssBudgetMb = 200;
private const int Keys = 2_000;
private const int DistinctSources = 50;
@@ -481,7 +487,7 @@ public sealed class ScriptRunnerPerfTests
}
///
- /// §8.2 "Shard restart → first result, 1,000 cached scripts ≤ 2 s", "Worker idle RSS ≤ 80 MB + scripts" and
+ /// §8.2 "Shard restart → first result, 1,000 cached scripts ≤ 2 s", "Worker idle RSS ≤ 200 MB + scripts" and
/// "Kill latency after T+G ≤ 100 ms". Report-only.
///
[Fact]
@@ -559,14 +565,20 @@ public sealed class ScriptRunnerPerfTests
}
}
+ static string RssCell(long bytes)
+ {
+ var mb = bytes / 1048576.0;
+ return $"{mb:N0} MB {(mb <= WorkerIdleRssBudgetMb ? "PASS" : "MISS")}";
+ }
+
report.AppendLine("| Measure | Result | §8.2 budget |")
.AppendLine("|---|---:|---|")
.AppendLine($"| {scripts} scripts, empty cache, all first results (compile + write) | {cold.TotalMilliseconds:N0} ms | — |")
.AppendLine($"| {scripts} scripts, fresh runner on a warm cache, all first results | {warmFresh.TotalMilliseconds:N0} ms | ≤ 2,000 ms |")
.AppendLine($"| kill both shards → every script's next correct result | {killRestart.TotalMilliseconds:N0} ms | ≤ 2,000 ms |")
- .AppendLine($"| worker RSS after first eval, compiled (max of 2 shards) | {rssIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
+ .AppendLine($"| worker RSS after first eval, compiled (max of 2 shards) | {RssCell(rssIdle)} | ≤ {WorkerIdleRssBudgetMb} MB + scripts |")
.AppendLine($"| worker RSS with {scripts / 2} scripts compiled per shard (max) | {rssLoaded / 1048576.0:N0} MB | — |")
- .AppendLine($"| worker RSS after first eval, warm PE cache, no compile (max) | {rssWarmIdle / 1048576.0:N0} MB | ≤ 80 MB + scripts |")
+ .AppendLine($"| worker RSS after first eval, warm PE cache, no compile (max) | {RssCell(rssWarmIdle)} | ≤ {WorkerIdleRssBudgetMb} MB + scripts |")
.AppendLine($"| worker RSS with {scripts / 2} scripts loaded from the cache per shard (max) | {rssWarmLoaded / 1048576.0:N0} MB | — |")
.AppendLine($"| kill latency after T+G (5 runs, ms) | {string.Join(" / ", killLatencies.Select(l => l.ToString("N0")))} | ≤ 100 ms |");
Report(report);