From b437c34f924a689177b3ab8b867f78ed6b5edb95 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:22:20 -0400 Subject: [PATCH 01/51] fix(controlplane): recover a lost dispatch instead of wedging the deploy pipeline A Deployment row left in Dispatching with no NodeDeploymentState rows (HandleDispatch threw before its SaveChanges, or the process died between the AdminOps row write and the coordinator's Tell) was recovered with an empty expected set; the deadline then hit the 0 == 0 "race" branch and returned without touching the row, so every later StartDeployment was refused AnotherDeploymentInFlight until someone hand-edited SQL. - PreStart: a recovered row with no expected-ack rows is a lost dispatch. Inside the apply deadline it is re-driven through HandleDispatch (bounded: CreatedAtUtc never moves); past it the row is closed TimedOut with a reason naming the condition. - HandleDispatch is idempotent on a re-dispatch (rows already seeded ARE the expected set). - HandleDeadline takes the race branch only when _expectedAcks.Count > 0. - The PreStart recovery read projects ArtifactBlob away (R03:3.5). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Coordinators/ConfigPublishCoordinator.cs | 99 ++++++-- .../ConfigPublishCoordinatorRecoveryTests.cs | 234 ++++++++++++++++++ 2 files changed, 316 insertions(+), 17 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index b621cc64..3c99e998 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -161,9 +161,12 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers DistributedPubSub.Get(Context.System).Mediator.Tell(new Subscribe(DeploymentAcksTopic, Self)); using var db = _dbFactory.CreateDbContext(); + // Projected: the row carries the whole ArtifactBlob (varbinary(max)), which this recovery never + // reads — and this query runs on every singleton start and hand-over. var inflight = db.Deployments .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.RevisionHash, d.CreatedAtUtc }) .FirstOrDefault(); if (inflight is null) return; @@ -173,6 +176,41 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers .AsNoTracking() .ToList(); + // Resume the deadline timer using the remaining time. The deadline runs from when the + // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough + // proxy — we don't track a separate "dispatched at" column). + var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc; + var remaining = _applyDeadline - elapsed; + + if (nodeStates.Count == 0) + { + // The expected set and the AwaitingApplyAcks flip are written in ONE SaveChanges by + // HandleDispatch, so a row with no NodeDeploymentState rows is a dispatch that never completed: + // HandleDispatch threw before its SaveChanges (this actor restarted and the message was + // dropped), the process died between AdminOps' row write and the coordinator's Tell, or the + // Tell was lost across a singleton hand-over. Recovering it with an EMPTY expected set used to + // let the deadline take the "everyone acked" race branch (0 == 0) and leave the row in flight + // forever, refusing every later StartDeployment. Inside the deadline the dispatch is simply + // re-driven through the normal handler (bounded: CreatedAtUtc never moves, so a dispatch that + // keeps throwing ends up on the other branch once the deadline passes); past it the row is + // closed as TimedOut with a reason that names the condition. + if (remaining > TimeSpan.Zero) + { + _log.Warning( + "Coordinator recovered deployment {Id} with NO expected-ack rows — the dispatch was lost " + + "before any node was expected (coordinator restart or singleton hand-over mid-dispatch); " + + "re-dispatching with {Remaining} of the apply deadline left", + _current, remaining); + Self.Tell(new DispatchDeployment( + _current.Value, RevisionHash.Parse(inflight.RevisionHash), CorrelationId.NewId())); + } + else + { + Self.Tell(new DeadlineElapsed(_current.Value)); + } + return; + } + _expectedAcks = nodeStates.Select(s => NodeId.Parse(s.NodeId)).ToHashSet(NodeIdComparer); foreach (var s in nodeStates.Where(s => s.Status != NodeDeploymentStatus.Applying)) { @@ -182,11 +220,6 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers if (IsDegraded(s.Status, s.FailureReason)) _degraded.Add(NodeId.Parse(s.NodeId)); } - // Resume the deadline timer using the remaining time. The deadline runs from when the - // deployment was first marked AwaitingApplyAcks (Deployment.CreatedAtUtc is a close enough - // proxy — we don't track a separate "dispatched at" column). - var elapsed = DateTime.UtcNow - inflight.CreatedAtUtc; - var remaining = _applyDeadline - elapsed; if (remaining <= TimeSpan.Zero) { Self.Tell(new DeadlineElapsed(_current.Value)); @@ -208,19 +241,34 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // Seed NodeDeploymentState rows so a failover knows which nodes were expected to ack. using (var db = _dbFactory.CreateDbContext()) { - // Discovery reuses this context rather than opening a second one — it reads the same - // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single - // context keeps the expected set and the seeded rows consistent by construction. - _expectedAcks = DiscoverDriverNodes(db); - - foreach (var node in _expectedAcks) + // Idempotent on a re-dispatch (PreStart re-drives a lost dispatch through this handler): rows + // already seeded for this deployment ARE the expected set — re-discovering would both duplicate + // the primary key and let a node toggled since the first attempt change who is expected. + var seeded = db.NodeDeploymentStates + .AsNoTracking() + .Where(x => x.DeploymentId == msg.DeploymentId.Value) + .Select(x => x.NodeId) + .ToList(); + if (seeded.Count > 0) { - db.NodeDeploymentStates.Add(new NodeDeploymentState + _expectedAcks = seeded.Select(NodeId.Parse).ToHashSet(NodeIdComparer); + } + else + { + // Discovery reuses this context rather than opening a second one — it reads the same + // ClusterNode table the NodeDeploymentState rows below are FK-bound to, so a single + // context keeps the expected set and the seeded rows consistent by construction. + _expectedAcks = DiscoverDriverNodes(db); + + foreach (var node in _expectedAcks) { - NodeId = node.Value, - DeploymentId = msg.DeploymentId.Value, - Status = NodeDeploymentStatus.Applying, - }); + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = node.Value, + DeploymentId = msg.DeploymentId.Value, + Status = NodeDeploymentStatus.Applying, + }); + } } UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks); db.SaveChanges(); @@ -360,7 +408,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers msg.DeploymentId, _current); return; } - if (_acks.Count == _expectedAcks.Count) + // Only a NON-EMPTY expected set can be "fully acked": with nothing expected, 0 == 0 is not the + // everyone-acked race, it is a lost dispatch (see PreStart) that must be closed, not skipped. + if (_expectedAcks.Count > 0 && _acks.Count == _expectedAcks.Count) { // Race: every node acked just as the deadline fired. Already sealed/failed elsewhere. return; @@ -368,6 +418,21 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers using var db = _dbFactory.CreateDbContext(); UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); + if (_expectedAcks.Count == 0) + { + var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); + if (d is not null) + d.FailureReason = Truncate( + "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + + "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."); + db.SaveChanges(); + _log.Warning( + "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " + + "expected and the {Deadline} apply deadline has passed. Closed as TimedOut — redeploy", + _current.Value, _applyDeadline); + ResetForNext(); + return; + } db.SaveChanges(); // Name the silent nodes, not just the counts. Since the expected set became DB-derived this // is the failure an operator meets after deploying while a configured node is switched off, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs new file mode 100644 index 00000000..86f6efa7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -0,0 +1,234 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Review 03 §2.2: a Deployment row left in Dispatching with NO NodeDeploymentState +/// rows — HandleDispatch threw before its SaveChanges, or the process died between the +/// AdminOps row write and the coordinator's Tell — used to be recovered with an EMPTY expected +/// set, at which point the deadline hit the 0 == 0 "race" branch and returned without touching +/// the row. Every later StartDeployment was then refused AnotherDeploymentInFlight forever. +/// +public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTestBase +{ + private static readonly RevisionHash TestRevision = RevisionHash.Parse(new string('c', 64)); + private static readonly TimeSpan NoTimeout = TimeSpan.FromMinutes(5); + + /// + /// A lost dispatch that is still inside the apply deadline is re-driven: PreStart re-dispatches it, + /// the expected set is seeded from the enabled rows, and the node's ack seals it. + /// + [Fact] + public void Recovered_row_with_no_expected_set_is_redispatched_and_then_seals() + { + var dbFactory = NewInMemoryDbFactory(); + var deploymentId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + SeedNodes(dbFactory, "site-a-1:4053"); + + // No DispatchDeployment is ever sent — PreStart alone has to recover it. + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.NodeDeploymentStates.Select(s => s.NodeId).ShouldBe(new[] { "site-a-1:4053" }); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + actor.Tell(new ApplyAck(deploymentId, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// + /// A lost dispatch that is already past the apply deadline is closed as TimedOut with a reason + /// naming the condition — never left in Dispatching — and the next deployment proceeds + /// (positive control: it is dispatched, acked and sealed through the same coordinator). + /// + [Fact] + public void Recovered_row_past_deadline_with_no_expected_set_is_timed_out_not_wedged() + { + var dbFactory = NewInMemoryDbFactory(); + var staleId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + SeedNodes(dbFactory, "site-a-1:4053"); + + var actor = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + var row = db.Deployments.Single(d => d.DeploymentId == staleId.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + }, duration: PresenceBudget); + + // Positive control: the coordinator is not wedged — a fresh deployment runs end to end. + var nextId = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + actor.Tell(new DispatchDeployment(nextId, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + actor.Tell(new ApplyAck(nextId, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == nextId.Value).Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// + /// An AwaitingApplyAcks row with no rows (the process died between the status flip and the + /// empty-fleet seal) is re-driven too, and with no enabled nodes it seals empty exactly as a fresh + /// dispatch would — the recovery keys on the missing expected set, not on the status value. + /// + [Fact] + public void Recovered_AwaitingApplyAcks_row_with_no_rows_and_no_fleet_seals_empty() + { + var dbFactory = NewInMemoryDbFactory(); + SeedDeployment(dbFactory, DeploymentStatus.AwaitingApplyAcks, DateTime.UtcNow); + Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + db.NodeDeploymentStates.ShouldBeEmpty(); + }, duration: PresenceBudget); + } + + /// + /// The brief's scenario end to end: HandleDispatch throws once (a transient fault seeding the + /// expected set). The coordinator restarts, recovers the Dispatching row, re-dispatches it and + /// the deployment completes; a second StartDeployment is then Accepted (positive control: a + /// normal deploy seals). + /// + [Fact] + public async Task HandleDispatch_throw_is_recovered_and_the_next_StartDeployment_succeeds() + { + var dbName = Guid.NewGuid().ToString("N"); + var adminFactory = NewInMemoryDbFactory(dbName); + var faulting = new OneShotFaultingDbFactory(NewInMemoryDbFactory(dbName)); + SeedNodes(adminFactory, "site-a-1:4053"); + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(faulting, NoTimeout)); + var admin = Sys.ActorOf(AdminOperationsActor.Props(adminFactory, coordinator, Enumerable.Empty())); + + // Arm the fault AFTER PreStart has run (it would otherwise fault the recovery read itself and + // the actor would be stopped, not restarted). Identify is processed after creation, so a + // resolved selection proves PreStart completed. + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); + faulting.FailNext(); + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var first = ExpectMsg(PresenceBudget); + first.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + first.DeploymentId.ShouldNotBeNull(); + + // The fault fired, the actor restarted, PreStart re-dispatched: the expected set lands. + AwaitAssert(() => + { + faulting.Faulted.ShouldBeTrue(); + using var db = adminFactory.CreateDbContext(); + db.NodeDeploymentStates.Count(s => s.DeploymentId == first.DeploymentId.Value.Value).ShouldBe(1); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + coordinator.Tell(new ApplyAck(first.DeploymentId.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = adminFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + + // Positive control: nothing is wedged — the next deploy is Accepted and seals too. + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var second = ExpectMsg(PresenceBudget); + second.Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.Tell(new ApplyAck(second.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = adminFactory.CreateDbContext(); + db.Deployments.Count(d => d.Status == DeploymentStatus.Sealed).ShouldBe(2); + }, duration: PresenceBudget); + } + + private static DeploymentId SeedDeployment( + IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) + { + var id = DeploymentId.NewId(); + using var db = dbFactory.CreateDbContext(); + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, + RevisionHash = TestRevision.Value, + Status = status, + CreatedBy = "test", + CreatedAtUtc = createdAtUtc, + }); + db.SaveChanges(); + return id; + } + + private static void SeedNodes(IDbContextFactory dbFactory, params string[] nodeIds) + { + using var db = dbFactory.CreateDbContext(); + db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = (byte)nodeIds.Length, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + foreach (var nodeId in nodeIds) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = nodeId, ClusterId = "SITE-A", Host = nodeId.Split(':')[0], + ApplicationUri = $"urn:OtOpcUa:{nodeId}", Enabled = true, CreatedBy = "test", + }); + } + db.SaveChanges(); + } + + /// Throws from exactly one CreateDbContext call after . + private sealed class OneShotFaultingDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private int _armed; + public bool Faulted { get; private set; } + + public void FailNext() => Interlocked.Exchange(ref _armed, 1); + + public OtOpcUaConfigDbContext CreateDbContext() + { + if (Interlocked.Exchange(ref _armed, 0) == 1) + { + Faulted = true; + throw new InvalidOperationException("injected transient fault seeding the expected set"); + } + return inner.CreateDbContext(); + } + } +} From 73e74e270ddf8b9be39c010935cc1aff9c8f91fe Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:36:12 -0400 Subject: [PATCH 02/51] fix(controlplane): validate and compose a deployment from ONE snapshot read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit StartDeployment read the whole config DB twice with no transaction — DraftSnapshotFactory for the validator, then ConfigComposer for the artifact, plus clusters/nodes and Script.SourceCode again in between. An AdminUI save landing between the two shipped an artifact the DraftValidator never saw. - DraftSnapshot carries ServerClusters, ClusterNodes and ApiKeys; DraftSnapshotFactory reads every table inside one SNAPSHOT transaction on a relational provider (no-op on InMemory or under an ambient transaction). - ConfigComposer.Compose(DraftSnapshot) is pure over the snapshot (ordinal-sorted keys); SnapshotAndFlattenAsync(db) delegates to it, byte-identical. - AdminOperationsActor validates topology, estimates compile cost and composes the artifact from the one snapshot it holds. - Migration EnableSnapshotIsolation: ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON (suppressTransaction; no model change). Tests: the SQL Server snapshot read ignores a row committed mid-read (positive control: a later read sees it); an edit injected between the Equipment and Tags reads is absent from the shipped artifact and rejected by the next deploy; Compose(snapshot) == SnapshotAndFlattenAsync(db). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- ...102459_EnableSnapshotIsolation.Designer.cs | 1618 +++++++++++++++++ .../20261022102459_EnableSnapshotIsolation.cs | 36 + .../Validation/DraftSnapshot.cs | 17 + .../Validation/DraftSnapshotFactory.cs | 31 + .../AdminOperations/AdminOperationsActor.cs | 29 +- .../AdminOperations/ConfigComposer.cs | 61 +- .../DraftSnapshotFactorySqlServerTests.cs | 117 ++ .../DraftSnapshotFactoryTests.cs | 30 + ...AdminOperationsActorSingleSnapshotTests.cs | 133 ++ .../ConfigComposerTests.cs | 41 + 10 files changed, 2077 insertions(+), 36 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs new file mode 100644 index 00000000..db54f3a9 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.Designer.cs @@ -0,0 +1,1618 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ZB.MOM.WW.OtOpcUa.Configuration; + +#nullable disable + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations +{ + [DbContext(typeof(OtOpcUaConfigDbContext))] + [Migration("20261022102459_EnableSnapshotIsolation")] + partial class EnableSnapshotIsolation + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.7") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Microsoft.AspNetCore.DataProtection.EntityFrameworkCore.DataProtectionKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("FriendlyName") + .HasColumnType("nvarchar(max)"); + + b.Property("Xml") + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("DataProtectionKeys", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ApiKey", b => + { + b.Property("ApiKeyRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ApiKeyId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedUtc") + .HasColumnType("datetime2"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("ExpiresUtc") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SecretHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.HasKey("ApiKeyRowId"); + + b.HasIndex("ApiKeyId") + .IsUnique() + .HasDatabaseName("UX_ApiKey_ApiKeyId"); + + b.HasIndex("Principal") + .HasDatabaseName("IX_ApiKey_Principal"); + + b.ToTable("ApiKey", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("AkkaPort") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasDefaultValue(4053); + + b.Property("ApplicationUri") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("DashboardPort") + .HasColumnType("int"); + + b.Property("DriverConfigOverridesJson") + .HasColumnType("nvarchar(max)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("GrpcPort") + .HasColumnType("int"); + + b.Property("Host") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("LastSeenAt") + .HasColumnType("datetime2(3)"); + + b.Property("MaintenanceMode") + .HasColumnType("bit"); + + b.Property("OpcUaPort") + .HasColumnType("int"); + + b.Property("ServiceLevelBase") + .HasColumnType("tinyint"); + + b.HasKey("NodeId"); + + b.HasIndex("ApplicationUri") + .IsUnique() + .HasDatabaseName("UX_ClusterNode_ApplicationUri"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_ClusterNode_ClusterId"); + + b.ToTable("ClusterNode", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNodeCredential", b => + { + b.Property("CredentialId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("NodeId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RotatedAt") + .HasColumnType("datetime2(3)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.HasKey("CredentialId"); + + b.HasIndex("Kind", "Value") + .IsUnique() + .HasDatabaseName("UX_ClusterNodeCredential_Value") + .HasFilter("[Enabled] = 1"); + + b.HasIndex("NodeId", "Enabled") + .HasDatabaseName("IX_ClusterNodeCredential_NodeId"); + + b.ToTable("ClusterNodeCredential", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ConfigAuditLog", b => + { + b.Property("AuditId") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("AuditId")); + + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CorrelationId") + .HasColumnType("uniqueidentifier"); + + b.Property("DetailsJson") + .HasColumnType("nvarchar(max)"); + + b.Property("EventId") + .HasColumnType("uniqueidentifier"); + + b.Property("EventType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("GenerationId") + .HasColumnType("bigint"); + + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Timestamp") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.HasKey("AuditId"); + + b.HasIndex("EventId") + .IsUnique() + .HasDatabaseName("UX_ConfigAuditLog_EventId") + .HasFilter("[EventId] IS NOT NULL"); + + b.HasIndex("GenerationId") + .HasDatabaseName("IX_ConfigAuditLog_Generation") + .HasFilter("[GenerationId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Timestamp") + .IsDescending(false, true) + .HasDatabaseName("IX_ConfigAuditLog_Cluster_Time"); + + b.ToTable("ConfigAuditLog", null, t => + { + t.HasCheckConstraint("CK_ConfigAuditLog_DetailsJson_IsJson", "DetailsJson IS NULL OR ISJSON(DetailsJson) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ConfigEdit", b => + { + b.Property("EditId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("EditedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("EditedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EntityId") + .HasColumnType("uniqueidentifier"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ExecutionId") + .HasColumnType("uniqueidentifier"); + + b.Property("FieldsJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SourceNode") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("EditId"); + + b.HasIndex("EditedAtUtc") + .HasDatabaseName("IX_ConfigEdit_EditedAt"); + + b.HasIndex("ExecutionId") + .HasDatabaseName("IX_ConfigEdit_Execution") + .HasFilter("[ExecutionId] IS NOT NULL"); + + b.HasIndex("EntityType", "EntityId") + .HasDatabaseName("IX_ConfigEdit_Entity"); + + b.ToTable("ConfigEdit", null, t => + { + t.HasCheckConstraint("CK_ConfigEdit_FieldsJson_IsJson", "ISJSON(FieldsJson) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Deployment", b => + { + b.Property("DeploymentId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ArtifactBlob") + .IsRequired() + .HasColumnType("varbinary(max)"); + + b.Property("CreatedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("FailureReason") + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("RevisionHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SealedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("DeploymentId"); + + b.HasIndex("CreatedAtUtc") + .HasDatabaseName("IX_Deployment_CreatedAt"); + + b.HasIndex("Status") + .HasDatabaseName("IX_Deployment_Status"); + + b.ToTable("Deployment", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Device", b => + { + b.Property("DeviceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DeviceConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DeviceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverInstanceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("DeviceRowId"); + + b.HasIndex("DeviceId") + .IsUnique() + .HasDatabaseName("UX_Device_LogicalId") + .HasFilter("[DeviceId] IS NOT NULL"); + + b.HasIndex("DriverInstanceId") + .HasDatabaseName("IX_Device_Driver"); + + b.HasIndex("DriverInstanceId", "Name") + .IsUnique() + .HasDatabaseName("UX_Device_Driver_Name"); + + b.ToTable("Device", null, t => + { + t.HasCheckConstraint("CK_Device_DeviceConfig_IsJson", "ISJSON(DeviceConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b => + { + b.Property("DriverInstanceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("DriverInstanceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DriverType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ResilienceConfig") + .HasColumnType("nvarchar(max)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("DriverInstanceRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_DriverInstance_Cluster"); + + b.HasIndex("DriverInstanceId") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_LogicalId") + .HasFilter("[DriverInstanceId] IS NOT NULL"); + + b.HasIndex("RawFolderId") + .HasDatabaseName("IX_DriverInstance_RawFolder"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_ClusterRoot_Name") + .HasFilter("[RawFolderId] IS NULL"); + + b.HasIndex("ClusterId", "RawFolderId", "Name") + .IsUnique() + .HasDatabaseName("UX_DriverInstance_Folder_Name") + .HasFilter("[RawFolderId] IS NOT NULL"); + + b.ToTable("DriverInstance", null, t => + { + t.HasCheckConstraint("CK_DriverInstance_DriverConfig_IsJson", "ISJSON(DriverConfig) = 1"); + + t.HasCheckConstraint("CK_DriverInstance_ResilienceConfig_IsJson", "ResilienceConfig IS NULL OR ISJSON(ResilienceConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Equipment", b => + { + b.Property("EquipmentRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AssetLocation") + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("DeviceManualUri") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentClassRef") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EquipmentId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("EquipmentUuid") + .HasColumnType("uniqueidentifier"); + + b.Property("HardwareRevision") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("MachineCode") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Manufacturer") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ManufacturerUri") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Model") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SAPID") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SerialNumber") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SoftwareRevision") + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("UnsLineId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("YearOfConstruction") + .HasColumnType("smallint"); + + b.Property("ZTag") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("EquipmentRowId"); + + b.HasIndex("EquipmentId") + .IsUnique() + .HasDatabaseName("UX_Equipment_LogicalId") + .HasFilter("[EquipmentId] IS NOT NULL"); + + b.HasIndex("EquipmentUuid") + .IsUnique() + .HasDatabaseName("UX_Equipment_Uuid"); + + b.HasIndex("MachineCode") + .HasDatabaseName("IX_Equipment_MachineCode"); + + b.HasIndex("SAPID") + .HasDatabaseName("IX_Equipment_SAPID") + .HasFilter("[SAPID] IS NOT NULL"); + + b.HasIndex("UnsLineId") + .HasDatabaseName("IX_Equipment_Line"); + + b.HasIndex("ZTag") + .HasDatabaseName("IX_Equipment_ZTag") + .HasFilter("[ZTag] IS NOT NULL"); + + b.HasIndex("UnsLineId", "Name") + .IsUnique() + .HasDatabaseName("UX_Equipment_LinePath"); + + b.ToTable("Equipment", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ExternalIdReservation", b => + { + b.Property("ReservationId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("EquipmentUuid") + .HasColumnType("uniqueidentifier"); + + b.Property("FirstPublishedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("FirstPublishedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("LastPublishedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("ReleaseReason") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("ReleasedAt") + .HasColumnType("datetime2(3)"); + + b.Property("ReleasedBy") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Value") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("ReservationId"); + + b.HasIndex("EquipmentUuid") + .HasDatabaseName("IX_ExternalIdReservation_Equipment"); + + b.HasIndex("Kind", "Value") + .IsUnique() + .HasDatabaseName("UX_ExternalIdReservation_KindValue_Active") + .HasFilter("[ReleasedAt] IS NULL"); + + b.ToTable("ExternalIdReservation", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.LdapGroupRoleMapping", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("IsSystemWide") + .HasColumnType("bit"); + + b.Property("LdapGroup") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("Role") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("Id"); + + b.HasIndex("ClusterId"); + + b.HasIndex("LdapGroup") + .HasDatabaseName("IX_LdapGroupRoleMapping_Group"); + + b.HasIndex("LdapGroup", "ClusterId") + .IsUnique() + .HasDatabaseName("UX_LdapGroupRoleMapping_Group_Cluster") + .HasFilter("[ClusterId] IS NOT NULL"); + + b.ToTable("LdapGroupRoleMapping", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeAcl", b => + { + b.Property("NodeAclRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("NodeAclId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("PermissionFlags") + .HasColumnType("int"); + + b.Property("Principal") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("nvarchar(256)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScopeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("ScopeKind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.HasKey("NodeAclRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_NodeAcl_Cluster"); + + b.HasIndex("NodeAclId") + .IsUnique() + .HasDatabaseName("UX_NodeAcl_LogicalId") + .HasFilter("[NodeAclId] IS NOT NULL"); + + b.HasIndex("Principal") + .HasDatabaseName("IX_NodeAcl_Principal"); + + b.HasIndex("ScopeKind", "ScopeId") + .HasDatabaseName("IX_NodeAcl_Scope") + .HasFilter("[ScopeId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Principal", "ScopeKind", "ScopeId") + .IsUnique() + .HasDatabaseName("UX_NodeAcl_PrincipalScope") + .HasFilter("[ScopeId] IS NOT NULL"); + + b.ToTable("NodeAcl", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState", b => + { + b.Property("NodeId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("DeploymentId") + .HasColumnType("uniqueidentifier"); + + b.Property("AppliedAtUtc") + .HasColumnType("datetime2(3)"); + + b.Property("FailureReason") + .HasMaxLength(2048) + .HasColumnType("nvarchar(2048)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("StartedAtUtc") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("NodeId", "DeploymentId"); + + b.HasIndex("DeploymentId") + .HasDatabaseName("IX_NodeDeploymentState_Deployment"); + + b.HasIndex("Status") + .HasDatabaseName("IX_NodeDeploymentState_Status"); + + b.ToTable("NodeDeploymentState", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.PollGroup", b => + { + b.Property("PollGroupRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DriverInstanceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("IntervalMs") + .HasColumnType("int"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PollGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.HasKey("PollGroupRowId"); + + b.HasIndex("DriverInstanceId") + .HasDatabaseName("IX_PollGroup_Driver"); + + b.HasIndex("PollGroupId") + .IsUnique() + .HasDatabaseName("UX_PollGroup_LogicalId") + .HasFilter("[PollGroupId] IS NOT NULL"); + + b.ToTable("PollGroup", null, t => + { + t.HasCheckConstraint("CK_PollGroup_IntervalMs_Min", "IntervalMs >= 50"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.RawFolder", b => + { + b.Property("RawFolderRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ParentRawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RawFolderId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.HasKey("RawFolderRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_RawFolder_Cluster"); + + b.HasIndex("RawFolderId") + .IsUnique() + .HasDatabaseName("UX_RawFolder_LogicalId") + .HasFilter("[RawFolderId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_RawFolder_ClusterRoot_Name") + .HasFilter("[ParentRawFolderId] IS NULL"); + + b.HasIndex("ClusterId", "ParentRawFolderId", "Name") + .IsUnique() + .HasDatabaseName("UX_RawFolder_Parent_Name") + .HasFilter("[ParentRawFolderId] IS NOT NULL"); + + b.ToTable("RawFolder", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Script", b => + { + b.Property("ScriptRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("Language") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("SourceCode") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SourceHash") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("ScriptRowId"); + + b.HasIndex("ScriptId") + .IsUnique() + .HasDatabaseName("UX_Script_LogicalId") + .HasFilter("[ScriptId] IS NOT NULL"); + + b.HasIndex("SourceHash") + .HasDatabaseName("IX_Script_SourceHash"); + + b.ToTable("Script", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ScriptedAlarm", b => + { + b.Property("ScriptedAlarmRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AlarmType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("HistorizeToAveva") + .HasColumnType("bit"); + + b.Property("MessageTemplate") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PredicateScriptId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Retain") + .HasColumnType("bit"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptedAlarmId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Severity") + .HasColumnType("int"); + + b.HasKey("ScriptedAlarmRowId"); + + b.HasIndex("PredicateScriptId") + .HasDatabaseName("IX_ScriptedAlarm_Script"); + + b.HasIndex("ScriptedAlarmId") + .IsUnique() + .HasDatabaseName("UX_ScriptedAlarm_LogicalId") + .HasFilter("[ScriptedAlarmId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "Name") + .IsUnique() + .HasDatabaseName("UX_ScriptedAlarm_EquipmentPath"); + + b.ToTable("ScriptedAlarm", null, t => + { + t.HasCheckConstraint("CK_ScriptedAlarm_AlarmType", "AlarmType IN ('AlarmCondition','LimitAlarm','OffNormalAlarm','DiscreteAlarm')"); + + t.HasCheckConstraint("CK_ScriptedAlarm_Severity_Range", "Severity BETWEEN 1 AND 1000"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", b => + { + b.Property("ClusterId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("CreatedAt") + .ValueGeneratedOnAdd() + .HasColumnType("datetime2(3)") + .HasDefaultValueSql("SYSUTCDATETIME()"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("Enterprise") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("ModifiedAt") + .HasColumnType("datetime2(3)"); + + b.Property("ModifiedBy") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("NodeCount") + .HasColumnType("tinyint"); + + b.Property("Notes") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("RedundancyMode") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("Site") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.HasKey("ClusterId"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("UX_ServerCluster_Name"); + + b.HasIndex("Site") + .HasDatabaseName("IX_ServerCluster_Site"); + + b.ToTable("ServerCluster", null, t => + { + t.HasCheckConstraint("CK_ServerCluster_RedundancyMode_NodeCount", "((NodeCount = 1 AND RedundancyMode = 'None') OR (NodeCount = 2 AND RedundancyMode IN ('Warm', 'Hot')))"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Tag", b => + { + b.Property("TagRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("AccessLevel") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("PollGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("TagConfig") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("TagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("TagId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("WriteIdempotent") + .HasColumnType("bit"); + + b.HasKey("TagRowId"); + + b.HasIndex("TagId") + .IsUnique() + .HasDatabaseName("UX_Tag_LogicalId") + .HasFilter("[TagId] IS NOT NULL"); + + b.HasIndex("DeviceId", "Name") + .IsUnique() + .HasDatabaseName("UX_Tag_Device_Name") + .HasFilter("[TagGroupId] IS NULL"); + + b.HasIndex("DeviceId", "TagGroupId") + .HasDatabaseName("IX_Tag_Device"); + + b.HasIndex("DeviceId", "TagGroupId", "Name") + .IsUnique() + .HasDatabaseName("UX_Tag_Group_Name") + .HasFilter("[TagGroupId] IS NOT NULL"); + + b.ToTable("Tag", null, t => + { + t.HasCheckConstraint("CK_Tag_TagConfig_IsJson", "ISJSON(TagConfig) = 1"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.TagGroup", b => + { + b.Property("TagGroupRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DeviceId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("ParentTagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.Property("TagGroupId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("TagGroupRowId"); + + b.HasIndex("DeviceId") + .HasDatabaseName("IX_TagGroup_Device"); + + b.HasIndex("TagGroupId") + .IsUnique() + .HasDatabaseName("UX_TagGroup_LogicalId") + .HasFilter("[TagGroupId] IS NOT NULL"); + + b.HasIndex("DeviceId", "Name") + .IsUnique() + .HasDatabaseName("UX_TagGroup_DeviceRoot_Name") + .HasFilter("[ParentTagGroupId] IS NULL"); + + b.HasIndex("DeviceId", "ParentTagGroupId", "Name") + .IsUnique() + .HasDatabaseName("UX_TagGroup_Parent_Name") + .HasFilter("[ParentTagGroupId] IS NOT NULL"); + + b.ToTable("TagGroup", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsArea", b => + { + b.Property("UnsAreaRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ClusterId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("UnsAreaId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsAreaRowId"); + + b.HasIndex("ClusterId") + .HasDatabaseName("IX_UnsArea_Cluster"); + + b.HasIndex("UnsAreaId") + .IsUnique() + .HasDatabaseName("UX_UnsArea_LogicalId") + .HasFilter("[UnsAreaId] IS NOT NULL"); + + b.HasIndex("ClusterId", "Name") + .IsUnique() + .HasDatabaseName("UX_UnsArea_ClusterName"); + + b.ToTable("UnsArea", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsLine", b => + { + b.Property("UnsLineRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Notes") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("UnsAreaId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("UnsLineId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsLineRowId"); + + b.HasIndex("UnsAreaId") + .HasDatabaseName("IX_UnsLine_Area"); + + b.HasIndex("UnsLineId") + .IsUnique() + .HasDatabaseName("UX_UnsLine_LogicalId") + .HasFilter("[UnsLineId] IS NOT NULL"); + + b.HasIndex("UnsAreaId", "Name") + .IsUnique() + .HasDatabaseName("UX_UnsLine_AreaName"); + + b.ToTable("UnsLine", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsTagReference", b => + { + b.Property("UnsTagReferenceRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("DisplayNameOverride") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("SortOrder") + .HasColumnType("int"); + + b.Property("TagId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("UnsTagReferenceId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("UnsTagReferenceRowId"); + + b.HasIndex("EquipmentId") + .HasDatabaseName("IX_UnsTagReference_Equipment"); + + b.HasIndex("TagId") + .HasDatabaseName("IX_UnsTagReference_Tag"); + + b.HasIndex("UnsTagReferenceId") + .IsUnique() + .HasDatabaseName("UX_UnsTagReference_LogicalId") + .HasFilter("[UnsTagReferenceId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "TagId") + .IsUnique() + .HasDatabaseName("UX_UnsTagReference_Equip_Tag"); + + b.ToTable("UnsTagReference", (string)null); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.VirtualTag", b => + { + b.Property("VirtualTagRowId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasDefaultValueSql("NEWSEQUENTIALID()"); + + b.Property("ChangeTriggered") + .HasColumnType("bit"); + + b.Property("DataType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("nvarchar(32)"); + + b.Property("Enabled") + .HasColumnType("bit"); + + b.Property("EquipmentId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("Historize") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .IsRequired() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("rowversion"); + + b.Property("ScriptId") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.Property("TimerIntervalMs") + .HasColumnType("int"); + + b.Property("VirtualTagId") + .HasMaxLength(64) + .HasColumnType("nvarchar(64)"); + + b.HasKey("VirtualTagRowId"); + + b.HasIndex("ScriptId") + .HasDatabaseName("IX_VirtualTag_Script"); + + b.HasIndex("VirtualTagId") + .IsUnique() + .HasDatabaseName("UX_VirtualTag_LogicalId") + .HasFilter("[VirtualTagId] IS NOT NULL"); + + b.HasIndex("EquipmentId", "Name") + .IsUnique() + .HasDatabaseName("UX_VirtualTag_EquipmentPath"); + + b.ToTable("VirtualTag", null, t => + { + t.HasCheckConstraint("CK_VirtualTag_TimerInterval_Min", "TimerIntervalMs IS NULL OR TimerIntervalMs >= 50"); + + t.HasCheckConstraint("CK_VirtualTag_Trigger_AtLeastOne", "ChangeTriggered = 1 OR TimerIntervalMs IS NOT NULL"); + }); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany("Nodes") + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNodeCredential", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", "Node") + .WithMany("Credentials") + .HasForeignKey("NodeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Node"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.DriverInstance", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.LdapGroupRoleMapping", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.NodeDeploymentState", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.Deployment", "Deployment") + .WithMany() + .HasForeignKey("DeploymentId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", "Node") + .WithMany() + .HasForeignKey("NodeId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Deployment"); + + b.Navigation("Node"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.RawFolder", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", null) + .WithMany("RawFolders") + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.UnsArea", b => + { + b.HasOne("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", "Cluster") + .WithMany() + .HasForeignKey("ClusterId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("Cluster"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ClusterNode", b => + { + b.Navigation("Credentials"); + }); + + modelBuilder.Entity("ZB.MOM.WW.OtOpcUa.Configuration.Entities.ServerCluster", b => + { + b.Navigation("Nodes"); + + b.Navigation("RawFolders"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs new file mode 100644 index 00000000..58cf9a7e --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Migrations/20261022102459_EnableSnapshotIsolation.cs @@ -0,0 +1,36 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Migrations +{ + /// + /// Enables ALLOW_SNAPSHOT_ISOLATION on the config database so the deploy path can read every + /// table inside one SNAPSHOT transaction (DraftSnapshotFactory.FromConfigDbAsync): + /// the validator and the artifact composer then see one consistent state instead of two reads + /// an AdminUI save can land between. No schema change — the model snapshot is untouched. + /// + /// + /// ALTER DATABASE cannot run inside the user transaction the migrator wraps each migration in, + /// hence suppressTransaction: true. Turning the option ON does not need exclusive access (unlike + /// READ_COMMITTED_SNAPSHOT); it waits for in-flight transactions and then takes effect. Down + /// leaves it ON: a database that has run with it is not harmed by keeping it, and switching it OFF + /// would fail any deploy running at that instant. + /// + public partial class EnableSnapshotIsolation : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + "ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON;", + suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + // Deliberately a no-op; see the class remarks. + } + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs index 0c0c47d9..fd7dc35a 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshot.cs @@ -66,6 +66,23 @@ public sealed class DraftSnapshot /// silently dropped when the permission trie is built, so the grant has to be caught here instead. public IReadOnlyList NodeAcls { get; init; } = []; + /// + /// Cluster rows. Not read by the row-level rules; carried so the deploy path validates the cluster + /// topology () and composes the artifact from + /// the SAME read as every other table — a second read between validate and compose is exactly how + /// an edit ships unvalidated. + /// + public IReadOnlyList ServerClusters { get; init; } = []; + + /// Cluster-node rows; see for why they ride the snapshot. + public IReadOnlyList ClusterNodes { get; init; } = []; + + /// + /// Data-plane API keys (peppered HMAC only, never a secret). Not validated; carried so the artifact + /// composer ships the keys a driver-only node authenticates against from the one consistent read. + /// + public IReadOnlyList ApiKeys { get; init; } = []; + /// Prior Equipment rows (any generation, same cluster) for stability checks. public IReadOnlyList PriorEquipment { get; init; } = []; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs index 3bfd0a5c..154d664c 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs @@ -1,3 +1,4 @@ +using System.Data; using Microsoft.EntityFrameworkCore; namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation; @@ -21,6 +22,16 @@ namespace ZB.MOM.WW.OtOpcUa.Configuration.Validation; /// / are left null so the path-length rule uses its /// conservative upper bound. /// +/// +/// One consistent read. On a relational provider the tables are read inside a +/// transaction, so a /uns or /raw save that +/// lands mid-read cannot leave half the tables on the new state and half on the old. The deploy path +/// validates AND composes its artifact from this one snapshot, which is what makes "the artifact the +/// validator never saw" unrepresentable. SQL Server requires ALLOW_SNAPSHOT_ISOLATION ON +/// (migration EnableSnapshotIsolation); without it the first query fails with error 3952 and +/// the deploy is rejected rather than silently read unisolated. A non-relational provider (the +/// InMemory test provider) and a context already inside a transaction read without one. +/// /// public static class DraftSnapshotFactory { @@ -29,6 +40,23 @@ public static class DraftSnapshotFactory /// Cancellation token. /// A snapshot populated from the live DB, ready for . public static async Task FromConfigDbAsync(OtOpcUaConfigDbContext db, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + + var ownsTransaction = db.Database.IsRelational() && db.Database.CurrentTransaction is null; + var tx = ownsTransaction + ? await db.Database.BeginTransactionAsync(IsolationLevel.Snapshot, ct).ConfigureAwait(false) + : null; + await using (tx) + { + var snapshot = await ReadAsync(db, ct).ConfigureAwait(false); + // Read-only: committing releases the snapshot's version-store reservation; nothing was written. + if (tx is not null) await tx.CommitAsync(ct).ConfigureAwait(false); + return snapshot; + } + } + + private static async Task ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct) => new DraftSnapshot { // GenerationId=0 and null Enterprise/Site are intentional conservative fallbacks at the @@ -36,6 +64,8 @@ public static class DraftSnapshotFactory // EquipmentUuid checks run in separate validator passes — no rule here reads these fields. GenerationId = 0, // generation model dropped; placeholder (no rule reads it) ClusterId = string.Empty, // global snapshot; rules compare entity ClusterId fields, not this + ServerClusters = await db.ServerClusters.AsNoTracking().ToListAsync(ct), + ClusterNodes = await db.ClusterNodes.AsNoTracking().ToListAsync(ct), // v3: Namespace entity retired — no namespace list to materialize. The Raw tree // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length, // effective-leaf uniqueness). @@ -53,6 +83,7 @@ public static class DraftSnapshotFactory Scripts = await db.Scripts.AsNoTracking().ToListAsync(ct), PollGroups = await db.PollGroups.AsNoTracking().ToListAsync(ct), NodeAcls = await db.NodeAcls.AsNoTracking().ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().ToListAsync(ct), PriorEquipment = [], // intentional: no prior-generation table to diff against at this level ActiveReservations = await db.ExternalIdReservations .AsNoTracking() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index d569fd08..4aabe3db 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -223,18 +223,14 @@ public sealed class AdminOperationsActor : ReceiveActor // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at // Hot/Warm and the constraint stays green, which would boot the runtime into an - // InvalidTopology band. ValidateClusterTopology catches that drift, but it isn't carried on - // the generation-versioned DraftSnapshot (the cluster/node rows aren't versioned), so it must - // be run separately here against the live rows. Read-only (AsNoTracking); errors fold into the - // same reject summary alongside the snapshot rules so a deploy failing either check is - // rejected with both sets of messages. ClusterId-ordered for a deterministic summary. - var clusters = await db.ServerClusters - .AsNoTracking() - .OrderBy(c => c.ClusterId) - .ToListAsync(); - var nodesByCluster = (await db.ClusterNodes - .AsNoTracking() - .ToListAsync()) + // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of + // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. + // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing + // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. + // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT + // transaction), not a second query that could observe a later edit. + var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); + var nodesByCluster = draft.ClusterNodes .GroupBy(n => n.ClusterId, StringComparer.Ordinal) .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); foreach (var cluster in clusters) @@ -310,8 +306,9 @@ public sealed class AdminOperationsActor : ReceiveActor try { const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) - var scriptSources = await db.Scripts.AsNoTracking().Select(s => s.SourceCode).ToListAsync(); - var compiled = scriptSources + // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. + var compiled = draft.Scripts + .Select(s => s.SourceCode) .Where(src => !PassthroughScript.TryMatch(src, out _)) .Distinct(StringComparer.Ordinal) .Count(); @@ -331,7 +328,9 @@ public sealed class AdminOperationsActor : ReceiveActor advisory = null; } - var artifact = await ConfigComposer.SnapshotAndFlattenAsync(db); + // Composed from the validated snapshot, never from a second read: an edit that lands between + // validation and here is simply not in this artifact (R03:3.2). + var artifact = ConfigComposer.Compose(draft); var deploymentId = DeploymentId.NewId(); var revHash = RevisionHash.Parse(artifact.RevisionHash); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs index d73dd7b3..6f332143 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs @@ -1,9 +1,8 @@ using System.IO.Compression; using System.Security.Cryptography; using System.Text.Json; -using Microsoft.EntityFrameworkCore; using ZB.MOM.WW.OtOpcUa.Configuration; -using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; @@ -25,45 +24,65 @@ public static class ConfigComposer }; /// Reads the current configuration and returns a deterministic snapshot blob with revision hash. + /// + /// One consistent read (, a SNAPSHOT transaction on + /// SQL Server) handed to . The deploy path does not call this: it validates and + /// composes from the one it already holds, so nothing can land between the + /// validator's read and the composer's. This overload serves the drift probe and tests. + /// /// The configuration database context. /// The cancellation token for the operation. /// A deterministic containing the serialised snapshot bytes and its revision hash. public static async Task SnapshotAndFlattenAsync( OtOpcUaConfigDbContext db, CancellationToken ct = default) + => Compose(await DraftSnapshotFactory.FromConfigDbAsync(db, ct).ConfigureAwait(false)); + + /// + /// Serialises an already-read into the deterministic artifact. Pure over + /// its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now. + /// + /// The rows to compose — the SAME rows the validator was run over. + /// The compressed artifact blob and its revision hash. + public static ConfigArtifact Compose(DraftSnapshot snapshot) { + ArgumentNullException.ThrowIfNull(snapshot); + // v3 greenfield snapshot: the Raw tree (RawFolders → DriverInstances(+RawFolderId) → Devices(+DeviceConfig) // → TagGroups → Tags(DeviceId/TagGroupId, no EquipmentId/FolderPath/DriverInstanceId)) plus the UNS // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/ // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit). // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256. - var snapshot = new + // Every collection is sorted by its natural key with an ORDINAL comparer so the hash is independent of + // both EF row order and the process culture (a culture-aware sort could order "a-1"/"A-1" differently + // on two central nodes and make the same config hash twice). + var ordered = new { - Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), - Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), - RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), - DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), - Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), - TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), - Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), - Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), - UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), - PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), - UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), - UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), - NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + Clusters = snapshot.ServerClusters.OrderBy(x => x.ClusterId, StringComparer.Ordinal).ToList(), + Nodes = snapshot.ClusterNodes.OrderBy(x => x.NodeId, StringComparer.Ordinal).ToList(), + RawFolders = snapshot.RawFolders.OrderBy(x => x.RawFolderId, StringComparer.Ordinal).ToList(), + DriverInstances = snapshot.DriverInstances.OrderBy(x => x.DriverInstanceId, StringComparer.Ordinal).ToList(), + Devices = snapshot.Devices.OrderBy(x => x.DeviceId, StringComparer.Ordinal).ToList(), + TagGroups = snapshot.TagGroups.OrderBy(x => x.TagGroupId, StringComparer.Ordinal).ToList(), + Equipment = snapshot.Equipment.OrderBy(x => x.EquipmentId, StringComparer.Ordinal).ToList(), + Tags = snapshot.Tags.OrderBy(x => x.TagId, StringComparer.Ordinal).ToList(), + UnsTagReferences = snapshot.UnsTagReferences.OrderBy(x => x.UnsTagReferenceId, StringComparer.Ordinal).ToList(), + PollGroups = snapshot.PollGroups.OrderBy(x => x.PollGroupId, StringComparer.Ordinal).ToList(), + UnsAreas = snapshot.UnsAreas.OrderBy(x => x.UnsAreaId, StringComparer.Ordinal).ToList(), + UnsLines = snapshot.UnsLines.OrderBy(x => x.UnsLineId, StringComparer.Ordinal).ToList(), + NodeAcls = snapshot.NodeAcls.OrderBy(x => x.NodeAclId, StringComparer.Ordinal).ToList(), // the data-plane credentials travel with the config, because a driver-only node // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob // still cannot be turned into a working credential. This is also why revoking a key reaches a // driver-only node only on the next deployment; see the ApiKey entity's remarks. - ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), - Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), - VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), - ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + ApiKeys = snapshot.ApiKeys.OrderBy(x => x.ApiKeyId, StringComparer.Ordinal).ToList(), + Scripts = snapshot.Scripts.OrderBy(x => x.ScriptId, StringComparer.Ordinal).ToList(), + VirtualTags = snapshot.VirtualTags.OrderBy(x => x.VirtualTagId, StringComparer.Ordinal).ToList(), + ScriptedAlarms = snapshot.ScriptedAlarms.OrderBy(x => x.ScriptedAlarmId, StringComparer.Ordinal).ToList(), }; - return Pack(JsonSerializer.SerializeToUtf8Bytes(snapshot, JsonOptions)); + return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions)); } /// diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs new file mode 100644 index 00000000..aa16ead6 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactorySqlServerTests.cs @@ -0,0 +1,117 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; + +namespace ZB.MOM.WW.OtOpcUa.Configuration.Tests; + +/// +/// R03:3.2 on the real provider: the migrated config database allows SNAPSHOT isolation (migration +/// EnableSnapshotIsolation) and reads under +/// it. The InMemory tests cannot see either — the provider has no transactions — so this is the one +/// place the deploy path's "one consistent read" is proven against SQL Server. +/// +[Trait("Category", "SchemaCompliance")] +[Collection(nameof(SchemaComplianceCollection))] +public sealed class DraftSnapshotFactorySqlServerTests +{ + private readonly SchemaComplianceFixture _fixture; + + /// Initializes a new instance of the class. + /// The migrated test database. + public DraftSnapshotFactorySqlServerTests(SchemaComplianceFixture fixture) => _fixture = fixture; + + /// The migration leaves the database with snapshot isolation allowed. + [Fact] + public void Migrated_database_allows_snapshot_isolation() + { + using var conn = _fixture.OpenConnection(); + using var cmd = new SqlCommand( + "SELECT snapshot_isolation_state_desc FROM sys.databases WHERE name = DB_NAME();", conn); + cmd.ExecuteScalar().ShouldBe("ON"); + } + + /// + /// A row committed by another connection while the snapshot read is in progress is NOT observed by + /// the tables read after it — every table reflects the instant the transaction started. Positive + /// control: a read started after the commit does see the row. + /// + [Fact] + public async Task FromConfigDb_reads_every_table_as_of_one_instant() + { + var clusterId = $"SNAP-{Guid.NewGuid():N}"[..24]; + var scriptId = $"late-{Guid.NewGuid():N}"; + await using (var seed = NewContext()) + { + seed.ServerClusters.Add(new ServerCluster + { + ClusterId = clusterId, Name = "snapshot-read", Enterprise = "zb", Site = "test", + NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + await seed.SaveChangesAsync(); + } + + try + { + var options = new DbContextOptionsBuilder() + .UseSqlServer(_fixture.ConnectionString) + .AddInterceptors(new LateEditInterceptor(_fixture.ConnectionString, clusterId, scriptId)) + .Options; + + await using (var db = new OtOpcUaConfigDbContext(options)) + { + var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(db); + // The interceptor committed a Script while ServerClusters (the FIRST table read) was being + // materialised. Scripts is read later in the same transaction and must not see it. + snapshot.ServerClusters.ShouldContain(c => c.ClusterId == clusterId); + snapshot.Scripts.ShouldNotContain(s => s.ScriptId == scriptId); + } + + await using (var verify = NewContext()) + { + // Positive control: the edit really landed, and a read started afterwards sees it. + (await verify.Scripts.AnyAsync(s => s.ScriptId == scriptId)).ShouldBeTrue(); + var later = await DraftSnapshotFactory.FromConfigDbAsync(verify); + later.Scripts.ShouldContain(s => s.ScriptId == scriptId); + } + } + finally + { + await using var cleanup = NewContext(); + await cleanup.Scripts.Where(s => s.ScriptId == scriptId).ExecuteDeleteAsync(); + await cleanup.ServerClusters.Where(c => c.ClusterId == clusterId).ExecuteDeleteAsync(); + } + } + + private OtOpcUaConfigDbContext NewContext() => new( + new DbContextOptionsBuilder().UseSqlServer(_fixture.ConnectionString).Options); + + /// + /// Commits a Script on a SEPARATE connection the first time the snapshot read materialises the + /// seeded ServerCluster row — i.e. mid-read, after the first table and before Scripts. + /// + private sealed class LateEditInterceptor(string connectionString, string clusterId, string scriptId) + : IMaterializationInterceptor + { + private int _fired; + + public object InitializedInstance(MaterializationInterceptionData materializationData, object entity) + { + if (entity is ServerCluster c && c.ClusterId == clusterId && Interlocked.Exchange(ref _fired, 1) == 0) + { + using var other = new OtOpcUaConfigDbContext( + new DbContextOptionsBuilder().UseSqlServer(connectionString).Options); + other.Scripts.Add(new Script + { + ScriptId = scriptId, Name = "late edit", SourceCode = "return 1;", SourceHash = "late-hash", + }); + other.SaveChanges(); + } + return entity; + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs index 9ad5806a..df1fad5a 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftSnapshotFactoryTests.cs @@ -103,6 +103,36 @@ public sealed class DraftSnapshotFactoryTests : IDisposable snapshot.ActiveReservations[0].ReleasedAt.ShouldBeNull(); } + /// + /// R03:3.2 — the deploy path validates AND composes from ONE snapshot, so the tables the composer + /// needs beyond the validator's set (clusters, nodes, API keys) must ride the same read. + /// + [Fact] + public async Task FromConfigDb_carries_clusters_nodes_and_api_keys_for_the_composer() + { + _db.ServerClusters.Add(new ServerCluster + { + ClusterId = "SITE-A", Name = "Site A", Enterprise = "zb", Site = "site-a", + NodeCount = 1, RedundancyMode = RedundancyMode.None, CreatedBy = "test", + }); + _db.ClusterNodes.Add(new ClusterNode + { + NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1", + ApplicationUri = "urn:OtOpcUa:site-a-1", Enabled = true, CreatedBy = "test", + }); + _db.ApiKeys.Add(new ApiKey + { + ApiKeyId = "k1", Name = "scada", SecretHash = "hash", Principal = "scadabridge", Enabled = true, + }); + await _db.SaveChangesAsync(); + + var snapshot = await DraftSnapshotFactory.FromConfigDbAsync(_db); + + snapshot.ServerClusters.ShouldHaveSingleItem().ClusterId.ShouldBe("SITE-A"); + snapshot.ClusterNodes.ShouldHaveSingleItem().NodeId.ShouldBe("site-a-1:4053"); + snapshot.ApiKeys.ShouldHaveSingleItem().ApiKeyId.ShouldBe("k1"); + } + private void SeedEquipment(string equipmentId) { var uuid = Guid.NewGuid(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs new file mode 100644 index 00000000..aa5ded29 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorSingleSnapshotTests.cs @@ -0,0 +1,133 @@ +using System.IO.Compression; +using System.Text; +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Storage; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:3.2 — StartDeployment used to read the whole config DB twice: once for the validator, +/// once for the composer. An edit landing between the two shipped an artifact the validator never saw. +/// The deploy now validates and composes ONE snapshot, so an edit that lands mid-deploy is either +/// validated (it was in the snapshot) or absent from the artifact (it was not) — never shipped unseen. +/// +public sealed class AdminOperationsActorSingleSnapshotTests : ControlPlaneActorTestBase +{ + private const string LateEquipmentId = "EQ-late-and-NOT-canonical"; + + /// + /// An Equipment with a non-canonical id (the validator rejects those with EquipmentIdNotDerived) + /// is committed by another context while the snapshot read is in progress, AFTER the Equipment table + /// has been read. The validator cannot see it, so the deploy is Accepted — and the artifact must not + /// carry it either. Positive control: the row really landed in the DB, and a deploy started after it + /// is Rejected on exactly that rule. + /// + [Fact] + public void Edit_landing_between_validate_and_compose_cannot_ship_unvalidated() + { + // One shared root: an InMemory store is scoped to the internal service provider, and a context + // carrying an interceptor builds its own provider — without the shared root the intercepting + // context would read an EMPTY database and the whole test would be vacuous. + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var plain = new RootedDbFactory(dbName, root, interceptor: null); + using (var db = plain.CreateDbContext()) + { + // A Tag to materialise — Tags is read AFTER Equipment by the snapshot factory, so an insert + // fired on the first Tag lands "between" the two tables of one read. + db.DriverInstances.Add(new DriverInstance + { + DriverInstanceId = "d", ClusterId = "", Name = "drv", DriverType = "Modbus", DriverConfig = "{}", + }); + db.Devices.Add(new Device { DeviceId = "dev", DriverInstanceId = "d", Name = "dev", DeviceConfig = "{}" }); + db.Tags.Add(new Tag + { + TagId = "tag-speed", DeviceId = "dev", Name = "speed", DataType = "Float", + AccessLevel = TagAccessLevel.Read, TagConfig = "{}", + }); + db.SaveChanges(); + } + + var injecting = new RootedDbFactory(dbName, root, new LateEquipmentInterceptor(plain)); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(injecting, coordinator.Ref, Enumerable.Empty())); + + actor.Tell(new StartDeployment("joe", CorrelationId.NewId())); + + var reply = ExpectMsg(PresenceBudget); + reply.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, reply.Message); + coordinator.ExpectMsg(PresenceBudget); + + using (var verify = plain.CreateDbContext()) + { + // The injection happened... + verify.Equipment.Any(e => e.EquipmentId == LateEquipmentId).ShouldBeTrue(); + // ...and the shipped artifact does not contain what the validator never saw. + var row = verify.Deployments.Single(); + Inflate(row.ArtifactBlob).ShouldNotContain(LateEquipmentId); + // The coordinator is a probe here, so seal the row by hand to let the next deploy through. + row.Status = DeploymentStatus.Sealed; + verify.SaveChanges(); + } + + // Positive control: now that the row is visible to a fresh read, the gate rejects it. + actor.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var second = ExpectMsg(PresenceBudget); + second.Outcome.ShouldBe(StartDeploymentOutcome.Rejected); + second.Message.ShouldNotBeNull(); + second.Message.ShouldContain("EquipmentIdNotDerived"); + } + + private static string Inflate(byte[] blob) + { + using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + return reader.ReadToEnd(); + } + + /// On the first Tag materialised, commits a non-canonical Equipment through ANOTHER context. + private sealed class LateEquipmentInterceptor(IDbContextFactory other) + : IMaterializationInterceptor + { + private int _fired; + + public object InitializedInstance(MaterializationInterceptionData materializationData, object entity) + { + if (entity is Tag && Interlocked.Exchange(ref _fired, 1) == 0) + { + using var db = other.CreateDbContext(); + db.Equipment.Add(new Equipment + { + EquipmentUuid = Guid.NewGuid(), EquipmentId = LateEquipmentId, Name = "late", + UnsLineId = "line-a", MachineCode = "m", + }); + db.SaveChanges(); + } + return entity; + } + } + + private sealed class RootedDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor? interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() + { + var builder = new DbContextOptionsBuilder().UseInMemoryDatabase(dbName, root); + if (interceptor is not null) builder.AddInterceptors(interceptor); + return new OtOpcUaConfigDbContext(builder.Options); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs index 4e431151..674ececd 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerTests.cs @@ -222,6 +222,47 @@ public sealed class ConfigComposerTests : ControlPlaneActorTestBase withoutOverride.ResilienceConfig.ShouldBeNull("an unset override stays null ⇒ tier defaults at spawn"); } + /// + /// R03:3.2 — the deploy path composes from the it + /// validated rather than re-reading the DB. over the factory's + /// snapshot must therefore be byte-identical to the DB overload, or the drift probe (which still + /// uses the DB overload) would report every deployment as drifted. + /// + [Fact] + public async Task Compose_over_the_factory_snapshot_is_byte_identical_to_the_db_overload() + { + var f = NewInMemoryDbFactory(); + await using (var db = f.CreateDbContext()) + { + db.ServerClusters.Add(NewCluster("cluster-b")); + db.ServerClusters.Add(NewCluster("cluster-a")); + db.ApiKeys.Add(new ApiKey + { + ApiKeyId = "k1", Name = "scada", SecretHash = "h", Principal = "scadabridge", CreatedUtc = FixedTimestamp, + }); + db.Scripts.Add(new Script { ScriptId = "s-1", Name = "s", SourceCode = "return 1;", SourceHash = "x" }); + await db.SaveChangesAsync(); + } + + var viaDb = await ConfigComposer.SnapshotAndFlattenAsync(f.CreateDbContext()); + var snapshot = await Configuration.Validation.DraftSnapshotFactory.FromConfigDbAsync(f.CreateDbContext()); + var viaSnapshot = ConfigComposer.Compose(snapshot); + + viaSnapshot.RevisionHash.ShouldBe(viaDb.RevisionHash); + viaSnapshot.Blob.ShouldBe(viaDb.Blob); + // The composed content is the snapshot's, not something re-read: the keys and clusters are in it. + var json = Inflate(viaSnapshot.Blob); + json.ShouldContain("\"ApiKeyId\":\"k1\""); + json.IndexOf("cluster-a", StringComparison.Ordinal).ShouldBeLessThan(json.IndexOf("cluster-b", StringComparison.Ordinal)); + } + + private static string Inflate(byte[] blob) + { + using var gz = new GZipStream(new MemoryStream(blob), CompressionMode.Decompress); + using var reader = new StreamReader(gz, Encoding.UTF8); + return reader.ReadToEnd(); + } + private static readonly DateTime FixedTimestamp = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); private static ServerCluster NewCluster(string id) => new() From eb9269c337d5ce074deb075b83bec1b54351b438 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:39:43 -0400 Subject: [PATCH 03/51] feat(scripting): ScriptRunGuard quarantines runaway scripts across every seam (E1) A timed-out script is abandoned, not stopped, and every seam re-ran it on the next trigger, leaking one pool thread per trigger (R03:1.1, R05:2, R07:L1). A shared ScriptRunGuard, keyed per script instance, now: - refuses a run while that script's previous orphan is still executing (Bad "previous evaluation still running" + otopcua.script.run_refused); - quarantines after 3 consecutive non-completions or an allocation-budget breach (sampled with GC.GetAllocatedBytesForCurrentThread; default 64 MB): no further dispatch, Bad quality, one Error, otopcua.script.quarantined, cleared on source change / redeploy / reset; - starts the budget clock when the pool picks the run up, so starvation is no longer charged to the script (R07:P5), and cancels the evaluator token on timeout. Wired into RoslynVirtualTagEvaluator, CalculationEvaluator (new async EvaluateAsync; the driver's pump and timer ticks now await the run behind a SemaphoreSlim gate) and ScriptedAlarmEngine (a non-completing predicate holds state and annotates the condition BadInternalError). ScriptingOptions (Scripting:MaxAllocBytesPerRun, Scripting:QuarantineAfterConsecutiveTimeouts) carries the knobs; the Calculation driver config accepts the same two keys. Tests: guard unit suite (incl. the 30-trigger runaway leaving one orphan with a healthy sibling as positive control), per-seam refusal/quarantine/reset tests. The S12 race test now uses one vtag id per loop: twelve concurrent evaluations of ONE id is a shape production never produces and the guard refuses by design. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ScriptedAlarmEngine.cs | 82 +++- .../IScriptQuarantineOwner.cs | 15 + .../ScriptRunGuard.cs | 387 ++++++++++++++++++ .../ScriptingOptions.cs | 42 ++ .../TimedScriptEvaluator.cs | 10 +- .../CalculationDriver.cs | 83 ++-- .../CalculationDriverFactoryExtensions.cs | 17 + .../CalculationDriverOptions.cs | 21 +- .../CalculationEvaluator.cs | 75 +++- .../CalculationTimerScheduler.cs | 20 +- .../Engines/RoslynVirtualTagEvaluator.cs | 56 ++- .../VirtualTags/VirtualTagHostActor.cs | 6 + .../ScriptedAlarmEngineRunGuardTests.cs | 105 +++++ .../ScriptRunGuardTests.cs | 306 ++++++++++++++ .../CalculationEvaluatorRunGuardTests.cs | 86 ++++ .../RoslynVirtualTagEvaluatorRunGuardTests.cs | 66 +++ .../RoslynVirtualTagEvaluatorTests.cs | 10 +- 17 files changed, 1313 insertions(+), 74 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index 4ccc3592..808935ae 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -73,6 +73,14 @@ public sealed class ScriptedAlarmEngine : IDisposable private readonly Func _clock; private readonly TimeSpan _scriptTimeout; + /// + /// Per-alarm run quarantine (keyed by alarm id, source identity = predicate source). Refuses a + /// predicate run while the alarm's previous (timed-out, abandoned) run is still executing and + /// quarantines after three consecutive non-completions or an allocation-budget breach — one orphan + /// per alarm instead of one per upstream push. Reset on every (a redeploy). + /// + private readonly ScriptRunGuard _guard; + // ConcurrentDictionary, not a plain Dictionary: every mutation happens under // _evalGate, but four read paths (GetState, GetAllStates, LoadedAlarmIds, // RunShelvingCheck) touch _alarms from arbitrary threads (Admin UI request @@ -204,13 +212,15 @@ public sealed class ScriptedAlarmEngine : IDisposable /// The logger for engine-level diagnostics. /// Optional function providing the current UTC time; defaults to DateTime.UtcNow. /// Optional timeout for script execution; defaults to the evaluator's default timeout. + /// Run-guard budget + quarantine threshold; null ⇒ defaults. public ScriptedAlarmEngine( ITagUpstreamSource upstream, IAlarmStateStore store, ScriptLoggerFactory loggerFactory, ILogger engineLogger, Func? clock = null, - TimeSpan? scriptTimeout = null) + TimeSpan? scriptTimeout = null, + ScriptingOptions? scripting = null) { _upstream = upstream ?? throw new ArgumentNullException(nameof(upstream)); _store = store ?? throw new ArgumentNullException(nameof(store)); @@ -218,8 +228,20 @@ public sealed class ScriptedAlarmEngine : IDisposable _engineLogger = engineLogger ?? throw new ArgumentNullException(nameof(engineLogger)); _clock = clock ?? (() => DateTime.UtcNow); _scriptTimeout = scriptTimeout ?? TimedScriptEvaluator.DefaultTimeout; + _guard = new ScriptRunGuard(scripting, _engineLogger); } + /// The run guard every predicate evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + + /// + /// OPC UA BadInternalError — the quality a condition is annotated with while its predicate + /// cannot be evaluated (timed out, refused because the previous run is still executing, or + /// quarantined). The condition's state is frozen; the quality says why it cannot be trusted. The + /// next completed evaluation restores the input-derived quality through the normal bucket tracking. + /// + private const uint StatusBadInternalError = 0x80020000u; + /// Raised for every emission the Part9StateMachine produces that the engine should publish. public event EventHandler? OnEvent; @@ -268,6 +290,9 @@ public sealed class ScriptedAlarmEngine : IDisposable // recompile this one. Skipping this is what made the earlier fix a // no-op in production. _compileCache.Clear(); + // A (re)load is a deploy: lift every run-guard quarantine (an edited predicate already lifts + // its own through the guard's source identity; this covers "redeployed to retry"). + _guard.ResetAll(); var compileFailures = new List(); foreach (var def in definitions) @@ -284,15 +309,17 @@ public sealed class ScriptedAlarmEngine : IDisposable // Route through CompiledScriptCache so the emitted assembly's // collectible ALC participates in publish-replace cleanup. + // The compiled script is run through _guard (timeout + quarantine) at evaluation time — + // the per-alarm TimedScriptEvaluator this used to wrap gave the timeout but could not + // see that the run it abandoned was still executing. var evaluator = _compileCache.GetOrCompile(def.PredicateScriptSource); - var timed = new TimedScriptEvaluator(evaluator, _scriptTimeout); var logger = _loggerFactory.Create(def.AlarmId); var templateTokens = MessageTemplate.ExtractTokenPaths(def.MessageTemplate); var allInputs = new HashSet(extraction.Reads, StringComparer.Ordinal); foreach (var t in templateTokens) allInputs.Add(t); - _alarms[def.AlarmId] = new AlarmState(def, timed, extraction.Reads, templateTokens, logger, + _alarms[def.AlarmId] = new AlarmState(def, evaluator, extraction.Reads, templateTokens, logger, AlarmConditionState.Fresh(def.AlarmId, _clock())); foreach (var path in allInputs) @@ -767,22 +794,35 @@ public sealed class ScriptedAlarmEngine : IDisposable bool predicateTrue; try { - predicateTrue = await state.Evaluator.RunAsync(context, ct).ConfigureAwait(false); + predicateTrue = await _guard.RunAsync( + state.Definition.AlarmId, state.Definition.PredicateScriptSource, state.Evaluator, context, + _scriptTimeout, ct) + .ConfigureAwait(false); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } catch (ScriptTimeoutException tex) { - state.Logger.Warning("Alarm predicate timed out after {Timeout} — state unchanged", tex.Timeout); - return seed; + return PredicateDidNotComplete(state, seed, scratch, + $"timed out after {tex.Timeout.TotalMilliseconds:F0} ms", pendingEmissions); + } + catch (ScriptRunRefusedException ex) + { + return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions); + } + catch (ScriptQuarantinedException ex) + { + return PredicateDidNotComplete(state, seed, scratch, ex.Message, pendingEmissions); } catch (Exception ex) { + scratch.LastNonCompletion = null; state.Logger.Error(ex, "Alarm predicate threw — state unchanged"); return seed; } + scratch.LastNonCompletion = null; var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc); var transition = result.Emission != EmissionKind.None @@ -802,6 +842,28 @@ public sealed class ScriptedAlarmEngine : IDisposable return result.State; } + /// + /// The predicate did not return (timed out, refused because the previous run is still executing, or + /// quarantined): hold the condition at , log the reason once per change of + /// reason (a quarantined alarm is re-attempted on every push and would otherwise log every time), + /// and annotate the condition out of band — the frozen state + /// cannot be trusted and the quality is how the operator sees that. The next completed evaluation + /// restores the input-derived quality through the normal bucket tracking. + /// + private AlarmConditionState PredicateDidNotComplete( + AlarmState state, AlarmConditionState seed, AlarmScratch scratch, string reason, + List pendingEmissions) + { + if (!string.Equals(scratch.LastNonCompletion, reason, StringComparison.Ordinal)) + { + scratch.LastNonCompletion = reason; + state.Logger.Warning("Alarm predicate did not complete ({Reason}) — state unchanged, quality Bad", reason); + } + if (TrackQualityBucket(state.Definition.AlarmId, StatusBadInternalError)) + pendingEmissions.Add(BuildQualityEmission(state, seed, StatusBadInternalError)); + return seed; + } + /// /// Refill in place from _valueCache, falling /// back to a synchronous ITagUpstreamSource.ReadTag for paths whose @@ -1201,7 +1263,7 @@ public sealed class ScriptedAlarmEngine : IDisposable private sealed record AlarmState( ScriptedAlarmDefinition Definition, - TimedScriptEvaluator Evaluator, + ScriptEvaluator Evaluator, IReadOnlySet Inputs, IReadOnlyList TemplateTokens, ILogger Logger, @@ -1227,6 +1289,10 @@ public sealed class ScriptedAlarmEngine : IDisposable /// Gets the predicate evaluation context. public AlarmPredicateContext Context { get; } + /// The reason the last evaluation did not complete (timeout / refused / quarantined), or null + /// after a completed run — the once-per-change log gate for PredicateDidNotComplete. + public string? LastNonCompletion { get; set; } + /// /// Initializes a new AlarmScratch with the specified inputs, logger, and clock. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs new file mode 100644 index 00000000..3cf38fac --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/IScriptQuarantineOwner.cs @@ -0,0 +1,15 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Implemented by a script seam that runs its scripts through a , so the +/// host that drives its deploy boundary can lift every quarantine on a redeploy (the guard's own +/// source-change detection covers an edited script; this covers "the operator redeployed to retry"). +/// +public interface IScriptQuarantineOwner +{ + /// Lift every quarantine and forget every consecutive-timeout count (). + void ResetQuarantines(); + + /// The script keys currently quarantined. + IReadOnlyCollection QuarantinedScripts { get; } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs new file mode 100644 index 00000000..8f1acefd --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs @@ -0,0 +1,387 @@ +using System.Collections.Concurrent; +using System.Diagnostics.Metrics; +using Serilog; +using ZB.MOM.WW.OtOpcUa.Commons.Observability; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Per-script run quarantine shared by every script seam on the node. Wraps each evaluation of a +/// with the wall-clock timeout +/// used to provide, and adds what that wrapper +/// could not: knowledge of whether the script it abandoned is still running. +/// +/// +/// +/// The leak this closes. A timed-out script is abandoned, not stopped — a CPU-bound +/// while(true) keeps its thread-pool thread forever. Each seam re-evaluated on every trigger, +/// so a runaway script on a 1 Hz input leaked one pool thread per second until the pool starved and +/// unrelated work (Akka dispatchers, the OPC UA publish thread, gRPC) stalled. The guard keys every +/// run by script instance and: +/// +/// +/// Refuses to start a run while that script's previous run is still executing — +/// (), counted on +/// . One orphan per script, never one per trigger. +/// Quarantines after +/// consecutive non-completions (timeouts and still-running refusals both count; a run that +/// returns — a value or a throw — resets the count): no further dispatch, every attempt throws +/// , one Error log, one otopcua.script.quarantined +/// increment, one event. Lifted by a source change (the operator fixed the +/// script — a new source identity under the same key starts clean and the old orphan no longer blocks +/// it), by on redeploy, or by . +/// Quarantines on an allocation-budget breach: the run's own-thread allocation is sampled +/// with around the script delegate and compared +/// with . This is observation after the fact — a +/// single run that allocates enough to OOM the process is not stopped; that needs an out-of-process +/// runner. +/// +/// +/// The timeout clock starts when the script starts, not when it is queued. The run is pushed to +/// the pool (a CPU-bound script would otherwise execute inline and return an already-completed task +/// that no timeout can fire on), and the wait only begins once the pool has actually picked the run +/// up — so thread-pool starvation is charged to nobody's script. A caller-supplied cancellation wins +/// over the timeout (, never +/// ) so shutdown paths are not misread as timeouts. +/// +/// +/// Cancellation reaches the evaluator, not (yet) the script. On timeout the token handed to +/// is cancelled; a script can only observe +/// it once ScriptContext exposes it (follow-up — the context type lives in the Abstractions +/// project). +/// +/// +public sealed class ScriptRunGuard +{ + /// Failure reason a seam publishes when a run was refused because the previous one is still executing. + public const string PreviousRunStillRunningReason = "previous evaluation still running"; + + /// Prefix of every quarantine reason / message. + public const string QuarantinedReasonPrefix = "script quarantined: "; + + /// + /// Counter otopcua.script.quarantined (Prometheus: otopcua_script_quarantined): one + /// increment per script quarantined, tagged reason = timeouts | allocation. + /// + public static readonly Counter QuarantinedCounter = + OtOpcUaTelemetry.Meter.CreateCounter("otopcua.script.quarantined", unit: "{script}", + description: "Scripts quarantined by the run guard (reason: timeouts | allocation)."); + + /// + /// Counter otopcua.script.run_refused: runs refused because the script's previous run was + /// still executing. + /// + public static readonly Counter RefusedCounter = + OtOpcUaTelemetry.Meter.CreateCounter("otopcua.script.run_refused", unit: "{run}", + description: "Script runs refused because the previous run of the same script was still executing."); + + private readonly ConcurrentDictionary _keys = new(StringComparer.Ordinal); + private readonly long _maxAllocBytesPerRun; + private readonly int _quarantineThreshold; + private readonly ILogger _logger; + private int _inFlight; + private long _started; + private long _refused; + private long _quarantined; + + /// Initializes a new guard. + /// Budget + threshold; null ⇒ defaults. + /// Where the quarantine Error line goes; null ⇒ Serilog's global logger. + public ScriptRunGuard(ScriptingOptions? options = null, ILogger? logger = null) + { + options ??= new ScriptingOptions(); + _maxAllocBytesPerRun = options.MaxAllocBytesPerRun; + _quarantineThreshold = Math.Max(1, options.QuarantineAfterConsecutiveTimeouts); + _logger = logger ?? Log.Logger; + } + + /// Raised (on the thread that detected it) when a script is quarantined. + public event EventHandler? Quarantined; + + /// Runs currently executing on a pool thread — abandoned (timed-out) orphans included. + public int InFlightCount => Volatile.Read(ref _inFlight); + + /// Runs actually dispatched to the pool, lifetime. + public long StartedRunCount => Interlocked.Read(ref _started); + + /// Runs refused because the same script's previous run was still executing, lifetime. + public long RefusedWhileRunningCount => Interlocked.Read(ref _refused); + + /// Scripts quarantined, lifetime (a reset and re-quarantine counts twice). + public long QuarantinedCount => Interlocked.Read(ref _quarantined); + + /// The keys currently quarantined. + public IReadOnlyCollection QuarantinedKeys => + _keys.Where(kv => kv.Value.QuarantineReason is not null).Select(kv => kv.Key).ToArray(); + + /// Whether is quarantined, and why. + /// The script instance key. + /// The quarantine reason (prefixed ), or null. + /// true when quarantined. + public bool IsQuarantined(string key, out string? reason) + { + reason = _keys.TryGetValue(key, out var state) ? state.QuarantineReason : null; + return reason is not null; + } + + /// + /// Forget everything about : clears its quarantine and its consecutive count, and + /// abandons its in-flight slot (an orphan still running keeps counting on + /// until it finishes, but no longer blocks the key). + /// + /// The script instance key. + public void Reset(string key) => _keys.TryRemove(key, out _); + + /// + /// for every key — the redeploy / compile-cache-clear boundary. Orphans keep + /// counting on until they finish. + /// + public void ResetAll() => _keys.Clear(); + + /// + /// Run against under the guard. + /// + /// The script context type. + /// The script result type. + /// The script instance key (virtual-tag id, calc-tag RawPath, alarm id). + /// + /// The script source (or its hash). A change under the same key clears the key's quarantine and + /// abandons its orphan slot — the operator fixed the script. Null ⇒ not tracked. + /// + /// The compiled script. + /// The script's ctx. + /// Wall-clock budget, measured from the moment the pool starts the run. + /// Caller cancellation; wins over the timeout. + /// The script's result. + /// The key is (or just became) quarantined. + /// The key's previous run is still executing. + /// The run exceeded . + /// was cancelled. + public async Task RunAsync( + string key, + string? sourceIdentity, + ScriptEvaluator evaluator, + TContext context, + TimeSpan timeout, + CancellationToken ct = default) + where TContext : ScriptContext + { + ArgumentNullException.ThrowIfNull(key); + ArgumentNullException.ThrowIfNull(evaluator); + ArgumentNullException.ThrowIfNull(context); + if (timeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(timeout), "Timeout must be positive."); + + var state = Admit(key, sourceIdentity); + + // Slot acquisition is the whole mechanism: a second trigger landing while the previous run (orphan or + // not) still holds the slot is refused here, before anything is dispatched to the pool. + if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0) + { + Interlocked.Increment(ref _refused); + RefusedCounter.Add(1, new KeyValuePair("script", key)); + if (NoteNonCompletion(key, state, "refused: previous evaluation still running")) + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + throw new ScriptRunRefusedException(key); + } + + Interlocked.Increment(ref _inFlight); + Interlocked.Increment(ref _started); + + var slot = new RunSlot(); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var runCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + var runToken = runCts.Token; + + // Task.Run deliberately takes NO token: a pre-cancelled token would skip the lambda, `started` would + // never complete, and the slot would never be released. The evaluator observes the token instead. + var runTask = Task.Run(async () => + { + started.TrySetResult(); + var before = GC.GetAllocatedBytesForCurrentThread(); + try + { + // ScriptEvaluator.RunAsync executes the compiled delegate synchronously and returns a completed + // task, so the after-sample is taken on the same thread, right after the script returned. + var pending = evaluator.RunAsync(context, runToken); + slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before; + return await pending.ConfigureAwait(false); + } + catch + { + slot.AllocatedBytes = GC.GetAllocatedBytesForCurrentThread() - before; + throw; + } + }); + + // Release the slot when the run REALLY finishes — which for an abandoned run is long after this + // method has thrown ScriptTimeoutException. Registered before the wait so it fires regardless of + // which way the wait below comes out. A late-finishing orphan that breached the allocation budget + // still quarantines its key here. + _ = runTask.ContinueWith(t => + { + Interlocked.Exchange(ref state.InFlight, 0); + Interlocked.Decrement(ref _inFlight); + runCts.Dispose(); + if (t.IsFaulted) _ = t.Exception; // observed below by the awaiter when it is still listening + if (t.IsCompletedSuccessfully && BreachedBudget(slot.AllocatedBytes)) + Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation"); + }, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); + + // The budget clock starts when the pool starts the run, not when we queued it (P5): pool starvation + // is a host problem and must not be reported as "script timed out". + await started.Task.WaitAsync(ct).ConfigureAwait(false); + + TResult result; + try + { + result = await runTask.WaitAsync(timeout, ct).ConfigureAwait(false); + } + catch (TimeoutException) + { + // The caller's cancel wins even when it lands a few µs after the timeout fired. + if (ct.IsCancellationRequested) throw new OperationCanceledException(ct); + runCts.Cancel(); + if (NoteNonCompletion(key, state, $"timed out after {timeout.TotalMilliseconds:F1} ms")) + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + throw new ScriptTimeoutException(timeout); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception) + { + // The script returned control (by throwing) — that is a completion for quarantine purposes. + Interlocked.Exchange(ref state.ConsecutiveNonCompletions, 0); + throw; + } + + Interlocked.Exchange(ref state.ConsecutiveNonCompletions, 0); + if (BreachedBudget(slot.AllocatedBytes)) + { + // The continuation above has already (or will have) quarantined the key; make the reason visible + // to THIS caller too, so the breaching run itself publishes Bad rather than a suspect value. + Quarantine(key, state, AllocationReason(slot.AllocatedBytes), "allocation"); + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + } + return result; + } + + /// Resolve the key's state, rolling it over when the source identity changed, and refuse a quarantined key. + private KeyState Admit(string key, string? sourceIdentity) + { + while (true) + { + var state = _keys.GetOrAdd(key, static (_, src) => new KeyState(src), sourceIdentity); + if (sourceIdentity is not null && state.SourceIdentity is not null + && !string.Equals(state.SourceIdentity, sourceIdentity, StringComparison.Ordinal)) + { + // The script changed under this key: start from a clean slate. The old state's orphan (if any) + // releases the GLOBAL in-flight count through its own continuation; nothing else references it. + _keys.TryUpdate(key, new KeyState(sourceIdentity), state); + continue; + } + if (sourceIdentity is not null && state.SourceIdentity is null) + { + state.SourceIdentity = sourceIdentity; + } + if (state.QuarantineReason is { } reason) + throw new ScriptQuarantinedException(key, reason); + return state; + } + } + + /// Record a non-completion; returns true when it tipped the key into quarantine. + private bool NoteNonCompletion(string key, KeyState state, string what) + { + var count = Interlocked.Increment(ref state.ConsecutiveNonCompletions); + if (count < _quarantineThreshold) return false; + Quarantine(key, state, + $"{count} consecutive evaluation(s) did not complete (last: {what}); " + + "the script is not evaluated again until its source changes or it is redeployed", + "timeouts"); + return true; + } + + private bool BreachedBudget(long allocatedBytes) => + _maxAllocBytesPerRun > 0 && allocatedBytes > _maxAllocBytesPerRun; + + private string AllocationReason(long allocatedBytes) => + $"one evaluation allocated {allocatedBytes / (1024.0 * 1024.0):F1} MB, over the " + + $"{_maxAllocBytesPerRun / (1024.0 * 1024.0):F0} MB budget (Scripting:MaxAllocBytesPerRun); " + + "the script is not evaluated again until its source changes or it is redeployed"; + + private void Quarantine(string key, KeyState state, string why, string metricReason) + { + var reason = QuarantinedReasonPrefix + why; + // First writer wins: a late orphan and the awaiter can both reach here for the same run. + if (Interlocked.CompareExchange(ref state.QuarantineReason, reason, null) is not null) return; + + Interlocked.Increment(ref _quarantined); + QuarantinedCounter.Add(1, + new KeyValuePair("script", key), + new KeyValuePair("reason", metricReason)); + _logger.Error("Script {ScriptKey} quarantined: {Reason}", key, why); + try { Quarantined?.Invoke(this, new ScriptQuarantinedEventArgs(key, reason)); } + catch (Exception ex) + { + _logger.Warning(ex, "ScriptRunGuard Quarantined subscriber threw for {ScriptKey}", key); + } + } + + private sealed class KeyState(string? sourceIdentity) + { + public string? SourceIdentity = sourceIdentity; + public int InFlight; + public int ConsecutiveNonCompletions; + public string? QuarantineReason; + } + + private sealed class RunSlot + { + public long AllocatedBytes; + } +} + +/// Carried by . +/// The quarantined script instance key. +/// Why, prefixed . +public sealed record ScriptQuarantinedEventArgs(string Key, string Reason); + +/// +/// Thrown by when the script's previous run is +/// still executing (an abandoned, timed-out run that never returned). Nothing was dispatched; the seam +/// publishes Bad with . +/// +public sealed class ScriptRunRefusedException : Exception +{ + /// The script instance key. + public string Key { get; } + + /// Initializes the exception for . + /// The script instance key. + public ScriptRunRefusedException(string key) : base(ScriptRunGuard.PreviousRunStillRunningReason) + { + Key = key; + } +} + +/// +/// Thrown by for a quarantined script: repeated +/// non-completion or an allocation-budget breach. Nothing is dispatched while quarantined; the seam +/// publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or +/// redeploying. +/// +public sealed class ScriptQuarantinedException : Exception +{ + /// The script instance key. + public string Key { get; } + + /// Initializes the exception with the quarantine reason (already prefixed). + /// The script instance key. + /// The quarantine reason, prefixed . + public ScriptQuarantinedException(string key, string reason) : base(reason) + { + Key = key; + } +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs new file mode 100644 index 00000000..ba1399fa --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs @@ -0,0 +1,42 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Node-wide script runtime-safety knobs (the Scripting appsettings section), consumed by +/// . One instance is shared by every script seam on the node — the +/// virtual-tag evaluator, the Calculation evaluator and the scripted-alarm engine — so a single setting +/// bounds every authored script the same way. +/// +/// +/// +/// The in-process sandbox cannot pre-empt a script: a while(true) is abandoned on its pool +/// thread at the timeout, and an allocation is observed only after the run returns. These knobs bound +/// the damage — one orphan per script, no further runs until the source changes — not the +/// first occurrence. Pre-emptive CPU/memory budgeting needs an out-of-process script runner. +/// +/// +public sealed class ScriptingOptions +{ + /// The appsettings section this binds from. + public const string SectionName = "Scripting"; + + /// Default : 64 MB. + public const long DefaultMaxAllocBytesPerRun = 64L * 1024 * 1024; + + /// Default : 3. + public const int DefaultQuarantineAfterConsecutiveTimeouts = 3; + + /// + /// Managed bytes one script run may allocate on its own thread (sampled with + /// around the run) before the script is + /// quarantined. Zero or negative disables the budget. Default 64 MB. + /// + public long MaxAllocBytesPerRun { get; set; } = DefaultMaxAllocBytesPerRun; + + /// + /// Consecutive non-completions (a run that times out, or a run refused because the previous one is + /// still executing) after which the script is quarantined: no further runs are dispatched, every + /// evaluation reports Bad, and the quarantine lifts only on a source change, a redeploy or an explicit + /// reset. Default 3; values below 1 are treated as 1. + /// + public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts; +} diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs index 4adeccab..7a860436 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/TimedScriptEvaluator.cs @@ -19,10 +19,12 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// task continues running on a thread-pool thread until the Roslyn runtime /// returns. In the CPU-bound-infinite-loop case that's effectively "leaked" — /// the thread is tied up until the runtime decides to return, which it may -/// never do. Phase 7 plan Stream A.4 accepts this as a known trade-off; tighter -/// CPU budgeting would require an out-of-process script runner, which is a v3 -/// concern. In practice, the timeout + structured warning log surfaces the -/// offending script so the operator can fix it; the orphan thread is rare. +/// never do. This wrapper has no memory of that orphan, so re-running the same +/// script through it on every trigger leaks one more thread per trigger — which +/// is why the production seams (virtual tags, Calculation, scripted alarms) run +/// through instead: same timeout, plus a per-script +/// "previous run still executing" refusal and a quarantine. This class remains +/// for callers that evaluate once (tests, the authoring preview). /// /// /// Caller-supplied is honored — if the caller diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs index d556ccf6..dc0d8ba1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs @@ -92,7 +92,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // the timer ticks — the mutual exclusion _lock used to provide while it was held across the run, minus // the blocking. Taken ONLY by evaluation workers, so no host thread (mux callback, ReadAsync, // reinit) can ever wait on a script. Lock order is always _evalGate → _lock, never the reverse. - private readonly object _evalGate = new(); + // A SemaphoreSlim rather than a monitor because the holder AWAITS the script run (the pump is async — + // it no longer parks a second pool thread behind the one the script runs on, and the run-guard's + // budget clock starts only once the pool has picked the script up, so pool starvation is never + // reported as a script timeout). + private readonly SemaphoreSlim _evalGate = new(1, 1); + + // Cancels an awaited script run on Dispose (the only hard teardown — ShutdownAsync leaves an in-flight + // run to its RunTimeout, exactly as before, since the pump drops its result once _running is false). + private readonly CancellationTokenSource _evalCts = new(); // Change-triggered evaluations queued by OnDependencyValue and drained, in arrival order, by a single // pump task. Both fields are guarded by _lock; _pumpScheduled stays true until the pump observes an @@ -129,7 +137,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // _logger (ILogger) implements the non-generic ILogger, so it is passed straight // through — the evaluator's host-side diagnostics (compile errors, sandbox violations, execution // throws) land on the SAME logger the driver itself was constructed with, not a separate one. - _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout); + _evaluator = new CalculationEvaluator(_logger, _scriptRoot, _options.RunTimeout, _options.Scripting); RebuildTagTable(_options); } @@ -225,6 +233,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [], RunTimeout = _options.RunTimeout, + Scripting = _options.Scripting, }; } } @@ -235,7 +244,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba _driverInstanceId); } } - return new CalculationDriverOptions { RawTags = _options.RawTags, RunTimeout = _options.RunTimeout }; + return new CalculationDriverOptions + { + RawTags = _options.RawTags, RunTimeout = _options.RunTimeout, Scripting = _options.Scripting, + }; } /// Re-bind the tag table from — shared by @@ -491,13 +503,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { if (_pumpScheduled) return; _pumpScheduled = true; - _ = Task.Run(RunEvaluationPump); + _ = Task.Run(RunEvaluationPumpAsync); } - /// Drain the queued change-triggered evaluations, in arrival order, on a pool thread. At most - /// one pump runs at a time, so per-tag evaluation ordering is preserved exactly as it was when the - /// change path evaluated inline under the state lock. - private void RunEvaluationPump() + /// Drain the queued change-triggered evaluations, in arrival order. At most one pump runs at a + /// time, so per-tag evaluation ordering is preserved exactly as it was when the change path evaluated + /// inline under the state lock. Async: the pump AWAITS each script run instead of parking a pool thread + /// behind the one the script executes on (P5 — sync-over-async with a nested Task.Run manufactured + /// false timeouts under pool starvation). + private async Task RunEvaluationPumpAsync() { while (true) { @@ -515,39 +529,50 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still // evaluate one at a time while no host thread can block behind a script. - lock (_evalGate) + await _evalGate.WaitAsync().ConfigureAwait(false); + try { - try - { - RunEvaluation(next); - } - catch (Exception ex) - { - // The evaluator never throws for script faults, so this is a defect path only — it - // must not kill the pump and strand every queued evaluation behind it. - _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed", - _driverInstanceId, next.Definition.RawPath); - } + await RunEvaluationAsync(next).ConfigureAwait(false); + } + catch (Exception ex) + { + // The evaluator never throws for script faults, so this is a defect path only — it + // must not kill the pump and strand every queued evaluation behind it. + _logger.LogError(ex, "Calculation {Driver}: evaluation of {RawPath} failed", + _driverInstanceId, next.Definition.RawPath); + } + finally + { + _evalGate.Release(); } } } - /// Timer-trigger entry point — prepares and evaluates a single calc tag on the scheduler's - /// pool thread (the scheduler's per-group in-flight guard is what bounds outstanding timer work). - private void EvaluateTagFromTimer(string rawPath) + /// Timer-trigger entry point — prepares and evaluates a single calc tag (the scheduler's + /// per-group in-flight guard is what bounds outstanding timer work; it stays set until this task + /// completes). + private async Task EvaluateTagFromTimerAsync(string rawPath) { if (!_running) return; PendingEvaluation? prepared; lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow); if (prepared is null) return; - lock (_evalGate) RunEvaluation(prepared); + await _evalGate.WaitAsync().ConfigureAwait(false); + try + { + await RunEvaluationAsync(prepared).ConfigureAwait(false); + } + finally + { + _evalGate.Release(); + } } /// Run one prepared evaluation and publish the outcome. MUST be called under /// and NOT under : the script run is the whole point of the /// split, and the script log + fan-out both invoke subscriber code. /// The prepared evaluation. - private void RunEvaluation(PendingEvaluation pending) + private async Task RunEvaluationAsync(PendingEvaluation pending) { var def = pending.Definition; @@ -572,7 +597,8 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // Snapshots, not bare values: the script's ctx.GetTag(path).StatusCode / .SourceTimestampUtc are // then the input's real ones rather than a fabricated Good stamped at evaluation time. - var result = _evaluator.Evaluate(def.RawPath, def.ScriptSource, pending.Dependencies); + var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token) + .ConfigureAwait(false); string? failureReason = null; (object? Value, uint StatusCode)? toPublish; @@ -688,7 +714,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba { var timed = _tagsByRawPath.Values.Where(t => t.TimerInterval is not null).ToArray(); if (timed.Length == 0) return; - _timers = new CalculationTimerScheduler(EvaluateTagFromTimer, _logger); + _timers = new CalculationTimerScheduler(EvaluateTagFromTimerAsync, _logger); _timers.Start(timed); } @@ -723,6 +749,9 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba _running = false; StopTimers(); lock (_lock) _pending.Clear(); + // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop + // the compiled scripts. The CTS is not disposed: a pump mid-await may still read its token. + try { _evalCts.Cancel(); } catch (ObjectDisposedException) { } _evaluator.Dispose(); } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs index ab10f06d..349a0cf1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs @@ -71,11 +71,22 @@ public static class CalculationDriverFactoryExtensions ?? throw new InvalidOperationException( $"Calculation driver config for '{driverInstanceId}' deserialised to null"); + // Run-guard knobs are optional and default to the node-wide ScriptingOptions defaults; only an + // explicitly authored key overrides one (so a config carrying neither binds Scripting = null). + ScriptingOptions? scripting = null; + if (dto.MaxAllocBytesPerRun is not null || dto.QuarantineAfterConsecutiveTimeouts is not null) + { + scripting = new ScriptingOptions(); + if (dto.MaxAllocBytesPerRun is { } alloc) scripting.MaxAllocBytesPerRun = alloc; + if (dto.QuarantineAfterConsecutiveTimeouts is { } n) scripting.QuarantineAfterConsecutiveTimeouts = n; + } + var options = new CalculationDriverOptions { RawTags = dto.RawTags is { Count: > 0 } ? [.. dto.RawTags] : [], RunTimeout = dto.RunTimeoutMs is { } ms && ms > 0 ? TimeSpan.FromMilliseconds(ms) : TimeSpan.FromSeconds(2), + Scripting = scripting, }; return new CalculationDriver( @@ -92,5 +103,11 @@ public static class CalculationDriverFactoryExtensions /// Optional per-script evaluation wall-clock budget override, in milliseconds (default 2000). public int? RunTimeoutMs { get; init; } + + /// Optional run-guard allocation budget per script run, in bytes (default 64 MB; ≤ 0 disables). + public long? MaxAllocBytesPerRun { get; init; } + + /// Optional run-guard quarantine threshold: consecutive non-completions before a tag's script stops being evaluated (default 3). + public int? QuarantineAfterConsecutiveTimeouts { get; init; } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs index b6642c09..f00f50e4 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverOptions.cs @@ -1,20 +1,21 @@ using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; -/// -/// Bound options for a instance. The Calculation pseudo-driver has -/// no backend/endpoint config — its only input is the authored raw calc tags the deploy artifact -/// injects as the list (each carrying the tag's scriptId + resolved -/// scriptSource + trigger config inside its blob). -/// +/// Bound options for one instance. public sealed class CalculationDriverOptions { - /// The driver's authored raw calc tags (RawPath + TagConfig blob + WriteIdempotent), from - /// the deploy artifact. Each is mapped to a at construction. + /// The authored raw calc tags (RawPath + TagConfig JSON) this driver computes. public IReadOnlyList RawTags { get; init; } = Array.Empty(); - /// Per-script wall-clock evaluation budget handed to the . - /// Defaults to 2 seconds (parity with the VirtualTag evaluator). + /// Wall-clock budget per script run (driver-config runTimeoutMs); default 2 s. public TimeSpan RunTimeout { get; init; } = TimeSpan.FromSeconds(2); + + /// + /// Run-guard budget + quarantine threshold for this driver's scripts (driver-config + /// maxAllocBytesPerRun / quarantineAfterConsecutiveTimeouts). Null ⇒ the + /// defaults (64 MB, 3). + /// + public ScriptingOptions? Scripting { get; init; } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index 249c6958..4296b993 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -37,8 +37,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// apply-boundary clear, which is what stops collectible AssemblyLoadContexts accreting across /// script edits. /// +/// +/// Run quarantine. Every compiled run goes through one keyed by +/// calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous +/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive +/// non-completions or an allocation-budget breach — both surface as a +/// the driver maps to Bad. The driver's evaluation pump awaits rather than +/// blocking a pool thread on the sync , +/// and the guard starts the budget clock only once the pool has picked the script up, so pool starvation +/// is never reported as a script timeout. +/// /// -public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable +public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineOwner, IDisposable { // Source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and Clear()/Dispose() // that dispose each evaluator's collectible AssemblyLoadContext. ClearCompiledScripts() — driven by the @@ -47,6 +57,7 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable private readonly ILogger _logger; private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; + private readonly ScriptRunGuard _guard; private bool _disposed; /// Initializes a new . @@ -56,16 +67,22 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable /// implements this — instead of needing a second, separately-obtained logger instance. /// Root script logger; user ctx.Logger.* output flows through this to the Script-log page. /// Wall-clock budget per script run; defaults to 2 seconds (parity with the VT evaluator). + /// Run-guard budget + quarantine threshold; null ⇒ defaults. public CalculationEvaluator( ILogger logger, ScriptRootLogger scriptRoot, - TimeSpan? runTimeout = null) + TimeSpan? runTimeout = null, + ScriptingOptions? scripting = null) { _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger; _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2); + _guard = new ScriptRunGuard(scripting, _scriptRoot); } + /// The run guard every compiled evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + /// /// Evaluate for the calc tag against /// the snapshot in (RawPath → last value). Never throws — script @@ -112,8 +129,25 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable /// The C# script source producing the tag's value. /// Read-only snapshot of dependency values keyed by RawPath. /// The evaluation outcome. + /// Blocks the calling thread for the run. The driver's pump uses ; + /// this overload remains for callers with no async context — chiefly tests. public VirtualTagEvalResult Evaluate( string calcTagId, string scriptSource, IReadOnlyDictionary dependencies) + => EvaluateAsync(calcTagId, scriptSource, dependencies).GetAwaiter().GetResult(); + + /// + /// The async form of : + /// the compiled script runs on the pool under the and the caller awaits + /// it instead of blocking a second pool thread for the duration. Never throws for script faults. + /// + /// Identity of the calc tag (bound to the script log). + /// The C# script source producing the tag's value. + /// Read-only snapshot of dependency values keyed by RawPath. + /// Caller cancellation (driver shutdown); wins over the run timeout. + /// The evaluation outcome. + public async Task EvaluateAsync( + string calcTagId, string scriptSource, IReadOnlyDictionary dependencies, + CancellationToken ct = default) { if (_disposed) return VirtualTagEvalResult.Failure("evaluator disposed"); if (string.IsNullOrWhiteSpace(scriptSource)) return VirtualTagEvalResult.Failure("empty expression"); @@ -180,11 +214,12 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable try { - // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't - // interrupt a CPU-bound/infinite-loop Roslyn script, so the timed wrapper is what bounds a - // runaway script to the wall-clock budget instead of wedging the driver's dispatch. - var timed = new TimedScriptEvaluator(evaluator, _runTimeout); - var raw = timed.RunAsync(context).GetAwaiter().GetResult(); + // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by RawPath): a raw + // CancellationToken can't interrupt a CPU-bound/infinite-loop Roslyn script, so the wall-clock + // budget is what bounds a runaway script — and the guard is what keeps the next trigger from + // stacking another orphan behind it (one per tag, then quarantine). + var raw = await _guard.RunAsync(calcTagId, scriptSource, evaluator, context, _runTimeout, ct) + .ConfigureAwait(false); return VirtualTagEvalResult.Ok(raw); } catch (ScriptTimeoutException) @@ -192,6 +227,18 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable return VirtualTagEvalResult.Failure( $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s"); } + catch (ScriptRunRefusedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (ScriptQuarantinedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + return VirtualTagEvalResult.Failure("evaluation cancelled (driver shutting down)"); + } catch (ObjectDisposedException) when (!_disposed && attempt == 0) { // A concurrent ClearCompiledScripts disposed the evaluator between our GetOrCompile and the @@ -212,7 +259,19 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IDisposable } /// - public void ClearCompiledScripts() => _cache.Clear(); + public void ClearCompiledScripts() + { + _cache.Clear(); + // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine + // (an edited source already lifts its own through the guard's source identity). + _guard.ResetAll(); + } + + /// + public void ResetQuarantines() => _guard.ResetAll(); + + /// + public IReadOnlyCollection QuarantinedScripts => _guard.QuarantinedKeys; /// Disposes the evaluator, draining + disposing every cached compile's collectible ALC. public void Dispose() diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs index ef74da0e..694f7d8a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationTimerScheduler.cs @@ -7,11 +7,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// per distinct interval group (so the wire count stays low /// regardless of tag count) with a per-group in-flight flag that skips a tick when the prior tick for /// the same group is still running. Each tick invokes the driver's own evaluate -/// callback per tag in the group. +/// callback per tag in the group and AWAITS it — the in-flight flag is held across the awaited script +/// run, not just across the synchronous part of the callback, so a slow script still bounds its group +/// to one outstanding tick. /// internal sealed class CalculationTimerScheduler : IDisposable { - private readonly Action _evaluate; + private readonly Func _evaluate; private readonly ILogger _logger; private readonly List _timers = []; private readonly List _groups = []; @@ -20,9 +22,9 @@ internal sealed class CalculationTimerScheduler : IDisposable private bool _disposed; /// Initializes a new instance of the class. - /// Callback invoked with a calc tag's RawPath when its timer group ticks. + /// Callback invoked with a calc tag's RawPath when its timer group ticks; awaited. /// Logger for diagnostics. - public CalculationTimerScheduler(Action evaluate, ILogger logger) + public CalculationTimerScheduler(Func evaluate, ILogger logger) { _evaluate = evaluate ?? throw new ArgumentNullException(nameof(evaluate)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -69,12 +71,20 @@ internal sealed class CalculationTimerScheduler : IDisposable return; } + // The Timer callback is void; the tick itself is async so the awaited script runs do not pin this + // pool thread. TickAsync owns the in-flight flag and swallows everything — nothing can escape to the + // Timer (which would be an unobserved fault on a timer thread). + _ = TickAsync(ctx); + } + + private async Task TickAsync(TickGroup ctx) + { try { foreach (var path in ctx.Paths) { if (_cts.IsCancellationRequested) return; - try { _evaluate(path); } + try { await _evaluate(path).ConfigureAwait(false); } catch (Exception ex) { _logger.LogError(ex, "CalculationTimerScheduler evaluate failed for {Path}", path); } } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index d77631ea..c6823c41 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -31,8 +31,17 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines; /// into the fresh cache for one generation; P7's expression-set gate means no-op redeploys no /// longer open this window at all. /// +/// +/// Run quarantine. Every compiled run goes through one keyed by +/// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous +/// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive +/// non-completions or an allocation-budget breach. Both surface as +/// so VirtualTagActor publishes Bad exactly as it does for a timeout. A quarantine lifts when the +/// expression changes, on / (the deploy +/// boundary), never by itself. +/// /// -public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IDisposable +public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCacheOwner, IScriptQuarantineOwner, IDisposable { // Purpose-built source-hash-keyed compile cache: Lazy single-compile (no double-compile race) and // Clear()/Dispose() that dispose each evaluator's collectible AssemblyLoadContext. The apply-boundary @@ -42,22 +51,29 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac private readonly ILogger _logger; private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; + private readonly ScriptRunGuard _guard; private bool _disposed; /// Initializes a new RoslynVirtualTagEvaluator with the given loggers and optional timeout. /// Logger for recording compilation and execution errors (host diagnostics). /// Root script logger; user ctx.Logger.* output flows through this to the Script-log page. /// Maximum execution time for each script; defaults to 2 seconds if not specified. + /// Run-guard budget + quarantine threshold (the Scripting section); null ⇒ defaults. public RoslynVirtualTagEvaluator( ILogger logger, ScriptRootLogger scriptRoot, - TimeSpan? runTimeout = null) + TimeSpan? runTimeout = null, + ScriptingOptions? scripting = null) { _logger = logger; _scriptRoot = (scriptRoot ?? throw new ArgumentNullException(nameof(scriptRoot))).Logger; _runTimeout = runTimeout ?? TimeSpan.FromSeconds(2); + _guard = new ScriptRunGuard(scripting, _scriptRoot); } + /// The run guard every compiled evaluation goes through (diagnostics / tests). + public ScriptRunGuard RunGuard => _guard; + // Value-only overload. Every dependency reaches the script stamped Good with an // evaluation-time timestamp — bare values carry nothing better. The runtime calls the snapshot // overload instead; this one remains for callers that genuinely hold only @@ -150,13 +166,13 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac try { - // Route through TimedScriptEvaluator (Task.Run + WaitAsync): a raw CancellationToken can't - // interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll it), so the previous - // CTS-only path let one runaway script hang this actor forever. WaitAsync returns control when - // the wall-clock budget fires regardless of whether the inner task completes (the orphaned - // thread is the documented, accepted trade-off — TimedScriptEvaluator remarks). - var timed = new TimedScriptEvaluator(evaluator, _runTimeout); - var raw = timed.RunAsync(context).GetAwaiter().GetResult(); + // Route through the ScriptRunGuard (Task.Run + WaitAsync, keyed by virtual-tag id): a raw + // CancellationToken can't interrupt a CPU-bound/infinite-loop script (Roslyn scripts don't poll + // it), so the wall-clock budget is what returns control — and the guard is what stops the + // abandoned run from being joined by another on the next trigger (one orphan per tag, then + // quarantine), which is the leak the old per-evaluation TimedScriptEvaluator allowed. + var raw = _guard.RunAsync(virtualTagId, expression, evaluator, context, _runTimeout) + .GetAwaiter().GetResult(); return VirtualTagEvalResult.Ok(raw); } catch (ScriptTimeoutException) @@ -164,6 +180,14 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac return VirtualTagEvalResult.Failure( $"{ScriptEvalFailureReasons.TimedOutPrefix}{_runTimeout.TotalSeconds:F1}s"); } + catch (ScriptRunRefusedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } + catch (ScriptQuarantinedException ex) + { + return VirtualTagEvalResult.Failure(ex.Message); + } catch (ObjectDisposedException) when (!_disposed && attempt == 0) { // S12: the apply-boundary ClearCompiledScripts disposed the evaluator between our @@ -185,7 +209,19 @@ public sealed class RoslynVirtualTagEvaluator : IVirtualTagEvaluator, IScriptCac } /// - public void ClearCompiledScripts() => _cache.Clear(); + public void ClearCompiledScripts() + { + _cache.Clear(); + // The compile-cache clear IS the deploy boundary for this evaluator: a redeploy lifts every quarantine + // (an edited expression already lifts its own through the guard's source identity). + _guard.ResetAll(); + } + + /// + public void ResetQuarantines() => _guard.ResetAll(); + + /// + public IReadOnlyCollection QuarantinedScripts => _guard.QuarantinedKeys; /// Disposes the evaluator and cleans up all cached script evaluators. public void Dispose() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs index d00dfb36..f97fa387 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagHostActor.cs @@ -3,6 +3,7 @@ using Akka.Event; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Core.VirtualTags; using ZB.MOM.WW.OtOpcUa.OpcUaServer; using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; @@ -112,6 +113,11 @@ public sealed class VirtualTagHostActor : ReceiveActor (_evaluator as IScriptCacheOwner)?.ClearCompiledScripts(); } + // Every apply is a deploy, and a deploy lifts the run guard's quarantines unconditionally — "the + // operator redeployed to retry" is one of the two documented ways out of a quarantine (the other, + // an edited expression, the guard detects on its own). Cheap on a no-op redeploy (an empty map). + (_evaluator as IScriptQuarantineOwner)?.ResetQuarantines(); + var desired = new HashSet(msg.Plans.Select(p => p.VirtualTagId), StringComparer.Ordinal); // Stop + forget children whose vtagId is no longer desired. Stopping the child triggers its diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs new file mode 100644 index 00000000..4074d0d9 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs @@ -0,0 +1,105 @@ +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// +/// runs every predicate through a keyed by +/// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation +/// spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is +/// quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a +/// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine. +/// +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmEngineRunGuardTests +{ + private const uint StatusBadInternalError = 0x80020000u; + + private const string SpinPredicate = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return false;"; + + private static ScriptedAlarmDefinition Alarm(string id, string predicate) => + new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, + Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); + + private static ScriptedAlarmEngine Build(FakeUpstream up, ScriptingOptions? scripting = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + // 500 ms, not the 60 ms the guard's own tests use: the HEALTHY alarm's first run is JIT-cold and + // must finish inside the budget on a loaded CI box, or the positive control itself times out. The + // spin predicate outlives this by 12x, so the runaway alarm still cannot finish first. + return new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger, + scriptTimeout: TimeSpan.FromMilliseconds(500), scripting: scripting); + } + + [Fact] + public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up); + var events = new List(); + eng.OnEvent += (_, e) => { lock (events) events.Add(e); }; + + // The load itself evaluates once (startup recovery) — that is the first, abandoned run. + await eng.LoadAsync( + [Alarm("runaway", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"), + Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + // Two more pushes: refused (orphan still running), then quarantined. + up.Push("Temp", 60); + await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1); + up.Push("Temp", 70); + await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _)); + + eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one"); + eng.RunGuard.InFlightCount.ShouldBe(1); + + // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving. + ScriptedAlarmEvent[] snapshot; + lock (events) snapshot = events.ToArray(); + var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray(); + quality.ShouldNotBeEmpty(); + quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); + eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented"); + + // Positive control: the healthy sibling still transitions on the same engine. + up.Push("Temp", 150); + await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active); + } + + [Fact] + public async Task A_reload_lifts_the_quarantine() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + + await eng.LoadAsync([Alarm("a", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("a", out _).ShouldBeTrue(); + + // The operator fixed the predicate and redeployed. + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("a", out _).ShouldBeFalse(); + + up.Push("Temp", 150); + await WaitForAsync(() => eng.GetState("a")!.Active == AlarmActiveState.Active); + } + + private static async Task WaitForAsync(Func cond, int timeoutMs = 10_000) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); + while (DateTime.UtcNow < deadline) + { + if (cond()) return; + await Task.Delay(25); + } + throw new TimeoutException("Condition did not become true in time"); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs new file mode 100644 index 00000000..974e6a7c --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs @@ -0,0 +1,306 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// — the per-script run quarantine shared by the virtual-tag evaluator, the +/// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so +/// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard +/// refuses to start a run while the previous one is still executing, quarantines a script after three +/// consecutive non-completions, and quarantines a script whose single run allocates past the budget. +/// +[Trait("Category", "Unit")] +public sealed class ScriptRunGuardTests +{ + /// + /// A CPU-bound script that outlives every timeout in this file but still ENDS (so the orphaned pool + /// thread it leaves behind frees itself after the test). System.DateTime is outside the sandbox + /// deny-list. + /// + private const string SpinScript = + """ + var end = System.DateTime.UtcNow.AddSeconds(6); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + + private const string FastScript = "return 42;"; + + private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60); + + private static ScriptEvaluator Compile(string source) => + ScriptEvaluator.Compile(source); + + private static Task Run(ScriptRunGuard guard, string key, string source, + ScriptEvaluator evaluator, TimeSpan? timeout = null, CancellationToken ct = default) => + guard.RunAsync(key, source, evaluator, new FakeScriptContext(), timeout ?? TimeSpan.FromSeconds(5), ct); + + [Fact] + public async Task Completed_run_returns_the_value_and_releases_the_in_flight_slot() + { + var guard = new ScriptRunGuard(); + using var fast = Compile(FastScript); + + var value = await Run(guard, "vt-1", FastScript, fast, ct: TestContext.Current.CancellationToken); + + value.ShouldBe(42); + guard.InFlightCount.ShouldBe(0); + guard.StartedRunCount.ShouldBe(1); + guard.IsQuarantined("vt-1", out _).ShouldBeFalse(); + } + + [Fact] + public async Task A_timed_out_run_throws_ScriptTimeoutException_and_keeps_the_slot_held_by_the_orphan() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + + await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + // The abandoned script is still executing on its pool thread — the guard knows, the caller does not. + guard.InFlightCount.ShouldBe(1); + guard.StartedRunCount.ShouldBe(1); + } + + [Fact] + public async Task A_run_while_the_previous_orphan_is_still_executing_is_refused_without_starting_another() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + var ex = await Should.ThrowAsync(() => + Run(guard, "vt-1", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + ex.Message.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + guard.StartedRunCount.ShouldBe(1, "the refusal must not dispatch a second Task.Run"); + guard.InFlightCount.ShouldBe(1); + guard.RefusedWhileRunningCount.ShouldBe(1); + } + + [Fact] + public async Task Three_consecutive_non_completions_quarantine_the_script_and_a_sibling_keeps_evaluating() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + // 1: times out, 2: refused (orphan still running), 3: refused → threshold reached → quarantined. + await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + var quarantined = await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + + quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + quarantined.Message.ShouldContain("3 consecutive"); + guard.IsQuarantined("bad", out var reason).ShouldBeTrue(); + reason.ShouldNotBeNull(); + guard.QuarantinedCount.ShouldBe(1); + + // A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run. + await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + guard.StartedRunCount.ShouldBe(1); + + // Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key. + (await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42); + guard.IsQuarantined("good", out _).ShouldBeFalse(); + guard.StartedRunCount.ShouldBe(2); + } + + /// + /// The thread-leak regression. Pre-guard, a while(true) script on a 1 Hz trigger leaked one + /// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the + /// first trigger's orphan is the only one, every later trigger is refused or quarantined, and a healthy + /// sibling keeps evaluating throughout (the positive control an absence assertion needs). + /// + /// + /// The load-bearing assertions are on the guard's own bookkeeping ( + /// / ): moves with + /// every parallel test class on a loaded machine, so it is kept as a coarse secondary check only — + /// it must stay far below the 30 the leak would have produced. + /// + [Fact] + public async Task Runaway_script_leaves_at_most_one_orphan_after_30_triggers_while_a_sibling_keeps_evaluating() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + var threadsBefore = ThreadPool.ThreadCount; + + var healthyResults = new List(); + for (var trigger = 0; trigger < 30; trigger++) + { + try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); } + catch (ScriptTimeoutException) { } + catch (ScriptRunRefusedException) { } + catch (ScriptQuarantinedException) { } + + healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct)); + } + + guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing"); + guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling"); + guard.IsQuarantined("runaway", out _).ShouldBeTrue(); + healthyResults.ShouldAllBe(v => v == 42); + healthyResults.Count.ShouldBe(30); + + var threadDelta = ThreadPool.ThreadCount - threadsBefore; + threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads"); + } + + [Fact] + public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("vt", out _).ShouldBeTrue(); + + // The operator fixed the script: a different source identity under the same key starts clean. The + // old orphan is a sunk cost — it keeps running, but it must not block the FIXED script. + (await Run(guard, "vt", FastScript, fast, ct: ct)).ShouldBe(42); + guard.IsQuarantined("vt", out _).ShouldBeFalse(); + guard.InFlightCount.ShouldBe(1, "the abandoned orphan is still counted until it finishes"); + } + + [Fact] + public async Task Reset_and_ResetAll_clear_a_quarantine() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + using var spin = Compile(SpinScript); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "a", SpinScript, spin, ShortTimeout, ct)); + await Should.ThrowAsync(() => Run(guard, "b", SpinScript, spin, ShortTimeout, ct)); + guard.QuarantinedKeys.ShouldBe(new[] { "a", "b" }, ignoreOrder: true); + + guard.Reset("a"); + guard.IsQuarantined("a", out _).ShouldBeFalse(); + guard.IsQuarantined("b", out _).ShouldBeTrue(); + + guard.ResetAll(); + guard.IsQuarantined("b", out _).ShouldBeFalse(); + // The key is usable again (a fresh slot: the orphan from the quarantined run no longer blocks it). + (await Run(guard, "b", FastScript, fast, ct: ct)).ShouldBe(42); + } + + [Fact] + public async Task A_run_that_allocates_past_the_budget_is_quarantined() + { + var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 1024 * 1024 }); + // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the + // allocation as dead code (which is exactly what it does to the naive one-liner). + const string hungry = + "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;"; + using var evaluator = Compile(hungry); + using var fast = Compile(FastScript); + var ct = TestContext.Current.CancellationToken; + + var ex = await Should.ThrowAsync(() => Run(guard, "hog", hungry, evaluator, ct: ct)); + ex.Message.ShouldContain("allocat"); + guard.IsQuarantined("hog", out var reason).ShouldBeTrue(); + reason.ShouldNotBeNull(); + reason.ShouldContain("allocat"); + + // Never dispatched again, and the slot was released (the run DID complete). + await Should.ThrowAsync(() => Run(guard, "hog", hungry, evaluator, ct: ct)); + guard.StartedRunCount.ShouldBe(1); + guard.InFlightCount.ShouldBe(0); + + // Positive control: a modest allocation under the budget is fine. + (await Run(guard, "ok", FastScript, fast, ct: ct)).ShouldBe(42); + } + + [Fact] + public async Task A_disabled_allocation_budget_never_quarantines() + { + var guard = new ScriptRunGuard(new ScriptingOptions { MaxAllocBytesPerRun = 0 }); + // Writes into every array so the JIT cannot fold `new byte[N].Length` to a constant and drop the + // allocation as dead code (which is exactly what it does to the naive one-liner). + const string hungry = + "long total = 0; for (var i = 0; i < 32; i++) { var a = new byte[256 * 1024]; a[i] = 1; total += a[i] + a.Length; } return (int)total;"; + using var evaluator = Compile(hungry); + + (await Run(guard, "hog", hungry, evaluator, ct: TestContext.Current.CancellationToken)).ShouldBe(32 * (256 * 1024 + 1)); + guard.IsQuarantined("hog", out _).ShouldBeFalse(); + } + + [Fact] + public async Task A_script_that_throws_propagates_unwrapped_and_counts_as_a_completion() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 }); + const string throwing = "throw new System.InvalidOperationException(\"boom\");"; + using var thrower = Compile(throwing); + // A spin that outlives the timeout but finishes quickly, so the slot frees up between attempts. + const string briefSpin = + """ + var end = System.DateTime.UtcNow.AddMilliseconds(250); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + using var spin = Compile(briefSpin); + var ct = TestContext.Current.CancellationToken; + + // timeout #1 + await Should.ThrowAsync(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + + // A throw is a completion — the script returned control — so the consecutive counter resets… + var ex = await Should.ThrowAsync(() => Run(guard, "k", throwing, thrower, ct: ct)); + ex.Message.ShouldBe("boom"); + guard.InFlightCount.ShouldBe(0); + + // …which is why this second timeout does NOT reach the threshold of 2. + await Should.ThrowAsync(() => Run(guard, "k", briefSpin, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + await WaitForIdle(guard); + } + + [Fact] + public async Task Caller_cancellation_wins_over_the_timeout() + { + var guard = new ScriptRunGuard(); + using var spin = Compile(SpinScript); + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(80)); + + await Should.ThrowAsync(() => + Run(guard, "k", SpinScript, spin, TimeSpan.FromSeconds(5), cts.Token)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + } + + [Fact] + public async Task Quarantine_is_observable_through_the_event_and_the_counter() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + using var spin = Compile(SpinScript); + var seen = new List<(string Key, string Reason)>(); + guard.Quarantined += (_, e) => seen.Add((e.Key, e.Reason)); + + await Should.ThrowAsync(() => + Run(guard, "k", SpinScript, spin, ShortTimeout, TestContext.Current.CancellationToken)); + + seen.ShouldHaveSingleItem(); + seen[0].Key.ShouldBe("k"); + seen[0].Reason.ShouldContain("consecutive"); + guard.QuarantinedCount.ShouldBe(1); + } + + private static async Task WaitForIdle(ScriptRunGuard guard) + { + var deadline = DateTime.UtcNow.AddSeconds(5); + while (guard.InFlightCount != 0 && DateTime.UtcNow < deadline) + await Task.Delay(10); + guard.InFlightCount.ShouldBe(0, "the brief spin must have finished"); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs new file mode 100644 index 00000000..85943f16 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs @@ -0,0 +1,86 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; + +/// +/// runs every compiled script through a +/// keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every +/// later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now +/// the next trigger is refused ("previous evaluation still running"), the third consecutive +/// non-completion quarantines the tag, and a healthy sibling on the same evaluator keeps computing. +/// +public sealed class CalculationEvaluatorRunGuardTests +{ + private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger()); + + /// Outlives the 60 ms budget by a wide margin but still ends, so the orphan frees itself. + private const string SpinScript = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;"; + + private static readonly Dictionary NoDeps = new(); + + [Fact] + public async Task A_trigger_during_the_previous_orphan_is_refused_then_the_tag_is_quarantined_and_a_sibling_keeps_computing() + { + using var sut = new CalculationEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); + var ct = TestContext.Current.CancellationToken; + + var first = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct); + first.Success.ShouldBeFalse(); + first.Reason.ShouldNotBeNull(); + first.Reason.ShouldContain("timed out"); + + var second = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); + second.Success.ShouldBeFalse(); + second.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + + var third = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); + third.Success.ShouldBeFalse(); + third.Reason.ShouldNotBeNull(); + third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + sut.QuarantinedScripts.ShouldBe(new[] { "calc/loop" }); + + // Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger. + sut.RunGuard.StartedRunCount.ShouldBe(1); + sut.RunGuard.InFlightCount.ShouldBe(1); + + // Positive control: a sibling tag on the same evaluator still computes. + var sibling = await sut.EvaluateAsync("calc/ok", "return 42;", Snap(NoDeps), ct); + sibling.Success.ShouldBeTrue(sibling.Reason); + sibling.Value.ShouldBe(42); + } + + [Fact] + public async Task ClearCompiledScripts_lifts_the_quarantine_as_a_redeploy_does() + { + using var sut = new CalculationEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60), + new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + var ct = TestContext.Current.CancellationToken; + + var r = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct).WaitAsync(TimeSpan.FromSeconds(30), ct); + r.Reason.ShouldNotBeNull(); + r.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + + sut.ClearCompiledScripts(); + + sut.QuarantinedScripts.ShouldBeEmpty(); + // The fixed script (new source) runs; the old orphan is a sunk cost that no longer blocks the tag. + var fixedRun = await sut.EvaluateAsync("calc/loop", "return 7;", Snap(NoDeps), ct); + fixedRun.Success.ShouldBeTrue(fixedRun.Reason); + fixedRun.Value.ShouldBe(7); + } + + private static Dictionary Snap(Dictionary deps) + { + var now = DateTime.UtcNow; + var d = new Dictionary(StringComparer.Ordinal); + foreach (var kv in deps) d[kv.Key] = new Core.Abstractions.DataValueSnapshot(kv.Value, 0u, now, now); + return d; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs new file mode 100644 index 00000000..f0de4999 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Host.Engines; + +namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; + +/// +/// runs every compiled expression through a +/// keyed by virtual-tag id: a runaway expression leaks one orphaned pool +/// thread, not one per trigger, the tag is quarantined after three consecutive non-completions (surfacing +/// as a Failure the actor publishes Bad for), and the quarantine lifts at the deploy boundary. +/// +public sealed class RoslynVirtualTagEvaluatorRunGuardTests +{ + private static ScriptRootLogger NoOpScriptRoot() => new(new LoggerConfiguration().CreateLogger()); + + private const string SpinScript = + "var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 0;"; + + private static readonly Dictionary NoDeps = new(); + + [Fact] + public void Runaway_expression_is_refused_then_quarantined_and_a_sibling_keeps_evaluating() + { + using var sut = new RoslynVirtualTagEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); + + var first = sut.Evaluate("vt-loop", SpinScript, NoDeps); + first.Success.ShouldBeFalse(); + first.Reason.ShouldNotBeNull(); + first.Reason.ShouldContain("timed out"); + + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + + var third = sut.Evaluate("vt-loop", SpinScript, NoDeps); + third.Reason.ShouldNotBeNull(); + third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + sut.QuarantinedScripts.ShouldBe(new[] { "vt-loop" }); + sut.RunGuard.StartedRunCount.ShouldBe(1, "one orphan, not one per trigger"); + + var sibling = sut.Evaluate("vt-ok", "return 42;", NoDeps); + sibling.Success.ShouldBeTrue(sibling.Reason); + sibling.Value.ShouldBe(42); + } + + [Fact] + public void ResetQuarantines_lifts_the_quarantine_as_a_redeploy_does() + { + using var sut = new RoslynVirtualTagEvaluator( + NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60), + new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason!.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); + sut.QuarantinedScripts.ShouldNotBeEmpty(); + + sut.ResetQuarantines(); + + sut.QuarantinedScripts.ShouldBeEmpty(); + var fixedRun = sut.Evaluate("vt-loop", "return 7;", NoDeps); + fixedRun.Success.ShouldBeTrue(fixedRun.Reason); + fixedRun.Value.ShouldBe(7); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs index 004b7165..47ac617a 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorTests.cs @@ -489,12 +489,18 @@ public sealed class RoslynVirtualTagEvaluatorTests // most evaluations then hit the warm cache instead of recompiling. Neither the cadence nor the // iteration count is a correctness knob any more — the assertions below hold for any number of // clears landing inside a single evaluation, and for any number of evaluations completed. - var evalLoops = Enumerable.Range(0, loopCount).Select(_ => Task.Run(() => + // One virtual-tag id PER LOOP, sharing the expression. The race under test lives in the compile + // cache, which is keyed by source, so twelve ids on one expression contend for exactly the entry + // twelve copies of one id did — while twelve concurrent evaluations of ONE id is a shape production + // never produces (one VirtualTagActor per id serialises them) and exactly the shape the per-tag + // ScriptRunGuard refuses ("previous evaluation still running", then quarantine). + var evalLoops = Enumerable.Range(0, loopCount).Select(loop => Task.Run(() => { + var vtagId = $"vt-race-{loop}"; var done = 0; for (var i = 0; i < MaxIterationsPerLoop && Environment.TickCount64 < stopAtMs; i++) { - var r = sut.Evaluate("vt-race", expr, new Dictionary { ["a"] = i }); + var r = sut.Evaluate(vtagId, expr, new Dictionary { ["a"] = i }); done++; if (!r.Success) { From ae84253773481894b413052f3efcb1ad923f4e90 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:41:35 -0400 Subject: [PATCH 04/51] fix(controlplane): keep the admin singleton's mailbox free during Test Connect and deploy compose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReceiveAsync suspended the mailbox for the whole of every handler, so one Test Connect against a black-holed PLC (60 s clamp) or one large deploy compose stalled every other admin operation fleet-wide — AdminUI alarm acks/shelves, driver restarts, the next deploy. - TestDriverConnect runs the probe on the thread pool and pipes a TestDriverConnectResult straight to the requester (every path yields a result, never Status.Failure). - StartDeployment serialises only the in-flight check (DB row + an actor-local compose-in-flight flag, because the Deployment row is written at the END of the compose); the snapshot read, validation, gzip and row write run off-actor and return as DeployComposed / DeployComposeFailed, where the coordinator is told and the requester answered. Tests: an alarm ack completes in < 1 s while a 60 s probe is pending and while a compose is parked on a gated DbContext (positive controls: both still complete afterwards); a second StartDeployment during an in-flight compose is refused. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 454 +++++++++++------- .../AdminOperationsActorMailboxTests.cs | 159 ++++++ 2 files changed, 432 insertions(+), 181 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 4aabe3db..8b1c9ddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -36,6 +36,9 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly ILoggingAdapter _log = Context.GetLogger(); + /// The StartDeployment whose compose is running off-actor, or null. See . + private StartDeployment? _composeInFlight; + /// Creates actor props for the admin operations actor. /// Factory for creating config database contexts. /// Reference to the deployment coordinator actor. @@ -74,8 +77,12 @@ public sealed class AdminOperationsActor : ReceiveActor _tagConfigValidationMode = tagConfigValidationMode; _meshRouter = meshRouter; - ReceiveAsync(HandleStartDeploymentAsync); - ReceiveAsync(HandleTestDriverConnectAsync); + // Synchronous receives: the compose and the probe run off-actor and come back as messages, so + // the mailbox is never suspended for the length of a DB snapshot or a PLC connect timeout. + Receive(HandleStartDeployment); + Receive(HandleDeployComposed); + Receive(HandleDeployComposeFailed); + Receive(HandleTestDriverConnect); ReceiveAsync(HandleRestartDriverAsync); ReceiveAsync(HandleReconnectDriverAsync); Receive(HandleAcknowledgeAlarm); @@ -181,19 +188,42 @@ public sealed class AdminOperationsActor : ReceiveActor } } - private async Task HandleStartDeploymentAsync(StartDeployment msg) + /// + /// Serialised part of StartDeployment: the in-flight check. Everything after it — the + /// snapshot read, validation, gzip and the row write — runs on the thread pool and comes back as a + /// / message, so a large compose no + /// longer suspends the mailbox (R03:3.1) and alarm acks keep flowing while it runs. + /// + /// + /// is what keeps the check correct once the compose is off-actor: the + /// Deployment row is written at the END of the compose, so a second request arriving mid-compose + /// would pass a DB-only check and two artifacts would race for one coordinator. The flag is cleared + /// when the outcome message lands, on both branches. + /// + private void HandleStartDeployment(StartDeployment msg) { var replyTo = Sender; + if (_composeInFlight is { } inFlightRequest) + { + replyTo.Tell(new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: null, + RevisionHash: null, + Message: $"A deployment requested by {inFlightRequest.CreatedBy} is still being composed.", + msg.CorrelationId)); + return; + } + try { - await using var db = await _dbFactory.CreateDbContextAsync(); - // Refuse if any deployment is already in flight — keeps the coordinator's state - // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. - var inflight = await db.Deployments + // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. One indexed + // projection (IX_Deployment_Status), synchronous on purpose: it IS the serialised part. + using var db = _dbFactory.CreateDbContext(); + var inflight = db.Deployments .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) .Select(d => d.DeploymentId) - .FirstOrDefaultAsync(); + .FirstOrDefault(); if (inflight != Guid.Empty) { replyTo.Tell(new StartDeploymentResult( @@ -204,172 +234,6 @@ public sealed class AdminOperationsActor : ReceiveActor msg.CorrelationId)); return; } - - // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject - // on ANY validation error. The config is now canonical — the company overlay loader emits - // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId - // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions, - // …) gates the deploy. A green build in this repo does not prove the config is valid; this - // is the last guard before a bad address space (or a non-derived EquipmentId) ships. - // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can - // (rarely) be a false positive if an external-id reservation release has not yet been - // committed/visible when the snapshot is read — operators seeing a spurious one should - // check ExternalIdReservation state before re-submitting. - var draft = await DraftSnapshotFactory.FromConfigDbAsync(db); - var errors = DraftValidator.Validate(draft).ToList(); - - // Cluster-topology guard. The SQL - // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode) - // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an - // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at - // Hot/Warm and the constraint stays green, which would boot the runtime into an - // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of - // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. - // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing - // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. - // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT - // transaction), not a second query that could observe a later edit. - var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); - var nodesByCluster = draft.ClusterNodes - .GroupBy(n => n.ClusterId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); - foreach (var cluster in clusters) - { - var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns) - ? (IReadOnlyList)ns - : []; - errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes)); - } - - // Deploy-time TagConfig strictness inspection over equipment tags. The - // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would - // REJECT — a present-but-invalid enum value, an absent or malformed address field, a - // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads - // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum; - // it now rejects the tag, so these warnings describe rejections.) In Warn mode - // (default) these are non-blocking (logged + appended to the accepted result Message); in Error - // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed. - // Running servers are untouched either way — the gate only sees re-deploys. - // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId / - // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type. - var driverTypeById = draft.DriverInstances - .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal); - var driverByDevice = draft.Devices - .GroupBy(d => d.DeviceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal); - var deviceConfigById = draft.Devices - .GroupBy(d => d.DeviceId, StringComparer.Ordinal) - .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal); - var tagConfigWarnings = new List<(string TagId, string Text)>(); - foreach (var t in draft.Tags) - { - if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue; - if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue; - foreach (var w in EquipmentTagConfigInspector.Inspect( - driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId))) - tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}")); - } - if (tagConfigWarnings.Count > 0) - { - if (_tagConfigValidationMode == TagConfigValidationMode.Error) - errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId))); - else - foreach (var x in tagConfigWarnings) - _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text); - } - - if (errors.Count > 0) - { - var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}")); - _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}", - errors.Count, summary); - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Rejected, - DeploymentId: null, - RevisionHash: null, - Message: summary, - msg.CorrelationId)); - return; - } - - // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough) - // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc - // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct - // sources are counted (the compile cache keys on source, so duplicates collapse to one unit). - // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard - // reject; this advisory rides in the Accepted Message so the UI can show it. - // advisory is declared outside the inner try so the Accepted reply below can still read it even - // when the guardrail query throws (e.g. transient SQL). A failed estimate must NEVER block a - // valid deploy — the outer catch is reserved for genuine seal/save failures. - string? advisory = null; - try - { - const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) - // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. - var compiled = draft.Scripts - .Select(s => s.SourceCode) - .Where(src => !PassthroughScript.TryMatch(src, out _)) - .Distinct(StringComparer.Ordinal) - .Count(); - if (compiled > 0) - { - var estMiB = compiled * PerScriptMiB; - _log.Warning( - "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it", - compiled, estMiB); - advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)"; - } - } - catch (Exception ex) - { - // Guardrail is advisory-only — a failed estimate must never block a valid deploy. - _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped"); - advisory = null; - } - - // Composed from the validated snapshot, never from a second read: an edit that lands between - // validation and here is simply not in this artifact (R03:3.2). - var artifact = ConfigComposer.Compose(draft); - var deploymentId = DeploymentId.NewId(); - var revHash = RevisionHash.Parse(artifact.RevisionHash); - - db.Deployments.Add(new Deployment - { - DeploymentId = deploymentId.Value, - RevisionHash = artifact.RevisionHash, - Status = DeploymentStatus.Dispatching, - CreatedBy = msg.CreatedBy, - ArtifactBlob = artifact.Blob, - }); - - // Marker ConfigEdit row so the audit timeline shows the deployment snapshot. - db.ConfigEdits.Add(new ConfigEdit - { - EntityType = "Deployment", - EntityId = deploymentId.Value, - FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}", - EditedBy = msg.CreatedBy, - SourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown", - }); - - await db.SaveChangesAsync(); - - _coordinator.Tell(new DispatchDeployment(deploymentId, revHash, msg.CorrelationId)); - - // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so - // the operator sees the non-blocking strictness findings on a successful deploy. - var messageParts = new List(); - if (advisory is not null) messageParts.Add(advisory); - messageParts.AddRange(tagConfigWarnings.Select(x => x.Text)); - var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null; - - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Accepted, - deploymentId, - revHash, - Message: acceptedMessage, - msg.CorrelationId)); } catch (Exception ex) { @@ -380,10 +244,233 @@ public sealed class AdminOperationsActor : ReceiveActor RevisionHash: null, Message: ex.Message, msg.CorrelationId)); + return; } + + _composeInFlight = msg; + // Captured here: Context/Sender are not available on the thread-pool thread. + var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown"; + var self = Self; + Task.Run(() => ComposeDeploymentAsync(msg, sourceNode)) + .PipeTo(self, + success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch), + failure: ex => new DeployComposeFailed(msg, replyTo, ex)); } - private async Task HandleTestDriverConnectAsync(TestDriverConnect msg) + private void HandleDeployComposed(DeployComposed msg) + { + _composeInFlight = null; + // The row is committed (the compose task completed), so the coordinator can be told now; it reads + // the row back by id when it seeds the expected-ack set. + if (msg.Dispatch is { } dispatch) _coordinator.Tell(dispatch); + msg.ReplyTo.Tell(msg.Result); + } + + private void HandleDeployComposeFailed(DeployComposeFailed msg) + { + _composeInFlight = null; + _log.Error(msg.Error, "StartDeployment failed for {CreatedBy}", msg.Request.CreatedBy); + msg.ReplyTo.Tell(new StartDeploymentResult( + StartDeploymentOutcome.Rejected, + DeploymentId: null, + RevisionHash: null, + Message: msg.Error.Message, + msg.Request.CorrelationId)); + } + + /// Outcome of the off-actor compose: the reply to send, and the dispatch to issue when Accepted. + private sealed record ComposeOutcome(StartDeploymentResult Result, DispatchDeployment? Dispatch); + + private sealed record DeployComposed( + StartDeployment Request, IActorRef ReplyTo, StartDeploymentResult Result, DispatchDeployment? Dispatch); + + private sealed record DeployComposeFailed(StartDeployment Request, IActorRef ReplyTo, Exception Error); + + /// + /// The deploy compose, off the actor thread. Touches no actor state: everything it needs is in + /// , and the readonly fields captured at + /// construction (, , — + /// Akka's logging adapter is thread-safe). + /// + private async Task ComposeDeploymentAsync(StartDeployment msg, string sourceNode) + { + await using var db = await _dbFactory.CreateDbContextAsync(); + + // Full pre-seal gate: run the complete DraftValidator (one pass, all rules) and reject + // on ANY validation error. The config is now canonical — the company overlay loader emits + // canonical EquipmentIds and the seed is clean — so every rule (UNS segments, EquipmentId + // derivation, cross-cluster/namespace binding, driver-namespace compat, signal collisions, + // …) gates the deploy. A green build in this repo does not prove the config is valid; this + // is the last guard before a bad address space (or a non-derived EquipmentId) ships. + // NOTE: a BadDuplicateExternalIdentifier rejection from the reservation pre-flight can + // (rarely) be a false positive if an external-id reservation release has not yet been + // committed/visible when the snapshot is read — operators seeing a spurious one should + // check ExternalIdReservation state before re-submitting. + var draft = await DraftSnapshotFactory.FromConfigDbAsync(db); + var errors = DraftValidator.Validate(draft).ToList(); + + // Cluster-topology guard. The SQL + // CK_ServerCluster_RedundancyMode_NodeCount CHECK enforces the (NodeCount, RedundancyMode) + // pair on the row itself, but it cannot see the per-node ClusterNode.Enabled flag — an + // operator can disable a node (effective enabled-count = 1) while leaving RedundancyMode at + // Hot/Warm and the constraint stays green, which would boot the runtime into an + // InvalidTopology band. ValidateClusterTopology catches that drift; it is not part of + // DraftValidator.Validate, so it runs separately here over the snapshot's cluster/node rows. + // Errors fold into the same reject summary alongside the snapshot rules so a deploy failing + // either check is rejected with both sets of messages. ClusterId-ordered for a deterministic summary. + // R03:3.2 — both come off the SAME snapshot read as every other table (one SNAPSHOT + // transaction), not a second query that could observe a later edit. + var clusters = draft.ServerClusters.OrderBy(c => c.ClusterId, StringComparer.Ordinal).ToList(); + var nodesByCluster = draft.ClusterNodes + .GroupBy(n => n.ClusterId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); + foreach (var cluster in clusters) + { + var nodes = nodesByCluster.TryGetValue(cluster.ClusterId, out var ns) + ? (IReadOnlyList)ns + : []; + errors.AddRange(DraftValidator.ValidateClusterTopology(cluster, nodes)); + } + + // Deploy-time TagConfig strictness inspection over equipment tags. The + // driver-typed EquipmentTagConfigInspector surfaces every blob the driver's own reader would + // REJECT — a present-but-invalid enum value, an absent or malformed address field, a + // structurally-unparseable TagConfig — each of which makes the driver skip the tag so it reads + // BadNodeIdUnknown forever. (The runtime used to silently default a typo'd enum; + // it now rejects the tag, so these warnings describe rejections.) In Warn mode + // (default) these are non-blocking (logged + appended to the accepted result Message); in Error + // mode they fold into the reject list so a typo'd config cannot be re-deployed until fixed. + // Running servers are untouched either way — the gate only sees re-deploys. + // v3: a raw Tag's driver is resolved via its Device (Tag no longer carries EquipmentId / + // DriverInstanceId). Inspect EVERY raw tag's TagConfig against its device's driver type. + var driverTypeById = draft.DriverInstances + .GroupBy(d => d.DriverInstanceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DriverType, StringComparer.Ordinal); + var driverByDevice = draft.Devices + .GroupBy(d => d.DeviceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DriverInstanceId, StringComparer.Ordinal); + var deviceConfigById = draft.Devices + .GroupBy(d => d.DeviceId, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First().DeviceConfig, StringComparer.Ordinal); + var tagConfigWarnings = new List<(string TagId, string Text)>(); + foreach (var t in draft.Tags) + { + if (!driverByDevice.TryGetValue(t.DeviceId, out var driverInstanceId)) continue; + if (!driverTypeById.TryGetValue(driverInstanceId, out var driverType)) continue; + foreach (var w in EquipmentTagConfigInspector.Inspect( + driverType, t.TagConfig, deviceConfigById.GetValueOrDefault(t.DeviceId))) + tagConfigWarnings.Add((t.TagId, $"tag '{t.TagId}': {w}")); + } + if (tagConfigWarnings.Count > 0) + { + if (_tagConfigValidationMode == TagConfigValidationMode.Error) + errors.AddRange(tagConfigWarnings.Select(x => new ValidationError("TagConfigInvalid", x.Text, x.TagId))); + else + foreach (var x in tagConfigWarnings) + _log.Warning("StartDeployment: TagConfig warning (Warn mode, non-blocking): {Warning}", x.Text); + } + + if (errors.Count > 0) + { + var summary = string.Join("; ", errors.Select(e => $"[{e.Code}] {e.Message}")); + _log.Warning("StartDeployment rejected ({Count} validation error(s)): {Summary}", + errors.Count, summary); + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.Rejected, + DeploymentId: null, + RevisionHash: null, + Message: summary, + msg.CorrelationId), + Dispatch: null); + } + + // Warn-only compile-cost guardrail (NEVER blocks). Each genuinely-compiled (non-passthrough) + // script costs ~1.66 MiB RSS per node to materialize via Roslyn (measured post-A0; design doc + // 2026-06-07). Passthrough "mirror" scripts compile to ~nothing, so they are excluded; distinct + // sources are counted (the compile cache keys on source, so duplicates collapse to one unit). + // This only surfaces the estimate to the operator — the DraftValidator gate above is the hard + // reject; this advisory rides in the Accepted Message so the UI can show it. + // advisory is declared outside the inner try so the Accepted reply below can still read it even + // when the guardrail throws. A failed estimate must NEVER block a valid deploy — the caller's + // failure branch is reserved for genuine seal/save failures. + string? advisory = null; + try + { + const double PerScriptMiB = 1.66; // measured post-A0 per-script RSS (design doc 2026-06-07) + // Projected from the snapshot already in hand — Script.SourceCode used to be read three times. + var compiled = draft.Scripts + .Select(s => s.SourceCode) + .Where(src => !PassthroughScript.TryMatch(src, out _)) + .Distinct(StringComparer.Ordinal) + .Count(); + if (compiled > 0) + { + var estMiB = compiled * PerScriptMiB; + _log.Warning( + "StartDeployment: {Compiled} script(s) will compile (~{EstMiB:F0} MiB RSS per node); ensure node mem_limit covers it", + compiled, estMiB); + advisory = $"{compiled} script(s) will compile (~{estMiB:F0} MiB/node)"; + } + } + catch (Exception ex) + { + // Guardrail is advisory-only — a failed estimate must never block a valid deploy. + _log.Warning(ex, "StartDeployment: script compile-cost estimate failed; advisory skipped"); + advisory = null; + } + + // Composed from the validated snapshot, never from a second read: an edit that lands between + // validation and here is simply not in this artifact (R03:3.2). + var artifact = ConfigComposer.Compose(draft); + var deploymentId = DeploymentId.NewId(); + var revHash = RevisionHash.Parse(artifact.RevisionHash); + + db.Deployments.Add(new Deployment + { + DeploymentId = deploymentId.Value, + RevisionHash = artifact.RevisionHash, + Status = DeploymentStatus.Dispatching, + CreatedBy = msg.CreatedBy, + ArtifactBlob = artifact.Blob, + }); + + // Marker ConfigEdit row so the audit timeline shows the deployment snapshot. + db.ConfigEdits.Add(new ConfigEdit + { + EntityType = "Deployment", + EntityId = deploymentId.Value, + FieldsJson = $"{{\"revisionHash\":\"{artifact.RevisionHash}\",\"sizeBytes\":{artifact.Blob.Length}}}", + EditedBy = msg.CreatedBy, + SourceNode = sourceNode, + }); + + await db.SaveChangesAsync(); + + // Accepted message = the compile-cost advisory (if any) + any Warn-mode TagConfig warnings, so + // the operator sees the non-blocking strictness findings on a successful deploy. + var messageParts = new List(); + if (advisory is not null) messageParts.Add(advisory); + messageParts.AddRange(tagConfigWarnings.Select(x => x.Text)); + var acceptedMessage = messageParts.Count > 0 ? string.Join("; ", messageParts) : null; + + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.Accepted, + deploymentId, + revHash, + Message: acceptedMessage, + msg.CorrelationId), + new DispatchDeployment(deploymentId, revHash, msg.CorrelationId)); + } + + /// + /// Test Connect runs on the thread pool and pipes its reply straight to the requester: a probe + /// against a black-holed PLC (60 s clamp) must not suspend the singleton's mailbox (R03:3.1). Every + /// path inside the task yields a , so the requester never + /// sees a Status.Failure. + /// + private void HandleTestDriverConnect(TestDriverConnect msg) { var replyTo = Sender; if (!_probesByType.TryGetValue(msg.DriverType, out var probe)) @@ -396,6 +483,11 @@ public sealed class AdminOperationsActor : ReceiveActor return; } + Task.Run(() => ProbeAsync(probe, msg)).PipeTo(replyTo); + } + + private async Task ProbeAsync(IDriverProbe probe, TestDriverConnect msg) + { var clampedSec = Math.Clamp(msg.TimeoutSeconds, 1, 60); var timeout = TimeSpan.FromSeconds(clampedSec); using var cts = new CancellationTokenSource(timeout); @@ -405,28 +497,28 @@ public sealed class AdminOperationsActor : ReceiveActor { var result = await probe.ProbeAsync(msg.ConfigJson, timeout, cts.Token); sw.Stop(); - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( result.Ok, result.Message, result.Ok ? sw.Elapsed.TotalMilliseconds : (double?)null, - msg.CorrelationId)); + msg.CorrelationId); } catch (OperationCanceledException) { - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( false, $"Probe timed out after {clampedSec}s.", null, - msg.CorrelationId)); + msg.CorrelationId); } catch (Exception ex) { _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType); - replyTo.Tell(new TestDriverConnectResult( + return new TestDriverConnectResult( false, ex.Message, null, - msg.CorrelationId)); + msg.CorrelationId); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs new file mode 100644 index 00000000..95a2b0fc --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorMailboxTests.cs @@ -0,0 +1,159 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:3.1 — the admin singleton's mailbox used to be suspended for the whole of every +/// ReceiveAsync: one Test Connect against a black-holed PLC (60 s clamp) or one large deploy +/// compose stalled every other admin operation fleet-wide, alarm acks included. Both now run off the +/// actor thread; only the "is one in flight" check stays serialised. +/// +public sealed class AdminOperationsActorMailboxTests : ControlPlaneActorTestBase +{ + private static readonly TimeSpan AckBudget = TimeSpan.FromSeconds(1); + + /// An alarm ack completes in under a second while a 60 s Test Connect is still pending. + [Fact] + public void Alarm_ack_completes_while_a_60s_test_connect_is_pending() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe("coord"); + var probe = new BlockingProbe(); + var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe })); + var connectClient = CreateTestProbe("connect-client"); + + connectClient.Send(actor, new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 60, Guid.NewGuid())); + probe.Started.Wait(PresenceBudget).ShouldBeTrue("the probe must be in flight before the ack is sent"); + + actor.Tell(new AcknowledgeAlarmCommand("alarm-1", "op", null, CorrelationId.NewId())); + var ack = ExpectMsg(AckBudget); + ack.Ok.ShouldBeTrue(); + + // Positive control: the probe reply still lands, with the right sender, once the probe returns. + connectClient.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + probe.Release(new DriverProbeResult(true, null, TimeSpan.FromMilliseconds(5))); + var result = connectClient.ExpectMsg(PresenceBudget); + result.Ok.ShouldBeTrue(); + result.LatencyMs.ShouldNotBeNull(); + } + + /// A probe that times out still answers its own sender with the clamped timeout message. + [Fact] + public void Test_connect_timeout_is_reported_to_its_sender() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe("coord"); + var probe = new BlockingProbe(); // never released: the CTS fires at the 1 s clamp floor + var actor = Sys.ActorOf(AdminOperationsActor.Props(dbFactory, coordinator.Ref, new IDriverProbe[] { probe })); + + actor.Tell(new TestDriverConnect("Blocking", "{}", TimeoutSeconds: 1, Guid.NewGuid())); + + var result = ExpectMsg(PresenceBudget); + result.Ok.ShouldBeFalse(); + result.Message.ShouldNotBeNull(); + result.Message.ShouldContain("timed out"); + } + + /// + /// The deploy compose (snapshot + validate + gzip + row write) runs off the actor: an alarm ack + /// completes while the compose is parked on a gated DB context, and the deploy still completes + /// (Accepted + dispatched) once the gate opens. + /// + [Fact] + public void Alarm_ack_completes_while_a_deploy_compose_is_in_flight() + { + var gated = new GatedDbFactory(NewInMemoryDbFactory()); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty())); + var deployer = CreateTestProbe("deployer"); + + gated.Hold(); + deployer.Send(actor, new StartDeployment("joe", CorrelationId.NewId())); + gated.Entered.Wait(PresenceBudget).ShouldBeTrue("the compose must be parked before the ack is sent"); + + actor.Tell(new AcknowledgeAlarmCommand("alarm-2", "op", null, CorrelationId.NewId())); + ExpectMsg(AckBudget).Ok.ShouldBeTrue(); + + // Positive control: the deploy completes once released. + deployer.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); + gated.Release(); + deployer.ExpectMsg(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.ExpectMsg(PresenceBudget); + } + + /// + /// Only the in-flight check is serialised — and it must still catch a second request that arrives + /// while the first compose has not yet written its row (the DB alone would let both through). + /// + [Fact] + public void Second_StartDeployment_during_an_in_flight_compose_is_refused() + { + var gated = new GatedDbFactory(NewInMemoryDbFactory()); + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(gated, coordinator.Ref, Enumerable.Empty())); + var first = CreateTestProbe("first"); + var second = CreateTestProbe("second"); + + gated.Hold(); + first.Send(actor, new StartDeployment("joe", CorrelationId.NewId())); + gated.Entered.Wait(PresenceBudget).ShouldBeTrue(); + + second.Send(actor, new StartDeployment("kim", CorrelationId.NewId())); + second.ExpectMsg(AckBudget).Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + + gated.Release(); + first.ExpectMsg(PresenceBudget).Outcome.ShouldBe(StartDeploymentOutcome.Accepted); + coordinator.ExpectMsg(PresenceBudget); + } + + private sealed class BlockingProbe : IDriverProbe + { + private readonly TaskCompletionSource _tcs = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ManualResetEventSlim Started { get; } = new(); + public string DriverType => "Blocking"; + + public async Task ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) + { + Started.Set(); + return await _tcs.Task.WaitAsync(ct); + } + + public void Release(DriverProbeResult result) => _tcs.TrySetResult(result); + } + + /// + /// Passes synchronous context creation through (the serialised in-flight check) and parks the + /// asynchronous one (the off-actor compose) on a gate while is in effect. + /// + private sealed class GatedDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private volatile TaskCompletionSource? _gate; + + public ManualResetEventSlim Entered { get; } = new(); + + public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _gate?.TrySetResult(); + + public OtOpcUaConfigDbContext CreateDbContext() => inner.CreateDbContext(); + + public async Task CreateDbContextAsync(CancellationToken cancellationToken = default) + { + Entered.Set(); + if (_gate is { } gate) await gate.Task.WaitAsync(cancellationToken); + return inner.CreateDbContext(); + } + } +} From 2e9160317f23847f01cbc29325b39480d64eb7b5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 06:45:03 -0400 Subject: [PATCH 05/51] feat(scripting): deny System.Threading, Type reflection entry points and RuntimeHelpers in the sandbox (E2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R05:10 — the namespace deny-list covered System.Threading.Tasks but not System.Threading, so SemaphoreSlim / Monitor / SpinWait / Mutex / EventWaitHandle could park a script's pool thread past its budget (or touch a system-global kernel object); System.Type.GetType(string) / GetMethod* / InvokeMember and System.Runtime.CompilerServices.RuntimeHelpers resolved and reach a forbidden type without naming it syntactically. ForbiddenTypeAnalyzer now denies the whole System.Threading namespace with a type-granular allow-list (CancellationToken, Interlocked, Volatile — the sync-free primitives), adds a member deny-list for System.Type's reflection entry points (GetType/GetMethod(s)/InvokeMember/GetProperty(ies)/GetField(s)/ GetMember(s)/GetConstructor(s)/GetEvent(s)/GetDefaultMembers; instance object.GetType() stays allowed) and puts RuntimeHelpers on the type deny-list. Thread/ThreadPool/Timer leave the type list — the prefix rule reaches them. R05:11 — the allocation budget landed with E1's ScriptRunGuard (Scripting:MaxAllocBytesPerRun, default 64 MB): sampled around each run, breach quarantines. It is observation after the fact; a pre-emptive OOM guard still needs an out-of-process runner (follow-up issue in the lane report). The Monaco editor shares the analyzer, so DiagnoseTests pins every new form as an OTSCRIPT_FORBIDDEN marker and every allow-listed form as clean. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ForbiddenTypeAnalyzer.cs | 191 ++++++++++++------ .../ScriptSandboxHardeningTests.cs | 148 ++++++++++++++ .../ScriptAnalysis/DiagnoseTests.cs | 20 ++ 3 files changed, 300 insertions(+), 59 deletions(-) create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs index 005734b6..5f7d8140 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ForbiddenTypeAnalyzer.cs @@ -17,13 +17,15 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// /// /// -/// Deny-list is the authoritative Phase 7 plan set: +/// Deny-list is the authoritative Phase 7 plan set, hardened since: /// System.IO, System.Net, System.Diagnostics, -/// System.Reflection, System.Threading.Thread, -/// System.Threading.Tasks (scripts are synchronous predicates — no -/// legitimate need to start background tasks; a Task.Run fan-out outlives -/// the evaluation timeout entirely), System.Runtime.InteropServices, -/// Microsoft.Win32. +/// System.Reflection, System.Threading.Tasks and the whole of +/// System.Threading (scripts are synchronous predicates — no legitimate need +/// to start background work, and no blocking primitive may park the evaluation's pool +/// thread past its budget; CancellationToken / Interlocked / +/// Volatile are allow-listed back in), System.Runtime.InteropServices, +/// System.Runtime.Loader, Microsoft.Win32; plus the member-granular +/// (System.Type's reflection entry points). /// /// /// Deny-list prefix match. System.Net catches System.Net.Http, @@ -63,13 +65,19 @@ public static class ForbiddenTypeAnalyzer "System.Net", "System.Diagnostics", // catches Process, ProcessStartInfo, EventLog, Trace/Debug file sinks "System.Reflection", - // System.Threading.Thread is NOT in this list: Thread's containing namespace is - // "System.Threading" (not "System.Threading.Thread"), so a prefix check on - // "System.Threading.Thread" never matches. Thread is denied type-granularly via - // ForbiddenFullTypeNames instead so the check actually fires. "System.Threading.Tasks", // Task.Run / Parallel — scripts are synchronous predicates // and have no legitimate need to start background work; // a Task fan-out outlives the evaluation timeout entirely. + // Listed BEFORE the parent prefix so its rejection names it. + "System.Threading", // Thread / ThreadPool / Timer (background work that outlives + // the timeout) AND every blocking primitive — SemaphoreSlim, + // Monitor, SpinWait, ManualResetEvent(Slim), Mutex, + // EventWaitHandle — which park the script's pool thread past + // its budget (the same orphan ScriptRunGuard exists to bound) + // and, for the named kernel objects, touch system-global + // state. The few sync-free primitives a script may + // legitimately name are allow-listed in + // AllowedTypesInForbiddenNamespaces. "System.Runtime.InteropServices", "System.Runtime.Loader", // AssemblyLoadContext + AssemblyDependencyResolver — // arbitrary DLL load into the host process. @@ -79,11 +87,55 @@ public static class ForbiddenTypeAnalyzer "Microsoft.Win32", // registry ]; + /// + /// Types that stay reachable although their namespace is in + /// . Matched by exact fully-qualified name against the + /// resolved type symbol (generic constructions unwrapped). The whole of System.Threading is + /// denied; these three are the sync-free primitives with no blocking surface — a + /// is read-only state, and + /// are single-instruction memory operations. + /// + public static readonly IReadOnlyList AllowedTypesInForbiddenNamespaces = + [ + "System.Threading.CancellationToken", + "System.Threading.Interlocked", + "System.Threading.Volatile", + ]; + + /// + /// Members scripts are NOT allowed to call on otherwise-allowed types, as + /// (declaring type, member name). System.Type must stay reachable + /// (typeof, Name, IsValueType are ordinary metadata reads) but its reflection + /// entry points obtain a type or member by NAME — Type.GetType("System.IO.File") reaches a + /// forbidden type without ever naming it syntactically, and GetMethod* / GetProperty + /// / InvokeMember / … hand back a System.Reflection handle (which pass 2 catches when + /// declared, but not when consumed inline). Instance object.GetType() is declared on + /// System.Object and is not matched. + /// + public static readonly IReadOnlyList<(string Type, string Member)> ForbiddenMembers = + [ + ("System.Type", "GetType"), + ("System.Type", "GetMethod"), + ("System.Type", "GetMethods"), + ("System.Type", "InvokeMember"), + ("System.Type", "GetProperty"), + ("System.Type", "GetProperties"), + ("System.Type", "GetField"), + ("System.Type", "GetFields"), + ("System.Type", "GetMember"), + ("System.Type", "GetMembers"), + ("System.Type", "GetConstructor"), + ("System.Type", "GetConstructors"), + ("System.Type", "GetEvent"), + ("System.Type", "GetEvents"), + ("System.Type", "GetDefaultMembers"), + ]; + /// /// Fully-qualified type names scripts are NOT allowed to reference, regardless of /// namespace. Used when a dangerous type shares its namespace with legitimate types - /// (e.g. System.Threading hosts both CancellationToken and - /// Thread) so a namespace-prefix rule would over-block. Matched by exact + /// (e.g. System.Runtime.CompilerServices hosts both MethodImplOptions and + /// RuntimeHelpers) so a namespace-prefix rule would over-block. Matched by exact /// fully-qualified name against the resolved type symbol — every member of /// the type (including read-only ones) is therefore rejected. /// @@ -99,19 +151,22 @@ public static class ForbiddenTypeAnalyzer /// System.Activator — CreateInstance is a /// reflection-equivalent escape that constructs a forbidden type by name /// without ever naming it syntactically. - /// System.Threading.Thread — raw thread creation bypasses the - /// per-evaluation timeout; denied type-granularly because its containing - /// namespace is System.Threading (shared with allowed types like - /// CancellationToken), so a namespace-prefix rule cannot reach it - /// without blocking unrelated types. - /// System.Threading.ThreadPool / System.Threading.Timer — - /// background-work vectors that outlive the per-evaluation timeout; same - /// threat as Task.Run. /// System.Runtime.CompilerServices.Unsafe — exposes raw type- /// reinterpretation (As<TFrom,TTo>, As<T>(object)) /// that bypasses the CLR type system without requiring an unsafe context at /// the call site; enables type-confusion and managed heap corruption. + /// System.Runtime.CompilerServices.RuntimeHelpers — + /// GetUninitializedObject constructs an instance of any type (a forbidden + /// one included) with no constructor and no syntactic type name beyond a + /// Type value; PrepareMethod / RunClassConstructor reach + /// runtime internals. /// + /// + /// System.Threading.Thread / ThreadPool / Timer used to be listed + /// here; since the whole System.Threading namespace is denied (with a type-granular + /// allow-list, see ) the prefix rule reaches + /// them first. + /// /// public static readonly IReadOnlyList ForbiddenFullTypeNames = [ @@ -119,26 +174,6 @@ public static class ForbiddenTypeAnalyzer "System.AppDomain", "System.GC", "System.Activator", - // System.Threading.Thread lives in the System.Threading namespace (shared with - // CancellationToken, SemaphoreSlim, etc.), so a namespace-prefix deny-list cannot - // target it without blocking those legitimate types. Denied type-granularly here. - "System.Threading.Thread", - // Broadening the references list to the BCL trusted-platform-assemblies set - // re-exposed two background-work vectors the original deny-list missed. Both live - // in System.Threading (shared with allowed sync primitives like CancellationToken / - // SemaphoreSlim), so they must be denied type-granularly: - // - // System.Threading.ThreadPool — QueueUserWorkItem / UnsafeQueueUserWorkItem - // re-introduce the background-fanout threat - // closed against System.Threading.Tasks. - // System.Threading.Timer — Timer(callback, …) schedules unbounded work - // that outlives the per-evaluation timeout. - // - // System.Runtime.Loader.AssemblyLoadContext is also covered, but at the namespace- - // prefix level above (System.Runtime.Loader) so future BCL additions to that - // namespace are denied by default. - "System.Threading.ThreadPool", - "System.Threading.Timer", // System.Runtime.CompilerServices.Unsafe exposes raw type-reinterpretation // (Unsafe.As, Unsafe.As(object)) that bypasses the CLR type system // without requiring an 'unsafe' compilation context at the call site. In a script, @@ -148,6 +183,11 @@ public static class ForbiddenTypeAnalyzer // compile-time-only types (CallerMemberNameAttribute, MethodImplAttribute, // MethodImplOptions) that scripts may legitimately reference. "System.Runtime.CompilerServices.Unsafe", + // RuntimeHelpers.GetUninitializedObject(Type) builds an instance of ANY type — a + // forbidden one obtained through Type.GetType (itself denied via ForbiddenMembers) + // included — without a constructor or a syntactic type name. Same namespace caveat + // as Unsafe above, so type-granular. + "System.Runtime.CompilerServices.RuntimeHelpers", ]; /// @@ -277,28 +317,61 @@ public static class ForbiddenTypeAnalyzer if (rejections.Any(r => r.Span == span && r.TypeName == typeName)) return; - var ns = typeSymbol.ContainingNamespace?.ToDisplayString() ?? string.Empty; - foreach (var forbidden in ForbiddenNamespacePrefixes) - { - if (ns == forbidden || ns.StartsWith(forbidden + ".", StringComparison.Ordinal)) - { - rejections.Add(new ForbiddenTypeRejection( - Span: span, - TypeName: typeName, - Namespace: ns, - Message: $"Type '{typeName}' is in the forbidden namespace '{ns}'. " + - $"Scripts cannot reach {forbidden}* per Phase 7 sandbox rules.")); - return; - } - } - - // Type-granular deny-list — dangerous types that live in the allow-listed - // System namespace and so cannot be caught by ForbiddenNamespacePrefixes. - // Matched on the full type name; OriginalDefinition unwraps any generic - // construction before naming. + // Full name of the (unwrapped) type — shared by the allow-list, the type deny-list and the + // member deny-list below. OriginalDefinition unwraps any generic construction before naming. var fullTypeName = typeSymbol.OriginalDefinition.ToDisplayString( SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle( SymbolDisplayGlobalNamespaceStyle.Omitted)); + + var ns = typeSymbol.ContainingNamespace?.ToDisplayString() ?? string.Empty; + + // Member deny-list first: the containing type (System.Type) is allowed, so neither list + // below would ever look at the member name. + if (symbol is IMethodSymbol method) + { + var declaring = method.ContainingType.OriginalDefinition.ToDisplayString( + SymbolDisplayFormat.FullyQualifiedFormat.WithGlobalNamespaceStyle( + SymbolDisplayGlobalNamespaceStyle.Omitted)); + foreach (var (type, member) in ForbiddenMembers) + { + if (declaring == type && method.Name == member) + { + rejections.Add(new ForbiddenTypeRejection( + Span: span, + TypeName: $"{type}.{member}", + Namespace: ns, + Message: $"Member '{type}.{member}' is on the sandbox forbidden-member deny-list: " + + "it obtains a type or member by name, which bypasses the type allow-list " + + "(see ForbiddenTypeAnalyzer.ForbiddenMembers for the authoritative list).")); + return; + } + } + } + + // A type on the allow-list stays reachable even though its namespace is denied; it is still + // subject to the type-granular deny-list below (nothing is on both today). + var allowListed = AllowedTypesInForbiddenNamespaces.Contains(fullTypeName, StringComparer.Ordinal); + + if (!allowListed) + { + foreach (var forbidden in ForbiddenNamespacePrefixes) + { + if (ns == forbidden || ns.StartsWith(forbidden + ".", StringComparison.Ordinal)) + { + rejections.Add(new ForbiddenTypeRejection( + Span: span, + TypeName: typeName, + Namespace: ns, + Message: $"Type '{typeName}' is in the forbidden namespace '{ns}'. " + + $"Scripts cannot reach {forbidden}* per Phase 7 sandbox rules.")); + return; + } + } + } + + // Type-granular deny-list — dangerous types that live in an allow-listed namespace + // (System, System.Runtime.CompilerServices) and so cannot be caught by + // ForbiddenNamespacePrefixes. foreach (var forbiddenType in ForbiddenFullTypeNames) { if (fullTypeName == forbiddenType) diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs new file mode 100644 index 00000000..5c932832 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptSandboxHardeningTests.cs @@ -0,0 +1,148 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// Sandbox hardening (review R05:10): System.Threading is denied as a namespace with a +/// type-granular allow-list (CancellationToken, Interlocked, Volatile) so a +/// script can no longer park its pool thread on a blocking primitive or touch a system-global kernel +/// object; System.Type's reflection entry points (GetType(string), GetMethod*, +/// InvokeMember, …) and System.Runtime.CompilerServices.RuntimeHelpers are denied so a +/// forbidden type cannot be reached by name without ever being named syntactically. +/// +[Trait("Category", "Unit")] +public sealed class ScriptSandboxHardeningTests +{ + private static ScriptSandboxViolationException Rejects(string source) => + Should.Throw(() => + ScriptEvaluator.Compile(source)); + + // ── System.Threading blocking / synchronisation primitives ─────────────────────────────── + + [Fact] + public void Rejects_SemaphoreSlim_Wait() + { + var ex = Rejects("new System.Threading.SemaphoreSlim(0).Wait(); return 0;"); + ex.Rejections.ShouldContain(r => r.TypeName.Contains("SemaphoreSlim") && r.Namespace == "System.Threading"); + } + + [Fact] + public void Rejects_SpinWait_SpinUntil() + => Rejects("System.Threading.SpinWait.SpinUntil(() => false); return 0;"); + + [Fact] + public void Rejects_Monitor() + => Rejects("var o = new object(); System.Threading.Monitor.Enter(o); return 0;"); + + [Fact] + public void Rejects_named_Mutex_system_global_kernel_object() + => Rejects("using var m = new System.Threading.Mutex(false, \"Global\\\\otopcua\"); return 0;"); + + [Fact] + public void Rejects_ManualResetEventSlim() + => Rejects("var e = new System.Threading.ManualResetEventSlim(false); e.Wait(); return 0;"); + + [Fact] + public void Rejects_EventWaitHandle() + => Rejects("var e = new System.Threading.EventWaitHandle(false, System.Threading.EventResetMode.ManualReset); return 0;"); + + [Fact] + public void Rejects_Thread_ThreadPool_and_Timer_still() + { + Rejects("var t = new System.Threading.Thread(() => { }); return 0;"); + Rejects("System.Threading.ThreadPool.QueueUserWorkItem(_ => { }); return 0;"); + Rejects("var t = new System.Threading.Timer(_ => { }); return 0;"); + } + + [Fact] + public void Rejects_Thread_Sleep() + => Rejects("System.Threading.Thread.Sleep(10); return 0;"); + + // ── the allow-list inside System.Threading ─────────────────────────────────────────────── + + [Fact] + public async Task Allows_CancellationToken() + { + using var ev = ScriptEvaluator.Compile( + "var t = System.Threading.CancellationToken.None; return t.IsCancellationRequested ? 1 : 0;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(0); + } + + [Fact] + public async Task Allows_Interlocked() + { + using var ev = ScriptEvaluator.Compile( + "var x = 41; System.Threading.Interlocked.Increment(ref x); return x;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(42); + } + + [Fact] + public async Task Allows_Volatile() + { + using var ev = ScriptEvaluator.Compile( + "var x = 42; return System.Threading.Volatile.Read(ref x);"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(42); + } + + // ── System.Type reflection entry points ────────────────────────────────────────────────── + + [Fact] + public void Rejects_Type_GetType_by_name() + { + var ex = Rejects("var t = System.Type.GetType(\"System.IO.File\"); return t is null ? 0 : 1;"); + ex.Rejections.ShouldContain(r => r.Message.Contains("GetType")); + } + + [Fact] + public void Rejects_Type_GetMethod() + => Rejects("var m = typeof(string).GetMethod(\"Trim\"); return m is null ? 0 : 1;"); + + [Fact] + public void Rejects_Type_GetMethods() + => Rejects("return typeof(string).GetMethods().Length;"); + + [Fact] + public void Rejects_Type_InvokeMember() + => Rejects("""return (int)typeof(string).InvokeMember("Length", System.Reflection.BindingFlags.GetProperty, null, "abc", null)!;"""); + + [Fact] + public void Rejects_Type_GetProperty_GetField_GetMember_GetConstructor_GetEvent() + { + Rejects("return typeof(string).GetProperty(\"Length\") is null ? 0 : 1;"); + Rejects("return typeof(string).GetField(\"Empty\") is null ? 0 : 1;"); + Rejects("return typeof(string).GetMember(\"Length\").Length;"); + Rejects("return typeof(string).GetConstructors().Length;"); + Rejects("return typeof(string).GetEvent(\"x\") is null ? 0 : 1;"); + } + + [Fact] + public async Task Allows_instance_GetType_and_Type_metadata_reads() + { + // object.GetType() is declared on System.Object, and Type.Name / IsValueType are plain metadata + // reads — neither is a reflection entry point, and both are common in authored scripts. + using var ev = ScriptEvaluator.Compile( + "object o = 42; var t = o.GetType(); return t.IsValueType && typeof(int).Name == \"Int32\" ? 1 : 0;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(1); + } + + // ── RuntimeHelpers ─────────────────────────────────────────────────────────────────────── + + [Fact] + public void Rejects_RuntimeHelpers() + { + var ex = Rejects( + "var o = System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(string)); return o is null ? 0 : 1;"); + ex.Rejections.ShouldContain(r => r.TypeName.Contains("RuntimeHelpers")); + } + + [Fact] + public async Task Allows_compile_time_only_CompilerServices_attributes() + { + // The namespace stays open for the harmless compile-time types (MethodImplOptions etc.). + using var ev = ScriptEvaluator.Compile( + "var o = System.Runtime.CompilerServices.MethodImplOptions.NoInlining; return (int)o;"); + (await ev.RunAsync(new FakeScriptContext(), TestContext.Current.CancellationToken)).ShouldBe(8); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs index 21460637..94784a45 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/ScriptAnalysis/DiagnoseTests.cs @@ -31,6 +31,26 @@ public sealed class DiagnoseTests => Diag("""var c = new System.Net.Http.HttpClient(); return 0;""") .ShouldContain(x => x.Severity == 8); + // Sandbox hardening parity (R05:10): the editor runs the SAME ForbiddenTypeAnalyzer the publish gate + // runs, so every newly-denied form must surface as an OTSCRIPT_FORBIDDEN marker here — editor accepts + // ⇔ publish accepts. One case per deny mechanism (namespace, member, type), plus the allow-list. + [Theory] + [InlineData("""new System.Threading.SemaphoreSlim(0).Wait(); return 0;""")] // System.Threading namespace + [InlineData("""System.Threading.SpinWait.SpinUntil(() => false); return 0;""")] + [InlineData("""var t = System.Type.GetType("System.IO.File"); return t is null ? 0 : 1;""")] // forbidden member + [InlineData("""return typeof(string).GetMethods().Length;""")] + [InlineData("""var o = System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(string)); return 0;""")] // forbidden type + public void Hardened_sandbox_forms_are_flagged_in_the_editor(string code) + => Diag(code).ShouldContain(x => x.Code == "OTSCRIPT_FORBIDDEN" && x.Severity == 8); + + [Theory] + [InlineData("""var t = System.Threading.CancellationToken.None; return t.IsCancellationRequested ? 1 : 0;""")] + [InlineData("""var x = 41; System.Threading.Interlocked.Increment(ref x); return x;""")] + [InlineData("""var x = 42; return System.Threading.Volatile.Read(ref x);""")] + [InlineData("""object o = 42; return o.GetType().IsValueType ? 1 : 0;""")] + public void Allow_listed_forms_are_not_flagged_in_the_editor(string code) + => Diag(code).ShouldBeEmpty(); + [Fact] public void Dynamic_tag_path_is_flagged() => Diag("""var p = "A"; return ctx.GetTag(p).Value;""") .ShouldContain(x => x.Message.Contains("literal")); From f5579e6ef2ba48358814ea93f143d276d9a72867 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:15:05 -0400 Subject: [PATCH 06/51] fix(host): make a failed OPC UA SDK start visible and stop the SDK on StopAsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed SDK start (port in use, PKI unwritable, bad OpcUa:* override) was one Error line: the node stayed /health/ready 200, joined its mesh, could be elected driver Primary and acked deploys through the Null sink while every client got connection-refused. - IOpcUaEndpointState / OpcUaEndpointState: the start outcome is a fact the hosted service maintains (NotStarted / Started / Failed / Stopped + reason). - OpcUaEndpointHealthCheck ("opcua-endpoint", Ready + Active) is Unhealthy unless Started; registered by AddOtOpcUaHealth(hasAdmin, hasDriver) on driver-role nodes. - A driver-only node rethrows the start failure (Program.cs's fatal handler exits non-zero so the supervisor restarts it); a fused node keeps its AdminUI up but is visibly not ready. - StopAsync detaches the sinks and then STOPS the SDK (the port closes, clients get connection-refused rather than a stale, inert endpoint); DisposeAsync is idempotent with it. - The ServiceLevel calculator / redundancy election live in Cluster + Runtime (not this lane); the fact is exposed for the Wave 3 follow-up to consume. Tests: port-in-use ⇒ state Failed, probe Unhealthy, real /health/ready and /health/active 503 (positive control: 200 once Started); driver-only rethrows; StopAsync closes the port and DisposeAsync twice is a no-op; the probe registration is on both tiers for driver nodes only. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Health/HealthEndpoints.cs | 21 +- .../Health/OpcUaEndpointHealthCheck.cs | 45 ++++ .../OpcUa/IOpcUaEndpointState.cs | 73 ++++++ .../OpcUa/OtOpcUaServerHostedService.cs | 124 ++++++++-- src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 10 +- .../Health/ActiveTierRegistrationTests.cs | 28 +++ ...cUaServerHostedServiceStartFailureTests.cs | 213 ++++++++++++++++++ 7 files changed, 492 insertions(+), 22 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index f7a740bd..2f57158f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.LocalDb.Replication; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Host.Configuration; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; namespace ZB.MOM.WW.OtOpcUa.Host.Health; @@ -28,11 +29,29 @@ public static class HealthEndpoints /// factory, so probing it would throw when the readiness endpoint resolves the factory. Defaults to /// true so admin/fused callers (including the test harnesses) keep the probe unchanged. /// + /// + /// Whether this node carries the driver role and therefore serves an OPC UA endpoint. When + /// true the joins the ready + active tiers (R03:2.1), reading + /// the the host registers alongside OtOpcUaServerHostedService. + /// Defaults to false so admin-only callers and the existing harnesses register nothing new. + /// /// The same service collection, for chaining. - public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true) + public static IServiceCollection AddOtOpcUaHealth(this IServiceCollection services, bool hasAdmin = true, bool hasDriver = false) { var checks = services.AddHealthChecks(); + // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of + // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or + // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe. + if (hasDriver) + { + checks.Add(new HealthCheckRegistration( + OpcUaEndpointHealthCheck.Name, + sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService()), + failureStatus: null, + tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })); + } + // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config // from central via ConfigSource:Mode=FetchAndCache and never registers the ConfigDb factory, so // the type-activated DatabaseHealthCheck would throw resolving it. Admin + fused nodes: unchanged. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs new file mode 100644 index 00000000..8caa69f5 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Health; + +/// +/// Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is +/// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy +/// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets +/// connection-refused. +/// +/// +/// Registered on driver-role nodes only (); an +/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. +/// +public sealed class OpcUaEndpointHealthCheck : IHealthCheck +{ + /// The registration name, as it appears in the health JSON. + public const string Name = "opcua-endpoint"; + + private readonly IOpcUaEndpointState _state; + + /// Initializes a new instance of the class. + /// The endpoint lifecycle fact the hosted service maintains. + public OpcUaEndpointHealthCheck(IOpcUaEndpointState state) + { + ArgumentNullException.ThrowIfNull(state); + _state = state; + } + + /// + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + var data = new Dictionary { ["status"] = _state.Status.ToString() }; + var result = _state.Status switch + { + OpcUaEndpointStatus.Started => HealthCheckResult.Healthy("OPC UA endpoint is listening", data), + OpcUaEndpointStatus.NotStarted => HealthCheckResult.Unhealthy("OPC UA endpoint has not started yet", data: data), + OpcUaEndpointStatus.Stopped => HealthCheckResult.Unhealthy("OPC UA endpoint is stopped", data: data), + _ => HealthCheckResult.Unhealthy( + "OPC UA SDK failed to start: " + (_state.FailureReason ?? "no reason recorded"), data: data), + }; + return Task.FromResult(result); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs new file mode 100644 index 00000000..c2377ea1 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IOpcUaEndpointState.cs @@ -0,0 +1,73 @@ +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// Lifecycle of this node's OPC UA endpoint as drives it. +public enum OpcUaEndpointStatus +{ + /// The hosted service has not run StartAsync yet. + NotStarted, + + /// The SDK is listening and the address-space / ServiceLevel sinks are bound. + Started, + + /// The SDK failed to start (port in use, PKI unwritable, bad OpcUa:* override, …). + Failed, + + /// The hosted service stopped the SDK; the port is released. + Stopped, +} + +/// +/// The one place the "did the OPC UA SDK actually start?" fact lives (R03:2.1). Before it, a failed +/// start was one Error line: /health/ready stayed 200, the node joined its mesh, could be +/// elected driver Primary and acked deploys as Applied through the Null sink — while clients got +/// connection-refused from the node the fleet called healthy. +/// +/// +/// Consumed by OpcUaEndpointHealthCheck (Ready + Active tiers). The redundancy election and the +/// ServiceLevel calculator do not read it yet — they live outside the Host (Cluster / +/// Runtime); feeding them is the Wave 3 follow-up the lane report names. Until then a fused +/// node whose SDK failed is visible (503) but still Primary-eligible. +/// +public interface IOpcUaEndpointState +{ + /// The endpoint's current lifecycle status. + OpcUaEndpointStatus Status { get; } + + /// Why the start failed, when is . + string? FailureReason { get; } +} + +/// Mutable ; written by the hosted service, read by the health check. +public sealed class OpcUaEndpointState : IOpcUaEndpointState +{ + private volatile OpcUaEndpointStatus _status = OpcUaEndpointStatus.NotStarted; + private volatile string? _failureReason; + + /// + public OpcUaEndpointStatus Status => _status; + + /// + public string? FailureReason => _failureReason; + + /// Records a successful SDK start. + public void MarkStarted() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Started; + } + + /// Records a failed SDK start. + /// The failure, as the health probe should report it. + public void MarkFailed(string reason) + { + _failureReason = reason; + _status = OpcUaEndpointStatus.Failed; + } + + /// Records that the hosted service stopped the SDK. + public void MarkStopped() + { + _failureReason = null; + _status = OpcUaEndpointStatus.Stopped; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs index 79f62cd8..609b0087 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Authorization; @@ -48,6 +49,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl private readonly IDriverPrimaryRoleSource? _primaryRoleSource; private readonly HistorianUpdaterHolder _historianUpdater; private readonly IHistoryUpdateRoleView? _historyUpdateRoleView; + private readonly OpcUaEndpointState _endpointState; + private readonly bool _rethrowStartFailure; private OpcUaApplicationHost? _appHost; private OtOpcUaSdkServer? _server; @@ -103,6 +106,13 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl /// The HistoryUpdate verdict DriverHostActor publishes (the /// data-plane Primary verdict, closed until this node's first redundancy snapshot). Optional: registered on every /// driver-role node; an admin-only node has none and wires no HistoryUpdate Primary gate (allow). + /// The endpoint lifecycle fact this service maintains for the + /// opcua-endpoint health probe (R03:2.1). Optional: a private instance is kept when none is registered, + /// so a harness that never probes health is unchanged. + /// The node's resolved roles. A node with the driver role and NO admin role exists only + /// to serve this endpoint, so a failed SDK start RETHROWS there (the host's fatal handler exits non-zero and + /// the supervisor restarts the process); a fused node keeps its AdminUI up and reports the failure through + /// the health probe instead. Null (harnesses) behaves as fused. public OtOpcUaServerHostedService( IOptions options, DeferredAddressSpaceSink deferredSink, @@ -119,7 +129,9 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl ILoggerFactory loggerFactory, IDriverPrimaryRoleSource? primaryRoleSource = null, HistorianUpdaterHolder? historianUpdater = null, - IHistoryUpdateRoleView? historyUpdateRoleView = null) + IHistoryUpdateRoleView? historyUpdateRoleView = null, + OpcUaEndpointState? endpointState = null, + EffectiveRoles? roles = null) { ArgumentNullException.ThrowIfNull(aclOptions); ArgumentNullException.ThrowIfNull(time); @@ -144,6 +156,8 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl _primaryRoleSource = primaryRoleSource; _historianUpdater = historianUpdater ?? HistorianUpdaterHolder.None; _historyUpdateRoleView = historyUpdateRoleView; + _endpointState = endpointState ?? new OpcUaEndpointState(); + _rethrowStartFailure = roles is { HasDriver: true, HasAdmin: false }; _logger = loggerFactory.CreateLogger(); } @@ -166,11 +180,22 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } catch (Exception ex) { + // R03:2.1 — the failure is a FACT now, not a log line: the opcua-endpoint probe turns Unhealthy + // (ready + active 503) and a driver-only node, which exists only to serve this endpoint, exits + // so its supervisor restarts it. A fused node keeps its AdminUI up so the operator can correct + // the OpcUa:* config without a process bounce — but it is visibly not ready. + _endpointState.MarkFailed(ex.Message); + await ReleaseFailedStartAsync().ConfigureAwait(false); + if (_rethrowStartFailure) + { + _logger.LogCritical(ex, + "OtOpcUaServerHostedService: SDK start failed on a driver-only node — nothing else on this " + + "node justifies staying up; failing host start so the supervisor restarts the process"); + throw; + } _logger.LogError(ex, - "OtOpcUaServerHostedService: SDK start failed; OpcUaPublishActor writes will continue to no-op"); - // Don't rethrow — the rest of the host (admin UI, driver actors, etc.) can still boot. - // Operators see the failure via the logs + can correct config without a process bounce - // of the whole binary. + "OtOpcUaServerHostedService: SDK start failed; /health/ready reports Unhealthy and " + + "OpcUaPublishActor writes will continue to no-op until the host is restarted with a corrected OpcUa section"); return; } @@ -178,6 +203,7 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogWarning( "OtOpcUaServerHostedService: SDK reported started but NodeManager is null; sink stays Null"); + _endpointState.MarkFailed("SDK reported started but the node manager is null"); return; } @@ -352,22 +378,16 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl { _logger.LogError(ex, "OtOpcUaServerHostedService: ServiceLevel binding failed; stopping the OPC UA endpoint while the admin host continues"); + _endpointState.MarkFailed("ServiceLevel binding failed: " + ex.Message); try { - await StopAsync(cancellationToken).ConfigureAwait(false); + DetachSinks(); } catch (Exception cleanupError) { _logger.LogError(cleanupError, "OtOpcUaServerHostedService: failed to detach OPC UA bindings after startup failure"); } - var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( - async () => await _server.StopAsync(CancellationToken.None).ConfigureAwait(false), - () => _appHost.DisposeAsync(), _logger).ConfigureAwait(false); - if (released) - { - _appHost = null; - _server = null; - } + await ReleaseEndpointAsync().ConfigureAwait(false); return; } @@ -375,9 +395,52 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // attached, so the first tick cannot land on a half-wired server. StartRevalidationTimer(); + _endpointState.MarkStarted(); _logger.LogInformation("OtOpcUaServerHostedService: SDK started, address-space + ServiceLevel sinks bound"); } + /// + /// Releases an SDK whose StartAsync threw. The application instance may hold a half-opened + /// listener (the port-in-use case binds nothing, but a certificate or PKI failure can fire after the + /// transport is up); disposing it is the only way to be sure. The state stays Failed. + /// + private async Task ReleaseFailedStartAsync() + { + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is null) return; + try + { + await appHost.DisposeAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "OtOpcUaServerHostedService: disposing the application host after a failed start threw"); + } + } + + /// + /// Stops the SDK and disposes the application host, releasing endpoint ownership only when both + /// succeeded (): on a failure the + /// fields are retained so retries at host disposal. + /// + private async Task ReleaseEndpointAsync() + { + var server = _server; + var appHost = _appHost; + if (server is null && appHost is null) return; + var released = await FailedServiceLevelBindingCleanup.StopAndDisposeAsync( + async () => { if (server is not null) await server.StopAsync(CancellationToken.None).ConfigureAwait(false); }, + () => appHost?.DisposeAsync() ?? ValueTask.CompletedTask, + _logger).ConfigureAwait(false); + if (released) + { + _appHost = null; + _server = null; + } + } + /// /// The ServerHistorian:HonorReturnBounds kill switch StartAsync applies to the node manager /// (default true when the section or key is absent). Exposed so the wiring is testable without booting the SDK. @@ -441,11 +504,30 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl } /// - /// Stops the OPC UA server asynchronously. + /// Stops the OPC UA server asynchronously: detaches every sink, then STOPS the SDK so the port closes. /// + /// + /// R03:1.4 — the SDK used to stay listening from here until host disposal (Akka coordinated shutdown, + /// the other hosted services' StopAsync), serving the last-written values with an emptied authority: + /// reads succeeded on stale data while writes got "unavailable", and clients saw a server that was + /// up but inert instead of the connection-refused that triggers their failover. + /// stays idempotent with this: ownership is nulled once the stop succeeded. + /// /// Cancellation token. /// A task that represents the asynchronous operation. - public Task StopAsync(CancellationToken cancellationToken) + public async Task StopAsync(CancellationToken cancellationToken) + { + DetachSinks(); + await ReleaseEndpointAsync().ConfigureAwait(false); + if (_endpointState.Status != OpcUaEndpointStatus.Failed) _endpointState.MarkStopped(); + } + + /// + /// Reverts every seam on the node manager to its Null adapter so in-flight work from a stopping actor + /// or a late client call cannot hit a half-disposed server. Pure detach — the SDK keeps running + /// until . + /// + private void DetachSinks() { // FIRST, ahead of every detach below (arch-review 03/S19). A tick landing on a node manager whose // authority and gate have already been cleared is the shape this whole teardown block exists to @@ -472,7 +554,6 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl // request rather than let the seams fall back to their role checks — a stopping node should shed // enforcement in the same order it acquired it. _server?.SetDataPlaneAuthorizer(null); - return Task.CompletedTask; } /// @@ -482,10 +563,15 @@ public sealed class OtOpcUaServerHostedService : IHostedService, IAsyncDisposabl public async ValueTask DisposeAsync() { // Before the app host, and idempotent with StopAsync — a host that disposes without stopping must - // not leave a tick armed against a server that is about to go away. + // not leave a tick armed against a server that is about to go away. A StopAsync that released the + // endpoint left both fields null, so this is a no-op after it; a stop that FAILED to release left + // them set, and this is the retry. _revalidationTimer?.Dispose(); _revalidationTimer = null; - if (_appHost is not null) await _appHost.DisposeAsync(); + var appHost = _appHost; + _appHost = null; + _server = null; + if (appHost is not null) await appHost.DisposeAsync(); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index 6404504e..d326d1b1 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -452,6 +452,11 @@ if (hasDriver) // mediator (for the inbound alarm-command router) without racing Akka startup — same pattern the // DpsScriptLogPublisher above uses. TryAdd so a fused admin+driver node registers it exactly once. builder.Services.TryAddSingleton>(sp => () => sp.GetRequiredService()); + // The "did the SDK start?" fact (R03:2.1): written by the hosted service, read by the opcua-endpoint + // health probe (AddOtOpcUaHealth below, hasDriver). One instance behind both the class and the + // interface, so the writer and the probe cannot disagree. + builder.Services.AddSingleton(); + builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); } @@ -554,8 +559,9 @@ if (hasAdmin) builder.Services.AddOtOpcUaSecrets(builder.Configuration); // hasAdmin gates the configdb probe: a driver-only node registers no ConfigDb factory (see above), so -// probing it would throw when the readiness endpoint resolves the factory. -builder.Services.AddOtOpcUaHealth(hasAdmin); +// probing it would throw when the readiness endpoint resolves the factory. hasDriver adds the +// opcua-endpoint probe (ready + active): a driver node whose SDK failed to start is Unhealthy, not green. +builder.Services.AddOtOpcUaHealth(hasAdmin, hasDriver); builder.Services.AddOtOpcUaObservability(builder.Configuration); // gRPC server plumbing shared by three endpoints: the LocalDb passive sync endpoint (driver-role, diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs index 1a9730ce..d880e7b3 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs @@ -49,6 +49,34 @@ public sealed class ActiveTierRegistrationTests ClusterPrimaryRegistration(hasAdmin).Tags.ShouldContain(ZbHealthTags.Active); } + /// + /// R03:2.1 — the OPC UA endpoint probe is on BOTH tiers of a driver node (it cannot serve, and must not + /// be routed to) and absent from an admin-only node, which starts no endpoint to be degraded by. + /// + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Opcua_endpoint_probe_is_registered_on_ready_and_active_for_driver_nodes_only(bool hasDriver) + { + var registrations = new ServiceCollection() + .AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver) + .BuildServiceProvider() + .GetRequiredService>() + .Value.Registrations + .Where(r => r.Name == OpcUaEndpointHealthCheck.Name) + .ToList(); + + if (!hasDriver) + { + registrations.ShouldBeEmpty(); + return; + } + + var registration = registrations.ShouldHaveSingleItem(); + registration.Tags.ShouldContain(ZbHealthTags.Ready); + registration.Tags.ShouldContain(ZbHealthTags.Active); + } + [Fact] public void Active_tier_check_is_not_on_the_ready_tier() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs new file mode 100644 index 00000000..341595d7 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/OtOpcUaServerHostedServiceStartFailureTests.cs @@ -0,0 +1,213 @@ +using System.Net; +using System.Net.Sockets; +using Akka.Hosting; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Health; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:2.1 / 1.4 — a failed OPC UA SDK start used to be one Error line: the node stayed Ready, joined +/// its mesh, and acked deploys through the Null sink while clients got connection-refused. The start +/// outcome is now a fact () the ready/active tiers read, a driver-only +/// node rethrows so the process exits, and StopAsync actually stops the SDK. +/// +public sealed class OtOpcUaServerHostedServiceStartFailureTests +{ + private static readonly TimeSpan Deadline = TimeSpan.FromSeconds(30); + + /// + /// Port already bound ⇒ the SDK cannot listen ⇒ the state is Failed, the probe is Unhealthy and the + /// real /health/ready endpoint answers 503 — while StartAsync itself returns (fused node). + /// Positive control: the same endpoint answers 200 once the state is Started. + /// + [Fact] + public async Task Port_in_use_marks_the_endpoint_failed_and_health_ready_answers_503() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var fused = EffectiveRoles.Resolve(new[] { "admin", "driver" }, envRoles: null); + + var service = NewService(port, pki, state, fused); + try + { + await service.StartAsync(deadline.Token); // must NOT throw on a fused node + + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + state.FailureReason.ShouldNotBeNullOrWhiteSpace(); + + var probe = await new OpcUaEndpointHealthCheck(state).CheckHealthAsync(new HealthCheckContext()); + probe.Status.ShouldBe(HealthStatus.Unhealthy); + probe.Description.ShouldNotBeNull(); + probe.Description.ShouldContain("failed to start"); + + await using var web = BuildHealthHost(state); + await web.StartAsync(deadline.Token); + using var http = new HttpClient { BaseAddress = new Uri(web.Urls.First()) }; + + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + (await http.GetAsync("/health/active", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + + // Positive control on the endpoint: the same host answers 200 for a started endpoint, so the + // 503 above measured the probe and not a broken health pipeline. + state.MarkStarted(); + (await http.GetAsync("/health/ready", deadline.Token)).StatusCode.ShouldBe(HttpStatusCode.OK); + await web.StopAsync(deadline.Token); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// A driver-only node has nothing else to stay up for: the failed start propagates. + [Fact] + public async Task Driver_only_node_rethrows_when_the_sdk_fails_to_start() + { + using var deadline = new CancellationTokenSource(Deadline); + using var squatter = SquatAllInterfaces(out var port); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var driverOnly = EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null); + + var service = NewService(port, pki, state, driverOnly); + try + { + await Should.ThrowAsync(() => service.StartAsync(deadline.Token)); + state.Status.ShouldBe(OpcUaEndpointStatus.Failed); + } + finally + { + await service.DisposeAsync(); + squatter.Stop(); + Cleanup(pki); + } + } + + /// + /// A clean start is Started and listening; StopAsync closes the port (connection refused, not + /// a stale-but-listening endpoint) and marks Stopped; DisposeAsync twice afterwards is a no-op. + /// + [Fact] + public async Task StopAsync_closes_the_port_and_DisposeAsync_is_idempotent() + { + using var deadline = new CancellationTokenSource(Deadline); + var port = FreePort(); + var pki = NewPki(); + var state = new OpcUaEndpointState(); + var service = NewService(port, pki, state, EffectiveRoles.Resolve(new[] { "driver" }, envRoles: null)); + try + { + await service.StartAsync(deadline.Token); + state.Status.ShouldBe(OpcUaEndpointStatus.Started); + using (var client = new TcpClient()) + { + await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token); + client.Connected.ShouldBeTrue("positive control: the endpoint is listening after StartAsync"); + } + + await service.StopAsync(deadline.Token); + + state.Status.ShouldBe(OpcUaEndpointStatus.Stopped); + using (var client = new TcpClient()) + { + await Should.ThrowAsync(async () => await client.ConnectAsync(IPAddress.Loopback, port, deadline.Token)); + } + + await service.DisposeAsync(); + await service.DisposeAsync(); // idempotent + } + finally + { + await service.DisposeAsync(); + Cleanup(pki); + } + } + + private static OtOpcUaServerHostedService NewService(int port, string pki, OpcUaEndpointState state, EffectiveRoles roles) => + new( + Options.Create(new OpcUaApplicationHostOptions + { + ApplicationName = "OtOpcUa.StartFailure", + ApplicationUri = $"urn:OtOpcUa.StartFailure:{Guid.NewGuid():N}", + OpcUaPort = port, PublicHostname = "localhost", PkiStoreRoot = pki, + }), + new DeferredAddressSpaceSink(), new DeferredServiceLevelPublisher(), + NullOpcUaUserAuthenticator.Instance, NullHistorianDataSource.Instance, + EmptyDataPlaneAuthoritySource.Instance, new ConfigurationBuilder().Build(), + () => throw new NotSupportedException("no actor system required"), new ActorRegistry(), + null, Options.Create(new AclOptions()), TimeProvider.System, NullLoggerFactory.Instance, + endpointState: state, roles: roles); + + /// The production health registration (driver role) over the supplied state, on a loopback Kestrel. + private static WebApplication BuildHealthHost(OpcUaEndpointState state) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + builder.Logging.ClearProviders(); + builder.Services.AddSingleton(state); + builder.Services.AddOtOpcUaHealth(hasAdmin: false, hasDriver: true); + // The akka + cluster-primary probes need a running ActorSystem; keep only the probe under test so + // the status measured is the endpoint's. (The registration shape itself is pinned separately.) + builder.Services.Configure(o => + { + foreach (var r in o.Registrations.Where(r => r.Name != OpcUaEndpointHealthCheck.Name).ToList()) + o.Registrations.Remove(r); + }); + var app = builder.Build(); + app.MapOtOpcUaHealth(); + return app; + } + + /// + /// Holds a port on the IPv4 AND IPv6 wildcard addresses. The SDK binds its listener on the + /// wildcards, and on macOS a loopback-only squatter does not conflict with a wildcard bind (both + /// sockets carry SO_REUSEADDR), so a loopback squatter would let the SDK start — the first run of + /// this test proved exactly that. The SDK also only fails when NEITHER family binds. + /// + private static TcpListener SquatAllInterfaces(out int port) + { + var squatter = new TcpListener(IPAddress.IPv6Any, 0); + squatter.Server.DualMode = true; + squatter.ExclusiveAddressUse = true; + squatter.Start(); + port = ((IPEndPoint)squatter.LocalEndpoint).Port; + return squatter; + } + + private static int FreePort() + { + using var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + var port = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + return port; + } + + private static string NewPki() => Path.Combine(Path.GetTempPath(), $"otopcua-startfail-{Guid.NewGuid():N}"); + + private static void Cleanup(string pki) + { + if (Directory.Exists(pki)) Directory.Delete(pki, recursive: true); + } +} From 6cbf763719ba5eebb339947512a4ecafd395ac8c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:17:30 -0400 Subject: [PATCH 07/51] fix(alarms): engine cache prune + bounded dispose, source Warning, cached dead-letter depth, script-log rate limit (E3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R05:1/12 ScriptedAlarmEngine: _valueCache is cleared in LoadAsync (it is re-seeded for the referenced paths) so retired tags no longer accrete across deploys; OnUpstreamChange no longer writes the cache after Dispose. - R05:3 ScriptedAlarmEngine.Dispose drains in-flight work for at most 5 s (DisposeDrainTimeout), then cancels it through a shutdown token every DrainDirtyAsync/ShelvingCheckAsync now runs on, logs a Warning and gives one second of grace — a stalled IAlarmStateStore.SaveAsync can no longer hang host shutdown. A save that completes inside the bound is still drained (positive control test). - R05:6 ScriptedAlarmSource logs a throwing OnAlarmEvent subscriber at Warning (naming the alarm) instead of swallowing it. - R05:7 LocalDbStoreAndForwardSink keeps a cached dead-letter counter in step with every dead-letter/revive/purge path and resyncs it with the queued counter on the same cadence; GetStatus no longer runs a COUNT(*) scan. - R05:9 ScriptLogRateLimiter (20 per source per minute, fixed window, one summary entry naming the dropped count when the window rolls) behind ScriptLogTopicSink (key = script|vtag|alarm) and ScriptLogCompanionSink (key = script name). Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../LocalDbStoreAndForwardSink.cs | 59 +++++-- .../ScriptedAlarmEngine.cs | 53 +++++- .../ScriptedAlarmSource.cs | 19 ++- .../ScriptLogCompanionSink.cs | 23 ++- .../ScriptLogRateLimiter.cs | 123 ++++++++++++++ .../ScriptLogTopicSink.cs | 49 +++++- .../LocalDbStoreAndForwardSinkTests.cs | 34 ++++ .../ScriptedAlarmEngineLifecycleTests.cs | 159 ++++++++++++++++++ .../ScriptedAlarmSourceLoggingTests.cs | 59 +++++++ .../ScriptLogRateLimiterTests.cs | 137 +++++++++++++++ 10 files changed, 683 insertions(+), 32 deletions(-) create mode 100644 src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs create mode 100644 tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs index 8a9b9ec4..6d239f75 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/LocalDbStoreAndForwardSink.cs @@ -92,6 +92,12 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl private bool? _lastGateDecision; private long _queuedRowCount; + // Cached dead-letter depth (R05:7). GetStatus used to run a synchronous COUNT(*) table scan per + // call — one per /healthz probe and per AdminUI poll on the shared LocalDb file. Kept in step by + // every path that flips dead_lettered (drain outcomes, corrupt rows, RetryDeadLettered, retention + // purge) and resynced from storage on the same cadence as _queuedRowCount, so a row the pair peer + // replicates in is visible after the next resync rather than never. + private long _deadLetterRowCount; // Probe counter — incremented every time we actually issue a real COUNT(*) for // capacity enforcement. Public for test instrumentation only. private long _capacityProbeCount; @@ -145,9 +151,16 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl _clock = clock ?? (() => DateTime.UtcNow); _drainGate = drainGate ?? (static () => true); - _queuedRowCount = ProbeQueuedRowCount(); + _queuedRowCount = ProbeRowCount(deadLettered: false); + _deadLetterRowCount = ProbeRowCount(deadLettered: true); } + /// Test-only: re-sync both cached counters (queued + dead-lettered) from storage — the same + /// resync the enqueue cadence drives every enqueues. + /// Cancellation token. + /// A task that completes once both counters match storage. + public Task DebugResyncCountersAsync(CancellationToken ct) => ResyncRowCountsAsync(ct); + /// /// Start the background drain worker. Not started automatically so tests can /// drive deterministically. @@ -260,7 +273,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl // has a second way to fall behind that no local code path can observe. if (enqueuesSinceResync >= ResyncEnqueueInterval) { - await ResyncQueuedRowCountAsync(ct).ConfigureAwait(false); + await ResyncRowCountsAsync(ct).ConfigureAwait(false); cached = Interlocked.Read(ref _queuedRowCount); Interlocked.Exchange(ref _enqueuesSinceResync, 0); } @@ -273,22 +286,26 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl await EnforceCapacityAsync(ct).ConfigureAwait(false); } - /// Synchronously query COUNT(*) of non-dead-lettered rows. Used at startup. - private long ProbeQueuedRowCount() + /// Synchronously query COUNT(*) of queued or dead-lettered rows. Used at startup. + private long ProbeRowCount(bool deadLettered) { Interlocked.Increment(ref _capacityProbeCount); using var conn = _db.CreateConnection(); using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; + cmd.CommandText = deadLettered + ? "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1" + : "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 0"; return (long)(cmd.ExecuteScalar() ?? 0L); } - /// Re-sync the in-memory counter from storage (async path). - private async Task ResyncQueuedRowCountAsync(CancellationToken ct) + /// Re-sync both in-memory counters from storage (async path). + private async Task ResyncRowCountsAsync(CancellationToken ct) { Interlocked.Increment(ref _capacityProbeCount); var live = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); Interlocked.Exchange(ref _queuedRowCount, live); + var dead = await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false); + Interlocked.Exchange(ref _deadLetterRowCount, dead); } private async Task CountAsync(string predicate, CancellationToken ct) @@ -356,6 +373,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl await corruptTx.CommitAsync(ct).ConfigureAwait(false); } Interlocked.Add(ref _queuedRowCount, -corruptIds.Count); + Interlocked.Add(ref _deadLetterRowCount, corruptIds.Count); _logger.Warning( "Dead-lettered {Count} historian queue row(s) with un-deserializable payload", corruptIds.Count); @@ -405,6 +423,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl } int rowsLeavingQueue = 0; + int rowsDeadLettered = 0; await using (var tx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false)) { for (var i = 0; i < outcomes.Count; i++) @@ -423,6 +442,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl case HistorianWriteOutcome.PermanentFail: await DeadLetterRowAsync(tx, id, $"permanent fail at {_clock():O}", ct).ConfigureAwait(false); rowsLeavingQueue++; + rowsDeadLettered++; break; case HistorianWriteOutcome.RetryPlease: // finding 002: cap retries so a perpetually-RetryPlease (poison) @@ -434,6 +454,7 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl { await DeadLetterRowAsync(tx, id, $"max attempts ({_maxAttempts}) exceeded", ct).ConfigureAwait(false); rowsLeavingQueue++; + rowsDeadLettered++; } else { @@ -449,6 +470,8 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl // cap (finding 002) — keep the counter aligned. if (rowsLeavingQueue > 0) Interlocked.Add(ref _queuedRowCount, -rowsLeavingQueue); + if (rowsDeadLettered > 0) + Interlocked.Add(ref _deadLetterRowCount, rowsDeadLettered); var acks = outcomes.Count(o => o == HistorianWriteOutcome.Ack); lock (_statusLock) @@ -523,13 +546,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl var queued = Interlocked.Read(ref _queuedRowCount); if (queued < 0) queued = 0; - long deadlettered; - using (var conn = _db.CreateConnection()) - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = "SELECT COUNT(*) FROM alarm_sf_events WHERE dead_lettered = 1"; - deadlettered = (long)(cmd.ExecuteScalar() ?? 0L); - } + // Cached, never a live scan (R05:7) — see the field's remarks for the resync cadence. + var deadlettered = Interlocked.Read(ref _deadLetterRowCount); + if (deadlettered < 0) deadlettered = 0; DateTime? lastDrain, lastSuccess; string? lastError; @@ -568,7 +587,11 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl var revived = cmd.ExecuteNonQuery(); // Dead-lettered rows rejoin the non-dead-lettered queue — keep the in-memory // counter aligned. - if (revived > 0) Interlocked.Add(ref _queuedRowCount, revived); + if (revived > 0) + { + Interlocked.Add(ref _queuedRowCount, revived); + Interlocked.Add(ref _deadLetterRowCount, -revived); + } return revived; } @@ -646,8 +669,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl Interlocked.Increment(ref _capacityProbeCount); var count = await CountAsync("dead_lettered = 0", ct).ConfigureAwait(false); - // Resync the in-memory counter while we have a fresh number. + // Resync the in-memory counters while we have a fresh number. Interlocked.Exchange(ref _queuedRowCount, count); + Interlocked.Exchange(ref _deadLetterRowCount, await CountAsync("dead_lettered = 1", ct).ConfigureAwait(false)); if (count < _capacity) return; var toEvict = count - _capacity + 1; @@ -684,7 +708,10 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl ct).ConfigureAwait(false); if (purged > 0) + { + Interlocked.Add(ref _deadLetterRowCount, -purged); _logger.Information("Purged {Count} dead-lettered row(s) past retention window", purged); + } } private void BumpBackoff() => _backoffIndex = Math.Min(_backoffIndex + 1, BackoffLadder.Length - 1); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index 808935ae..9438132a 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -170,6 +170,24 @@ public sealed class ScriptedAlarmEngine : IDisposable => _scratchByAlarmId.TryGetValue(alarmId, out var s) ? s.Context : null; private readonly ConcurrentDictionary _valueCache = new(StringComparer.Ordinal); + + /// Test hook — the number of paths currently held in the value cache. + internal int ValueCacheCountForTest => _valueCache.Count; + + /// Test hook — whether is currently held in the value cache. + internal bool ValueCacheContainsForTest(string path) => _valueCache.ContainsKey(path); + + /// + /// How long waits for in-flight background work (a re-evaluation or shelving + /// check mid-SaveAsync) before cancelling it through and + /// returning. 5 s, like PollGroupEngine's drain; a hung store must not hang host shutdown. + /// + internal TimeSpan DisposeDrainTimeout { get; set; } = TimeSpan.FromSeconds(5); + + /// Cancelled by once the drain bound expires — every background task + /// (DrainDirtyAsync, ShelvingCheckAsync) runs on its token, so a store call that honours + /// cancellation returns instead of outliving the engine. + private readonly CancellationTokenSource _shutdownCts = new(); private readonly Dictionary> _alarmsReferencing = new(StringComparer.Ordinal); // tag path -> alarm ids @@ -281,6 +299,10 @@ public sealed class ScriptedAlarmEngine : IDisposable UnsubscribeFromUpstream(); _alarms.Clear(); _alarmsReferencing.Clear(); + // The value cache is re-seeded below for exactly the paths this generation references. It + // used to survive reloads untouched, so every path a retired alarm had ever referenced stayed + // cached forever — a slow leak over many deploys with a churning tag set. + _valueCache.Clear(); // Drop the prior generation's per-alarm scratch buffers — definitions may // have changed (different Inputs, different Logger), so any reuse would be // unsafe. @@ -573,8 +595,8 @@ public sealed class ScriptedAlarmEngine : IDisposable /// The new data value snapshot. internal void OnUpstreamChange(string path, DataValueSnapshot value) { + if (_disposed) return; // a disposed engine neither caches nor queues — nothing reads it again _valueCache[path] = value; - if (_disposed) return; // don't queue new work against a disposing engine if (!_alarmsReferencing.TryGetValue(path, out var alarmIds)) return; bool startDrain; @@ -589,7 +611,7 @@ public sealed class ScriptedAlarmEngine : IDisposable // Started outside the lock: DrainDirtyAsync runs synchronously up to its // first real await, and nothing it does should happen under _dirtyLock. - if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(CancellationToken.None)); + if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownCts.Token)); } /// @@ -732,6 +754,11 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } + catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + { + // Dispose cut the drain short after its bound — expected, not a failure. + return; + } catch (Exception ex) { _engineLogger.Error(ex, "ScriptedAlarmEngine reevaluate failed"); @@ -1048,7 +1075,7 @@ public sealed class ScriptedAlarmEngine : IDisposable { if (_disposed) return; var ids = _alarms.Keys.ToArray(); - TrackBackgroundTask(() => ShelvingCheckAsync(ids, CancellationToken.None)); + TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownCts.Token)); } /// @@ -1187,6 +1214,10 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } + catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + { + return; // Dispose cut the check short after its bound + } catch (Exception ex) { _engineLogger.Warning(ex, "ScriptedAlarmEngine shelving-check failed"); @@ -1227,11 +1258,25 @@ public sealed class ScriptedAlarmEngine : IDisposable // flight can outlive the engine and write to a (possibly disposed) store // after Dispose() has returned. The tasks re-check _disposed after // acquiring the gate and bail out, but the await still has to complete. + // BOUNDED (R05:3): the drained tasks end in IAlarmStateStore.SaveAsync, and a stalled store + // (connection stall, lock, replication hiccup) used to hang this — and the host shutdown that + // calls it — indefinitely. Wait DisposeDrainTimeout; past it, cancel the tasks' shared token (a + // store that honours cancellation returns promptly), log, and give them one short grace period. Task[] toAwait; lock (_inFlightLock) { toAwait = [.. _inFlight]; } if (toAwait.Length > 0) { - try { Task.WhenAll(toAwait).GetAwaiter().GetResult(); } + var all = Task.WhenAll(toAwait); + try { all.WaitAsync(DisposeDrainTimeout).GetAwaiter().GetResult(); } + catch (TimeoutException) + { + _engineLogger.Warning( + "ScriptedAlarmEngine dispose: {Count} background task(s) still running after {Timeout} — cancelling them (a store call is stalled)", + toAwait.Length, DisposeDrainTimeout); + _shutdownCts.Cancel(); + try { all.WaitAsync(TimeSpan.FromSeconds(1)).GetAwaiter().GetResult(); } + catch (Exception) { /* cancelled, faulted, or still stuck — we are leaving either way */ } + } catch (Exception ex) { // Background task failures already logged inside ReevaluateAsync / diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs index dc76923c..9d44aee5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmSource.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using Serilog; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; @@ -23,19 +24,27 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; /// adapter doesn't maintain per-subscription state beyond the filter set — it /// checks each emission against every live subscription. /// +/// +/// A throwing subscriber is logged at Warning (naming the alarm) and +/// does not crash the adapter — the same posture as the engine's own FireEvent. It used to +/// be swallowed silently: a lost alarm event with no trace. +/// /// public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable { private readonly ScriptedAlarmEngine _engine; + private readonly ILogger _logger; private readonly ConcurrentDictionary _subscriptions = new(StringComparer.Ordinal); private bool _disposed; /// Initializes a new instance of the class. /// The scripted alarm engine to expose. - public ScriptedAlarmSource(ScriptedAlarmEngine engine) + /// Where a throwing subscriber is reported; null ⇒ Serilog's global logger. + public ScriptedAlarmSource(ScriptedAlarmEngine engine, ILogger? logger = null) { _engine = engine ?? throw new ArgumentNullException(nameof(engine)); + _logger = logger ?? Log.Logger; _engine.OnEvent += OnEngineEvent; } @@ -98,7 +107,13 @@ public sealed class ScriptedAlarmSource : IAlarmSource, IDisposable Severity: evt.Severity, SourceTimestampUtc: evt.TimestampUtc); try { OnAlarmEvent?.Invoke(this, payload); } - catch { /* subscriber exceptions don't crash the adapter */ } + catch (Exception ex) + { + // Subscriber exceptions don't crash the adapter — but they are not silent either: a lost + // alarm event must leave a trace the operator can find. + _logger.Warning(ex, "ScriptedAlarmSource OnAlarmEvent subscriber threw for {AlarmId} ({Emission})", + evt.AlarmId, evt.Emission); + } } } diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs index fb1aff35..84bb0e04 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogCompanionSink.cs @@ -24,22 +24,32 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// Original exception (if any) is attached so the main log's diagnostics keep /// the full stack trace. /// +/// +/// Rate-limited per script. A predicate that throws on every push would otherwise mirror +/// one Warning per push into the main log; a keyed by script +/// name admits 20 per minute and mirrors one summary line for the rest when the window rolls. The +/// scripts-*.log file sink is unaffected and keeps the full stream. +/// /// public sealed class ScriptLogCompanionSink : ILogEventSink { private readonly ILogger _mainLogger; private readonly LogEventLevel _minMirrorLevel; + private readonly ScriptLogRateLimiter _limiter; /// Initializes a new instance of the ScriptLogCompanionSink. /// The companion logger to mirror events to. /// The minimum log level to mirror (default: Error). - public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error) + /// Per-script rate limiter; null ⇒ the default 20 per minute. + public ScriptLogCompanionSink(ILogger mainLogger, LogEventLevel minMirrorLevel = LogEventLevel.Error, + ScriptLogRateLimiter? limiter = null) { _mainLogger = mainLogger ?? throw new ArgumentNullException(nameof(mainLogger)); _minMirrorLevel = minMirrorLevel; + _limiter = limiter ?? new ScriptLogRateLimiter(); } - /// Emits a log event to the companion logger if it meets the minimum level. + /// Emits a log event to the companion logger if it meets the minimum level and the script is within its per-minute budget. /// The log event to emit. public void Emit(LogEvent logEvent) { @@ -54,6 +64,15 @@ public sealed class ScriptLogCompanionSink : ILogEventSink scriptName = s; } + var admission = _limiter.Admit(scriptName); + if (admission.SuppressedInPreviousWindow > 0) + { + _mainLogger.Warning( + "[Script] {ScriptName}: {SuppressedCount} further script error(s) were suppressed in the previous {WindowSeconds:F0} s; see scripts-*.log", + scriptName, admission.SuppressedInPreviousWindow, _limiter.Window.TotalSeconds); + } + if (!admission.Admitted) return; + var rendered = logEvent.RenderMessage(); if (logEvent.Exception is not null) { diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs new file mode 100644 index 00000000..17a6f2f6 --- /dev/null +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs @@ -0,0 +1,123 @@ +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; + +/// +/// Per-source fixed-window rate limit for the script-log fan-out: the first +/// entries a source emits in a are +/// admitted, the rest are counted and dropped, and the count is handed back exactly once — on the first +/// admission after the window rolls — so the caller can emit one summary line in the dropped entries' +/// place. Nothing is silently lost (the operator sees "N suppressed"), and one chatty script cannot +/// shed every other script's entries out of the bounded telemetry channels (drop-oldest) or flood the +/// cluster bus / main log. +/// +/// +/// Used by (key = script + virtual-tag + alarm identity), +/// (key = script name) and the virtual-tag actor's per-failure +/// log (one key per tag). Thread-safe; Serilog invokes a sink on whichever thread logged. Idle sources +/// are swept every admissions so a churning tag set cannot grow the table +/// without bound. +/// +public sealed class ScriptLogRateLimiter +{ + /// Entries admitted per source per window by default: 20. + public const int DefaultLimitPerWindow = 20; + + /// The default window: one minute. + public static readonly TimeSpan DefaultWindow = TimeSpan.FromMinutes(1); + + private const int SweepEvery = 256; + + private readonly Dictionary _sources = new(StringComparer.Ordinal); + private readonly int _limit; + private readonly TimeSpan _window; + private readonly Func _clock; + private int _admitsSinceSweep; + + /// Initializes a limiter. + /// Entries admitted per source per window; values below 1 are treated as 1. + /// The fixed window; null ⇒ . + /// UTC clock; null ⇒ . + public ScriptLogRateLimiter(int limitPerWindow = DefaultLimitPerWindow, TimeSpan? window = null, Func? clock = null) + { + _limit = Math.Max(1, limitPerWindow); + _window = window is { } w && w > TimeSpan.Zero ? w : DefaultWindow; + _clock = clock ?? (static () => DateTime.UtcNow); + } + + /// The window length. + public TimeSpan Window => _window; + + /// + /// Ask whether 's next entry may be emitted. When the window has rolled + /// since the last call, carries the + /// number of entries dropped in the window that just closed (zero otherwise) — the caller emits one + /// summary for them BEFORE the admitted entry. + /// + /// Identity of the emitting script instance. + /// The admission decision. + public ScriptLogAdmission Admit(string sourceKey) + { + ArgumentNullException.ThrowIfNull(sourceKey); + var now = _clock(); + lock (_sources) + { + if (++_admitsSinceSweep >= SweepEvery) + { + _admitsSinceSweep = 0; + Sweep(now); + } + + if (!_sources.TryGetValue(sourceKey, out var source)) + { + source = new Source { WindowStart = now }; + _sources[sourceKey] = source; + } + + long summary = 0; + if (now - source.WindowStart >= _window) + { + summary = source.Suppressed; + source.WindowStart = now; + source.Admitted = 0; + source.Suppressed = 0; + } + + source.LastSeen = now; + if (source.Admitted < _limit) + { + source.Admitted++; + return new ScriptLogAdmission(true, summary); + } + + source.Suppressed++; + return new ScriptLogAdmission(false, summary); + } + } + + private void Sweep(DateTime now) + { + var idle = now - (_window + _window); + List? dead = null; + foreach (var (key, source) in _sources) + { + // A source that still has a pending summary is kept: its next entry must report the count. + if (source.LastSeen < idle && source.Suppressed == 0) + (dead ??= []).Add(key); + } + if (dead is null) return; + foreach (var key in dead) _sources.Remove(key); + } + + private sealed class Source + { + public DateTime WindowStart; + public DateTime LastSeen; + public int Admitted; + public long Suppressed; + } +} + +/// Result of . +/// Whether the entry may be emitted. +/// Entries dropped in the window that just closed — non-zero at +/// most once per window roll; the caller emits one summary line for them. +public readonly record struct ScriptLogAdmission(bool Admitted, long SuppressedInPreviousWindow); diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs index fe9e7f3e..a7b8f03e 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs @@ -25,25 +25,37 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// property so pipelines that pre-date this sink continue to work correctly. If /// neither property is present ScriptId defaults to "unknown". /// +/// +/// Rate-limited per source. A script that logs at Information on every evaluation of a 1 Hz +/// input put one rendered string per second per script onto the cluster bus; 200 such scripts shed +/// everyone's entries out of the bounded telemetry channels. A +/// keyed by the entry's identity (script + virtual tag + alarm) admits 20 entries per minute per +/// source and publishes one summary entry naming the dropped count when the window rolls. +/// /// public sealed class ScriptLogTopicSink : ILogEventSink { private readonly IScriptLogPublisher _publisher; private readonly LogEventLevel _min; + private readonly ScriptLogRateLimiter _limiter; /// Initializes a new instance of the class. /// The publisher that routes entries to the cluster bus. Must not be null. /// Minimum log level to forward. Events below this level are discarded (default: ). + /// Per-source rate limiter; null ⇒ the default 20 per minute. /// Thrown when is null. - public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information) + public ScriptLogTopicSink(IScriptLogPublisher publisher, LogEventLevel min = LogEventLevel.Information, + ScriptLogRateLimiter? limiter = null) { _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); _min = min; + _limiter = limiter ?? new ScriptLogRateLimiter(); } /// /// Converts the to a and - /// publishes it, unless the event is below the configured minimum level. + /// publishes it, unless the event is below the configured minimum level or the source is over its + /// per-minute budget. /// /// The Serilog event to process. Silently ignored when null. public void Emit(LogEvent logEvent) @@ -56,15 +68,36 @@ public sealed class ScriptLogTopicSink : ILogEventSink ? s : null; + var scriptId = P(ScriptLoggerFactory.ScriptIdProperty) + ?? P(ScriptLoggerFactory.ScriptNameProperty) + ?? "unknown"; + var virtualTagId = P(ScriptLoggerFactory.VirtualTagIdProperty); + var alarmId = P(ScriptLoggerFactory.AlarmIdProperty); + var equipmentId = P(ScriptLoggerFactory.EquipmentIdProperty); + + var admission = _limiter.Admit(string.Concat(scriptId, "|", virtualTagId, "|", alarmId)); + if (admission.SuppressedInPreviousWindow > 0) + { + _publisher.Publish(new ScriptLogEntry( + ScriptId: scriptId, + Level: nameof(LogEventLevel.Warning), + Message: $"{admission.SuppressedInPreviousWindow} script-log entr{(admission.SuppressedInPreviousWindow == 1 ? "y" : "ies")} " + + $"from this source were suppressed in the previous {_limiter.Window.TotalSeconds:F0} s " + + $"(limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); see scripts-*.log for the full stream", + TimestampUtc: logEvent.Timestamp.UtcDateTime, + VirtualTagId: virtualTagId, + AlarmId: alarmId, + EquipmentId: equipmentId)); + } + if (!admission.Admitted) return; + _publisher.Publish(new ScriptLogEntry( - ScriptId: P(ScriptLoggerFactory.ScriptIdProperty) - ?? P(ScriptLoggerFactory.ScriptNameProperty) - ?? "unknown", + ScriptId: scriptId, Level: logEvent.Level.ToString(), Message: logEvent.RenderMessage(), TimestampUtc: logEvent.Timestamp.UtcDateTime, - VirtualTagId: P(ScriptLoggerFactory.VirtualTagIdProperty), - AlarmId: P(ScriptLoggerFactory.AlarmIdProperty), - EquipmentId: P(ScriptLoggerFactory.EquipmentIdProperty))); + VirtualTagId: virtualTagId, + AlarmId: alarmId, + EquipmentId: equipmentId)); } } diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs index 9c4c6b11..97d68679 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian.Tests/LocalDbStoreAndForwardSinkTests.cs @@ -342,6 +342,40 @@ public sealed class LocalDbStoreAndForwardSinkTests : IDisposable sink.GetStatus().DeadLetterDepth.ShouldBe(0, "purged past retention"); } + /// + /// Review R05:7 — GetStatus no longer runs a COUNT(*) table scan per call for the + /// dead-letter depth: it reads a cached counter kept in step by every dead-letter / revive / purge + /// path (the other tests in this class pin those), and resynced from storage on the same cadence as + /// the queued-row counter. A row flipped underneath the sink (the peer replicating a dead-letter in) + /// is therefore invisible until the next resync — that is the documented trade, and the resync is + /// what makes it bounded rather than permanent. + /// + [Fact] + public async Task DeadLetterDepth_is_served_from_a_cached_counter_that_the_periodic_resync_corrects() + { + var writer = new FakeWriter(); + using var sink = new LocalDbStoreAndForwardSink(Db, writer, _log); + await sink.EnqueueAsync(Event("A1"), CancellationToken.None); + await sink.EnqueueAsync(Event("A2"), CancellationToken.None); + sink.GetStatus().DeadLetterDepth.ShouldBe(0); + var probesBefore = sink.DebugCapacityProbeCount; + + // Drift the table underneath the sink, exactly as a replicated dead-letter from the pair peer would. + await Db.ExecuteAsync( + "UPDATE alarm_sf_events SET dead_lettered = 1 WHERE id IN (SELECT id FROM alarm_sf_events LIMIT 1)", + null, CancellationToken.None); + + // Cached: GetStatus did not go to the table, so it still reports the pre-drift depth… + sink.GetStatus().DeadLetterDepth.ShouldBe(0, "GetStatus serves the cached counter, not a live COUNT(*)"); + sink.DebugCapacityProbeCount.ShouldBe(probesBefore, "GetStatus must not probe storage"); + + // …and the resync (the one the enqueue cadence drives) brings both counters back in line. + await sink.DebugResyncCountersAsync(CancellationToken.None); + var status = sink.GetStatus(); + status.DeadLetterDepth.ShouldBe(1); + status.QueueDepth.ShouldBe(1); + } + /// Verifies that RetryDeadLettered requeues dead-lettered rows for retry. [Fact] public async Task RetryDeadLettered_requeues_for_retry() diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs new file mode 100644 index 00000000..7afea07a --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs @@ -0,0 +1,159 @@ +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// +/// Review R05:1 / R05:12 / R05:3 — the engine's value cache is pruned on every reload (it used to keep +/// every path ever referenced, forever), is no longer written after Dispose, and Dispose's in-flight +/// drain is bounded (a hung store no longer hangs host shutdown). +/// +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmEngineLifecycleTests +{ + private static ScriptedAlarmDefinition Alarm(string id, string predicate) => + new(AlarmId: id, EquipmentPath: "Plant/Line1/Reactor", AlarmName: id, Kind: AlarmKind.AlarmCondition, + Severity: AlarmSeverity.High, MessageTemplate: "condition", PredicateScriptSource: predicate); + + private static ScriptedAlarmEngine Build(FakeUpstream up, IAlarmStateStore? store = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + return new ScriptedAlarmEngine(up, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + } + + [Fact] + public async Task Reload_prunes_value_cache_entries_for_paths_the_new_alarm_set_no_longer_references() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + up.Set("Pressure", 5); + using var eng = Build(up); + var ct = TestContext.Current.CancellationToken; + + await eng.LoadAsync( + [Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100 || (int)ctx.GetTag("Pressure").Value > 10;""")], ct); + eng.ValueCacheCountForTest.ShouldBe(2); + + // The redeploy drops Pressure from every predicate: its entry must go with it. + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], ct); + eng.ValueCacheCountForTest.ShouldBe(1); + eng.ValueCacheContainsForTest("Pressure").ShouldBeFalse(); + eng.ValueCacheContainsForTest("Temp").ShouldBeTrue(); + } + + [Fact] + public async Task A_push_after_Dispose_does_not_write_the_value_cache() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + var eng = Build(up); + await eng.LoadAsync([Alarm("a", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + eng.Dispose(); + eng.ValueCacheCountForTest.ShouldBe(1); + + eng.OnUpstreamChange("Late", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow)); + + eng.ValueCacheContainsForTest("Late").ShouldBeFalse("a disposed engine must not grow its cache"); + } + + [Fact] + public async Task Dispose_gives_up_on_a_store_that_never_returns_after_the_drain_bound_and_cancels_it() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + var store = new NeverReturningSaveStore(); + var eng = Build(up, store); + eng.DisposeDrainTimeout = TimeSpan.FromMilliseconds(300); + await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + store.BlockNext = true; + up.Push("Temp", 150); // Activated → SaveAsync → blocks forever (until cancelled) + await WaitForAsync(() => store.SaveInProgress); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + eng.Dispose(); + sw.Stop(); + + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the drain is bounded; a hung store cannot hang shutdown"); + await WaitForAsync(() => store.ObservedCancellation, timeoutMs: 2000); + store.ObservedCancellation.ShouldBeTrue("the in-flight save was cancelled through the engine's shutdown token"); + } + + [Fact] + public async Task Dispose_still_waits_for_a_save_that_completes_inside_the_bound() + { + // Positive control for the bound: a save that finishes promptly is drained, not cut off. + var up = new FakeUpstream(); + up.Set("Temp", 50); + var store = new NeverReturningSaveStore { ReleaseAfter = TimeSpan.FromMilliseconds(100) }; + var eng = Build(up, store); + eng.DisposeDrainTimeout = TimeSpan.FromSeconds(5); + await eng.LoadAsync([Alarm("A", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + + store.BlockNext = true; + up.Push("Temp", 150); + await WaitForAsync(() => store.SaveInProgress); + + eng.Dispose(); + + store.SaveInProgress.ShouldBeFalse(); + store.ObservedCancellation.ShouldBeFalse("a save that completes within the bound is never cancelled"); + store.Saved.ShouldContain("A"); + } + + private static async Task WaitForAsync(Func cond, int timeoutMs = 5000) + { + var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); + while (DateTime.UtcNow < deadline) + { + if (cond()) return; + await Task.Delay(20); + } + throw new TimeoutException("Condition did not become true in time"); + } + + /// A store whose next SaveAsync blocks until cancelled (or until ). + private sealed class NeverReturningSaveStore : IAlarmStateStore + { + private readonly InMemoryAlarmStateStore _inner = new(); + public volatile bool BlockNext; + public volatile bool SaveInProgress; + public volatile bool ObservedCancellation; + public TimeSpan? ReleaseAfter { get; set; } + public List Saved { get; } = []; + + public Task LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct); + public Task> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct); + public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct); + + public async Task SaveAsync(AlarmConditionState state, CancellationToken ct) + { + if (BlockNext) + { + BlockNext = false; + SaveInProgress = true; + try + { + await Task.Delay(ReleaseAfter ?? Timeout.InfiniteTimeSpan, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + ObservedCancellation = true; + throw; + } + finally + { + SaveInProgress = false; + } + } + lock (Saved) Saved.Add(state.AlarmId); + await _inner.SaveAsync(state, ct).ConfigureAwait(false); + } + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs new file mode 100644 index 00000000..94c0408c --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmSourceLoggingTests.cs @@ -0,0 +1,59 @@ +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; + +/// Review R05:6 — a throwing subscriber used to be +/// swallowed with no diagnostic: a lost alarm event with no trace. It is now logged at Warning. +[Trait("Category", "Unit")] +public sealed class ScriptedAlarmSourceLoggingTests +{ + private sealed class CapturingSink : ILogEventSink + { + public List Events { get; } = []; + public void Emit(LogEvent logEvent) { lock (Events) Events.Add(logEvent); } + } + + [Fact] + public async Task A_throwing_subscriber_is_logged_at_Warning_and_does_not_crash_the_adapter() + { + var sinkCapture = new CapturingSink(); + var logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sinkCapture).CreateLogger(); + var up = new FakeUpstream(); + up.Set("Temp", 50); + var engine = new ScriptedAlarmEngine(up, new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + await engine.LoadAsync( + [new ScriptedAlarmDefinition("Plant/Line1::HighTemp", "Plant/Line1", "HighTemp", AlarmKind.LimitAlarm, + AlarmSeverity.High, "Temp {Temp}C", """return (int)ctx.GetTag("Temp").Value > 100;""")], + TestContext.Current.CancellationToken); + using var _e = engine; + using var source = new ScriptedAlarmSource(engine, logger); + await source.SubscribeAlarmsAsync(Array.Empty(), TestContext.Current.CancellationToken); + source.OnAlarmEvent += (_, _) => throw new InvalidOperationException("subscriber broke"); + + up.Push("Temp", 150); + + var deadline = DateTime.UtcNow.AddSeconds(10); + while (DateTime.UtcNow < deadline) + { + lock (sinkCapture.Events) + { + if (sinkCapture.Events.Any(e => e.Level == LogEventLevel.Warning && e.Exception?.Message == "subscriber broke")) + break; + } + await Task.Delay(25); + } + + LogEvent warning; + lock (sinkCapture.Events) + warning = sinkCapture.Events.Single(e => e.Level == LogEventLevel.Warning && e.Exception?.Message == "subscriber broke"); + warning.RenderMessage().ShouldContain("Plant/Line1::HighTemp"); + // The adapter survived: the engine's state still moved. + engine.GetState("Plant/Line1::HighTemp")!.Active.ShouldBe(AlarmActiveState.Active); + } +} diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs new file mode 100644 index 00000000..3f560e40 --- /dev/null +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs @@ -0,0 +1,137 @@ +using Serilog; +using Serilog.Core; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; + +namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; + +/// +/// Per-source rate limit on the script-log fan-out (review R05:9 / R02:P6): a script that logs — or +/// fails — on every evaluation of a 1 Hz input used to put one entry per second per script onto the +/// cluster bus and the main log. The limiter admits the first 20 entries per source per minute, drops +/// the rest, and surfaces one summary line (with the dropped count) when the next entry arrives after +/// the window rolled — so nothing is silently lost, and a chatty script cannot shed everyone else's +/// entries out of the bounded telemetry channels. +/// +[Trait("Category", "Unit")] +public sealed class ScriptLogRateLimiterTests +{ + private sealed class Clock + { + public DateTime Now = new(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc); + public void Advance(TimeSpan by) => Now += by; + } + + [Fact] + public void Admits_the_first_N_per_window_then_suppresses_and_summarises_once_the_window_rolls() + { + var clock = new Clock(); + var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => clock.Now); + + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeTrue(); + limiter.Admit("s1").Admitted.ShouldBeFalse(); + limiter.Admit("s1").Admitted.ShouldBeFalse(); + limiter.Admit("s1").SuppressedInPreviousWindow.ShouldBe(0, "no summary until the window rolls"); + + clock.Advance(TimeSpan.FromMinutes(1)); + var first = limiter.Admit("s1"); + first.Admitted.ShouldBeTrue("a fresh window admits again"); + first.SuppressedInPreviousWindow.ShouldBe(3, "the caller emits one summary for the three dropped entries"); + limiter.Admit("s1").SuppressedInPreviousWindow.ShouldBe(0, "the summary is reported exactly once"); + } + + [Fact] + public void Sources_are_independent() + { + var limiter = new ScriptLogRateLimiter(limitPerWindow: 1); + limiter.Admit("a").Admitted.ShouldBeTrue(); + limiter.Admit("a").Admitted.ShouldBeFalse(); + limiter.Admit("b").Admitted.ShouldBeTrue("a chatty source must not shed another source's entries"); + } + + [Fact] + public void Defaults_are_20_per_minute() + { + ScriptLogRateLimiter.DefaultLimitPerWindow.ShouldBe(20); + ScriptLogRateLimiter.DefaultWindow.ShouldBe(TimeSpan.FromMinutes(1)); + var limiter = new ScriptLogRateLimiter(); + for (var i = 0; i < 20; i++) limiter.Admit("s").Admitted.ShouldBeTrue(); + limiter.Admit("s").Admitted.ShouldBeFalse(); + } + + // ── the two sinks ──────────────────────────────────────────────────────────────────────── + + private sealed class FakePublisher : IScriptLogPublisher + { + public List Published { get; } = []; + public void Publish(ScriptLogEntry entry) => Published.Add(entry); + } + + private static LogEvent Evt(LogEventLevel level, string template, params (string Key, string Val)[] props) + { + var tmpl = new Serilog.Parsing.MessageTemplateParser().Parse(template); + var logProps = props.Select(p => new LogEventProperty(p.Key, new ScalarValue(p.Val))); + return new LogEvent(DateTimeOffset.UtcNow, level, exception: null, tmpl, logProps); + } + + [Fact] + public void Topic_sink_rate_limits_per_source_and_publishes_a_summary_entry() + { + var clock = new Clock(); + var publisher = new FakePublisher(); + var sink = new ScriptLogTopicSink(publisher, LogEventLevel.Information, + new ScriptLogRateLimiter(limitPerWindow: 2, clock: () => clock.Now)); + + for (var i = 0; i < 5; i++) + sink.Emit(Evt(LogEventLevel.Warning, "chatty", (ScriptLoggerFactory.ScriptIdProperty, "S1"), (ScriptLoggerFactory.VirtualTagIdProperty, "V1"))); + sink.Emit(Evt(LogEventLevel.Warning, "other", (ScriptLoggerFactory.ScriptIdProperty, "S2"))); + + publisher.Published.Count(e => e.ScriptId == "S1").ShouldBe(2, "the limit is per source"); + publisher.Published.Count(e => e.ScriptId == "S2").ShouldBe(1, "another source is unaffected"); + + clock.Advance(TimeSpan.FromMinutes(1)); + sink.Emit(Evt(LogEventLevel.Warning, "chatty again", (ScriptLoggerFactory.ScriptIdProperty, "S1"), (ScriptLoggerFactory.VirtualTagIdProperty, "V1"))); + + var s1 = publisher.Published.Where(e => e.ScriptId == "S1").ToList(); + s1.Count.ShouldBe(4, "2 admitted + 1 summary + 1 admitted in the new window"); + var summary = s1[2]; + summary.Message.ShouldContain("3"); + summary.Message.ShouldContain("suppressed"); + summary.VirtualTagId.ShouldBe("V1", "the summary carries the source's identity so the page attributes it"); + summary.Level.ShouldBe("Warning"); + s1[3].Message.ShouldBe("chatty again"); + } + + private sealed class CapturingSink : ILogEventSink + { + public List Events { get; } = []; + public void Emit(LogEvent logEvent) => Events.Add(logEvent); + } + + [Fact] + public void Companion_sink_rate_limits_per_script_and_mirrors_a_summary_line() + { + var clock = new Clock(); + var main = new CapturingSink(); + var mainLogger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(main).CreateLogger(); + var sink = new ScriptLogCompanionSink(mainLogger, LogEventLevel.Error, + new ScriptLogRateLimiter(limitPerWindow: 2, clock: () => clock.Now)); + + for (var i = 0; i < 5; i++) + sink.Emit(Evt(LogEventLevel.Error, "boom", (ScriptLoggerFactory.ScriptNameProperty, "HighTemp"))); + main.Events.Count.ShouldBe(2); + + clock.Advance(TimeSpan.FromMinutes(1)); + sink.Emit(Evt(LogEventLevel.Error, "boom", (ScriptLoggerFactory.ScriptNameProperty, "HighTemp"))); + + main.Events.Count.ShouldBe(4); + var summary = main.Events[2].RenderMessage(); + summary.ShouldContain("HighTemp"); + summary.ShouldContain("3"); + summary.ShouldContain("suppressed"); + } +} From f72af76e99e241ad1d36b90a8d0952c7d74a17d1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:23:14 -0400 Subject: [PATCH 08/51] fix(security): validate the API-key pepper at start, finish JWT bearer, circuit-aware audit actor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ApiKeyOptionsValidator (ValidateOnStart on any driver/admin node): a blank or placeholder Security:ApiKeys:Pepper fails host start instead of rejecting every key after a green boot (R03:2.3). - JWT is finished and opt-in by key (R03:2.4): with Security:Jwt:SigningKey set, a Bearer scheme (JwtBearerAuthenticationHandler over JwtTokenService.TryValidate / BuildValidationParameters, fail closed) accepts the tokens /auth/token mints; the default scheme is a cookie-or-bearer policy scheme and the fallback policy names it, so /api/* and every RequireAuthorization endpoint take a browser session OR a machine token. JwtOptions are validated at start (JwtOptionsValidator). Without a key, /auth/token is not mapped (404, not the former constructor 500) and no bearer scheme exists. A direct AddJwtBearer needs Microsoft.AspNetCore.Authentication.JwtBearer in Directory.Packages.props (not this lane's file) — the handler is a drop-in to swap. - AuthenticationStateAuditActorAccessor is the registered IAuditActorAccessor: the Blazor circuit's principal first, HttpContext as the fallback (R03:2.5). HttpAuditActorAccessor shares its resolver. - CookieAuthenticationStateProvider (unregistered, fire-and-forget ping loop) is deleted (R03:1.5/2.5). - /auth/login logs the LDAP exception at Warning before answering 503 (R03:2.7). - Admin operations reply with a fixed message + correlation id instead of ex.Message (R03:4.1). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 40 ++++-- .../Configuration/ApiKeyOptionsValidator.cs | 41 ++++++ src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs | 13 +- .../AuthenticationStateAuditActorAccessor.cs | 75 +++++++++++ .../Audit/HttpAuditActorAccessor.cs | 26 +--- .../Auth/JwtBearerAuthenticationHandler.cs | 99 ++++++++++++++ .../CookieAuthenticationStateProvider.cs | 79 ----------- .../Endpoints/AuthEndpoints.cs | 17 ++- .../Jwt/JwtOptions.cs | 22 +++ .../Jwt/JwtOptionsValidator.cs | 19 +++ .../ServiceCollectionExtensions.cs | 48 +++++-- ...nOperationsActorServerFaultMessageTests.cs | 68 ++++++++++ .../ApiKeyOptionsValidatorTests.cs | 54 ++++++++ ...henticationStateAuditActorAccessorTests.cs | 80 +++++++++++ .../AuthEndpointsIntegrationTests.cs | 67 +++++++++ .../CapturingLoggerProvider.cs | 25 ++++ .../JwtConfigurationGateTests.cs | 127 ++++++++++++++++++ 17 files changed, 773 insertions(+), 127 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs delete mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 8b1c9ddb..e78c926e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -89,6 +89,18 @@ public sealed class AdminOperationsActor : ReceiveActor Receive(HandleShelveAlarm); } + /// + /// The reply text for an unexpected server-side failure (R03:4.1): a fixed sentence plus the request's + /// correlation id, never ex.Message — a SqlException or UriFormatException message carries + /// connection-string fragments and host names, and the reply reaches any ConfigEditor browser session. + /// The detail is in the central log under the same correlation id. + /// + /// The admin operation that failed. + /// The request's correlation token, as logged alongside the exception. + /// The operator-facing message. + internal static string ServerFault(string operation, object correlationId) => + $"{operation} failed on the server; see the central node log for correlation {correlationId}."; + /// /// Hands a central→node command to the mesh transport router, or publishes it directly when /// no router is wired. @@ -136,8 +148,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId}", msg.AlarmId); - replyTo.Tell(new AcknowledgeAlarmResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: Acknowledge failed for alarm {AlarmId} (correlation {CorrelationId})", msg.AlarmId, msg.CorrelationId); + replyTo.Tell(new AcknowledgeAlarmResult(false, ServerFault("Acknowledge", msg.CorrelationId), msg.CorrelationId)); } } @@ -183,8 +195,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId}", msg.Kind, msg.AlarmId); - replyTo.Tell(new ShelveAlarmResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: Shelve ({Kind}) failed for alarm {AlarmId} (correlation {CorrelationId})", msg.Kind, msg.AlarmId, msg.CorrelationId); + replyTo.Tell(new ShelveAlarmResult(false, ServerFault("Shelve", msg.CorrelationId), msg.CorrelationId)); } } @@ -237,12 +249,12 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "StartDeployment failed for {CreatedBy}", msg.CreatedBy); + _log.Error(ex, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.CreatedBy, msg.CorrelationId); replyTo.Tell(new StartDeploymentResult( StartDeploymentOutcome.Rejected, DeploymentId: null, RevisionHash: null, - Message: ex.Message, + Message: ServerFault("StartDeployment", msg.CorrelationId), msg.CorrelationId)); return; } @@ -269,12 +281,12 @@ public sealed class AdminOperationsActor : ReceiveActor private void HandleDeployComposeFailed(DeployComposeFailed msg) { _composeInFlight = null; - _log.Error(msg.Error, "StartDeployment failed for {CreatedBy}", msg.Request.CreatedBy); + _log.Error(msg.Error, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.Request.CreatedBy, msg.Request.CorrelationId); msg.ReplyTo.Tell(new StartDeploymentResult( StartDeploymentOutcome.Rejected, DeploymentId: null, RevisionHash: null, - Message: msg.Error.Message, + Message: ServerFault("StartDeployment", msg.Request.CorrelationId), msg.Request.CorrelationId)); } @@ -513,10 +525,10 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "Probe for {DriverType} threw", msg.DriverType); + _log.Error(ex, "Probe for {DriverType} threw (correlation {CorrelationId})", msg.DriverType, msg.CorrelationId); return new TestDriverConnectResult( false, - ex.Message, + ServerFault("Test Connect", msg.CorrelationId), null, msg.CorrelationId); } @@ -548,8 +560,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId}", msg.DriverInstanceId); - replyTo.Tell(new RestartDriverResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: RestartDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId); + replyTo.Tell(new RestartDriverResult(false, ServerFault("RestartDriver", msg.CorrelationId), msg.CorrelationId)); } } @@ -578,8 +590,8 @@ public sealed class AdminOperationsActor : ReceiveActor } catch (Exception ex) { - _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId}", msg.DriverInstanceId); - replyTo.Tell(new ReconnectDriverResult(false, ex.Message, msg.CorrelationId)); + _log.Error(ex, "AdminOps: ReconnectDriver failed for {DriverInstanceId} (correlation {CorrelationId})", msg.DriverInstanceId, msg.CorrelationId); + replyTo.Tell(new ReconnectDriverResult(false, ServerFault("ReconnectDriver", msg.CorrelationId), msg.CorrelationId)); } } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs new file mode 100644 index 00000000..eced66a8 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ApiKeyOptionsValidator.cs @@ -0,0 +1,41 @@ +using ZB.MOM.WW.Configuration; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; + +namespace ZB.MOM.WW.OtOpcUa.Host.Configuration; + +/// +/// Fail-fast startup validator for (R03:2.3). The pepper used to be +/// checked only when the first key was verified — ApiKeySecret.Verify throws on an empty one — +/// so a site node deployed without Security__ApiKeys__Pepper booted green, passed health, +/// applied deployments and denied every machine client with the same opaque "Invalid API key" until +/// someone read the log; a central node with a wrong pepper minted keys that could verify nowhere. +/// Every other security section already fails host start on a bad value; this one now does too. +/// +/// +/// Registered for hasDriver || hasAdmin: a driver node needs the pepper to VERIFY keys, an admin +/// node to MINT them. The minimum length is a floor against a placeholder ("x") passing as +/// configured, not a strength claim — the pepper is a server-side secret, not a password. +/// +public sealed class ApiKeyOptionsValidator : OptionsValidatorBase +{ + /// The shortest pepper accepted (characters, after trimming). + public const int MinPepperLength = 16; + + /// + protected override void Validate(ValidationBuilder builder, ApiKeyOptions options) + { + var pepper = options.Pepper?.Trim() ?? string.Empty; + builder.RequireThat(pepper.Length > 0, + $"{ApiKeyOptions.SectionName}:Pepper is required on any node with the driver or admin role — " + + "supply it via Security__ApiKeys__Pepper or a ${secret:} reference. An empty pepper would reject " + + "every API key (driver) or mint keys that can never verify (admin)."); + builder.RequireThat(pepper.Length == 0 || pepper.Length >= MinPepperLength, + $"{ApiKeyOptions.SectionName}:Pepper must be at least {MinPepperLength} characters."); + builder.RequireThat(!string.IsNullOrWhiteSpace(options.Username), + $"{ApiKeyOptions.SectionName}:Username must not be blank."); + builder.RequireThat(options.StoreTimeoutMs >= 0, + $"{ApiKeyOptions.SectionName}:StoreTimeoutMs must be >= 0 (0 disables the deadline)."); + builder.RequireThat(options.StoreCacheSeconds >= 0, + $"{ApiKeyOptions.SectionName}:StoreCacheSeconds must be >= 0 (0 disables the cache)."); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs index d326d1b1..b2300ab6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs @@ -162,10 +162,15 @@ builder.AddZbSerilog(o => o.ServiceName = "otopcua"); // central over the fetch-and-cache path (ConfigSource:Mode=FetchAndCache). Registering it // unconditionally would throw on a driver-only node, since AddOtOpcUaConfigDb requires the string. // A fused admin+driver node keeps ConfigDb via its admin role — byte-for-byte unaffected. -// bound unconditionally: a driver node needs the pepper to VERIFY keys, an admin node needs it -// to MINT them, and getting this wrong on either side is a silent authentication failure rather than a -// startup error. -builder.Services.Configure(builder.Configuration.GetSection(ApiKeyOptions.SectionName)); +// A driver node needs the pepper to VERIFY keys, an admin node needs it to MINT them, and getting this +// wrong on either side used to be a silent authentication failure rather than a startup error (R03:2.3): +// ApiKeyOptionsValidator now fails host start on a blank or placeholder pepper wherever either role is +// present. A node with neither role (dev-only) has no key path and keeps the plain bind. +if (hasDriver || hasAdmin) + builder.Services.AddValidatedOptions( + builder.Configuration, ApiKeyOptions.SectionName); +else + builder.Services.Configure(builder.Configuration.GetSection(ApiKeyOptions.SectionName)); // Security:Acl:Enabled. Default false; see AclOptions for why the flip must be a config // change rather than a redeploy, and what it swaps rather than adds. builder.Services.Configure(builder.Configuration.GetSection(AclOptions.SectionName)); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs new file mode 100644 index 00000000..d689b85e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs @@ -0,0 +1,75 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.AspNetCore.Http; +using ZB.MOM.WW.Auth.AspNetCore; + +namespace ZB.MOM.WW.OtOpcUa.Security.Audit; + +/// +/// that reads the Blazor circuit's principal first (R03:2.5). +/// +/// +/// +/// reads IHttpContextAccessor, which is right for a +/// minimal-API request but wrong inside a Blazor Server circuit: after the first render there is no +/// current HttpContext on the circuit's threads, so every audit row written from an interactive +/// page would have been stamped "system". The circuit's identity lives in the scoped +/// , so that is consulted first and the HTTP context is the +/// fallback for non-circuit scopes (where the provider is not registered at all). +/// +/// +/// CurrentActor is synchronous by contract; the server-side provider's +/// is already completed in a +/// circuit, and an incomplete task is treated as "no circuit identity" rather than blocked on. +/// +/// +public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor +{ + private readonly IHttpContextAccessor _httpContextAccessor; + private readonly AuthenticationStateProvider? _authenticationState; + + /// Initializes a new instance of the class. + /// The HTTP context accessor (minimal-API fallback). + /// The circuit's authentication state, when resolved inside a Blazor circuit. + public AuthenticationStateAuditActorAccessor( + IHttpContextAccessor httpContextAccessor, + AuthenticationStateProvider? authenticationState = null) + { + ArgumentNullException.ThrowIfNull(httpContextAccessor); + _httpContextAccessor = httpContextAccessor; + _authenticationState = authenticationState; + } + + /// + public string? CurrentActor + { + get + { + if (_authenticationState is not null) + { + var task = _authenticationState.GetAuthenticationStateAsync(); + if (task.IsCompletedSuccessfully) + { + var actor = ResolveActor(task.Result.User); + if (actor is not null) return actor; + } + } + return ResolveActor(_httpContextAccessor.HttpContext?.User); + } + } + + /// The claim-resolution rule shared with . + /// The principal, or null. + /// The actor string, or null when unauthenticated. + internal static string? ResolveActor(ClaimsPrincipal? user) + { + if (user?.Identity?.IsAuthenticated != true) + return null; + + // Prefer the canonical login-name claim; fall back to the Name claim or + // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name). + return user.FindFirst(ZbClaimTypes.Username)?.Value + ?? user.FindFirst(ZbClaimTypes.Name)?.Value + ?? user.Identity.Name; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs index 9a86339f..529504b3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/HttpAuditActorAccessor.cs @@ -1,5 +1,4 @@ using Microsoft.AspNetCore.Http; -using ZB.MOM.WW.Auth.AspNetCore; namespace ZB.MOM.WW.OtOpcUa.Security.Audit; @@ -17,10 +16,10 @@ namespace ZB.MOM.WW.OtOpcUa.Security.Audit; /// , in that order. /// /// -/// Registered as scoped in -/// so that it correctly follows the request scope used by Blazor Server interactive components -/// and minimal-API endpoints. IHttpContextAccessor is registered by -/// AddOtOpcUaAuth via services.AddHttpContextAccessor(). +/// No longer the registered implementation: is, because +/// a Blazor circuit has no HttpContext after its first render and this accessor would stamp every row +/// written from an interactive page "system". Kept as the explicit HTTP-only variant for non-circuit +/// hosts and tests. /// /// public sealed class HttpAuditActorAccessor : IAuditActorAccessor @@ -35,19 +34,6 @@ public sealed class HttpAuditActorAccessor : IAuditActorAccessor } /// - public string? CurrentActor - { - get - { - var user = _httpContextAccessor.HttpContext?.User; - if (user?.Identity?.IsAuthenticated != true) - return null; - - // Prefer the canonical login-name claim; fall back to the Name claim or - // Identity.Name (both of which map to ClaimTypes.Name / ZbClaimTypes.Name). - return user.FindFirst(ZbClaimTypes.Username)?.Value - ?? user.FindFirst(ZbClaimTypes.Name)?.Value - ?? user.Identity.Name; - } - } + public string? CurrentActor => + AuthenticationStateAuditActorAccessor.ResolveActor(_httpContextAccessor.HttpContext?.User); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs new file mode 100644 index 00000000..cc996758 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs @@ -0,0 +1,99 @@ +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Primitives; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; + +namespace ZB.MOM.WW.OtOpcUa.Security.Auth; + +/// The authentication scheme names OtOpcUa registers. +public static class OtOpcUaAuthSchemes +{ + /// + /// The bearer scheme that validates the HS256 tokens /auth/token mints, through + /// . Registered only when + /// . + /// + public const string Bearer = "Bearer"; + + /// + /// The default scheme: a policy scheme that forwards to when the request + /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one + /// RequireAuthorization() / policy accepts a browser session AND a machine token. + /// + public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; + + /// Whether the request presents a bearer token (the selector's whole rule). + /// The request. + /// when an Authorization: Bearer … header is present. + public static bool HasBearerToken(HttpContext context) => + TryGetBearerToken(context.Request.Headers.Authorization, out _); + + internal static bool TryGetBearerToken(StringValues authorization, out string token) + { + token = string.Empty; + var header = authorization.ToString(); + const string prefix = "Bearer "; + if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; + token = header[prefix.Length..].Trim(); + return token.Length > 0; + } +} + +/// +/// Validates an Authorization: Bearer token minted by /auth/token (R03:2.4). The JWT +/// feature used to be a dead end: tokens were issued and nothing in the product accepted them. This +/// handler is the consuming side, built on so the validation +/// parameters can never drift from the minting side ( +/// sets RoleClaimType/NameClaimType to the minted keys, so [Authorize(Roles)] and +/// IsInRole resolve). +/// +/// +/// Fail closed: no header ⇒ NoResult (the cookie scheme is not consulted by this handler — the +/// policy scheme selects one or the other); a malformed, expired, tampered or wrong-issuer token ⇒ +/// Fail, never a fall-through to the cookie. A direct AddJwtBearer needs the +/// Microsoft.AspNetCore.Authentication.JwtBearer package, which the central package list does +/// not carry; this handler is ~40 lines over the already-referenced System.IdentityModel.Tokens.Jwt +/// and is a drop-in to replace once that package is pinned. +/// +public sealed class JwtBearerAuthenticationHandler : AuthenticationHandler +{ + private readonly JwtTokenService _jwt; + + /// Initializes a new instance of the class. + /// The scheme options monitor. + /// The logger factory. + /// The URL encoder. + /// The token service whose validation parameters mirror its minting side. + public JwtBearerAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder, + JwtTokenService jwt) + : base(options, logger, encoder) + { + _jwt = jwt; + } + + /// + protected override Task HandleAuthenticateAsync() + { + if (!OtOpcUaAuthSchemes.TryGetBearerToken(Request.Headers.Authorization, out var token)) + return Task.FromResult(AuthenticateResult.NoResult()); + + if (!_jwt.TryValidate(token, out var principal) || principal is null) + return Task.FromResult(AuthenticateResult.Fail("Invalid bearer token.")); + + return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name))); + } + + /// + protected override Task HandleChallengeAsync(AuthenticationProperties properties) + { + Response.StatusCode = StatusCodes.Status401Unauthorized; + Response.Headers.WWWAuthenticate = OtOpcUaAuthSchemes.Bearer; + return Task.CompletedTask; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs deleted file mode 100644 index 0e42b789..00000000 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Blazor/CookieAuthenticationStateProvider.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System.Net.Http.Json; -using System.Security.Claims; -using Microsoft.AspNetCore.Components.Authorization; -using Microsoft.Extensions.Logging; - -namespace ZB.MOM.WW.OtOpcUa.Security.Blazor; - -/// -/// Blazor Server that snapshots the cookie-backed -/// principal supplied at circuit boot and polls /auth/ping every 60 seconds to detect -/// expiry. Mirrors ScadaLink's CentralUI implementation. -/// -public sealed class CookieAuthenticationStateProvider : AuthenticationStateProvider, IAsyncDisposable -{ - private static readonly TimeSpan PingInterval = TimeSpan.FromSeconds(60); - - private readonly HttpClient _http; - private readonly ILogger _logger; - private readonly CancellationTokenSource _cts = new(); - private ClaimsPrincipal _current; - private Task? _pingLoop; - - /// Initializes a new instance of the CookieAuthenticationStateProvider class. - /// The initial claims principal from circuit boot. - /// The HTTP client for authentication ping requests. - /// The logger for diagnostic messages. - public CookieAuthenticationStateProvider( - ClaimsPrincipal initial, - HttpClient http, - ILogger logger) - { - _current = initial; - _http = http; - _logger = logger; - } - - /// - public override Task GetAuthenticationStateAsync() - { - _pingLoop ??= Task.Run(() => PingLoopAsync(_cts.Token)); - return Task.FromResult(new AuthenticationState(_current)); - } - - private async Task PingLoopAsync(CancellationToken ct) - { - try - { - while (!ct.IsCancellationRequested) - { - await Task.Delay(PingInterval, ct).ConfigureAwait(false); - var resp = await _http.GetAsync("/auth/ping", ct).ConfigureAwait(false); - if (!resp.IsSuccessStatusCode && _current.Identity?.IsAuthenticated == true) - { - _logger.LogInformation("/auth/ping returned {Code}; notifying circuit", (int)resp.StatusCode); - _current = new ClaimsPrincipal(new ClaimsIdentity()); - NotifyAuthenticationStateChanged( - Task.FromResult(new AuthenticationState(_current))); - } - } - } - catch (OperationCanceledException) { /* expected on shutdown */ } - catch (Exception ex) - { - _logger.LogWarning(ex, "Auth ping loop terminated unexpectedly"); - } - } - - /// Cancels the background ping loop, awaits its completion, and disposes the cancellation token source. - /// A task that represents the asynchronous operation. - public async ValueTask DisposeAsync() - { - _cts.Cancel(); - if (_pingLoop is not null) - { - try { await _pingLoop.ConfigureAwait(false); } catch { /* swallow shutdown errors */ } - } - _cts.Dispose(); - } -} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs index 56ac6ef7..6b0876d7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs @@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using ZB.MOM.WW.Auth.AspNetCore; using ZB.MOM.WW.Auth.Abstractions.Roles; using ZB.MOM.WW.OtOpcUa.Security.Jwt; @@ -36,7 +37,14 @@ public static class AuthEndpoints // AddOtOpcUaAuth fallback policy that would otherwise 401 the request. app.MapPost("/auth/login", (Delegate)LoginAsync).AllowAnonymous().DisableAntiforgery(); app.MapGet("/auth/ping", (Delegate)Ping).AllowAnonymous(); - app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured + // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call. + if (app.ServiceProvider.GetRequiredService>().Value.IsConfigured) + app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + else + app.ServiceProvider.GetService()? + .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") + .LogInformation("Security:Jwt:SigningKey is not configured — /auth/token is not mapped and bearer tokens are not accepted"); app.MapPost("/auth/logout", (Delegate)LogoutAsync).RequireAuthorization(); return app; } @@ -73,8 +81,13 @@ public static class AuthEndpoints { result = await ldap.AuthenticateAsync(username, password, ct); } - catch (Exception) + catch (Exception ex) when (ex is not OperationCanceledException) { + // R03:2.7 — a directory outage used to leave NOTHING in the server log from this path; the + // operator had to infer every 503 from the directory side. + http.RequestServices.GetService()? + .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") + .LogWarning(ex, "LDAP authentication failed for {User}; answering 503", username); return Results.StatusCode(StatusCodes.Status503ServiceUnavailable); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs index 07cbdda4..f523a033 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptions.cs @@ -15,4 +15,26 @@ public sealed class JwtOptions /// Default token expiry. Mirrors ScadaLink (15 min). public int ExpiryMinutes { get; set; } = 15; + + /// + /// Whether the JWT feature is on: a non-blank . When off, /auth/token + /// is not mapped and no bearer scheme is registered, so an admin node that never configured a key + /// keeps booting exactly as before; when on, the key is validated at host start + /// (ValidateOnStart) instead of throwing HTTP 500 on the first token request. + /// + public bool IsConfigured => !string.IsNullOrWhiteSpace(SigningKey); + + /// The validation rule shared by the startup validator and the tests: why the options are invalid, or null. + /// A failure message, or when valid. + public string? Validate() + { + if (!IsConfigured) return null; + var keyBytes = System.Text.Encoding.UTF8.GetByteCount(SigningKey); + if (keyBytes < MinSigningKeyBytes) + return $"{SectionName}:SigningKey must be at least {MinSigningKeyBytes} bytes (256 bits) UTF-8 for HMAC-SHA256; the configured key is {keyBytes} byte(s)."; + if (string.IsNullOrWhiteSpace(Issuer)) return $"{SectionName}:Issuer must not be blank."; + if (string.IsNullOrWhiteSpace(Audience)) return $"{SectionName}:Audience must not be blank."; + if (ExpiryMinutes <= 0) return $"{SectionName}:ExpiryMinutes must be > 0."; + return null; + } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs new file mode 100644 index 00000000..c3805267 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtOptionsValidator.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.Options; + +namespace ZB.MOM.WW.OtOpcUa.Security.Jwt; + +/// +/// Startup validator for (R03:2.4): when a signing key is configured it must +/// be usable, and the host must learn that at start (ValidateOnStart) rather than from an HTTP 500 +/// on the first /auth/token. An unconfigured section (no key) is valid — the feature is simply off. +/// +public sealed class JwtOptionsValidator : IValidateOptions +{ + /// + public ValidateOptionsResult Validate(string? name, JwtOptions options) + { + ArgumentNullException.ThrowIfNull(options); + var failure = options.Validate(); + return failure is null ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failure); + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs index 6df24cad..d58fbcf3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs @@ -19,9 +19,12 @@ using ZB.MOM.WW.OtOpcUa.Security.Ldap; namespace ZB.MOM.WW.OtOpcUa.Security; /// -/// DI registration for OtOpcUa auth. Single Cookie scheme (the JWT lives inside the -/// cookie as its credential payload); no JwtBearer parallel scheme. Matches ScadaBridge -/// structurally — see docs/plans/2026-05-29-auth-alignment-design.md. +/// DI registration for OtOpcUa auth. The cookie scheme carries the browser session; when +/// Security:Jwt:SigningKey is configured a scheme +/// accepts the tokens /auth/token mints (R03:2.4), and the default scheme is a policy scheme +/// that forwards to whichever the request presents — so /api/* and every +/// RequireAuthorization endpoint take a browser session OR a machine token. Matches +/// ScadaBridge structurally — see docs/plans/2026-05-29-auth-alignment-design.md. /// public static class ServiceCollectionExtensions { @@ -36,7 +39,14 @@ public static class ServiceCollectionExtensions /// The same service collection, for chaining. public static IServiceCollection AddOtOpcUaAuth(this IServiceCollection services, IConfiguration configuration) { - services.AddOptions().Bind(configuration.GetSection(JwtOptions.SectionName)); + // JWT is opt-in by key: with no SigningKey the feature is off (no /auth/token, no bearer scheme) + // and an admin node boots as before; with one, the key is validated at host START (R03:2.4) — + // the JwtTokenService constructor used to throw on the first /auth/token call (HTTP 500). + services.AddOptions() + .Bind(configuration.GetSection(JwtOptions.SectionName)) + .ValidateOnStart(); + services.AddSingleton, JwtOptionsValidator>(); + var jwtConfigured = configuration.GetSection(JwtOptions.SectionName).Get()?.IsConfigured ?? false; services.AddOptions().Bind(configuration.GetSection(OtOpcUaCookieOptions.SectionName)); services.AddOptions().Bind(configuration.GetSection(LdapOptions.SectionName)); services.AddOptions().Bind(configuration.GetSection(AuthDisableLoginOptions.SectionName)); @@ -53,7 +63,10 @@ public static class ServiceCollectionExtensions // as the Actor field when constructing a canonical ZB.MOM.WW.Audit.AuditEvent. Registered // Scoped so it correctly follows the request scope used by Blazor Server and minimal-API // endpoints. - services.TryAddScoped(); + // AuthenticationStateAuditActorAccessor reads the Blazor circuit's principal first (no HttpContext + // exists on a circuit's threads after the first render) and falls back to the HTTP context for + // minimal-API scopes, where AuthenticationStateProvider is not registered (R03:2.5). + services.TryAddScoped(); // Singleton — OtOpcUaLdapAuthService is stateless (the shared-library directory client it // wraps opens/disposes an LdapConnection per call) and must be consumable by the Singleton @@ -78,7 +91,26 @@ public static class ServiceCollectionExtensions .GetSection(AuthDisableLoginOptions.SectionName) .GetValue(nameof(AuthDisableLoginOptions.DisableLogin)); - var authBuilder = services.AddAuthentication(CookieAuthenticationDefaults.AuthenticationScheme); + // The default scheme is the cookie-or-bearer selector; sign-in/out stay on the cookie scheme, which is + // the only one that can issue a session. With JWT off the selector always picks the cookie. + var authBuilder = services.AddAuthentication(o => + { + o.DefaultScheme = OtOpcUaAuthSchemes.CookieOrBearer; + o.DefaultSignInScheme = CookieAuthenticationDefaults.AuthenticationScheme; + o.DefaultSignOutScheme = CookieAuthenticationDefaults.AuthenticationScheme; + }); + authBuilder.AddPolicyScheme(OtOpcUaAuthSchemes.CookieOrBearer, "Cookie or Bearer", o => + { + o.ForwardDefaultSelector = ctx => + jwtConfigured && OtOpcUaAuthSchemes.HasBearerToken(ctx) + ? OtOpcUaAuthSchemes.Bearer + : CookieAuthenticationDefaults.AuthenticationScheme; + }); + if (jwtConfigured) + { + authBuilder.AddScheme( + OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + } if (disableLogin) { // DEV/TEST ONLY: replace the cookie handler with an always-succeeding handler registered @@ -142,8 +174,8 @@ public static class ServiceCollectionExtensions services.AddAuthorization(o => { - o.FallbackPolicy = new AuthorizationPolicyBuilder( - CookieAuthenticationDefaults.AuthenticationScheme) + // Named on the selector, not the cookie scheme, so the fallback accepts a bearer token too. + o.FallbackPolicy = new AuthorizationPolicyBuilder(OtOpcUaAuthSchemes.CookieOrBearer) .RequireAuthenticatedUser() .Build(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs new file mode 100644 index 00000000..f4e6fd22 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AdminOperationsActorServerFaultMessageTests.cs @@ -0,0 +1,68 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin; +using ZB.MOM.WW.OtOpcUa.Commons.Types; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:4.1 — admin operations echoed ex.Message to the UI, so a SqlException's connection-string +/// fragment reached any ConfigEditor browser session. The reply is now a fixed sentence plus the request's +/// correlation id; the detail stays in the central log. +/// +public sealed class AdminOperationsActorServerFaultMessageTests : ControlPlaneActorTestBase +{ + private const string Secret = "Server=sql-prod-01;Password=hunter2"; + + [Fact] + public void StartDeployment_server_fault_reply_carries_the_correlation_id_and_not_the_exception_text() + { + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(new ThrowingDbFactory(), coordinator.Ref, Enumerable.Empty())); + var correlationId = CorrelationId.NewId(); + + actor.Tell(new StartDeployment("joe", correlationId)); + + var reply = ExpectMsg(PresenceBudget); + reply.Outcome.ShouldBe(StartDeploymentOutcome.Rejected); + reply.Message.ShouldNotBeNull(); + reply.Message.ShouldNotContain("hunter2"); + reply.Message.ShouldNotContain("sql-prod-01"); + reply.Message.ShouldContain(correlationId.ToString()); // the operator can find the log line + } + + [Fact] + public void TestDriverConnect_probe_throw_reply_carries_the_correlation_id_and_not_the_exception_text() + { + var coordinator = CreateTestProbe("coord"); + var actor = Sys.ActorOf(AdminOperationsActor.Props(NewInMemoryDbFactory(), coordinator.Ref, new IDriverProbe[] { new ThrowingProbe() })); + var correlationId = Guid.NewGuid(); + + actor.Tell(new TestDriverConnect("Throwing", "{}", 5, correlationId)); + + var reply = ExpectMsg(PresenceBudget); + reply.Ok.ShouldBeFalse(); + reply.Message.ShouldNotBeNull(); + reply.Message.ShouldNotContain("hunter2"); + reply.Message.ShouldContain(correlationId.ToString()); + } + + private sealed class ThrowingDbFactory : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => + throw new InvalidOperationException("Cannot open database: " + Secret); + } + + private sealed class ThrowingProbe : IDriverProbe + { + public string DriverType => "Throwing"; + public Task ProbeAsync(string configJson, TimeSpan timeout, CancellationToken ct) => + throw new InvalidOperationException("probe blew up: " + Secret); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs new file mode 100644 index 00000000..e91fbda0 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/ApiKeyOptionsValidatorTests.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.Configuration; + +/// +/// R03:2.3 — an unset or placeholder pepper must fail host start, not surface as "Invalid API key" +/// on the first activation. +/// +public sealed class ApiKeyOptionsValidatorTests +{ + private static ValidateOptionsResult Validate(ApiKeyOptions options) => + new ApiKeyOptionsValidator().Validate(Options.DefaultName, options); + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void Blank_pepper_fails(string pepper) + { + var result = Validate(new ApiKeyOptions { Pepper = pepper }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain("Pepper is required"); + } + + [Fact] + public void Short_pepper_fails() + { + var result = Validate(new ApiKeyOptions { Pepper = "x" }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain($"at least {ApiKeyOptionsValidator.MinPepperLength}"); + } + + [Fact] + public void Blank_username_fails() + { + var result = Validate(new ApiKeyOptions { Pepper = new string('p', 32), Username = " " }); + result.Failed.ShouldBeTrue(); + result.FailureMessage.ShouldNotBeNull(); + result.FailureMessage.ShouldContain("Username"); + } + + /// Positive control: the docker-dev rig's pepper and the defaults pass unchanged. + [Fact] + public void Configured_pepper_passes() + { + Validate(new ApiKeyOptions { Pepper = "docker-dev-apikey-pepper-not-for-production" }) + .Succeeded.ShouldBeTrue(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs new file mode 100644 index 00000000..5dc1f391 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs @@ -0,0 +1,80 @@ +using System.Security.Claims; +using Microsoft.AspNetCore.Components.Authorization; +using Microsoft.AspNetCore.Http; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Auth.AspNetCore; +using ZB.MOM.WW.OtOpcUa.Security.Audit; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests.Audit; + +/// +/// R03:2.5 — inside a Blazor circuit there is no HttpContext after the first render, so an +/// HTTP-only accessor stamps every audit row from an interactive page "system". The circuit's +/// is consulted first; the HTTP context is the fallback. +/// +public sealed class AuthenticationStateAuditActorAccessorTests +{ + [Fact] + public void Circuit_principal_wins_when_there_is_no_http_context() + { + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new FixedStateProvider(Authenticated(new Claim(ZbClaimTypes.Username, "alice")))); + + accessor.CurrentActor.ShouldBe("alice"); + } + + [Fact] + public void Http_principal_is_the_fallback_outside_a_circuit() + { + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "bob")) }; + var accessor = new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), authenticationState: null); + + accessor.CurrentActor.ShouldBe("bob"); + } + + [Fact] + public void Unauthenticated_circuit_falls_through_to_http_then_null() + { + var anonymousCircuit = new FixedStateProvider(new ClaimsPrincipal(new ClaimsIdentity())); + + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(null), anonymousCircuit) + .CurrentActor.ShouldBeNull(); + + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Name, "carol")) }; + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), anonymousCircuit) + .CurrentActor.ShouldBe("carol"); + } + + /// An incomplete state task is "no circuit identity", never a block. + [Fact] + public void Pending_authentication_state_is_not_awaited() + { + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new PendingStateProvider()); + + accessor.CurrentActor.ShouldBeNull(); + } + + private static ClaimsPrincipal Authenticated(params Claim[] claims) => + new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role)); + + private sealed class HttpContextAccessorStub(HttpContext? context) : IHttpContextAccessor + { + public HttpContext? HttpContext { get; set; } = context; + } + + private sealed class FixedStateProvider(ClaimsPrincipal user) : AuthenticationStateProvider + { + public override Task GetAuthenticationStateAsync() => + Task.FromResult(new AuthenticationState(user)); + } + + private sealed class PendingStateProvider : AuthenticationStateProvider + { + public override Task GetAuthenticationStateAsync() => + new TaskCompletionSource().Task; + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs index 9cf8ab11..e5e3b620 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs @@ -11,6 +11,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Shouldly; using Xunit; using ZB.MOM.WW.Auth.AspNetCore; @@ -37,6 +38,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime private IHost _host = null!; private TestServer _server = null!; private readonly StubLdapGroupRoleMappingService _roleMappings = new(); + private readonly CapturingLoggerProvider _logs = new(); private static CancellationToken Ct => TestContext.Current.CancellationToken; @@ -70,6 +72,7 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime ["Security:Ldap:GroupToRole:ReadOnly"] = "Viewer", }).Build(); services.AddOtOpcUaAuth(configuration); + services.AddLogging(l => l.AddProvider(_logs)); services.AddSingleton(); // The login handler now resolves the DB role-map service via DI to merge // DB-backed grants on top of the appsettings baseline. Register the stub so @@ -96,6 +99,10 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime .ToArray(); return Results.Ok(claims); }).RequireAuthorization(); + // A role-gated machine endpoint: proves a bearer principal's roles resolve through + // IsInRole (the validation parameters pair RoleClaimType with the minted key). + e.MapGet("/api/viewer-only", () => Results.Ok("viewer")) + .RequireAuthorization(p => p.RequireRole("Viewer")); }); }); }) @@ -153,6 +160,66 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime new AuthEndpoints.LoginRequest("ldap-down", "anything"), Ct); response.StatusCode.ShouldBe(HttpStatusCode.ServiceUnavailable); + // R03:2.7 — the outage is named in the server log, not inferred from the directory side. + _logs.Entries.Any(e => + e.Level == LogLevel.Warning && e.Message.Contains("LDAP authentication failed") && e.Exception != null) + .ShouldBeTrue("a Warning naming the LDAP failure must be logged"); + } + + /// + /// R03:2.4 — a token from /auth/token authenticates a request carrying it as a bearer: + /// the fallback-protected root AND a role-gated endpoint (the bearer principal's "Role" claims + /// resolve through IsInRole). No cookie is sent. + /// + [Fact] + public async Task Bearer_token_authenticates_api_requests_without_a_cookie() + { + var token = await LoginAndIssueTokenAsync(); + var client = NewClient(); + + var root = new HttpRequestMessage(HttpMethod.Get, "/"); + root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + var gated = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + gated.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + var gatedResponse = await client.SendAsync(gated, Ct); + gatedResponse.StatusCode.ShouldBe(HttpStatusCode.OK); + (await gatedResponse.Content.ReadAsStringAsync(Ct)).ShouldContain("viewer"); + } + + /// A tampered or garbage bearer fails closed (401), it never falls through to anonymous. + [Theory] + [InlineData("not-a-jwt")] + [InlineData("tampered")] + public async Task Bad_bearer_token_is_rejected(string kind) + { + var token = kind == "tampered" + ? (await LoginAndIssueTokenAsync()) + "x" + : kind; + var client = NewClient(); + var req = new HttpRequestMessage(HttpMethod.Get, "/"); + req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + req.Headers.Add("X-Requested-With", "XMLHttpRequest"); + + var response = await client.SendAsync(req, Ct); + + response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); + } + + private async Task LoginAndIssueTokenAsync() + { + var client = NewClient(); + var loginResponse = await client.PostAsJsonAsync("/auth/login", + new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + loginResponse.EnsureSuccessStatusCode(); + var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + AttachCookies(tokenReq, loginResponse); + var response = await client.SendAsync(tokenReq, Ct); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadFromJsonAsync(Ct); + return payload.GetProperty("token").GetString()!; } /// Tests that ping anonymous returns 401. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs new file mode 100644 index 00000000..a79ab569 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/CapturingLoggerProvider.cs @@ -0,0 +1,25 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests; + +/// Collects every log entry written through the host's logger factory, for assertions on what was logged. +internal sealed class CapturingLoggerProvider : ILoggerProvider +{ + public ConcurrentQueue Entries { get; } = new(); + + public ILogger CreateLogger(string categoryName) => new Logger(categoryName, Entries); + + public void Dispose() { } + + internal sealed record Entry(string Category, LogLevel Level, string Message, Exception? Exception); + + private sealed class Logger(string category, ConcurrentQueue entries) : ILogger + { + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + entries.Enqueue(new Entry(category, logLevel, formatter(state, exception), exception)); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs new file mode 100644 index 00000000..96fc88ca --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs @@ -0,0 +1,127 @@ +using System.Net; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Services; +using ZB.MOM.WW.OtOpcUa.Security.Auth; +using ZB.MOM.WW.OtOpcUa.Security.Endpoints; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; +using ZB.MOM.WW.OtOpcUa.Security.Ldap; + +namespace ZB.MOM.WW.OtOpcUa.Security.Tests; + +/// +/// R03:2.4 — JWT is opt-in by signing key. Unconfigured: /auth/token is not mapped (it used to +/// answer HTTP 500 from the service constructor) and no bearer scheme exists. Configured badly: the +/// host refuses to START rather than failing on the first token request. +/// +public sealed class JwtConfigurationGateTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Without_a_signing_key_auth_token_is_not_mapped_and_bearer_is_ignored() + { + using var host = await StartHostAsync(signingKey: null); + var server = host.GetTestServer(); + var client = server.CreateClient(); + + // The login still works — the cookie path is untouched. + var login = await client.PostAsJsonAsync("/auth/login", new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + login.StatusCode.ShouldBe(HttpStatusCode.NoContent); + + // /auth/token is simply absent (404), not a 500. + var tokenReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + foreach (var c in login.Headers.GetValues("Set-Cookie")) tokenReq.Headers.Add("Cookie", c.Split(';')[0]); + (await client.SendAsync(tokenReq, Ct)).StatusCode.ShouldBe(HttpStatusCode.NotFound); + + // No bearer scheme is registered, so a bearer header is ignored and the cookie session still wins. + var schemes = host.Services.GetRequiredService(); + (await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer)).ShouldBeNull(); + var withBogusBearer = new HttpRequestMessage(HttpMethod.Get, "/"); + foreach (var c in login.Headers.GetValues("Set-Cookie")) withBogusBearer.Headers.Add("Cookie", c.Split(';')[0]); + withBogusBearer.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", "garbage"); + (await client.SendAsync(withBogusBearer, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + await host.StopAsync(Ct); + } + + /// Positive control for the gate: with a key, the bearer scheme IS registered. + [Fact] + public async Task With_a_signing_key_the_bearer_scheme_is_registered() + { + using var host = await StartHostAsync(signingKey: "test-signing-key-with-at-least-32-bytes-of-utf8-content"); + var schemes = host.Services.GetRequiredService(); + var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer); + bearer.ShouldNotBeNull(); + bearer.HandlerType.ShouldBe(typeof(JwtBearerAuthenticationHandler)); + await host.StopAsync(Ct); + } + + /// A configured-but-short key fails at host start (ValidateOnStart), not on the first /auth/token. + [Fact] + public async Task A_short_signing_key_fails_host_start() + { + var ex = await Should.ThrowAsync(async () => + { + using var host = await StartHostAsync(signingKey: "too-short"); + }); + ex.Message.ShouldContain("SigningKey"); + } + + private static async Task StartHostAsync(string? signingKey) + { + var dbName = $"jwt-gate-{Guid.NewGuid():N}"; + var settings = new Dictionary(); + if (signingKey is not null) settings["Security:Jwt:SigningKey"] = signingKey; + var configuration = new ConfigurationBuilder().AddInMemoryCollection(settings).Build(); + + var host = new HostBuilder() + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.ConfigureServices(services => + { + services.AddDbContextFactory(opt => opt.UseInMemoryDatabase(dbName)); + services.AddDbContext(opt => opt.UseInMemoryDatabase(dbName)); + services.AddRouting(); + services.AddOtOpcUaAuth(configuration); + services.AddSingleton(); + services.AddSingleton(); + }); + web.Configure(app => + { + app.UseRouting(); + app.UseAuthentication(); + app.UseAuthorization(); + app.UseEndpoints(e => + { + e.MapOtOpcUaAuth(); + e.MapGet("/", () => Results.Ok("authenticated")).RequireAuthorization(); + }); + }); + }) + .Build(); + await host.StartAsync(Ct); + return host; + } + + private sealed class AlwaysAliceLdap : ILdapAuthService + { + public Task AuthenticateAsync(string username, string password, CancellationToken ct = default) => + Task.FromResult(new LdapAuthResult( + Success: password == "valid-password", DisplayName: "Alice", Username: username, + Groups: [], Roles: [], Error: null)); + } +} From 4d3fc9c8167b075fbaee1bfa0121ef6070efba1c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:27:49 -0400 Subject: [PATCH 09/51] fix(runtime): restart-safe alarm host, mux last-value replay, VT log rate limit, Calculation remediation (E4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - R02:H1 ScriptedAlarmHostActor gains a factory Props (Func<(upstream, engine)>) so every incarnation builds and owns its own engine; a supervision restart no longer re-creates the actor around a disposed engine. The legacy instance-taking Props stays source-compatible and is now restart-safe in the weaker sense (PreRestart marks the stop as a restart, PostStop then detaches instead of disposing). DriverHostActor still spawns through the legacy overload (not owned by this lane — F3). - R02:H3 OnAlarmCommand runs every engine op under a 10 s token linked to the actor's lifetime (DefaultCommandTimeout / Props commandTimeout) and logs a Warning on timeout instead of suspending the mailbox on a stalled store. - R02:L3 DependencyMuxTagUpstreamSource.RetainOnly(loaded dep-ref union), called after every successful load. - R02:P6 VirtualTagActor's per-failure ScriptLogEntry is rate-limited per tag (ScriptLogRateLimiter, 20/min + one summary); the Bad publish and the metric are untouched. - R07:H6 DependencyMuxActor retains the last (value, status, ts) per ever-registered ref (through unregisters) and replays them in one batch to a RegisterInterest(ReplayRetained: true) — VirtualTagActor, the Calculation mux adapter and the alarm host opt in; the historization recorder does not (a replay there would record a stale sample per deploy). CalculationDriver records inputs regardless of _running so a replay landing before InitializeAsync still satisfies the change-gate. - R07:P4 CalculationEvaluator negative-caches compile failures by source hash (one Roslyn attempt + one Warning per source until redeploy); the driver's script-log entry fires on the Good→Bad transition or a change of reason, matching the once-per-transition Bad publish. - R07:H8 array results dedup structurally. - R07:L2 ShutdownAsync drops the compiled scripts (collectible ALCs). - R07:U5 DraftValidator rule CalculationNoEffectiveTrigger: a tag with no timer whose change trigger is off or whose script has no literal ctx.GetTag read can never evaluate. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Validation/DraftValidator.cs | 48 +++++ .../CalculationDriver.cs | 48 ++++- .../CalculationEvaluator.cs | 39 +++- .../DependencyMuxTagUpstreamSource.cs | 28 ++- .../ScriptedAlarms/ScriptedAlarmHostActor.cs | 161 ++++++++++++++--- .../DependencyConsumerMuxAdapter.cs | 5 +- .../VirtualTags/DependencyMuxActor.cs | 68 ++++--- .../VirtualTags/VirtualTagActor.cs | 36 +++- .../DraftValidatorCalculationTests.cs | 56 +++++- .../CalculationDriverRemediationTests.cs | 166 ++++++++++++++++++ .../CalculationDriverTests.cs | 6 +- .../DependencyMuxTagUpstreamSourceTests.cs | 22 +++ .../ScriptedAlarmHostActorRestartTests.cs | 145 +++++++++++++++ .../VirtualTags/DependencyMuxReplayTests.cs | 101 +++++++++++ ...VirtualTagActorFailureLogRateLimitTests.cs | 55 ++++++ 15 files changed, 921 insertions(+), 63 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs index 04e08fe9..3b8d0bb1 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs @@ -452,6 +452,54 @@ public static class DraftValidator "Calculation tags form a dependency cycle (members: " + string.Join(", ", cycle) + "); a calc→calc cycle is a live oscillation loop and must be broken before deploy", cycle.Count > 0 ? cycle[0] : null)); + + // Rule 3 — effective-trigger gate (R07:U5). A calc tag evaluates on a declared dependency + // changing (change-trigger: the literal ctx.GetTag paths) and/or on a timer. A change-triggered + // tag whose script has NO literal reads (`return ctx.Now.Hour;`, a dynamic-path read) and no timer + // never evaluates at all: it sits BadWaitingForInitialData forever while the driver reports + // Healthy, and nothing warned. Same for a tag that is neither change- nor timer-triggered. + foreach (var (t, _, scriptId) in calcTags) + { + var (changeTriggered, timerIntervalMs) = ParseTriggers(t.TagConfig); + if (timerIntervalMs is > 0) continue; + var source = scriptId is not null && scriptSourceById.TryGetValue(scriptId, out var s) ? s : null; + var depCount = EquipmentScriptPaths.ExtractDependencyRefs(source).Count; + if (changeTriggered && depCount > 0) continue; + errors.Add(new("CalculationNoEffectiveTrigger", + changeTriggered + ? $"Calculation tag '{t.TagId}' is change-triggered but its script has no literal ctx.GetTag read, " + + "and it has no timerIntervalMs — nothing would ever evaluate it. Add a dependency read or a timer." + : $"Calculation tag '{t.TagId}' has changeTriggered=false and no timerIntervalMs — nothing would ever evaluate it. " + + "Enable the change trigger or add a timer.", + t.TagId)); + } + } + + /// Read the trigger fields from a calc tag's TagConfig: changeTriggered + /// (default true, the driver's own default) and timerIntervalMs (null when absent or not a + /// number). Never throws — malformed JSON yields the defaults. + private static (bool ChangeTriggered, int? TimerIntervalMs) ParseTriggers(string? tagConfig) + { + if (string.IsNullOrWhiteSpace(tagConfig)) return (true, null); + try + { + using var doc = System.Text.Json.JsonDocument.Parse(tagConfig); + var root = doc.RootElement; + if (root.ValueKind != System.Text.Json.JsonValueKind.Object) return (true, null); + var changeTriggered = true; + if (root.TryGetProperty("changeTriggered", out var ct) + && ct.ValueKind is System.Text.Json.JsonValueKind.False or System.Text.Json.JsonValueKind.True) + changeTriggered = ct.GetBoolean(); + int? timer = null; + if (root.TryGetProperty("timerIntervalMs", out var tm) + && tm.ValueKind == System.Text.Json.JsonValueKind.Number && tm.TryGetInt32(out var ms)) + timer = ms; + return (changeTriggered, timer); + } + catch (System.Text.Json.JsonException) + { + return (true, null); + } } /// Extract the scriptId string from a calc tag's schemaless TagConfig JSON. Never diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs index dc0d8ba1..905fff13 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs @@ -338,6 +338,11 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba StopTimers(); // The pump exits at its next iteration; an evaluation already in flight is bounded by RunTimeout. lock (_lock) _pending.Clear(); + // R07:L2 — nothing in the runtime disposes a driver, and every config change is a respawn, so + // the compiled scripts' collectible ALCs survived each generation until GC got round to the whole + // driver graph. Drop them here (idempotent; a later InitializeAsync recompiles on first use). An + // in-flight run that races this is already handled by the evaluator's retry-once path. + _evaluator.ClearCompiledScripts(); return Task.CompletedTask; } @@ -363,11 +368,15 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// public void OnDependencyValue(string rawPath, object? value, uint statusCode, DateTime timestampUtc) { - if (!_running) return; lock (_lock) { + // Recorded REGARDLESS of _running (R07:H6): the mux adapter registers interest at construction, + // so inputs — the mux's retained-value replay in particular — can land before InitializeAsync. + // Dropping them left every calc tag waiting for each input to change again; keeping them means + // the first trigger after init evaluates from current inputs. Only the EVALUATION is gated. _depValues[rawPath] = new DataValueSnapshot(value, statusCode, timestampUtc, timestampUtc); _arrivedDeps.Add(rawPath); + if (!_running) return; if (!_changeDependents.TryGetValue(rawPath, out var dependents) || dependents.Count == 0) return; // QUEUE each change-triggered dependent — never evaluate here. This @@ -600,7 +609,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba var result = await _evaluator.EvaluateAsync(def.RawPath, def.ScriptSource, pending.Dependencies, _evalCts.Token) .ConfigureAwait(false); - string? failureReason = null; + string? failureToLog = null; (object? Value, uint StatusCode)? toPublish; lock (_lock) { @@ -610,16 +619,24 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba if (!result.Success) { - failureReason = result.Reason ?? "evaluator failure"; + var reason = result.Reason ?? "evaluator failure"; toPublish = MarkEvaluationFailed(st, pending.TimestampUtc); + // The script-log entry fires on the Good→Bad TRANSITION, or when the failure changes + // (R07:P4) — not on every trigger of a tag that is already Bad for the same reason, which + // flooded the script log at the input's cadence. The Bad publish was already once per + // transition; the log now matches it. + if (toPublish is not null || !string.Equals(st.LastFailureReason, reason, StringComparison.Ordinal)) + failureToLog = reason; + st.LastFailureReason = reason; } else { + st.LastFailureReason = null; toPublish = MarkEvaluationSucceeded(st, result.Value, pending.WorstInputStatus, pending.TimestampUtc); } } - if (failureReason is not null) PublishScriptLog(def, failureReason); + if (failureToLog is not null) PublishScriptLog(def, failureToLog); if (toPublish is { } p) Publish(def.RawPath, p.Value, p.StatusCode, pending.TimestampUtc); } @@ -638,7 +655,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba private (object? Value, uint StatusCode)? MarkEvaluationSucceeded( TagState st, object? value, uint statusCode, DateTime timestampUtc) { - if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && Equals(st.LastValue, value)) + if (st.HasPublished && st.LastPublishedStatus == statusCode && st.HasValue && ValuesEqual(st.LastValue, value)) return null; st.HasValue = true; @@ -650,6 +667,17 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba return (value, statusCode); } + /// Value equality for the dedup: structural for arrays (R07:H8 — object.Equals is reference + /// equality for a double[], so an array-returning timer script republished every tick through + /// the dual-namespace fan-out and the historian), ordinary + /// otherwise. + private static bool ValuesEqual(object? a, object? b) + { + if (a is Array x && b is Array y) + return System.Collections.StructuralComparisons.StructuralEqualityComparer.Equals(x, y); + return Equals(a, b); + } + /// Record that this tag's INPUTS are Bad, so no evaluation ran. Publishes the /// worst input's status — carrying the last known value — once per change of that status: a /// repeat of the same Bad is silent, while BadNoCommunication arriving over an earlier @@ -727,6 +755,12 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// Test/diagnostic accessor: number of timer ticks skipped for in-flight groups. internal long TimerSkippedTickCount => _timers?.SkippedTickCount ?? 0; + /// Test/diagnostic accessor: sources currently remembered as failing to compile. + internal int CompileFailureCountForTest => _evaluator.CompileFailureCount; + + /// Test/diagnostic accessor: compiled scripts currently held by the evaluator's cache. + internal int CompiledScriptCountForTest => _evaluator.CompiledScriptCount; + /// Test/diagnostic accessor: when no change-triggered evaluation is /// queued or in flight. A test that used to assert straight after — /// which evaluated inline — waits on this instead, so the assertion still runs against a settled @@ -770,6 +804,10 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba /// now carries the failing INPUT's sub-code rather than a fixed BadInternalError. public uint LastPublishedStatus; public DateTime LastTimestampUtc; + + /// The reason of the last failed evaluation (null after a success / inputs-Bad hold) — the + /// gate that keeps the script-log entry to the transition and to a CHANGE of reason. + public string? LastFailureReason; } private sealed record CalculationSubscriptionHandle(string DiagnosticId) : ISubscriptionHandle; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index 4296b993..ba4931c7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using Microsoft.Extensions.Logging; using ZB.MOM.WW.OtOpcUa.Commons.Engines; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; @@ -58,6 +59,13 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO private readonly SerilogLogger _scriptRoot; private readonly TimeSpan _runTimeout; private readonly ScriptRunGuard _guard; + + // Negative compile cache (R07:P4), keyed by source hash. A compile error is a property of the SOURCE, + // which never changes between triggers (it is fixed in the deploy artifact) — yet every trigger of a + // broken tag used to re-run a full Roslyn compile under the driver-wide evaluation gate, starving the + // healthy tags behind it, and log the same Warning each time. The failure is now remembered until the + // next ClearCompiledScripts (a redeploy), and the host Warning fires once per source. + private readonly ConcurrentDictionary _compileFailures = new(StringComparer.Ordinal); private bool _disposed; /// Initializes a new . @@ -180,6 +188,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO calcTagId, path), logger: scriptLog); + // A source that already failed to compile fails the same way without touching Roslyn (R07:P4). + var sourceKey = HashSource(scriptSource); + if (_compileFailures.TryGetValue(sourceKey, out var knownFailure)) + return VirtualTagEvalResult.Failure(knownFailure); + // Fetch + run inside a bounded loop so an ObjectDisposedException caused by a concurrent // ClearCompiledScripts (which disposed the evaluator between GetOrCompile and the run's // disposed guard) re-fetches and retries exactly once. The loop runs at most twice. @@ -192,13 +205,11 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO } catch (CompilationErrorException ex) { - _logger.LogWarning(ex, "Calculation {Id}: Roslyn compile failed", calcTagId); - return VirtualTagEvalResult.Failure($"compile error: {ex.Message}"); + return RememberCompileFailure(sourceKey, calcTagId, $"compile error: {ex.Message}", ex, "Roslyn compile failed"); } catch (ScriptSandboxViolationException ex) { - _logger.LogWarning(ex, "Calculation {Id}: sandbox violation", calcTagId); - return VirtualTagEvalResult.Failure($"sandbox violation: {ex.Message}"); + return RememberCompileFailure(sourceKey, calcTagId, $"sandbox violation: {ex.Message}", ex, "sandbox violation"); } catch (ObjectDisposedException) { @@ -258,10 +269,29 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO } } + /// Record a compile failure for and log it ONCE — the next trigger + /// of the same source returns the remembered reason without a Roslyn round-trip or a second Warning. + private VirtualTagEvalResult RememberCompileFailure(string sourceKey, string calcTagId, string reason, Exception ex, string what) + { + if (_compileFailures.TryAdd(sourceKey, reason)) + _logger.LogWarning(ex, "Calculation {Id}: {What} (remembered until the next redeploy)", calcTagId, what); + return VirtualTagEvalResult.Failure(reason); + } + + private static string HashSource(string source) + => Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(source))); + + /// Test/diagnostic: the number of sources currently remembered as failing to compile. + public int CompileFailureCount => _compileFailures.Count; + + /// Test/diagnostic: the number of compiled scripts currently cached. + public int CompiledScriptCount => _cache.Count; + /// public void ClearCompiledScripts() { _cache.Clear(); + _compileFailures.Clear(); // The compile-cache clear is this evaluator's deploy boundary — a redeploy lifts every quarantine // (an edited source already lifts its own through the guard's source identity). _guard.ResetAll(); @@ -279,5 +309,6 @@ public sealed class CalculationEvaluator : IScriptCacheOwner, IScriptQuarantineO if (_disposed) return; _disposed = true; _cache.Dispose(); + _compileFailures.Clear(); } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs index 9c7c741f..5466ab49 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/DependencyMuxTagUpstreamSource.cs @@ -42,8 +42,10 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource // publish. private const uint StatusNeverPushed = OpcUaStatus.BadWaitingForInitialData; - // Intentionally never pruned: cold-start semantics depend on retained last values so - // ReadTag can answer synchronously for any path that has ever been pushed. + // Retained last values: ReadTag answers synchronously for any path that has been pushed. Pruned + // ONLY through RetainOnly — the host calls it with the loaded dependency-ref union after every + // (re)load, so a renamed/removed raw path (a RawTree rename is a surgical remove+add) stops + // costing a boxed value + two DateTimes for the process lifetime (R02:L3). private readonly ConcurrentDictionary _cache = new(StringComparer.Ordinal); private readonly ConcurrentDictionary>> _observers @@ -91,6 +93,28 @@ public sealed class DependencyMuxTagUpstreamSource : ITagUpstreamSource } } + /// + /// Drop every cached snapshot whose path is not in . Observers are + /// untouched. Called by the host after each successful engine (re)load with the union of the loaded + /// alarms' dependency refs — exactly the set the mux will push from now on — so paths no alarm + /// references any more stop being retained. + /// + /// The paths to keep. + /// The number of cached snapshots dropped. + public int RetainOnly(IReadOnlySet paths) + { + ArgumentNullException.ThrowIfNull(paths); + var dropped = 0; + foreach (var key in _cache.Keys) + { + if (!paths.Contains(key) && _cache.TryRemove(key, out _)) dropped++; + } + return dropped; + } + + /// Test/diagnostic: the number of paths currently retained. + public int RetainedPathCount => _cache.Count; + /// public DataValueSnapshot ReadTag(string path) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs index ffa5dc4f..210a84ca 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/ScriptedAlarmHostActor.cs @@ -23,10 +23,16 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; /// cluster alerts DistributedPubSub topic. /// /// -/// The host is the engine's lifecycle owner: the caller (the driver-role startup) builds -/// the , constructs a -/// around that same upstream, and passes both here. The host disposes the engine in -/// . +/// The host is the engine's lifecycle owner. The restart-safe shape (R02:H1) is the +/// factory : +/// every incarnation builds its own + a +/// around it, and disposes them in . +/// Akka's default restart runs PostStop then re-creates the actor from the SAME Props, so the +/// legacy instance-taking Props — a fixed engine captured in the Props — restarted around a +/// disposed engine and every later LoadAsync threw: the node's scripted alarms +/// silently died. The legacy overload is kept source-compatible but is restart-safe only in the +/// weaker sense that a restart now DETACHES rather than disposes its engine (see +/// ); production should spawn through the factory. /// /// /// @@ -99,6 +105,9 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// Warning on every clean shutdown. private sealed record AlarmsLoadCanceled; + /// Default bound on one inbound alarm command's engine op (): 10 s. + public static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(10); + private readonly IActorRef _publishActor; private readonly IActorRef? _mux; private readonly DependencyMuxTagUpstreamSource _upstream; @@ -107,6 +116,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor private readonly ILoggingAdapter _log = Context.GetLogger(); private readonly CancellationTokenSource _cts = new(); private readonly EventHandler _onEngineEvent; + private readonly TimeSpan _commandTimeout; + + /// True when THIS incarnation built its engine (factory Props) and therefore disposes it on + /// every stop, restart included. False for the legacy instance-taking Props: the engine outlives the + /// incarnation, so a restart only detaches from it and a final stop disposes it. + private readonly bool _ownsEngine; + + /// Set by so the it triggers can tell a restart + /// (the next incarnation reuses a legacy engine) from a genuine stop. + private bool _restarting; /// Cached local from the latest /// snapshot (null = unknown until the first snapshot arrives, or no wired). The @@ -152,6 +171,8 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor /// Optional node-local live-telemetry hub (Phase 5) each cluster-wide /// alerts transition is also emitted into; null (tests) skips the emit. /// The used to instantiate the actor. + /// Legacy, instance-taking shape — see the class remarks: a supervision restart re-creates the + /// actor around these SAME instances, so prefer the factory overload in production. public static Props Props( IActorRef publishActor, IActorRef? mux, @@ -162,6 +183,60 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor ITelemetryLocalHub? telemetryHub = null) => Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, upstream, engine, localNode, driverMemberCountProvider, telemetryHub)); + /// + /// Restart-safe factory Props (R02:H1): each incarnation of the actor calls + /// in its constructor and owns what it returns — the engine must + /// be constructed around the returned upstream — so a supervision restart gets a fresh, + /// undisposed engine and the next loads cleanly. + /// + /// The OPC UA publish actor emissions are bridged to. + /// Optional dependency multiplexer the host registers interest with. + /// Builds a fresh (upstream, engine) pair; called once per incarnation. + /// The local cluster node id for the Primary alerts-emit gate; null ⇒ default-emit. + /// Test seam for the alerts-emit gate's driver-member count. + /// Optional node-local live-telemetry hub. + /// Bound on one inbound alarm command's engine op; null ⇒ . + /// The used to instantiate the actor. + public static Props Props( + IActorRef publishActor, + IActorRef? mux, + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory, + NodeId? localNode = null, + Func? driverMemberCountProvider = null, + ITelemetryLocalHub? telemetryHub = null, + TimeSpan? commandTimeout = null) => + Akka.Actor.Props.Create(() => new ScriptedAlarmHostActor(publishActor, mux, engineFactory, localNode, driverMemberCountProvider, telemetryHub, commandTimeout)); + + /// Factory-built incarnation: owns (and on every stop disposes) the engine it builds. + /// The OPC UA publish actor emissions are bridged to. + /// Optional dependency multiplexer the host registers interest with. + /// Builds this incarnation's (upstream, engine) pair. + /// The local cluster node id for the Primary alerts-emit gate. + /// Test seam for the alerts-emit gate's driver-member count. + /// Optional node-local live-telemetry hub. + /// Bound on one inbound alarm command's engine op. + public ScriptedAlarmHostActor( + IActorRef publishActor, + IActorRef? mux, + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory, + NodeId? localNode = null, + Func? driverMemberCountProvider = null, + ITelemetryLocalHub? telemetryHub = null, + TimeSpan? commandTimeout = null) + : this(publishActor, mux, Build(engineFactory), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: true, commandTimeout) + { + } + + private static (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) Build( + Func<(DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine)> engineFactory) + { + ArgumentNullException.ThrowIfNull(engineFactory); + var pair = engineFactory(); + ArgumentNullException.ThrowIfNull(pair.Upstream, "engineFactory().Upstream"); + ArgumentNullException.ThrowIfNull(pair.Engine, "engineFactory().Engine"); + return pair; + } + /// Initializes a new instance of the class. /// The OPC UA publish actor emissions are bridged to. /// Optional dependency multiplexer the host registers dependency interest with. @@ -183,14 +258,29 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor NodeId? localNode = null, Func? driverMemberCountProvider = null, ITelemetryLocalHub? telemetryHub = null) + : this(publishActor, mux, (upstream, engine), localNode, driverMemberCountProvider, telemetryHub, ownsEngine: false, commandTimeout: null) + { + } + + private ScriptedAlarmHostActor( + IActorRef publishActor, + IActorRef? mux, + (DependencyMuxTagUpstreamSource Upstream, ScriptedAlarmEngine Engine) pair, + NodeId? localNode, + Func? driverMemberCountProvider, + ITelemetryLocalHub? telemetryHub, + bool ownsEngine, + TimeSpan? commandTimeout) { ArgumentNullException.ThrowIfNull(publishActor); - ArgumentNullException.ThrowIfNull(upstream); - ArgumentNullException.ThrowIfNull(engine); + ArgumentNullException.ThrowIfNull(pair.Upstream); + ArgumentNullException.ThrowIfNull(pair.Engine); _publishActor = publishActor; _mux = mux; - _upstream = upstream; - _engine = engine; + _upstream = pair.Upstream; + _engine = pair.Engine; + _ownsEngine = ownsEngine; + _commandTimeout = commandTimeout is { } t && t > TimeSpan.Zero ? t : DefaultCommandTimeout; _localNode = localNode; _driverMemberCountProvider = driverMemberCountProvider; _telemetryHub = telemetryHub; @@ -294,8 +384,15 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor // subscriptions exist, so any DependencyValueChanged the mux delivers will be observed by the // engine. RegisterInterest replaces a subscriber's prior interest set (see DependencyMuxActor), // so a re-Apply with a fresh union simply supersedes the old one — no explicit unregister needed. - _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self)); - _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs", msg.DepRefs.Count); + // ReplayRetained: the mux replays the last value it saw for each ref, so a fresh engine (a + // supervision restart builds one around an EMPTY upstream) converges immediately instead of + // waiting for every input to change again. + _mux?.Tell(new DependencyMuxActor.RegisterInterest(msg.DepRefs, Self, ReplayRetained: true)); + // Prune the upstream's retained snapshots to the loaded set (R02:L3): a path no alarm references + // any more will never be pushed again, so its last value is dead weight for the process lifetime. + var dropped = _upstream.RetainOnly(new HashSet(msg.DepRefs, StringComparer.Ordinal)); + _log.Debug("ScriptedAlarmHost: loaded; registered mux interest for {Count} dep refs (pruned {Dropped} retained path(s))", + msg.DepRefs.Count, dropped); ReassertConditionNodes(); } @@ -591,18 +688,25 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor return; } + // Bounded (R02:H3): every op ends in IAlarmStateStore.SaveAsync (LocalDb/SQLite), and this is a + // ReceiveAsync — the mailbox is suspended until the op returns, so an unbounded await on a hung + // file lock stalled dependency-value ingestion for the duration. Linked to the actor's lifetime + // token so a stop still cancels promptly. + using var cts = CancellationTokenSource.CreateLinkedTokenSource(_cts.Token); + cts.CancelAfter(_commandTimeout); + var ct = cts.Token; try { switch (cmd.Operation) { case "Acknowledge": - await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.AcknowledgeAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; case "Confirm": - await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.ConfirmAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; case "OneShotShelve": - await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.OneShotShelveAsync(cmd.AlarmId, cmd.User, ct); break; case "TimedShelve": // A timed shelve needs the absolute unshelve instant. T18 derives it from the OPC UA @@ -614,16 +718,16 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor cmd.AlarmId); return; } - await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, CancellationToken.None); + await _engine.TimedShelveAsync(cmd.AlarmId, cmd.User, unshelveAt, ct); break; case "Unshelve": - await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.UnshelveAsync(cmd.AlarmId, cmd.User, ct); break; case "Enable": - await _engine.EnableAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.EnableAsync(cmd.AlarmId, cmd.User, ct); break; case "Disable": - await _engine.DisableAsync(cmd.AlarmId, cmd.User, CancellationToken.None); + await _engine.DisableAsync(cmd.AlarmId, cmd.User, ct); break; case "AddComment": if (string.IsNullOrWhiteSpace(cmd.Comment)) @@ -631,7 +735,7 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor _log.Warning("ScriptedAlarmHost: rejecting AddComment for {AlarmId} — empty comment text", cmd.AlarmId); return; } - await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, CancellationToken.None); + await _engine.AddCommentAsync(cmd.AlarmId, cmd.User, cmd.Comment, ct); break; default: _log.Warning("ScriptedAlarmHost: ignoring AlarmCommand with unknown operation {Op} for {AlarmId}", @@ -639,6 +743,11 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor break; } } + catch (OperationCanceledException) when (!_cts.IsCancellationRequested) + { + _log.Warning("ScriptedAlarmHost: engine op {Op} for alarm {AlarmId} timed out after {Timeout} — the condition-state store is stalled", + cmd.Operation, cmd.AlarmId, _commandTimeout); + } catch (Exception ex) { // A failing engine op must not crash the actor — mirror OnLoadFailed's log-and-stay-inert style. @@ -672,16 +781,28 @@ public sealed class ScriptedAlarmHostActor : ReceiveActor base.PreStart(); } + /// + /// Akka's default PreRestart stops the children and calls . The flag + /// lets that keep a legacy (Props-captured) engine alive for the next incarnation + /// instead of disposing the instance it is about to be re-created around. + protected override void PreRestart(Exception reason, object message) + { + _restarting = true; + base.PreRestart(reason, message); + } + /// protected override void PostStop() { // Unregister mux interest first so no further DependencyValueChanged arrives while we tear down. _mux?.Tell(new DependencyMuxActor.UnregisterInterest(Self)); // Cancel any in-flight LoadAsync, detach the OnEvent handler (so a late engine emission can't - // Tell a stopping actor), then dispose the engine (which drains its background work + clears). + // Tell a stopping actor), then dispose the engine (which drains its background work + clears) — + // unless this is a RESTART around a legacy Props-captured engine, which the next incarnation will + // re-attach to (R02:H1). _cts.Cancel(); _engine.OnEvent -= _onEngineEvent; - _engine.Dispose(); + if (_ownsEngine || !_restarting) _engine.Dispose(); _cts.Dispose(); base.PostStop(); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs index 7011492d..b8dc818f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyConsumerMuxAdapter.cs @@ -62,7 +62,10 @@ public sealed class DependencyConsumerMuxAdapter : ReceiveActor var refs = _consumer.DependencyRefs.ToList(); // A RegisterInterest replaces the prior interest set for this subscriber (mux is idempotent on // re-register), so re-registering after an apply cleanly adopts an added/removed ref. - _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self)); + // ReplayRetained (R07:H6): a driver Restart is a NEW consumer behind a new adapter; the mux replays + // the last value of every ref it retained so the calc tags compute from current inputs at once + // instead of sitting BadWaitingForInitialData until each input happens to change. + _mux.Tell(new DependencyMuxActor.RegisterInterest(refs, Self, ReplayRetained: true)); _log.Debug("DependencyConsumerMuxAdapter: registered {Count} dependency ref(s)", refs.Count); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs index fe57da4e..3af36d6a 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs @@ -26,6 +26,20 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; /// receive as well, for direct senders and tests. /// The mux is deliberately not a DPS subscriber — virtual-tag evaluation is local to each node and /// would over-emit if it spanned the cluster. +/// +/// +/// Last-value retention + replay (R07:H6). For every ref that has EVER been registered the +/// mux retains the last (value, status, timestamp) it forwarded — through unregisters too, +/// because the case that matters is a subscriber that went away and came back: a Calculation +/// driver Restart is a new driver instance + a new mux adapter, and before this every calc tag sat +/// BadWaitingForInitialData until each of its inputs happened to change again (a +/// scripted-alarm host restart has the same shape). A with +/// receives one batch of the retained values for its +/// refs, in the same shape a driver burst would arrive in. Opt-in per registration: the +/// continuous-historization recorder re-registers on every deploy and a replay there would +/// record a stale sample per tag per deploy. Memory is bounded by the number of refs scripts have +/// ever named on this node, not by the address space. +/// /// public sealed class DependencyMuxActor : ReceiveActor { @@ -33,14 +47,23 @@ public sealed class DependencyMuxActor : ReceiveActor /// Register a subscriber's interest in a set of tag refs. Idempotent on re-register — /// the second call replaces the prior interest set for that subscriber. - public sealed record RegisterInterest(IReadOnlyList TagRefs, IActorRef Subscriber); + /// The refs the subscriber wants. + /// The subscriber. + /// When true, the mux immediately sends the subscriber one + /// holding the last retained value of every ref + /// in it has one for (nothing when it has none). + public sealed record RegisterInterest(IReadOnlyList TagRefs, IActorRef Subscriber, bool ReplayRetained = false); /// Unregister every interest held by . Sent on PostStop by /// the subscriber, or by Terminated handling when the mux watches. public sealed record UnregisterInterest(IActorRef Subscriber); + // A ref stays in _byRef once registered — with an EMPTY set after its last subscriber leaves — so + // the hot path's single lookup per item also decides retention. _byRef.Count == 0 keeps meaning + // "nothing was ever registered", which the batch path early-outs on. private readonly Dictionary> _byRef = new(StringComparer.Ordinal); private readonly Dictionary> _bySubscriber = new(); + private readonly Dictionary _last = new(StringComparer.Ordinal); private readonly ILoggingAdapter _log = Context.GetLogger(); /// Creates props for the DependencyMuxActor. @@ -64,16 +87,13 @@ public sealed class DependencyMuxActor : ReceiveActor { foreach (var r in priorRefs) { - if (_byRef.TryGetValue(r, out var set)) - { - set.Remove(msg.Subscriber); - if (set.Count == 0) _byRef.Remove(r); - } + if (_byRef.TryGetValue(r, out var set)) set.Remove(msg.Subscriber); } } var refs = new HashSet(msg.TagRefs, StringComparer.Ordinal); _bySubscriber[msg.Subscriber] = refs; + List? replay = null; foreach (var r in refs) { if (!_byRef.TryGetValue(r, out var set)) @@ -82,9 +102,19 @@ public sealed class DependencyMuxActor : ReceiveActor _byRef[r] = set; } set.Add(msg.Subscriber); + if (msg.ReplayRetained && _last.TryGetValue(r, out var retained)) + (replay ??= []).Add(retained); } Context.Watch(msg.Subscriber); _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count); + + // One batch, after the registration is complete, in exactly the shape a driver burst arrives in — + // so the subscriber's hot path handles it with no special case. + if (replay is not null) + { + msg.Subscriber.Tell(new VirtualTagActor.DependencyValueBatch(replay)); + _log.Debug("DependencyMux: replayed {Count} retained value(s) to {Sub}", replay.Count, msg.Subscriber); + } } private void RemoveSubscriber(IActorRef subscriber) @@ -92,11 +122,9 @@ public sealed class DependencyMuxActor : ReceiveActor if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return; foreach (var r in refs) { - if (_byRef.TryGetValue(r, out var set)) - { - set.Remove(subscriber); - if (set.Count == 0) _byRef.Remove(r); - } + // The ref's entry stays (possibly empty) so its last value keeps being retained for a + // subscriber that comes back — see the class remarks. + if (_byRef.TryGetValue(r, out var set)) set.Remove(subscriber); } _bySubscriber.Remove(subscriber); Context.Unwatch(subscriber); @@ -105,13 +133,14 @@ public sealed class DependencyMuxActor : ReceiveActor private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg) { - if (!_byRef.TryGetValue(msg.FullReference, out var subscribers) || subscribers.Count == 0) + if (!_byRef.TryGetValue(msg.FullReference, out var subscribers)) { - // No virtual tag cares about this ref — drop. Common in normal operation; the address + // No virtual tag ever cared about this ref — drop. Common in normal operation; the address // space carries thousands of tags and only a fraction feed virtual-tag expressions. return; } var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode); + _last[msg.FullReference] = dep; foreach (var sub in subscribers) { // A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs @@ -122,15 +151,6 @@ public sealed class DependencyMuxActor : ReceiveActor } } - /// Filters one batched driver production burst per item and delivers each subscriber's matched - /// slice as ONE . Items nobody registered for are - /// dropped (the common case: only a fraction of an address space feeds virtual-tag expressions), so a - /// burst of N values costs one inbound message plus one outbound message per genuinely interested - /// SUBSCRIBER — not per (item, subscriber) pair as it did when this fanned out per value. That is what - /// lets evaluate its script once per burst instead of once per input. - /// The registered set is NOT just virtual-tag inputs: ContinuousHistorizationRecorder - /// registers interest in every historized ref, so on a historizing node can be - /// large and a sizeable fraction of each burst resolves. private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg) { // Nobody has registered anything — skip the whole burst rather than probing an empty map per item. @@ -146,8 +166,10 @@ public sealed class DependencyMuxActor : ReceiveActor for (var i = 0; i < count; i++) { var item = msg.Items[i]; - if (!_byRef.TryGetValue(item.FullReference, out var subscribers) || subscribers.Count == 0) continue; + if (!_byRef.TryGetValue(item.FullReference, out var subscribers)) continue; var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode); + _last[item.FullReference] = dep; + if (subscribers.Count == 0) continue; // retained for a returning subscriber; nobody to forward to now perSubscriber ??= new Dictionary>(); foreach (var sub in subscribers) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs index 3d2a3058..63ea8736 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs @@ -7,6 +7,7 @@ using ZB.MOM.WW.OtOpcUa.Commons.Observability; using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; using ZB.MOM.WW.OtOpcUa.Commons.Types; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; using ZB.MOM.WW.OtOpcUa.Runtime.Telemetry; namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; @@ -76,6 +77,10 @@ public sealed class VirtualTagActor : ReceiveActor private readonly IReadOnlyList _dependencyRefs; private readonly IActorRef? _mux; private readonly ITelemetryLocalHub? _telemetryHub; + // R02:P6 — a script failing on every trigger of a 1 Hz input published one cluster-wide DPS + // ScriptLogEntry per second per tag; 200 such tags shed everyone's entries out of the bounded + // telemetry channels. The per-failure entry is now rate-limited per tag (20/min + one summary). + private readonly ScriptLogRateLimiter _failureLogLimiter; private readonly ILoggingAdapter _log = Context.GetLogger(); // Dependency SNAPSHOTS, not bare values (archreview 02/S14): the status code is what distinguishes a // comms-lost input from a genuine Good null, and it is what the script reads through @@ -104,6 +109,8 @@ public sealed class VirtualTagActor : ReceiveActor /// Optional reference to a dependency multiplexer actor. /// Optional node-local live-telemetry hub each script-log entry is also /// emitted into (Phase 5); null (tests) simply skips the emit. + /// Rate limiter for this tag's per-failure script-log entries; null ⇒ + /// the default 20 per minute + one summary. /// A configured for creating the actor. public static Props Props( string virtualTagId, @@ -113,7 +120,8 @@ public sealed class VirtualTagActor : ReceiveActor Func? publisherFactory = null, IReadOnlyList? dependencyRefs = null, IActorRef? mux = null, - ITelemetryLocalHub? telemetryHub = null) => + ITelemetryLocalHub? telemetryHub = null, + ScriptLogRateLimiter? failureLogLimiter = null) => Akka.Actor.Props.Create(() => new VirtualTagActor( virtualTagId, expression, evaluator ?? NullVirtualTagEvaluator.Instance, @@ -121,7 +129,8 @@ public sealed class VirtualTagActor : ReceiveActor publisherFactory, dependencyRefs ?? Array.Empty(), mux, - telemetryHub)); + telemetryHub, + failureLogLimiter)); /// Initializes a virtual tag actor with the given configuration and dependencies. /// Unique identifier for the virtual tag. @@ -133,6 +142,8 @@ public sealed class VirtualTagActor : ReceiveActor /// Optional reference to a dependency multiplexer actor. /// Optional node-local live-telemetry hub each script-log entry is also /// emitted into (Phase 5); null (tests) simply skips the emit. + /// Rate limiter for this tag's per-failure script-log entries; null ⇒ + /// the default 20 per minute + one summary. public VirtualTagActor( string virtualTagId, string expression, @@ -141,7 +152,8 @@ public sealed class VirtualTagActor : ReceiveActor Func? publisherFactory, IReadOnlyList dependencyRefs, IActorRef? mux, - ITelemetryLocalHub? telemetryHub = null) + ITelemetryLocalHub? telemetryHub = null, + ScriptLogRateLimiter? failureLogLimiter = null) { _virtualTagId = virtualTagId; _scriptId = scriptId; @@ -151,6 +163,7 @@ public sealed class VirtualTagActor : ReceiveActor _dependencyRefs = dependencyRefs; _mux = mux; _telemetryHub = telemetryHub; + _failureLogLimiter = failureLogLimiter ?? new ScriptLogRateLimiter(); Receive(OnDependencyChanged); Receive(OnDependencyBatch); @@ -162,7 +175,9 @@ public sealed class VirtualTagActor : ReceiveActor { if (_mux is not null && _dependencyRefs.Count > 0) { - _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self)); + // ReplayRetained: a respawned child (plan edit, supervision restart) evaluates from the mux's + // last-known inputs at once instead of waiting for each to change again. + _mux.Tell(new DependencyMuxActor.RegisterInterest(_dependencyRefs, Self, ReplayRetained: true)); } } @@ -353,7 +368,18 @@ public sealed class VirtualTagActor : ReceiveActor private void OnEvaluationFailed(string reason, string level, DateTime timestampUtc) { OtOpcUaTelemetry.VirtualTagEval.Add(1, new KeyValuePair("outcome", "fail")); - PublishLog(level, reason); + + // Rate-limited per tag (R02:P6): the first 20 failures a minute publish as before; the rest are + // counted and surfaced as ONE summary entry when the window rolls. The metric above and the Bad + // publish below are not limited — only the cluster-wide log fan-out is. + var admission = _failureLogLimiter.Admit(_virtualTagId); + if (admission.SuppressedInPreviousWindow > 0) + { + PublishLog("Warning", + $"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " + + $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); last: {reason}"); + } + if (admission.Admitted) PublishLog(level, reason); if ((_hasPublished && _lastPublishedStatus == OpcUaStatus.BadInternalError) || !AllDependenciesArrived()) { diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs index 0b20f490..b542871f 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs @@ -82,7 +82,9 @@ public sealed class DraftValidatorCalculationTests DriverInstances = [CalcDriver()], Devices = [Device("dev-1", "di-calc", "engine")], Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))], - Scripts = [ScriptRow("s1", "return 42.0;")], + // A read, so the change trigger is effective (a constant change-triggered script with no timer + // is now itself a deploy error — CalculationNoEffectiveTrigger). + Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value + 42.0;")], }; DraftValidator.Validate(draft).ShouldNotContain(e => e.Code.StartsWith("Calculation")); @@ -132,6 +134,58 @@ public sealed class DraftValidatorCalculationTests msg.ShouldContain("calc/engine/b"); } + // ── effective-trigger gate (R07:U5) ───────────────────────────────────────────────────── + + [Fact] + public void Change_triggered_tag_with_no_literal_read_and_no_timer_is_rejected() + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", Cfg("s1"))], + Scripts = [ScriptRow("s1", "return ctx.Now.Hour;")], + }; + + DraftValidator.Validate(draft).ShouldContain(e => + e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1" && e.Message.Contains("no literal ctx.GetTag")); + } + + [Fact] + public void Tag_with_change_trigger_off_and_no_timer_is_rejected() + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"scriptId\":\"s1\",\"changeTriggered\":false}")], + Scripts = [ScriptRow("s1", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")], + }; + + DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationNoEffectiveTrigger" && e.Context == "t-1"); + } + + [Theory] + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true,\"timerIntervalMs\":1000}", "return ctx.Now.Hour;")] // timer rescues a read-less script + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":false,\"timerIntervalMs\":500}", "return 1.0;")] // timer-only tag + [InlineData("{\"scriptId\":\"s1\",\"changeTriggered\":true}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // change trigger with a read + [InlineData("{\"scriptId\":\"s1\"}", "return (double)ctx.GetTag(\"mb/plc/reg\").Value;")] // changeTriggered defaults to true + public void A_tag_with_an_effective_trigger_is_accepted(string cfg, string source) + { + var draft = new DraftSnapshot + { + GenerationId = 1, ClusterId = "c", + DriverInstances = [CalcDriver()], + Devices = [Device("dev-1", "di-calc", "engine")], + Tags = [CalcTagRow("t-1", "dev-1", "a", cfg)], + Scripts = [ScriptRow("s1", source)], + }; + + DraftValidator.Validate(draft).ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger"); + } + [Fact] public void Cross_driver_ref_is_terminal_not_an_edge() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs new file mode 100644 index 00000000..88015ffc --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs @@ -0,0 +1,166 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Serilog; +using Serilog.Events; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; + +/// +/// Review 07 Calculation remediation: a compile failure is negative-cached by source hash and its +/// Warning/script-log fires on the Good→Bad transition only (P4); inputs are recorded before +/// InitializeAsync so a Restart computes from current inputs (H6); array results dedup +/// structurally (H8); ShutdownAsync drops the compiled scripts (L2). +/// +public sealed class CalculationDriverRemediationTests +{ + private const uint StatusGood = 0u; + private const uint StatusBadInternalError = 0x80020000u; + + private static RawTagEntry CalcTag(string rawPath, string source, bool changeTriggered = true, int? timerMs = null) + { + var timer = timerMs is { } ms ? $",\"timerIntervalMs\":{ms}" : ""; + var json = $"{{\"scriptId\":\"s-{rawPath}\",\"changeTriggered\":{(changeTriggered ? "true" : "false")}{timer}," + + $"\"scriptSource\":{System.Text.Json.JsonSerializer.Serialize(source)}}}"; + return new RawTagEntry(rawPath, json, WriteIdempotent: false); + } + + private static (CalculationDriver Driver, ConcurrentQueue Published, List Logs, CapturingLogger HostLog) + Build(params RawTagEntry[] tags) + { + var logs = new List(); + var sink = new ScriptLogTopicSink(new CapturingPublisher(logs), LogEventLevel.Information); + var scriptRoot = new ScriptRootLogger(new LoggerConfiguration().WriteTo.Sink(sink).CreateLogger()); + var hostLog = new CapturingLogger(); + var driver = new CalculationDriver(new CalculationDriverOptions { RawTags = tags }, "calc-drv", scriptRoot, hostLog); + var published = new ConcurrentQueue(); + driver.OnDataChange += (_, e) => published.Enqueue(e); + return (driver, published, logs, hostLog); + } + + private static async Task Settle(CalculationDriver driver) + { + var deadline = DateTime.UtcNow.AddSeconds(30); + while (!driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(10); + driver.IsEvaluationIdle.ShouldBeTrue("the calc driver's evaluation pump should have drained"); + } + + [Fact] + public async Task A_compile_failure_is_remembered_and_logged_once_across_repeated_triggers() + { + var (driver, published, logs, hostLog) = Build( + CalcTag("calc/e/broken", "var x = (double)ctx.GetTag(\"src/a\").Value; this is not valid C#;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + for (var i = 0; i < 5; i++) + { + driver.OnDependencyValue("src/a", i, StatusGood, DateTime.UtcNow); + await Settle(driver); + } + + published.Count.ShouldBe(1, "Bad is published once per transition"); + published.TryDequeue(out var bad).ShouldBeTrue(); + bad!.Snapshot.StatusCode.ShouldBe(StatusBadInternalError); + logs.Count.ShouldBe(1, "the script-log entry fires on the Good→Bad transition only"); + hostLog.Entries.Count(e => e.Level == LogLevel.Warning && e.Message.Contains("compile failed")).ShouldBe(1, + "Roslyn is consulted once per source; later triggers hit the negative cache"); + driver.CompileFailureCountForTest.ShouldBe(1); + } + + [Fact] + public async Task A_changed_failure_reason_is_logged_even_while_already_Bad() + { + // Throws with a different message depending on the input, so the failure REASON changes while the + // published status stays Bad — the operator must see the new reason. + var (driver, published, logs, _) = Build( + CalcTag("calc/e/thrower", "throw new System.InvalidOperationException(\"boom \" + (int)ctx.GetTag(\"src/a\").Value);")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); + await Settle(driver); + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); // same reason → no new log + await Settle(driver); + driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow); // new reason → logged + await Settle(driver); + + published.Count.ShouldBe(1); + logs.Count.ShouldBe(2); + logs[0].Message.ShouldContain("boom 1"); + logs[1].Message.ShouldContain("boom 2"); + } + + [Fact] + public async Task Inputs_delivered_before_InitializeAsync_are_kept_so_the_first_trigger_computes() + { + // H6: a Restart is a new driver whose adapter registers interest at construction; the mux's + // retained-value replay therefore lands BEFORE InitializeAsync. It used to be dropped. + var (driver, published, _, _) = Build( + CalcTag("calc/e/sum", "return (double)ctx.GetTag(\"src/a\").Value + (double)ctx.GetTag(\"src/b\").Value;")); + + driver.OnDependencyValue("src/a", 10.0, StatusGood, DateTime.UtcNow); // replayed, pre-init + driver.OnDependencyValue("src/b", 20.0, StatusGood, DateTime.UtcNow); + published.ShouldBeEmpty("evaluation is gated on running"); + + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 12.0, StatusGood, DateTime.UtcNow); // only ONE input moves after init + await Settle(driver); + + published.Count.ShouldBe(1, "src/b's pre-init value satisfied the change-gate"); + published.TryDequeue(out var e).ShouldBeTrue(); + e!.Snapshot.Value.ShouldBe(32.0); + } + + [Fact] + public async Task An_equal_array_result_is_deduped_structurally() + { + var (driver, published, _, _) = Build( + CalcTag("calc/e/arr", "var v = (double)ctx.GetTag(\"src/a\").Value; return new double[] { v, v * 2 };")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); // same array contents + await Settle(driver); + published.Count.ShouldBe(1, "a structurally equal array is not a change"); + + driver.OnDependencyValue("src/a", 2.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + published.Count.ShouldBe(2, "a different array still publishes"); + published.Last().Snapshot.Value.ShouldBe(new double[] { 2.0, 4.0 }); + } + + [Fact] + public async Task ShutdownAsync_drops_the_compiled_scripts() + { + var (driver, published, _, _) = Build(CalcTag("calc/e/x", "return (double)ctx.GetTag(\"src/a\").Value * 2.0;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 1.0, StatusGood, DateTime.UtcNow); + await Settle(driver); + published.Count.ShouldBe(1); + driver.CompiledScriptCountForTest.ShouldBe(1); + + await driver.ShutdownAsync(TestContext.Current.CancellationToken); + + driver.CompiledScriptCountForTest.ShouldBe(0, "the collectible ALCs must not survive the generation"); + } + + private sealed class CapturingPublisher(List entries) : IScriptLogPublisher + { + public void Publish(ScriptLogEntry entry) { lock (entries) entries.Add(entry); } + } + + internal sealed class CapturingLogger : ILogger + { + public ConcurrentBag<(LogLevel Level, string Message)> Entries { get; } = new(); + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) + => Entries.Add((logLevel, formatter(state, exception))); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs index 5eed4c6b..cf22038b 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverTests.cs @@ -229,11 +229,13 @@ public sealed class CalculationDriverTests logs[0].ScriptId.ShouldBe("s-calc/e/div"); logs[0].VirtualTagId.ShouldBe("calc/e/div"); - // a = 0 again → still failing, but NOT a fresh transition → no second Bad publish (script-log still emits). + // a = 0 again → still failing for the SAME reason, NOT a fresh transition → no second Bad publish, + // and (R07:P4) no second script-log entry either: the log fires on the transition or on a change of + // reason, not on every trigger of a tag that is already Bad. driver.OnDependencyValue("src/a", 0, StatusGood, DateTime.UtcNow); await Settle(driver); published.ShouldBeEmpty(); - logs.Count.ShouldBe(2); + logs.Count.ShouldBe(1); // a = 5 → 20 → Good recovery is force-published. driver.OnDependencyValue("src/a", 5, StatusGood, DateTime.UtcNow); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs index d020d118..a3861648 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/DependencyMuxTagUpstreamSourceTests.cs @@ -15,6 +15,28 @@ public sealed class DependencyMuxTagUpstreamSourceTests private static DataValueSnapshot Good(object? value, DateTime ts) => new(value, 0u, ts, ts); + /// R02:L3 — the host prunes the retained snapshots to the loaded dependency-ref union after each + /// (re)load, so a renamed/removed raw path stops being retained; observers are untouched. + [Fact] + public void RetainOnly_drops_snapshots_outside_the_set_and_keeps_observers() + { + var src = new DependencyMuxTagUpstreamSource(); + var ts = DateTime.UtcNow; + src.Push("keep", Good(1, ts)); + src.Push("old/renamed", Good(2, ts)); + var seen = new List(); + using var sub = src.SubscribeTag("old/renamed", (p, _) => seen.Add(p)); + + src.RetainOnly(new HashSet(StringComparer.Ordinal) { "keep" }).ShouldBe(1); + + src.RetainedPathCount.ShouldBe(1); + src.ReadTag("keep").Value.ShouldBe(1); + src.ReadTag("old/renamed").StatusCode.ShouldBe(OpcUaStatus.BadWaitingForInitialData); + src.Push("old/renamed", Good(3, ts)); + // Pruning retained values does not unsubscribe anyone. + seen.ShouldBe(new[] { "old/renamed" }); + } + [Fact] public void Push_then_ReadTag_returns_the_pushed_snapshot() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs new file mode 100644 index 00000000..c4063d57 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/ScriptedAlarmHostActorRestartTests.cs @@ -0,0 +1,145 @@ +using Akka.Actor; +using Akka.TestKit; +using Serilog; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa; +using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.ScriptedAlarms; + +/// +/// R02:H1 — a supervision restart of used to dispose the +/// Props-captured engine and re-create the actor around the disposed instance, so every later +/// ApplyScriptedAlarms failed and the node's scripted alarms silently died. With the factory +/// Props each incarnation builds its own engine; with the legacy Props a restart only detaches. Both +/// shapes must load cleanly after a throw inside a +/// handler. Also R02:H3 — an inbound alarm command's engine op is bounded. +/// +public sealed class ScriptedAlarmHostActorRestartTests : RuntimeActorTestBase +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30); + + private static EquipmentScriptedAlarmPlan Plan(string id = "alm-1", string depRef = "M.T") => + new(ScriptedAlarmId: id, EquipmentId: "Plant/Line1/M", Name: "HighTemp", AlarmType: "AlarmCondition", + Severity: 800, MessageTemplate: "condition", PredicateScriptId: $"{id}-script", + PredicateSource: $"return (int)ctx.GetTag(\"{depRef}\").Value > 90;", + DependencyRefs: new[] { depRef }, HistorizeToAveva: true, Retain: true, Enabled: true); + + private static ScriptedAlarmEngine BuildEngine(DependencyMuxTagUpstreamSource upstream, IAlarmStateStore? store = null) + { + var logger = new LoggerConfiguration().CreateLogger(); + return new ScriptedAlarmEngine(upstream, store ?? new InMemoryAlarmStateStore(), new ScriptLoggerFactory(logger), logger); + } + + private static (DependencyMuxTagUpstreamSource, ScriptedAlarmEngine) Pair() + { + var up = new DependencyMuxTagUpstreamSource(); + return (up, BuildEngine(up)); + } + + /// Apply, crash the handler (a null TagId makes Push throw → restart), apply again + /// and prove the second load registers interest and the alarm still activates. + private void RestartThenReload(IActorRef host, TestProbe publish, TestProbe mux) + { + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + mux.ExpectMsg(Timeout); + + // The crash: a batch whose item has a null TagId makes the upstream Push throw out of the handler + // (the documented, deliberately unguarded path) → supervision restarts the actor. + host.Tell(new VirtualTagActor.DependencyValueBatch(new[] + { + new VirtualTagActor.DependencyValueChanged(null!, 1, DateTime.UtcNow), + })); + mux.ExpectMsg(Timeout); // PostStop of the crashed incarnation + + // A later apply must load cleanly on the restarted incarnation… + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + var reg = mux.ExpectMsg(Timeout); + reg.TagRefs.ShouldContain("M.T"); + + // …and the alarm must still work end to end. + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + var state = publish.ExpectMsg(Timeout); + state.AlarmNodeId.ShouldBe("alm-1"); + state.State.Active.ShouldBeTrue(); + } + + [Fact] + public void Factory_Props_survives_a_handler_throw_and_loads_cleanly_afterwards() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var built = 0; + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, () => { built++; return Pair(); })); + + RestartThenReload(host, publish, mux); + + built.ShouldBe(2, "the restarted incarnation built its own engine"); + } + + [Fact] + public void Legacy_instance_Props_survives_a_handler_throw_because_a_restart_only_detaches() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var (upstream, engine) = Pair(); + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, upstream, engine)); + + RestartThenReload(host, publish, mux); + } + + [Fact] + public void An_alarm_command_whose_store_hangs_is_bounded_and_the_mailbox_moves_on() + { + var publish = CreateTestProbe(); + var mux = CreateTestProbe(); + var store = new HangingSaveStore(); + var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(publish.Ref, mux.Ref, + () => { var up = new DependencyMuxTagUpstreamSource(); return (up, BuildEngine(up, store)); }, + commandTimeout: TimeSpan.FromMilliseconds(300))); + + host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan() })); + mux.ExpectMsg(Timeout); + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow)); + publish.ExpectMsg(Timeout).State.Active.ShouldBeTrue(); + + store.HangNext = true; + host.Tell(new AlarmCommand("alm-1", "Acknowledge", "alice", "ack", null)); + + // The hung save would have suspended the mailbox indefinitely; the bound releases it, so the + // next dependency value is processed and the alarm clears. + host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 10, DateTime.UtcNow)); + publish.ExpectMsg(Timeout).State.Active.ShouldBeFalse(); + store.ObservedCancellation.ShouldBeTrue(); + } + + /// Store whose next SaveAsync blocks until its token is cancelled. + private sealed class HangingSaveStore : IAlarmStateStore + { + private readonly InMemoryAlarmStateStore _inner = new(); + public volatile bool HangNext; + public volatile bool ObservedCancellation; + + public Task LoadAsync(string alarmId, CancellationToken ct) => _inner.LoadAsync(alarmId, ct); + public Task> LoadAllAsync(CancellationToken ct) => _inner.LoadAllAsync(ct); + public Task RemoveAsync(string alarmId, CancellationToken ct) => _inner.RemoveAsync(alarmId, ct); + + public async Task SaveAsync(AlarmConditionState state, CancellationToken ct) + { + if (HangNext) + { + HangNext = false; + try { await Task.Delay(System.Threading.Timeout.InfiniteTimeSpan, ct); } + catch (OperationCanceledException) { ObservedCancellation = true; throw; } + } + await _inner.SaveAsync(state, ct); + } + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs new file mode 100644 index 00000000..ae42edb9 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs @@ -0,0 +1,101 @@ +using System.Collections.Concurrent; +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Calculation; +using ZB.MOM.WW.OtOpcUa.Runtime.Drivers; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags; + +/// +/// R07:H6 — the mux retains the last value per ever-registered ref and replays it to a registration +/// that asks (). A Calculation driver +/// Restart (new driver + new adapter) therefore republishes from current inputs instead of sitting +/// BadWaitingForInitialData until every input happens to change. +/// +public sealed class DependencyMuxReplayTests : RuntimeActorTestBase +{ + private static DriverInstanceActor.AttributeValueBatchPublished Burst(params (string Ref, object? Value)[] items) => + new("driver-1", items.Select(i => new DriverInstanceActor.AttributeValueItem(i.Ref, i.Value, OpcUaStatus.Good, DateTime.UtcNow)).ToArray()); + + [Fact] + public void RegisterInterest_with_replay_receives_the_retained_values_for_its_refs_only() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var first = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref)); + mux.Tell(Burst(("a", 1), ("b", 2), ("c", 3))); + first.ExpectMsg(PresenceBudget).Items.Count.ShouldBe(2); + + // The first subscriber goes away; retention must survive the unregister (that is the Restart shape). + mux.Tell(new DependencyMuxActor.UnregisterInterest(first.Ref)); + + var second = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "c" }, second.Ref, ReplayRetained: true)); + + var replay = second.ExpectMsg(PresenceBudget); + replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by anyone, so it is not retained; 'b' is not in this subscriber's refs"); + replay.Items[0].Value.ShouldBe(1); + } + + [Fact] + public void RegisterInterest_without_replay_receives_nothing_and_a_fresh_ref_replays_nothing() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var seed = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, seed.Ref)); + mux.Tell(Burst(("a", 1))); + seed.ExpectMsg(PresenceBudget); + + var opted = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, opted.Ref)); // default: no replay + opted.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + + var fresh = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "never-published" }, fresh.Ref, ReplayRetained: true)); + fresh.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + /// The user-facing scenario: a calc tag computes, the driver is restarted (new instance + new + /// adapter) with NO input changing afterwards, and the new instance republishes the current value. + [Fact] + public async Task A_Calculation_driver_restart_republishes_from_the_current_inputs() + { + var mux = Sys.ActorOf(DependencyMuxActor.Props()); + var tag = new RawTagEntry("calc/e/sum", + "{\"scriptId\":\"s\",\"changeTriggered\":true,\"scriptSource\":\"return (double)ctx.GetTag(\\\"src/a\\\").Value + (double)ctx.GetTag(\\\"src/b\\\").Value;\"}", + WriteIdempotent: false); + + // Generation 1: wire the driver through the real adapter and feed a burst. + var gen1 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv"); + var published1 = new ConcurrentQueue(); + gen1.OnDataChange += (_, e) => published1.Enqueue(e); + await gen1.InitializeAsync("{}", CancellationToken.None); + var adapter1 = Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen1, mux)); + AwaitAssert(() => + { + mux.Tell(Burst(("src/a", 10.0), ("src/b", 20.0))); + published1.Count.ShouldBeGreaterThan(0); + }, PresenceBudget); + published1.Last().Snapshot.Value.ShouldBe(30.0); + + // Restart: the old instance goes away; a NEW driver + adapter come up. Nothing changes upstream. + Sys.Stop(adapter1); + await gen1.ShutdownAsync(CancellationToken.None); + var gen2 = new CalculationDriver(new CalculationDriverOptions { RawTags = [tag] }, "calc-drv"); + var published2 = new ConcurrentQueue(); + gen2.OnDataChange += (_, e) => published2.Enqueue(e); + Sys.ActorOf(DependencyConsumerMuxAdapter.Props(gen2, mux)); // registers with ReplayRetained + await gen2.InitializeAsync("{}", CancellationToken.None); + + // The replayed inputs satisfied the change-gate; the first trigger after init is the replay itself + // (if it lands post-init) or the next change (here: none) — so publish happens from the replay. + AwaitAssert(() => published2.Count.ShouldBeGreaterThan(0), PresenceBudget); + published2.Last().Snapshot.Value.ShouldBe(30.0); + published2.Last().Snapshot.StatusCode.ShouldBe(OpcUaStatus.Good); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs new file mode 100644 index 00000000..74ee2e32 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs @@ -0,0 +1,55 @@ +using System.Collections.Concurrent; +using Akka.Actor; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Commons.Engines; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Logging; +using ZB.MOM.WW.OtOpcUa.Core.Scripting; +using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness; +using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; + +namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags; + +/// R02:P6 — the per-failure cluster-wide is rate-limited per tag +/// (first N per window, then one summary); the Bad publish and the metric are unaffected. +public sealed class VirtualTagActorFailureLogRateLimitTests : RuntimeActorTestBase +{ + [Fact] + public void Repeated_failures_publish_at_most_the_limit_then_one_summary_when_the_window_rolls() + { + var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc); + var limiter = new ScriptLogRateLimiter(limitPerWindow: 3, window: TimeSpan.FromMinutes(1), clock: () => now); + var capture = new ConcurrentQueue(); + var parent = CreateTestProbe(); + var actor = parent.ChildActorOf(VirtualTagActor.Props( + "vt-chatty", "broken", + evaluator: new FailingEvaluator("boom"), + publisherFactory: () => new DPSPublisher((_, p) => capture.Enqueue((ScriptLogEntry)p)), + failureLogLimiter: limiter)); + + for (var i = 0; i < 10; i++) + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", i, DateTime.UtcNow)); + + // Positive control: the FIRST failure still degrades the node (once per transition, as before). + parent.ExpectMsg(PresenceBudget).StatusCode.ShouldBe(Commons.OpcUa.OpcUaStatus.BadInternalError); + AwaitAssert(() => capture.Count.ShouldBe(3), PresenceBudget); + parent.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + capture.Count.ShouldBe(3, "7 of 10 failures were suppressed from the cluster-wide log"); + + now = now.AddMinutes(1); + actor.Tell(new VirtualTagActor.DependencyValueChanged("a", 99, DateTime.UtcNow)); + + AwaitAssert(() => capture.Count.ShouldBe(5), PresenceBudget); + var entries = capture.ToArray(); + entries[3].Message.ShouldContain("7"); + entries[3].Message.ShouldContain("suppressed"); + entries[3].VirtualTagId.ShouldBe("vt-chatty"); + entries[4].Message.ShouldBe("boom"); + } + + private sealed class FailingEvaluator(string reason) : IVirtualTagEvaluator + { + public VirtualTagEvalResult Evaluate(string id, string expr, IReadOnlyDictionary deps) + => VirtualTagEvalResult.Failure(reason); + } +} From 1bf5ef916cedd6ab532edca25e6492320df048e3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:27:59 -0400 Subject: [PATCH 10/51] feat(controlplane): deployment artifact retention on the admin singleton MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every deploy stored the whole gzip'd fleet snapshot in Deployment.ArtifactBlob forever (R03:1.2). DeploymentRetentionActor (admin singleton, proxy-less) sweeps every Deployments:Retention: IntervalMinutes (60): terminal rows (Sealed / PartiallyFailed / TimedOut) older than the newest KeepSealed (50) and not referenced by a NodeDeploymentState started/applied within ReferencedDays (30) get their blob EMPTIED — never deleted, never NULLed: the column stays NOT NULL and every reader already treats zero bytes as "no artifact" (#485), so no schema migration and no reader change. In-flight rows are never touched; MaxRowsPerSweep (200) bounds one sweep. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Entities/Deployment.cs | 6 +- .../Retention/DeploymentRetentionActor.cs | 177 ++++++++++++++++++ .../Retention/DeploymentRetentionOptions.cs | 54 ++++++ .../ServiceCollectionExtensions.cs | 18 ++ .../DeploymentRetentionTests.cs | 174 +++++++++++++++++ 5 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs index 19d83728..d3044244 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Entities/Deployment.cs @@ -26,7 +26,11 @@ public sealed class Deployment public DateTime CreatedAtUtc { get; init; } = DateTime.UtcNow; /// Gets or sets the serialized artifact blob containing the configuration. - public byte[] ArtifactBlob { get; init; } = Array.Empty(); + /// + /// Settable (not init-only) so the retention sweep can EMPTY it on an old terminal row. An empty blob + /// is "no artifact" to every reader (#485), so the audit row survives with its bytes retired. + /// + public byte[] ArtifactBlob { get; set; } = Array.Empty(); /// Gets or sets the row version for optimistic concurrency control. public byte[] RowVersion { get; set; } = Array.Empty(); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs new file mode 100644 index 00000000..55ffd7dc --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs @@ -0,0 +1,177 @@ +using Akka.Actor; +using Akka.Event; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; + +/// +/// Admin-role cluster singleton that periodically empties the ArtifactBlob of old, terminal +/// deployments (R03:1.2). See for the policy and +/// for the rule, which is pure over a context so it is unit-testable. +/// +/// +/// +/// Empties, never deletes, and never nulls. The column is NOT NULL varbinary(max), and +/// every reader (DriverHostActor.LoadArtifactBlob, OpcUaPublishActor, +/// DeploymentArtifactService) already treats a zero-length blob as "no artifact" (#485: empty +/// bytes are not an empty configuration — the apply fails and the node keeps last-known-good). An +/// empty blob therefore needs no schema change and no reader change, where a NULL would need both. +/// +/// +/// Never touches an in-flight row (Dispatching / AwaitingApplyAcks): a node may still be +/// fetching it. The newest terminal rows keep +/// their blob whatever their age, so the "latest sealed" every boot path reads is always intact. +/// +/// +public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers +{ + private readonly IDbContextFactory _dbFactory; + private readonly DeploymentRetentionOptions _options; + private readonly ILoggingAdapter _log = Context.GetLogger(); + private const string SweepTimerKey = "retention-sweep"; + + /// Gets or sets the timer scheduler. + public ITimerScheduler Timers { get; set; } = null!; + + /// Creates the actor props. + /// The config DB factory. + /// The retention policy. + /// The props. + public static Props Props(IDbContextFactory dbFactory, DeploymentRetentionOptions options) => + Akka.Actor.Props.Create(() => new DeploymentRetentionActor(dbFactory, options)); + + /// Initializes a new instance of the class. + /// The config DB factory. + /// The retention policy. + public DeploymentRetentionActor(IDbContextFactory dbFactory, DeploymentRetentionOptions options) + { + ArgumentNullException.ThrowIfNull(dbFactory); + ArgumentNullException.ThrowIfNull(options); + _dbFactory = dbFactory; + _options = options; + + Receive(_ => RunSweep()); + Receive(msg => + { + if (msg.Pruned > 0) + _log.Info("Deployment retention emptied the artifact blob on {Count} old deployment(s)", msg.Pruned); + else + _log.Debug("Deployment retention sweep found nothing to prune"); + if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg); + }); + Receive(msg => + { + _log.Warning(msg.Error, "Deployment retention sweep failed; the next sweep retries"); + if (msg.ReplyTo is not null) msg.ReplyTo.Tell(msg); + }); + } + + /// + protected override void PreStart() + { + if (!_options.Enabled) + { + _log.Info("Deployment retention is disabled (Deployments:Retention:Enabled=false); artifact blobs are kept forever"); + return; + } + var interval = TimeSpan.FromMinutes(Math.Max(1, _options.IntervalMinutes)); + var initial = TimeSpan.FromSeconds(Math.Max(0, _options.InitialDelaySeconds)); + Timers.StartPeriodicTimer(SweepTimerKey, Sweep.Instance, initial, interval); + } + + private void RunSweep() + { + var replyTo = Sender.IsNobody() ? null : Sender; + var self = Self; + var now = DateTime.UtcNow; + Task.Run(async () => + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + return await SweepAsync(db, _options, now).ConfigureAwait(false); + }) + .PipeTo(self, + success: pruned => new SweepCompleted(pruned, replyTo), + failure: ex => new SweepFailed(ex, replyTo)); + } + + /// + /// The retention rule. Empties the blob of every terminal deployment that is (a) older than the + /// newest terminal rows, (b) not referenced by a + /// NodeDeploymentState row started or applied within + /// , and (c) still carrying bytes — at most + /// per call. + /// + /// The config DB. + /// The policy. + /// The sweep's clock. + /// Cancellation. + /// How many rows were emptied. + public static async Task SweepAsync( + OtOpcUaConfigDbContext db, DeploymentRetentionOptions options, DateTime utcNow, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(options); + + var keep = Math.Max(1, options.KeepSealed); + var referenceFloor = utcNow.AddDays(-Math.Max(0, options.ReferencedDays)); + var batch = Math.Max(1, options.MaxRowsPerSweep); + + // Projected: ids only. The candidate scan must not materialise blobs for rows it will keep. + var terminal = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Sealed + || d.Status == DeploymentStatus.PartiallyFailed + || d.Status == DeploymentStatus.TimedOut) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, HasBlob = d.ArtifactBlob.Length > 0 }) + .ToListAsync(ct) + .ConfigureAwait(false); + + var aged = terminal.Skip(keep).Where(d => d.HasBlob).Select(d => d.DeploymentId).ToList(); + if (aged.Count == 0) return 0; + + var referenced = (await db.NodeDeploymentStates + .AsNoTracking() + .Where(s => aged.Contains(s.DeploymentId) + && (s.StartedAtUtc >= referenceFloor + || (s.AppliedAtUtc != null && s.AppliedAtUtc >= referenceFloor))) + .Select(s => s.DeploymentId) + .Distinct() + .ToListAsync(ct) + .ConfigureAwait(false)) + .ToHashSet(); + + var targets = aged.Where(id => !referenced.Contains(id)).Take(batch).ToList(); + if (targets.Count == 0) return 0; + + // Tracked load of ONLY the rows being emptied: the update rides the row's real RowVersion on + // both providers, and the blob read is the one-time cost of retiring that row. + var rows = await db.Deployments + .Where(d => targets.Contains(d.DeploymentId)) + .ToListAsync(ct) + .ConfigureAwait(false); + foreach (var row in rows) row.ArtifactBlob = []; + await db.SaveChangesAsync(ct).ConfigureAwait(false); + return rows.Count; + } + + /// Asks for a sweep now; the sender (if any) receives the / . + public sealed class Sweep + { + /// The singleton instance. + public static readonly Sweep Instance = new(); + private Sweep() { } + } + + /// A sweep finished. + /// How many rows were emptied. + /// Who asked, when the sweep was requested rather than timer-driven. + public sealed record SweepCompleted(int Pruned, IActorRef? ReplyTo); + + /// A sweep threw. + /// The failure. + /// Who asked, when the sweep was requested rather than timer-driven. + public sealed record SweepFailed(Exception Error, IActorRef? ReplyTo); +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs new file mode 100644 index 00000000..6654ec1e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; + +/// +/// Deployments:Retention — how long a sealed deployment keeps its ArtifactBlob (R03:1.2). +/// Every deploy stored the whole gzip'd fleet snapshot forever; nothing pruned it, so a year of routine +/// edits was gigabytes of blobs. The retention sweep EMPTIES the blob on old rows and never deletes a +/// row: the audit record (who deployed what revision, when, and how every node acked) survives. +/// +public sealed class DeploymentRetentionOptions +{ + /// The configuration section name. + public const string SectionName = "Deployments:Retention"; + + /// Master switch; keeps every blob forever (the previous behaviour). + public bool Enabled { get; set; } = true; + + /// + /// How many of the most recent terminal (Sealed / PartiallyFailed / TimedOut) deployments keep their + /// blob regardless of age. A node booting from central SQL reads the LATEST sealed row's blob, so this + /// must stay well above 1; 50 also keeps a comfortable re-deploy history. + /// + public int KeepSealed { get; set; } = 50; + + /// + /// A deployment any NodeDeploymentState row started or applied within this many days keeps its + /// blob even when older than the window — a node that just applied it may + /// still replay it on restart. + /// + public int ReferencedDays { get; set; } = 30; + + /// How often the sweep runs. + public int IntervalMinutes { get; set; } = 60; + + /// Delay before the first sweep after the singleton starts, so a boot is not a blob scan. + public int InitialDelaySeconds { get; set; } = 60; + + /// + /// Rows emptied per sweep. The sweep loads each candidate row (blob included) to update it under its + /// RowVersion, so this bounds one sweep's memory and transaction size; a backlog drains over sweeps. + /// + public int MaxRowsPerSweep { get; set; } = 200; + + /// Binds the section from configuration, with defaults for anything absent. + /// The app configuration; null yields the defaults. + /// The bound options. + public static DeploymentRetentionOptions Bind(IConfiguration? configuration) + { + var options = new DeploymentRetentionOptions(); + configuration?.GetSection(SectionName).Bind(options); + return options; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index cb8d605a..f838a004 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -15,6 +15,7 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Commons.Interfaces; using ZB.MOM.WW.OtOpcUa.ControlPlane.Fleet; using ZB.MOM.WW.OtOpcUa.ControlPlane.Redundancy; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.ControlPlane; @@ -30,6 +31,7 @@ public static class ServiceCollectionExtensions public const string AuditWriterSingletonName = "audit-writer"; public const string RedundancyStateSingletonName = "redundancy-state"; public const string ClusterNodeAddressReconcilerSingletonName = "cluster-node-address-reconciler"; + public const string DeploymentRetentionSingletonName = "deployment-retention"; /// /// Registers the admin-role cluster singletons on the AkkaConfigurationBuilder — three with @@ -181,6 +183,21 @@ public static class ServiceCollectionExtensions singletonOptions, createProxyToo: false); + // Deployment artifact retention (R03:1.2): empties ArtifactBlob on old terminal deployments so the + // Deployment table stops growing by one fleet snapshot per edit. Admin singleton for the same + // reason as the reconciler (one sweeper per fleet, and only central holds the ConfigDb); + // proxy-less because nothing addresses it in production — a test sends Sweep directly. + builder.WithSingleton( + DeploymentRetentionSingletonName, + (system, registry, resolver) => + { + var dbFactory = resolver.GetService>(); + var options = DeploymentRetentionOptions.Bind(resolver.GetService()); + return DeploymentRetentionActor.Props(dbFactory, options); + }, + singletonOptions, + createProxyToo: false); + return builder; } @@ -256,4 +273,5 @@ public sealed class AdminOperationsActorKey { } public sealed class AuditWriterActorKey { } public sealed class RedundancyStateActorKey { } public sealed class ClusterNodeAddressReconcilerKey { } +public sealed class DeploymentRetentionKey { } public sealed class CentralCommunicationActorKey { } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs new file mode 100644 index 00000000..ecd397c4 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs @@ -0,0 +1,174 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Retention; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:1.2 — every deploy stored the whole fleet snapshot forever. The retention sweep empties the +/// blob on old terminal rows (never deletes a row, never touches an in-flight one, never the newest N, +/// never one a node applied recently). +/// +public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase +{ + private static readonly DateTime SweepNow = new(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc); + + [Fact] + public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 12); // ids[0] newest … ids[11] oldest + var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 }; + + int pruned; + await using (var db = f.CreateDbContext()) + pruned = await DeploymentRetentionActor.SweepAsync(db, options, SweepNow); + + pruned.ShouldBe(7); + await using var verify = f.CreateDbContext(); + foreach (var id in ids.Take(5)) + (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.Length.ShouldBeGreaterThan(0, "newest 5 keep their blob"); + foreach (var id in ids.Skip(5)) + (await verify.Deployments.SingleAsync(d => d.DeploymentId == id)).ArtifactBlob.ShouldBeEmpty("older rows are emptied"); + // Rows survive: the audit record is intact. + (await verify.Deployments.CountAsync()).ShouldBe(12); + + // Idempotent: a second sweep finds nothing. + await using var again = f.CreateDbContext(); + (await DeploymentRetentionActor.SweepAsync(again, options, SweepNow)).ShouldBe(0); + } + + [Fact] + public async Task Sweep_keeps_a_row_a_node_applied_recently_and_never_touches_an_in_flight_row() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 4); + var inFlight = Guid.NewGuid(); + await using (var db = f.CreateDbContext()) + { + db.ClusterNodes.Add(new ClusterNode + { + NodeId = "site-a-1:4053", ClusterId = "SITE-A", Host = "site-a-1", + ApplicationUri = "urn:x", Enabled = true, CreatedBy = "test", + }); + // ids[3] is the oldest; a node applied it yesterday, so it stays. + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = "site-a-1:4053", DeploymentId = ids[3], Status = NodeDeploymentStatus.Applied, + StartedAtUtc = SweepNow.AddDays(-1), AppliedAtUtc = SweepNow.AddDays(-1), + }); + // ids[2]: applied long ago — not protected. + db.NodeDeploymentStates.Add(new NodeDeploymentState + { + NodeId = "site-a-1:4053", DeploymentId = ids[2], Status = NodeDeploymentStatus.Applied, + StartedAtUtc = SweepNow.AddDays(-90), AppliedAtUtc = SweepNow.AddDays(-90), + }); + // An ancient in-flight row with a blob — never touched. + db.Deployments.Add(new Deployment + { + DeploymentId = inFlight, RevisionHash = new string('f', 64), Status = DeploymentStatus.AwaitingApplyAcks, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-400), ArtifactBlob = new byte[] { 1, 2, 3 }, + }); + await db.SaveChangesAsync(); + } + var options = new DeploymentRetentionOptions { KeepSealed = 1, ReferencedDays = 30 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); // ids[1], ids[2] + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty("newest"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[1])).ArtifactBlob.ShouldBeEmpty(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[2])).ArtifactBlob.ShouldBeEmpty("applied 90 days ago"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[3])).ArtifactBlob.ShouldNotBeEmpty("applied yesterday"); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight"); + } + + [Fact] + public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls() + { + var f = NewInMemoryDbFactory(); + Seed(f, sealedCount: 10); + var options = new DeploymentRetentionOptions { KeepSealed = 2, MaxRowsPerSweep = 3 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(3); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(2); + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(0); + } + + /// The actor runs the same rule off its thread and reports back. + [Fact] + public void Actor_sweeps_on_request_and_reports_the_count() + { + var f = NewInMemoryDbFactory(); + Seed(f, sealedCount: 6); + var actor = Sys.ActorOf(DeploymentRetentionActor.Props(f, new DeploymentRetentionOptions + { + KeepSealed = 4, InitialDelaySeconds = 3600, IntervalMinutes = 60, + })); + + actor.Tell(DeploymentRetentionActor.Sweep.Instance); + + ExpectMsg(PresenceBudget).Pruned.ShouldBe(2); + using var verify = f.CreateDbContext(); + verify.Deployments.Count(d => d.ArtifactBlob.Length == 0).ShouldBe(2); + } + + [Fact] + public void Options_bind_from_configuration_with_defaults() + { + var defaults = DeploymentRetentionOptions.Bind(null); + defaults.Enabled.ShouldBeTrue(); + defaults.KeepSealed.ShouldBe(50); + defaults.ReferencedDays.ShouldBe(30); + defaults.IntervalMinutes.ShouldBe(60); + + var config = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Deployments:Retention:KeepSealed"] = "7", + ["Deployments:Retention:Enabled"] = "false", + }) + .Build(); + var bound = DeploymentRetentionOptions.Bind(config); + bound.KeepSealed.ShouldBe(7); + bound.Enabled.ShouldBeFalse(); + bound.ReferencedDays.ShouldBe(30); + } + + /// Seeds sealed deployments, newest first; each one hour older than the previous. + private static List Seed(IDbContextFactory f, int sealedCount) + { + var ids = new List(); + using var db = f.CreateDbContext(); + for (var i = 0; i < sealedCount; i++) + { + var id = Guid.NewGuid(); + ids.Add(id); + db.Deployments.Add(new Deployment + { + DeploymentId = id, + RevisionHash = new string('a', 64), + Status = i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, + CreatedBy = "test", + CreatedAtUtc = SweepNow.AddHours(-i), + SealedAtUtc = SweepNow.AddHours(-i), + ArtifactBlob = new byte[] { 0x1f, 0x8b, (byte)i }, + }); + } + db.SaveChanges(); + return ids; + } +} From 18352442ab93f207acba2986a0c310968490578c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:30:03 -0400 Subject: [PATCH 11/51] fix(controlplane): audit writer drops only the rejected row, not the whole batch AuditWriterActor documented per-row dedup on the UX_ConfigAuditLog_EventId index but flushed one statement batch: a single duplicate EventId (any at-least-once producer across a singleton hand-over) threw for all 500 rows (R03:2.6). SaveChanges now detaches the entries a DbUpdateException names (2601/2627 logged at Debug as the expected duplicate, anything else at Warning) and re-saves, bounded by the batch size; a failure naming no entry stays a whole-batch failure. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Audit/AuditWriterActor.cs | 57 ++++++++- .../AuditWriterActorRejectedRowTests.cs | 110 ++++++++++++++++++ 2 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs index 2ca3026c..0e20da2f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs @@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; /// Dedup is two-layer: in-buffer (the below collapses /// duplicate EventIds before flush) and at the database via the filtered unique index /// UX_ConfigAuditLog_EventId (cross-restart safety — a retry of an already-flushed -/// batch hits the constraint and we drop the duplicate insert without losing the rest of -/// the batch). +/// batch hits the constraint; detaches the rejected row +/// and re-saves, so the duplicate is dropped without losing the rest of the batch). /// /// Implements the shared seam: is a /// best-effort, never-throwing entry point that simply Tells this actor and returns @@ -107,8 +107,8 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter Outcome = evt.Outcome.ToString(), }); } - db.SaveChanges(); - _log.Debug("AuditWriter flushed {Count} events", snapshot.Count); + var dropped = SaveDetachingRejectedRows(db); + _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped); } catch (Exception ex) { @@ -116,6 +116,55 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter } } + /// + /// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One + /// statement batch used to carry the whole flush: a single duplicate EventId (any at-least-once + /// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a + /// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran. + /// + /// The context holding the batch. + /// How many rows were dropped. + /// + /// Only a that NAMES its entries is handled here — that is the shape a + /// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the + /// connection died, a deadlock) is the whole-batch failure it always was and propagates to the + /// caller's catch. Bounded by the batch size so a store that rejects every row still terminates. + /// + private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db) + { + var dropped = 0; + var attempts = 0; + var maxAttempts = db.ChangeTracker.Entries().Count() + 1; + while (true) + { + try + { + db.SaveChanges(); + return dropped; + } + catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts) + { + var duplicate = IsUniqueViolation(ex); + foreach (var entry in ex.Entries) + { + var eventId = (entry.Entity as ConfigAuditLog)?.EventId; + if (duplicate) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId); + else + _log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId); + entry.State = EntityState.Detached; + dropped++; + } + } + } + } + + /// SQL Server 2601 (unique index) / 2627 (unique constraint), the dedup contract's expected rejection. + /// The update failure. + /// for a duplicate-key rejection. + private static bool IsUniqueViolation(DbUpdateException ex) => + ex.InnerException is Microsoft.Data.SqlClient.SqlException { Number: 2601 or 2627 }; + /// protected override void PreRestart(Exception reason, object message) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs new file mode 100644 index 00000000..bd64161a --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs @@ -0,0 +1,110 @@ +using Akka.Actor; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.EntityFrameworkCore.Update; +using Shouldly; +using Xunit; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// R03:2.6 — one constraint hit used to drop the whole flush. The writer now detaches the rejected +/// row(s) and re-saves, so a duplicate EventId costs exactly that row. +/// +public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase +{ + [Fact] + public void A_rejected_row_is_dropped_and_the_rest_of_the_batch_is_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var duplicate = Guid.NewGuid(); + var rejecting = new RejectingDbFactory(dbName, root, new RejectOnce(duplicate)); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + + Watch(actor); + actor.Tell(PoisonPill.Instance); // PostStop flushes + ExpectTerminated(actor); + + using var db = rejecting.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(20, "every row but the rejected one lands"); + persisted.ShouldNotContain(duplicate); + // Positive control: the rows on either side of the rejected one are present. + persisted.ShouldContain(ids[6]); + persisted.ShouldContain(ids[8]); + } + + /// A failure that names no entry is a whole-batch failure, exactly as before. + [Fact] + public void A_connection_level_failure_still_drops_the_batch_without_looping() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysFailWithoutEntries()); + var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); + + for (var i = 0; i < 5; i++) actor.Tell(NewEvent(Guid.NewGuid())); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); // terminates ⇒ the retry loop did not spin + + using var db = rejecting.CreateDbContext(); + db.ConfigAuditLogs.Count().ShouldBe(0); + } + + private static AuditEvent NewEvent(Guid eventId) => new() + { + EventId = eventId, Category = "Config", Action = "Edit", Actor = "joe", + OccurredAtUtc = DateTimeOffset.UtcNow, Outcome = AuditOutcome.Success, + DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(), + }; + + /// Throws a naming the entry for one EventId, the first time it is saved. + private sealed class RejectOnce(Guid eventId) : ISaveChangesInterceptor + { + private int _fired; + + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (_fired == 1 || eventData.Context is null) return result; + var entry = eventData.Context.ChangeTracker.Entries() + .FirstOrDefault(e => e.Entity.EventId == eventId); + if (entry is null) return result; + _fired = 1; + // GetInfrastructure() hands back the pubternal InternalEntityEntry, which is the IUpdateEntry the + // real provider puts on the exception; the test only needs that one cast. +#pragma warning disable EF1001 + var updateEntry = (IUpdateEntry)entry.GetInfrastructure(); +#pragma warning restore EF1001 + throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601)", new[] { updateEntry }); + } + } + + private sealed class AlwaysFailWithoutEntries : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) => + throw new DbUpdateException("connection lost (simulated)"); + } + + private sealed class RejectingDbFactory(string dbName, InMemoryDatabaseRoot root, IInterceptor interceptor) + : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => new( + new DbContextOptionsBuilder() + .UseInMemoryDatabase(dbName, root) + .AddInterceptors(interceptor) + .Options); + } +} From 63714e430a75dadb1aa48c4a6106d6aec5165b48 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:34:39 -0400 Subject: [PATCH 12/51] fix(controlplane): evict retired mesh channels and refresh PSKs off the request path - MeshGrpcCommandDialer.RetainOnly (IMeshCommandDialer, default no-op): CentralCommunicationActor hands the dialer the current endpoint set on every contact refresh, so a renamed host or moved port releases its cached GrpcChannel and idle HTTP/2 connection instead of living for the process (R03:1.3). - IMeshPskProvider.RefreshAsync (default = the old InvalidateAll): MeshPskProvider pre-resolves every known cluster's key concurrently, bounded like the sync path, and swaps an immutable snapshot in atomically; a failed read keeps that cluster's previous key. The actor calls it from ContactsLoaded (after SetKnownClusters, off-actor via PipeTo) instead of InvalidateAll, so the first ack per cluster per ContactRefreshSeconds no longer blocks a gRPC thread on a 3 s-bounded store read or gets denied past the bound (R03:3.4). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../CentralCommunicationActor.cs | 57 +++++++- .../Communication/IMeshPskProvider.cs | 117 +++++++++++++++- .../Communication/MeshGrpcCommandDialer.cs | 33 +++++ .../CentralCommunicationActorTests.cs | 41 ++++++ .../MeshGrpcCommandDialerTests.cs | 29 ++++ .../MeshPskProviderRefreshTests.cs | 129 ++++++++++++++++++ 6 files changed, 404 insertions(+), 2 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs index cb1ea253..16f7b5d4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/CentralCommunicationActor.cs @@ -201,6 +201,9 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers LoadContactsFromDb(); }); Receive(HandleContactsLoaded); + Receive(msg => _log.Debug("Mesh PSK refresh completed for {Count} cluster(s)", msg.ClusterCount)); + Receive(msg => _log.Warning(msg.Error, + "Mesh PSK refresh failed; previously resolved keys stay in service until the next refresh")); Receive(HandleCommandDispatched); // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler @@ -710,6 +713,20 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers _targets = msg.Targets; + // R03:1.3 — release the channels of endpoints that left the dispatch set (host renamed, port + // moved). Only the resolved dialer is asked; nothing is resolved just to prune it. + if (_dialer is { } dialer) + { + try + { + dialer.RetainOnly(_targets.Select(t => t.Endpoint).ToHashSet(StringComparer.Ordinal)); + } + catch (Exception ex) + { + _log.Warning(ex, "Mesh dialer failed to release channels for retired endpoints; they stay cached until the next refresh"); + } + } + if (_targets.Count == 0) { _log.Warning( @@ -780,7 +797,37 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers } pskProvider.SetKnownClusters(msg.ClusterIds); - pskProvider.InvalidateAll(); + + // R03:3.4 — refresh the per-cluster keys OFF the request path and swap them in atomically, + // instead of dropping the cache and making the first ack per cluster re-read the store on a + // gRPC thread. A provider that does not pre-resolve (the interface default, and every fake) + // still drops its cache here, synchronously, exactly as before. + CancellationToken ct; + try + { + ct = _lifecycle.Token; + } + catch (ObjectDisposedException) + { + return; // Stopping. + } + var self = Self; + Task refresh; + try + { + refresh = pskProvider.RefreshAsync(msg.ClusterIds, ct); + } + catch (Exception ex) + { + _log.Warning(ex, "Mesh PSK refresh threw synchronously; cached keys are unchanged until the next refresh"); + refresh = Task.CompletedTask; + } + if (!refresh.IsCompletedSuccessfully) + { + refresh.PipeTo(self, + success: () => new PskRefreshed(msg.ClusterIds.Count), + failure: ex => new PskRefreshFailed(ex)); + } if (!_gatePublished) { @@ -795,6 +842,14 @@ public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers /// Internal tick asking for a contact-point refresh. public sealed record RefreshContacts; + /// The asynchronous PSK refresh (R03:3.4) completed. + /// How many clusters were refreshed. + public sealed record PskRefreshed(int ClusterCount); + + /// The asynchronous PSK refresh (R03:3.4) threw; the previous keys stay in service. + /// The failure. + public sealed record PskRefreshFailed(Exception Error); + /// Result of a contact-point load, piped back from the DB read. /// /// The gRPC dispatch set: one entry per enabled, non-maintenance row that carries a diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs index a7282694..bdab7ba7 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs @@ -106,6 +106,26 @@ public interface IMeshPskProvider /// Drops every cached key. void InvalidateAll(); + + /// + /// Re-reads the keys for from the store OFF the request path and + /// swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep + /// serving the previous key; a cluster whose read fails keeps its previous key too. Replaces + /// "invalidate on every refresh", which made the first inbound ack per cluster per + /// ContactRefreshSeconds block a gRPC thread on a synchronous, 3 s-bounded store read. + /// + /// The clusters central currently knows (the same set given to ). + /// Cancels the refresh. + /// Completes when the swap has happened. + /// + /// The default implementation is the pre-R03:3.4 behaviour — drop the cache and let the next lookup + /// re-read — so a fake or a non-caching provider needs nothing more. + /// + Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) + { + InvalidateAll(); + return Task.CompletedTask; + } } /// @@ -144,6 +164,16 @@ public sealed class MeshPskProvider : IMeshPskProvider private readonly ConcurrentDictionary> _cache = new(StringComparer.OrdinalIgnoreCase); + /// + /// Keys pre-resolved by , consulted after the config map and before the + /// lazy . Swapped wholesale (volatile reference to an immutable dictionary), + /// never mutated, so a lookup racing a refresh sees the whole previous snapshot or the whole next + /// one. A cluster missing from it — never refreshed, or whose read failed — falls through to the + /// lazy path exactly as before, which is what keeps "refresh failed" from meaning "no key". + /// + private volatile IReadOnlyDictionary _refreshed = + new Dictionary(StringComparer.OrdinalIgnoreCase); + /// /// The known-cluster gate's membership set, or when the gate is /// inactive (never published, or published empty — see ). @@ -210,6 +240,13 @@ public sealed class MeshPskProvider : IMeshPskProvider return true; } + // Pre-resolved by the asynchronous refresh: no store read on this (gRPC request) thread at all. + if (_refreshed.TryGetValue(clusterId, out var refreshed)) + { + psk = refreshed; + return true; + } + var entry = _cache.GetOrAdd( clusterId, static (id, self) => new Lazy( @@ -288,7 +325,85 @@ public sealed class MeshPskProvider : IMeshPskProvider } /// - public void InvalidateAll() => _cache.Clear(); + public void InvalidateAll() + { + _cache.Clear(); + _refreshed = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + /// + public async Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(clusterIds); + + var previous = _refreshed; + var next = new Dictionary(StringComparer.OrdinalIgnoreCase); + var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + + // Resolve every cluster concurrently — each read bounded like the synchronous path — and carry a + // failed cluster's PREVIOUS refreshed key forward: a transient store fault must not turn into a + // window in which that cluster has no key at all. + var reads = ids.Select(async id => + { + var value = await ResolveFromStoreAsync(id, ct).ConfigureAwait(false); + return (Id: id, Value: value); + }).ToList(); + foreach (var (id, value) in await Task.WhenAll(reads).ConfigureAwait(false)) + { + if (!string.IsNullOrWhiteSpace(value)) next[id] = value; + else if (previous.TryGetValue(id, out var kept)) next[id] = kept; + } + + // The swap. Then drop the lazy entries for the clusters now covered, so a key rotated between + // refreshes is served from the new snapshot rather than from a stale lazy resolution; entries + // for clusters NOT covered (their read failed and they had no previous value) are left alone. + _refreshed = next; + foreach (var id in next.Keys) _cache.TryRemove(id, out _); + } + + /// + /// The asynchronous twin of : same name derivation, same bound, same + /// "anything but a usable key is null" contract — awaited instead of blocked on, because this one + /// runs on a refresh rather than inside a gRPC interceptor. + /// + /// The application Cluster id. + /// The caller's token, linked under the store-read bound. + /// The stored key, or null. + private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) + { + SecretName name; + try + { + name = new SecretName(SecretNamePrefix + clusterId); + } + catch (ArgumentException ex) + { + _logger.LogWarning( + ex, + "Cluster {ClusterId} cannot be expressed as a secret name, so its mesh PSK cannot be " + + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or " + + "rename the cluster to [a-z0-9._/-].", + clusterId); + return null; + } + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + cts.CancelAfter(StoreReadTimeout); + try + { + return await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Failed to refresh the mesh PSK for cluster {ClusterId} from secret {SecretName}; the " + + "previously resolved key (if any) stays in service until the next refresh.", + clusterId, + name.Value); + return null; + } + } /// /// Reads OT-MESH-PSK-{clusterId} from the Secrets store, converting every failure mode diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs index a10d228a..31a19b13 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/MeshGrpcCommandDialer.cs @@ -34,6 +34,15 @@ public interface IMeshCommandDialer /// rather than dialled unauthenticated. /// Task DispatchAsync(string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct); + + /// + /// Releases any per-endpoint resources for endpoints NOT in + /// (R03:1.3). Central calls it on every contact refresh with the current dispatch set, so a + /// renamed host or moved port does not leave the old channel and its idle HTTP/2 connection + /// alive for the life of the process. Default no-op for dialers that cache nothing. + /// + /// The endpoints still in the dispatch set. + void RetainOnly(IReadOnlyCollection nodeEndpoints) { } } /// @@ -181,6 +190,30 @@ public sealed class MeshGrpcCommandDialer : IMeshCommandDialer, IDisposable await call.ResponseAsync.ConfigureAwait(false); } + /// + public void RetainOnly(IReadOnlyCollection nodeEndpoints) + { + ArgumentNullException.ThrowIfNull(nodeEndpoints); + if (_disposed) return; + + var keep = nodeEndpoints as IReadOnlySet ?? nodeEndpoints.ToHashSet(StringComparer.Ordinal); + foreach (var endpoint in _channels.Keys) + { + if (keep.Contains(endpoint)) continue; + // A dial that already took this channel out of the cache before the removal fails once, as + // a named per-node RpcException the actor logs; the next dial to that endpoint (if it ever + // returns to the set) builds a fresh channel. Bounded to the dials in flight at the instant + // of a contact refresh, the same accepted race Dispose documents. + if (_channels.TryRemove(endpoint, out var channel)) + { + channel.Dispose(); + _log.LogInformation( + "Mesh dialer released the cached channel for {Endpoint}: it is no longer in the dispatch set", + endpoint); + } + } + } + /// Marks the dialer disposed and disposes every cached gRPC channel. public void Dispose() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs index 9843c8fe..51dcd046 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/CentralCommunicationActorTests.cs @@ -930,6 +930,34 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase /// A spawned gRPC-mode actor plus the two seams the tests observe it through. private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk); + /// + /// R03:1.3 — every contact refresh hands the dialer the CURRENT endpoint set so it can release the + /// channels of endpoints that left it. + /// + [Fact] + public void Each_contact_refresh_tells_the_dialer_which_endpoints_remain() + { + var factory = NewInMemoryDbFactory(); + SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false), ("host-b:4053", "host-b", 4056, true, false)); + var rig = SpawnGrpcRig(factory); + AwaitContactsLoaded(rig); + // The dialer is resolved lazily on first use; a dispatch forces that, and the NEXT refresh prunes. + rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())); + AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5)); + + using (var db = factory.CreateDbContext()) + { + db.ClusterNodes.Single(n => n.NodeId == "host-b:4053").MaintenanceMode = true; + db.SaveChanges(); + } + rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts()); + + AwaitCondition( + () => rig.Dialer.RetainCalls.Any(set => set.Count == 1 && set.Single().Contains("host-a")), + TimeSpan.FromSeconds(5)); + rig.Dialer.RetainCalls.Last().ShouldBe(new[] { "http://host-a:4056" }); + } + private GrpcRig SpawnGrpcRig( IDbContextFactory factory, RecordingDialer? dialer = null, @@ -1014,6 +1042,19 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase : Task.CompletedTask; } + /// Every endpoint set handed to , in order. + public IReadOnlyList> RetainCalls + { + get { lock (_gate) return _retainCalls.ToList(); } + } + + private readonly List> _retainCalls = []; + + public void RetainOnly(IReadOnlyCollection nodeEndpoints) + { + lock (_gate) _retainCalls.Add(nodeEndpoints.ToList()); + } + /// One recorded dial. public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId); } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs index 2d110eb6..fc177361 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshGrpcCommandDialerTests.cs @@ -224,6 +224,35 @@ public sealed class MeshGrpcCommandDialerTests Should.NotThrow(dialer.Dispose); } + /// + /// R03:1.3 — a channel for an endpoint that left the dispatch set (host renamed, port moved) is + /// released on the next refresh instead of living, with its idle connection, for the life of central. + /// Positive control: a dial to that endpoint afterwards rebuilds a channel. + /// + [Fact] + public async Task RetainOnly_releases_channels_for_endpoints_that_left_the_dispatch_set() + { + var dialer = new MeshGrpcCommandDialer( + new FakePskProvider(("SITE-A", "psk-site-a")), + Options.Create(new MeshTransportOptions { DispatchTimeoutSeconds = 1 }), + NullLogger.Instance); + foreach (var endpoint in new[] { "http://127.0.0.1:1", "http://127.0.0.1:2" }) + { + await Should.ThrowAsync( + () => dialer.DispatchAsync(endpoint, Envelope(), "SITE-A", CancellationToken.None)); + } + dialer.CachedChannelCount.ShouldBe(2); + + dialer.RetainOnly(new[] { "http://127.0.0.1:1" }); + + dialer.CachedChannelCount.ShouldBe(1); + + await Should.ThrowAsync( + () => dialer.DispatchAsync("http://127.0.0.1:2", Envelope(), "SITE-A", CancellationToken.None)); + dialer.CachedChannelCount.ShouldBe(2, "a returning endpoint gets a fresh channel"); + Should.NotThrow(dialer.Dispose); + } + [Fact] public void It_activates_from_the_container_despite_the_optional_client_factory_parameter() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs new file mode 100644 index 00000000..e727c153 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs @@ -0,0 +1,129 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; +using ZB.MOM.WW.Secrets.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication; + +/// +/// R03:3.4 — central invalidated every cached key on every contact refresh (default 60 s), so the +/// first inbound ack per cluster per minute blocked a gRPC thread on a synchronous, 3 s-bounded store +/// read and was denied past the bound. pre-resolves the +/// keys off the request path and swaps them in atomically. +/// +public sealed class MeshPskProviderRefreshTests +{ + private static MeshPskProvider Provider(ISecretResolver resolver) => + new(Options.Create(new MeshTransportOptions()), resolver, NullLogger.Instance); + + [Fact] + public async Task After_a_refresh_the_lookup_is_served_without_touching_the_store() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + resolver.Calls.ShouldBe(1); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-1"); + resolver.Calls.ShouldBe(1, "the request path did not consult the store"); + provider.Matches("SITE-A", "k-1").ShouldBeTrue(); + } + + /// + /// While a refresh is in flight the PREVIOUS key is still served (no invalidate-then-reread window); + /// once it completes the rotated key is served. Positive control on both sides of the swap. + /// + [Fact] + public async Task A_rotated_key_is_swapped_in_atomically_with_the_old_key_served_until_then() + { + var resolver = new GatedResolver { Value = "old" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + provider.TryGetPsk("SITE-A", out var before).ShouldBeTrue(); + before.ShouldBe("old"); + + resolver.Value = "new"; + resolver.Hold(); + var refresh = provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + resolver.Entered.Wait(TimeSpan.FromSeconds(5)).ShouldBeTrue(); + + // Mid-refresh: the old key, from memory — the store is NOT re-read on this path. + var callsMidRefresh = resolver.Calls; + provider.TryGetPsk("SITE-A", out var during).ShouldBeTrue(); + during.ShouldBe("old"); + resolver.Calls.ShouldBe(callsMidRefresh); + + resolver.Release(); + await refresh; + provider.TryGetPsk("SITE-A", out var after).ShouldBeTrue(); + after.ShouldBe("new"); + } + + [Fact] + public async Task A_cluster_whose_refresh_fails_keeps_its_previous_key() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + resolver.Throw = new InvalidOperationException("store offline"); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-1"); + } + + /// A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour). + [Fact] + public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily() + { + var resolver = new GatedResolver { Value = "k-b" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-B", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-b"); + resolver.Calls.ShouldBe(2); + } + + [Fact] + public async Task InvalidateAll_drops_refreshed_keys_too() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.InvalidateAll(); + resolver.Value = "k-2"; + provider.TryGetPsk("SITE-A", out var psk).ShouldBeTrue(); + psk.ShouldBe("k-2"); + } + + private sealed class GatedResolver : ISecretResolver + { + private int _calls; + private volatile TaskCompletionSource? _gate; + + public string? Value { get; set; } + public Exception? Throw { get; set; } + public int Calls => Volatile.Read(ref _calls); + public ManualResetEventSlim Entered { get; } = new(); + + public void Hold() => _gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + public void Release() => _gate?.TrySetResult(); + + public async Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + Entered.Set(); + if (_gate is { } gate) await gate.Task.WaitAsync(ct); + if (Throw is not null) throw Throw; + return Value; + } + } +} From e04156d8f7006067518955358281566f199939e1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 10:39:12 -0400 Subject: [PATCH 13/51] fix(host): refuse unknown API-key ids in memory and budget store misses; document the mesh gate's boot window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BoundedApiKeyStore's negative cache was keyed by key id, so a flood of DISTINCT well-formed otua__x credentials missed it every time and cost one SQL query per attempt under the SDK's SessionManager-wide lock, no credential needed (R03:3.3). - IApiKeyStore.ListKeyIdsAsync (default null = unsupported; ConfigDbApiKeyStore lists every ApiKeyId): the bounded store keeps a known-id set refreshed every StoreCacheSeconds (single-flight, failed refresh keeps the previous set) and refuses an unknown id without a store read ({outcome=unknown_id}). - A process-wide miss budget (token bucket, Security:ApiKeys:MissBurst=100 / MissRefillPerSecond=20, ApiKeyStoreGuardOptions) bounds lookups that reach the store and miss while no set is available; a hit refunds its token. Per-endpoint accounting needs the OpcUaServer impersonation seam to carry the client address — follow-up. - docs/security.md: the new knobs, and the known-cluster gate's documented fail-open window at central boot (R03:4.2) plus the async PSK refresh (R03:3.4). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/security.md | 44 +++++ .../DataPlaneAuthRegistration.cs | 5 +- .../OpcUa/ApiKeyStoreGuardOptions.cs | 56 ++++++ .../OpcUa/BoundedApiKeyStore.cs | 117 +++++++++++- .../OpcUa/ConfigDbApiKeyStore.cs | 10 + .../OpcUa/IApiKeyStore.cs | 10 + .../OpcUa/BoundedApiKeyStoreKnownIdTests.cs | 173 ++++++++++++++++++ .../OpcUa/BoundedApiKeyStoreTests.cs | 11 +- 8 files changed, 421 insertions(+), 5 deletions(-) create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs diff --git a/docs/security.md b/docs/security.md index 87ac8d22..6e44569a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -245,6 +245,25 @@ with two `Security:ApiKeys` knobs: a completed lookup (a hit or an authoritative miss) is stored, and a cached hit whose `ExpiresUtc` has since passed is re-read rather than served, so the TTL can never resurrect an expired key. +#### Pre-authentication guards (`KnownIdSet` / `MissBurst` / `MissRefillPerSecond`) + +The negative cache above is keyed by **key id**, so a flood of *distinct*, well-formed `otua__x` +credentials missed it every time and cost one indexed SQL query per attempt — under the SDK's +`SessionManager`-wide lock, with no credential required (review 03 §3.3). `BoundedApiKeyStore` now puts two +guards in front of the store, bound from the same `Security:ApiKeys` section +(`ApiKeyStoreGuardOptions`, Host-owned): + +| Key | Default | Meaning | +|---|---|---| +| `KnownIdSet` | `true` | Keep an in-memory set of every `ApiKeyId` the ConfigDb holds, refreshed every `StoreCacheSeconds` (single-flight; a failed refresh keeps the previous set). An id outside it is refused in memory — `otopcua.opcua.apikey_store.bounded{outcome=unknown_id}` — and the store is never asked. A key minted less than `StoreCacheSeconds` ago is refused until the next refresh (the same window `Enabled` revocation already has). `StoreCacheSeconds = 0` disables the set with the cache. | +| `MissBurst` | `100` | Token bucket on lookups that **reach** the store and **miss**, for when the set is unavailable (not loaded yet, or the store cannot list ids). Exhausted ⇒ unknown ids are refused without a store read (`{outcome=miss_budget}`, a Warning-free deny through the authenticator's catch-all). A hit refunds its token, so legitimate clients are never charged. `0` disables the budget. | +| `MissRefillPerSecond` | `20` | Refill rate of the miss bucket. | + +The budget is **process-wide, not per remote endpoint**: the OPC UA SDK's impersonation callback (the +`IOpcUaUserAuthenticator` seam) does not carry the client's address, so per-endpoint accounting needs that +seam widened in the OpcUaServer project (a Wave 3 follow-up). Neither guard applies to a driver-only node: +its `ArtifactApiKeyStore` is two in-memory dictionary probes and is never wrapped. + ### LDAP bind flow (`OtOpcUaLdapAuthService`) LDAP is configured under the `Security:Ldap` section (bound to `LdapOptions`, `src/Server/ZB.MOM.WW.OtOpcUa.Security/Ldap/LdapOptions.cs`, `SectionName = "Security:Ldap"`). The app authenticator is `OtOpcUaLdapAuthService` — a thin wrapper around the shared `ZB.MOM.WW.Auth.Ldap` directory client that adds two app-only concerns the shared library deliberately does not model: the `Enabled` master switch and `DevStubMode`. Since #520 it serves the **Admin UI cookie login (`/auth/login`) only** — `LdapOpcUaUserAuthenticator` is deleted and the OPC UA UserName path authenticates API keys instead (see [API-key identity](#api-key-identity-opc-ua-data-plane)). @@ -1116,6 +1135,31 @@ See `archreview/plans/R3-01-mesh-grpc-allowanonymous-plan.md` (R3-01) for the fu record, and `docs/Configuration.md` §`MeshTransport` / §`ConfigSource`-`ConfigServe` and `docs/Telemetry.md` for the affected listener ports. +### The known-cluster gate is fail-OPEN until central's first ClusterNode load (review 03 §4.2) + +`CentralCommandAuthInterceptor` verifies an inbound deploy ack against the PSK of the cluster named in +`x-otopcua-cluster`. `MeshPskProvider` consults a **known-cluster gate** first — the set of cluster ids +central read from `ClusterNode` on its last contact refresh — and refuses anything outside it in O(1), +touching neither the config map nor the secret store. That gate is **inactive until the first successful +load** (`CentralCommunicationActor` publishes it from `ContactsLoaded`; an empty publication also +deactivates it, deliberately, so "the DB told me nothing" never becomes a fleet-wide authentication +outage). During a central boot with the ConfigDb down, therefore: + +- an unauthenticated caller can name *any* `x-otopcua-cluster` value and make central perform one bounded + (3 s) secret-store read plus one Warning line per request — the cost/oracle the gate exists to remove; +- **keys are still required**: nothing is admitted without the matching PSK, so this is a cost exposure, + not an authentication bypass; +- the window is bounded by the bootstrap retry (15 × 2 s) and the `ContactRefreshSeconds` (60 s) periodic + refresh, and closes on the first load that succeeds. The actor logs the condition on every failed load + (*"The known-cluster gate was NOT refreshed either … it is INACTIVE"*). + +Since review 03 §3.4 the per-cluster keys are **refreshed asynchronously** from the same handler +(`IMeshPskProvider.RefreshAsync`: concurrent bounded reads, atomic snapshot swap, a failed read keeps the +previous key) instead of being invalidated on every refresh, so the first ack per cluster per minute no +longer pays a synchronous store read on a gRPC thread. Accepted as documented, bounded behaviour; the +alternative — refusing every cluster until SQL answers — would turn a ConfigDb outage into a deploy-ack +outage on top of it. + --- ## OTOPCUA0001 Analyzer — Compile-Time Guard diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs index e7c301e4..eace8505 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/DataPlaneAuthRegistration.cs @@ -80,7 +80,10 @@ public static class DataPlaneAuthRegistration sp.GetRequiredService>().Value)), sp.GetRequiredService>(), sp.GetRequiredService().CreateLogger(), - sp.GetRequiredService())); + sp.GetRequiredService(), + // R03:3.3 — the known-id set + miss budget in front of the SQL lookup; same section as + // ApiKeyOptions, Host-owned knobs. + ApiKeyStoreGuardOptions.Bind(sp.GetService()))); services.AddSingleton(); // Minting surface for the AdminUI. Same node condition as the store above: both need a ConfigDb. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs new file mode 100644 index 00000000..3103b982 --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyStoreGuardOptions.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +/// +/// The pre-authentication guards puts in front of the ConfigDb key +/// lookup (R03:3.3). Bound from the same Security:ApiKeys section as ApiKeyOptions but +/// owned by the Host, where the store lives. +/// +/// +/// +/// Why. The negative cache was keyed by key id, so a flood of DISTINCT well-formed +/// otua_<random>_x credentials missed it every time and cost one indexed SQL query per +/// attempt — under the SDK's SessionManager-wide lock — with no credential needed. Two guards close +/// that: a known-id set (every ApiKeyId the ConfigDb holds, refreshed every +/// StoreCacheSeconds) rejects an unknown id in memory, and a miss budget (token +/// bucket) bounds how many lookups that reach the store and MISS are allowed per second when the +/// set is unavailable. +/// +/// +/// The budget is process-wide, not per remote endpoint: the OPC UA SDK's impersonation callback +/// (the IOpcUaUserAuthenticator seam) does not carry the client's address. Per-endpoint +/// accounting needs that seam widened in the OpcUaServer project — a follow-up, not this lane. +/// +/// +public sealed class ApiKeyStoreGuardOptions +{ + /// The configuration section the knobs live in (shared with ApiKeyOptions). + public const string SectionName = "Security:ApiKeys"; + + /// + /// Whether the known-id set is consulted. restores a store read per unknown + /// id. Only meaningful when the inner store can list ids (the ConfigDb store can; the artifact + /// store is never wrapped). + /// + public bool KnownIdSet { get; set; } = true; + + /// + /// How many store MISSES (lookups that reach the inner store and find no key) may burst before + /// further lookups for unknown ids are refused without a store read. 0 disables the budget. + /// + public int MissBurst { get; set; } = 100; + + /// How many miss tokens are restored per second, up to . + public double MissRefillPerSecond { get; set; } = 20; + + /// Binds the section, with defaults for anything absent. + /// The app configuration; null yields the defaults. + /// The bound options. + public static ApiKeyStoreGuardOptions Bind(IConfiguration? configuration) + { + var options = new ApiKeyStoreGuardOptions(); + configuration?.GetSection(SectionName).Bind(options); + return options; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index af0fca3f..bb3154ed 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -56,6 +56,21 @@ public sealed class BoundedApiKeyStore : IApiKeyStore private readonly ConcurrentDictionary> _keys = new(StringComparer.Ordinal); private readonly ConcurrentDictionary>> _grants = new(StringComparer.Ordinal); + private readonly ApiKeyStoreGuardOptions _guard; + + // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds + // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first + // successful load, or when the inner store cannot list ids — then the per-id path runs as before. + private volatile IReadOnlySet? _knownIds; + private DateTimeOffset _knownIdsStampedUtc = DateTimeOffset.MinValue; + private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1); + + // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards + // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the + // configured rate instead of one SQL round trip per attempt under the SessionManager lock. + private readonly object _bucketGate = new(); + private double _missTokens; + private DateTimeOffset _missRefilledUtc; /// Initializes a new instance of the class. /// The store being bounded. @@ -66,11 +81,13 @@ public sealed class BoundedApiKeyStore : IApiKeyStore /// DataPlaneAuthRegistration for why a second clock here would be a hazard rather than a /// detail. /// + /// The pre-authentication guards (R03:3.3); defaults when omitted. public BoundedApiKeyStore( IApiKeyStore inner, IOptions options, ILogger log, - TimeProvider time) + TimeProvider time, + ApiKeyStoreGuardOptions? guard = null) { ArgumentNullException.ThrowIfNull(inner); ArgumentNullException.ThrowIfNull(options); @@ -81,8 +98,14 @@ public sealed class BoundedApiKeyStore : IApiKeyStore _options = options; _log = log; _time = time; + _guard = guard ?? new ApiKeyStoreGuardOptions(); + _missTokens = Math.Max(0, _guard.MissBurst); + _missRefilledUtc = time.GetUtcNow(); } + /// The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards. + public IReadOnlySet? KnownKeyIds => _knownIds; + /// The store being bounded. /// /// Exposed so the composition root's wiring can be asserted. A decorator registered in one branch @@ -115,12 +138,104 @@ public sealed class BoundedApiKeyStore : IApiKeyStore return cached; } + // R03:3.3, guard 1 — an id the ConfigDb does not hold is refused in memory; the store is never asked. + var known = await KnownIdsAsync(ttl, ct).ConfigureAwait(false); + if (known is not null && !known.Contains(apiKeyId)) + { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "unknown_id")); + return null; + } + + // R03:3.3, guard 2 — a lookup that will reach the store needs a miss token unless the id is known + // to exist (a known id is a real key being verified, never a guess). + if (known is null && !TryTakeMissToken()) + { + OtOpcUaTelemetry.ApiKeyStoreBounded.Add(1, new KeyValuePair("outcome", "miss_budget")); + throw new InvalidOperationException( + "ApiKeyStore: the unknown-key miss budget is exhausted (Security:ApiKeys:MissBurst / " + + "MissRefillPerSecond); refusing this lookup without a store read."); + } + var fresh = await BoundAsync(token => _inner.FindAsync(apiKeyId, utcNow, token), ct) .ConfigureAwait(false); + if (fresh is not null && known is null) RefundMissToken(); if (ttl > TimeSpan.Zero) Store(_keys, apiKeyId, fresh); return fresh; } + /// + /// The known-id set, refreshed when older than . Single-flight: concurrent + /// callers during a refresh use the previous set (or, before the first load, the per-id path). A + /// refresh that fails or exceeds the deadline logs and keeps the previous set. + /// + /// The refresh interval (the cache TTL); non-positive disables the set. + /// The caller's token. + /// The set, or null. + private async Task?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct) + { + if (!_guard.KnownIdSet || ttl <= TimeSpan.Zero) return null; + + var now = _time.GetUtcNow(); + var current = _knownIds; + if (current is not null && now - _knownIdsStampedUtc < ttl) return current; + + if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current; + try + { + // Re-check under the gate: another caller may have refreshed while this one waited. + if (_knownIds is { } refreshed && now - _knownIdsStampedUtc < ttl) return refreshed; + var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false); + if (ids is null) + { + // Unsupported by the inner store: say so once and stop asking. + _guard.KnownIdSet = false; + return null; + } + _knownIds = ids; + _knownIdsStampedUtc = _time.GetUtcNow(); + return ids; + } + catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) + { + _log.LogWarning(ex, + "ApiKeyStore: refreshing the known key-id set failed; {Mode}", + current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force"); + return current; + } + finally + { + _knownIdsRefresh.Release(); + } + } + + /// Takes one miss token, refilling by elapsed time first; always succeeds when the budget is disabled. + /// when the budget is exhausted. + private bool TryTakeMissToken() + { + var burst = _guard.MissBurst; + if (burst <= 0) return true; + lock (_bucketGate) + { + var now = _time.GetUtcNow(); + var elapsed = (now - _missRefilledUtc).TotalSeconds; + if (elapsed > 0) + { + _missTokens = Math.Min(burst, _missTokens + elapsed * Math.Max(0, _guard.MissRefillPerSecond)); + _missRefilledUtc = now; + } + if (_missTokens < 1) return false; + _missTokens -= 1; + return true; + } + } + + /// A lookup that reached the store and HIT was not a miss: its token goes back. + private void RefundMissToken() + { + if (_guard.MissBurst <= 0) return; + lock (_bucketGate) _missTokens = Math.Min(_guard.MissBurst, _missTokens + 1); + } + // Keyed by principal — the anonymous activation path, which runs on every unauthenticated connect. /// public async Task> FindGrantsAsync(string principal, CancellationToken ct) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs index c3f9c1f5..52fcbe51 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ConfigDbApiKeyStore.cs @@ -125,6 +125,16 @@ public sealed class ConfigDbApiKeyStore : IApiKeyStore return await LoadGrantsAsync(db, principal, ct).ConfigureAwait(false); } + /// + public async Task?> ListKeyIdsAsync(CancellationToken ct) + { + await using var db = await _factory.CreateDbContextAsync(ct).ConfigureAwait(false); + // One projection over the whole (small) table — the unique index on ApiKeyId makes this an index + // scan — so an unknown id can be refused in memory instead of costing a query per attempt. + var ids = await db.ApiKeys.AsNoTracking().Select(k => k.ApiKeyId).ToListAsync(ct).ConfigureAwait(false); + return ids.ToHashSet(StringComparer.Ordinal); + } + /// Loads a principal's grants, confined to this node's cluster. /// /// The full grant set, not just the Cluster-scoped rows. The old diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs index c2f82dae..93057153 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/IApiKeyStore.cs @@ -35,6 +35,16 @@ public interface IApiKeyStore /// The cancellation token. /// The matching ACL rows; empty when none. Task> FindGrantsAsync(string principal, CancellationToken ct); + + /// + /// Every ApiKeyId the store holds (enabled or not), for the pre-authentication known-id set + /// (R03:3.3); when the store cannot cheaply list them, in which case callers + /// fall back to a per-id lookup. The set only answers "does this id exist at all" — enabled, expiry + /// and the secret are still decided by . + /// + /// The cancellation token. + /// The id set, or null when unsupported. + Task?> ListKeyIdsAsync(CancellationToken ct) => Task.FromResult?>(null); } /// An API key resolved for authentication, with the ACL grants naming its principal. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs new file mode 100644 index 00000000..7051b151 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs @@ -0,0 +1,173 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Host.OpcUa; + +namespace ZB.MOM.WW.OtOpcUa.Host.Tests.OpcUa; + +/// +/// R03:3.3 — the negative cache was keyed by key id, so a flood of DISTINCT well-formed ids cost one +/// SQL query per attempt under the SessionManager lock, no credential needed. Two guards: +/// the known-id set (unknown ids refused in memory) and the miss budget (a token bucket on lookups +/// that reach the store and miss, for when the set is unavailable). +/// +public sealed class BoundedApiKeyStoreKnownIdTests +{ + private static readonly DateTimeOffset T0 = new(2026, 10, 22, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public async Task An_unknown_id_is_refused_without_reaching_the_store_and_a_known_one_still_does() + { + var inner = new ListingStore("k-real"); + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + (await sut.FindAsync("k-guess-1", T0.UtcDateTime, default)).ShouldBeNull(); + (await sut.FindAsync("k-guess-2", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(0, "unknown ids never reach the store"); + inner.ListCalls.ShouldBe(1, "one set load serves every unknown id inside the TTL"); + + // Positive control: the real id reaches the store and is found. + (await sut.FindAsync("k-real", T0.UtcDateTime, default)).ShouldNotBeNull(); + inner.FindCalls.ShouldBe(1); + } + + [Fact] + public async Task The_set_is_refreshed_after_the_TTL_so_a_newly_minted_key_becomes_visible() + { + var inner = new ListingStore("k-old"); + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + (await sut.FindAsync("k-new", T0.UtcDateTime, default)).ShouldBeNull(); + inner.Mint("k-new"); + // Still inside the TTL: the stale set refuses it (the documented window). + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(0); + + clock.Advance(TimeSpan.FromSeconds(3)); + (await sut.FindAsync("k-new", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + inner.ListCalls.ShouldBe(2); + } + + [Fact] + public async Task A_failed_set_load_falls_back_to_the_per_id_lookup_and_keeps_an_older_set() + { + var inner = new ListingStore("k-real") { ListThrows = true }; + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 5); + + // No set yet and the load fails: the per-id path runs (old behaviour), nothing is wedged. + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(1); + sut.KnownKeyIds.ShouldBeNull(); + + inner.ListThrows = false; + clock.Advance(TimeSpan.FromSeconds(6)); + (await sut.FindAsync("k-guess-2", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + sut.KnownKeyIds.ShouldNotBeNull(); + inner.FindCalls.ShouldBe(1, "once the set loaded, the guess was refused in memory"); + + // A later failed refresh keeps the previous set in force. + inner.ListThrows = true; + clock.Advance(TimeSpan.FromSeconds(6)); + (await sut.FindAsync("k-guess-3", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(1); + } + + /// With no id set available, misses are rate-limited; hits are not charged; time refills. + [Fact] + public async Task The_miss_budget_bounds_store_misses_when_no_set_is_available() + { + var inner = new ListingStore("k-real") { SupportsListing = false }; + var clock = new FakeClock(T0); + var sut = Bounded(inner, clock, cacheSeconds: 0, missBurst: 3, refillPerSecond: 1); + + for (var i = 0; i < 3; i++) + (await sut.FindAsync($"k-guess-{i}", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(3); + + // Budget exhausted: refused without a store read, as a throw the authenticator denies on. + await Should.ThrowAsync(() => sut.FindAsync("k-guess-9", T0.UtcDateTime, default)); + inner.FindCalls.ShouldBe(3); + + // Refill: one second restores one token. + clock.Advance(TimeSpan.FromSeconds(1)); + (await sut.FindAsync("k-guess-10", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(4); + + // A hit is not a miss: it reaches the store even with the bucket empty? No — it needs a token to + // be attempted, but the token is refunded, so the next attempt still has it. Prove the refund. + clock.Advance(TimeSpan.FromSeconds(1)); + (await sut.FindAsync("k-real", clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); + (await sut.FindAsync("k-guess-11", clock.GetUtcNow().UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(6); + } + + [Fact] + public async Task The_guards_are_off_when_the_cache_is_off_restoring_a_read_per_lookup() + { + var inner = new ListingStore("k-real"); + var sut = Bounded(inner, new FakeClock(T0), cacheSeconds: 0, missBurst: 0); + + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + (await sut.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + inner.FindCalls.ShouldBe(2); + inner.ListCalls.ShouldBe(0); + } + + private static BoundedApiKeyStore Bounded( + IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new( + inner, + Options.Create(new ApiKeyOptions + { + Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = cacheSeconds, + }), + NullLogger.Instance, + clock, + new ApiKeyStoreGuardOptions { MissBurst = missBurst, MissRefillPerSecond = refillPerSecond }); + + private sealed class ListingStore(params string[] ids) : IApiKeyStore + { + private readonly HashSet _ids = ids.ToHashSet(StringComparer.Ordinal); + private int _findCalls; + private int _listCalls; + + public bool SupportsListing { get; set; } = true; + public bool ListThrows { get; set; } + public int FindCalls => _findCalls; + public int ListCalls => _listCalls; + + public void Mint(string id) => _ids.Add(id); + + public Task FindAsync(string apiKeyId, DateTime utcNow, CancellationToken ct) + { + Interlocked.Increment(ref _findCalls); + return Task.FromResult(_ids.Contains(apiKeyId) + ? new ApiKeyLookup(new ApiKey { ApiKeyId = apiKeyId, Name = "k", SecretHash = "h", Principal = "p" }, []) + : null); + } + + public Task> FindGrantsAsync(string principal, CancellationToken ct) => + Task.FromResult>([]); + + public Task?> ListKeyIdsAsync(CancellationToken ct) + { + if (!SupportsListing) return Task.FromResult?>(null); + Interlocked.Increment(ref _listCalls); + if (ListThrows) throw new InvalidOperationException("ConfigDb unreachable (simulated)"); + return Task.FromResult?>(_ids.ToHashSet(StringComparer.Ordinal)); + } + } + + private sealed class FakeClock(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _now = utcNow; + public override DateTimeOffset GetUtcNow() => _now; + public void Advance(TimeSpan by) => _now = _now.Add(by); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs index e8081db1..65201905 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreTests.cs @@ -363,7 +363,10 @@ public sealed class BoundedApiKeyStoreTests { var (_, inner) = SetUp(); var clock = new FakeClock(new DateTimeOffset(2026, 8, 20, 12, 0, 0, TimeSpan.Zero)); - var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock); + // The miss budget (R03:3.3) is OFF here on purpose: this fact floods 1025 distinct unknown ids — the + // exact shape the budget refuses — to pin the cache CEILING, which is a separate rule. + var sut = Bounded(inner, timeoutMs: 0, cacheSeconds: 600, clock, + guard: new ApiKeyStoreGuardOptions { MissBurst = 0 }); // Seed the real key, then flood past the ceiling with ids that do not exist. (await sut.FindAsync(inner.KeyId, clock.GetUtcNow().UtcDateTime, default)).ShouldNotBeNull(); @@ -477,7 +480,8 @@ public sealed class BoundedApiKeyStoreTests int timeoutMs, int cacheSeconds, TimeProvider? clock = null, - ILogger? log = null) => new( + ILogger? log = null, + ApiKeyStoreGuardOptions? guard = null) => new( inner, Options.Create(new ApiKeyOptions { @@ -487,7 +491,8 @@ public sealed class BoundedApiKeyStoreTests StoreCacheSeconds = cacheSeconds, }), log ?? NullLogger.Instance, - clock ?? TimeProvider.System); + clock ?? TimeProvider.System, + guard); private static ApiKeyOpcUaUserAuthenticator Auth(IApiKeyStore store) => new( store, From 6aa5edaf309cc94aa2786783691674688f9dcd05 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:26:12 -0400 Subject: [PATCH 14/51] fix(scripting): only timeouts are quarantine strikes; alarm Bad annotation once per bucket change (fix round 1 I1, I3) - I1 ScriptRunGuard: a run refused because the previous one is still executing is a Bad publish plus otopcua.script.run_refused, and no longer a quarantine strike. Only a timeout (and an allocation breach) counts; a refusal neither adds to nor resets the timeout streak; a completed run (value or throw) resets it. On a 1 Hz trigger with a 2 s budget a single 4 s overrun is one timeout plus two refusals and must not quarantine. Tests updated across the guard, the alarm engine, the Calculation evaluator and the VT evaluator, plus Refusals_neither_count_toward_nor_reset_the_timeout_streak. - I3 ScriptedAlarmEngine: the quality bucket is tracked only after the predicate completes (or in the inputs-not-ready path), so the tracked bucket is the last one annotated. A refused or quarantined predicate used to re-track the input bucket Good before the attempt and then flip it Bad, emitting QualityChanged on every push. The run-guard tests now count emissions (exactly 1 across N pushes, refused and quarantined cases), with a healthy sibling's dispatch count as the positive control that every push was processed. Both were red before the fix. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ScriptedAlarmEngine.cs | 11 ++- .../ScriptRunGuard.cs | 31 +++++---- .../ScriptingOptions.cs | 7 +- .../CalculationDriverFactoryExtensions.cs | 2 +- .../CalculationEvaluator.cs | 2 +- .../Engines/RoslynVirtualTagEvaluator.cs | 2 +- .../ScriptedAlarmEngineRunGuardTests.cs | 67 +++++++++++++++--- .../ScriptRunGuardTests.cs | 69 +++++++++++++++---- .../CalculationEvaluatorRunGuardTests.cs | 25 +++---- .../RoslynVirtualTagEvaluatorRunGuardTests.cs | 12 ++-- 10 files changed, 161 insertions(+), 67 deletions(-) diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index 9438132a..ae6352f5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -76,7 +76,7 @@ public sealed class ScriptedAlarmEngine : IDisposable /// /// Per-alarm run quarantine (keyed by alarm id, source identity = predicate source). Refuses a /// predicate run while the alarm's previous (timed-out, abandoned) run is still executing and - /// quarantines after three consecutive non-completions or an allocation-budget breach — one orphan + /// quarantines after three consecutive timeouts (a refusal is not a strike) or an allocation-budget breach — one orphan /// per alarm instead of one per upstream push. Reset on every (a redeploy). /// private readonly ScriptRunGuard _guard; @@ -798,7 +798,6 @@ public sealed class ScriptedAlarmEngine : IDisposable // short-circuit so an outright-Bad input is still observed. A bucket change with no state // transition is delivered out of band as a QualityChanged emission (see below). var worstStatus = WorstInputStatus(scratch.ReadCache); - var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus); // Cold-start guard — skip the predicate when any referenced upstream tag has no // cached value yet (the upstream subscription hasn't delivered its first push). @@ -811,7 +810,7 @@ public sealed class ScriptedAlarmEngine : IDisposable // The condition is frozen (can't trust its state), but its source quality just changed // bucket — annotate it out of band so a comms-lost / Bad-input scripted condition reports // Bad, mirroring the native OT path. - if (qualityBucketChanged) + if (TrackQualityBucket(state.Definition.AlarmId, worstStatus)) pendingEmissions.Add(BuildQualityEmission(state, seed, worstStatus)); return seed; } @@ -851,6 +850,12 @@ public sealed class ScriptedAlarmEngine : IDisposable } scratch.LastNonCompletion = null; + // Track the bucket only once the predicate has COMPLETED (I3). The tracked bucket is the last + // bucket this alarm was annotated with; tracking the input bucket before the attempt flipped it to + // Good on every push, so a refused / quarantined predicate (which annotates Bad) re-emitted + // QualityChanged on every push instead of once per real bucket change. + var qualityBucketChanged = TrackQualityBucket(state.Definition.AlarmId, worstStatus); + var result = Part9StateMachine.ApplyPredicate(seed, predicateTrue, nowUtc); var transition = result.Emission != EmissionKind.None ? BuildEmission(state, result.State, result.Emission, worstStatus) diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs index 8f1acefd..dc308ce6 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs @@ -24,8 +24,10 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// (), counted on /// . One orphan per script, never one per trigger. /// Quarantines after -/// consecutive non-completions (timeouts and still-running refusals both count; a run that -/// returns — a value or a throw — resets the count): no further dispatch, every attempt throws +/// consecutive timeouts (only a timeout is a strike; a refusal is a Bad publish plus its counter +/// and leaves the streak untouched — on a 1 Hz trigger with a 2 s budget one 4 s overrun is a +/// timeout plus two refusals and must not quarantine; a run that returns — a value or a throw — +/// resets the streak): no further dispatch, every attempt throws /// , one Error log, one otopcua.script.quarantined /// increment, one event. Lifted by a source change (the operator fixed the /// script — a new source identity under the same key starts clean and the old orphan no longer blocks @@ -177,10 +179,12 @@ public sealed class ScriptRunGuard // not) still holds the slot is refused here, before anything is dispatched to the pool. if (Interlocked.CompareExchange(ref state.InFlight, 1, 0) != 0) { + // A refusal is a Bad publish plus its counter — NOT a strike. Only a timeout (and an allocation + // breach) counts toward quarantine: on a 1 Hz trigger with a 2 s budget a single 4 s overrun is + // one timeout followed by two refusals, and counting those would quarantine a script that merely + // ran long once. The refusal also leaves the timeout streak untouched. Interlocked.Increment(ref _refused); RefusedCounter.Add(1, new KeyValuePair("script", key)); - if (NoteNonCompletion(key, state, "refused: previous evaluation still running")) - throw new ScriptQuarantinedException(key, state.QuarantineReason!); throw new ScriptRunRefusedException(key); } @@ -241,7 +245,7 @@ public sealed class ScriptRunGuard // The caller's cancel wins even when it lands a few µs after the timeout fired. if (ct.IsCancellationRequested) throw new OperationCanceledException(ct); runCts.Cancel(); - if (NoteNonCompletion(key, state, $"timed out after {timeout.TotalMilliseconds:F1} ms")) + if (NoteTimeout(key, state, timeout)) throw new ScriptQuarantinedException(key, state.QuarantineReason!); throw new ScriptTimeoutException(timeout); } @@ -252,11 +256,11 @@ public sealed class ScriptRunGuard catch (Exception) { // The script returned control (by throwing) — that is a completion for quarantine purposes. - Interlocked.Exchange(ref state.ConsecutiveNonCompletions, 0); + Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0); throw; } - Interlocked.Exchange(ref state.ConsecutiveNonCompletions, 0); + Interlocked.Exchange(ref state.ConsecutiveTimeouts, 0); if (BreachedBudget(slot.AllocatedBytes)) { // The continuation above has already (or will have) quarantined the key; make the reason visible @@ -291,13 +295,14 @@ public sealed class ScriptRunGuard } } - /// Record a non-completion; returns true when it tipped the key into quarantine. - private bool NoteNonCompletion(string key, KeyState state, string what) + /// Record a timeout strike; returns true when it tipped the key into quarantine. A completed run + /// (value or throw) resets the streak; a refusal neither counts nor resets. + private bool NoteTimeout(string key, KeyState state, TimeSpan timeout) { - var count = Interlocked.Increment(ref state.ConsecutiveNonCompletions); + var count = Interlocked.Increment(ref state.ConsecutiveTimeouts); if (count < _quarantineThreshold) return false; Quarantine(key, state, - $"{count} consecutive evaluation(s) did not complete (last: {what}); " + + $"{count} consecutive evaluation(s) timed out (budget {timeout.TotalMilliseconds:F0} ms); " + "the script is not evaluated again until its source changes or it is redeployed", "timeouts"); return true; @@ -333,7 +338,7 @@ public sealed class ScriptRunGuard { public string? SourceIdentity = sourceIdentity; public int InFlight; - public int ConsecutiveNonCompletions; + public int ConsecutiveTimeouts; public string? QuarantineReason; } @@ -368,7 +373,7 @@ public sealed class ScriptRunRefusedException : Exception /// /// Thrown by for a quarantined script: repeated -/// non-completion or an allocation-budget breach. Nothing is dispatched while quarantined; the seam +/// timeouts or an allocation-budget breach. Nothing is dispatched while quarantined; the seam /// publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or /// redeploying. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs index ba1399fa..f101a0e1 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs @@ -33,10 +33,11 @@ public sealed class ScriptingOptions public long MaxAllocBytesPerRun { get; set; } = DefaultMaxAllocBytesPerRun; /// - /// Consecutive non-completions (a run that times out, or a run refused because the previous one is - /// still executing) after which the script is quarantined: no further runs are dispatched, every + /// Consecutive timeouts after which the script is quarantined: no further runs are dispatched, every /// evaluation reports Bad, and the quarantine lifts only on a source change, a redeploy or an explicit - /// reset. Default 3; values below 1 are treated as 1. + /// reset. Only a timeout is a strike — a run refused because the previous one is still executing is + /// reported Bad and counted, but neither adds to nor resets the streak; a completed run (value or + /// throw) resets it. Default 3; values below 1 are treated as 1. /// public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs index 349a0cf1..1a589d65 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriverFactoryExtensions.cs @@ -107,7 +107,7 @@ public static class CalculationDriverFactoryExtensions /// Optional run-guard allocation budget per script run, in bytes (default 64 MB; ≤ 0 disables). public long? MaxAllocBytesPerRun { get; init; } - /// Optional run-guard quarantine threshold: consecutive non-completions before a tag's script stops being evaluated (default 3). + /// Optional run-guard quarantine threshold: consecutive timeouts (refusals do not count) before a tag's script stops being evaluated (default 3). public int? QuarantineAfterConsecutiveTimeouts { get; init; } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index ba4931c7..e47cc745 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -42,7 +42,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// Run quarantine. Every compiled run goes through one keyed by /// calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive -/// non-completions or an allocation-budget breach — both surface as a +/// timeouts (a refusal is Bad but not a strike) or an allocation-budget breach — both surface as a /// the driver maps to Bad. The driver's evaluation pump awaits rather than /// blocking a pool thread on the sync , /// and the guard starts the budget clock only once the pool has picked the script up, so pool starvation diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index c6823c41..f0bbd150 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -35,7 +35,7 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines; /// Run quarantine. Every compiled run goes through one keyed by /// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive -/// non-completions or an allocation-budget breach. Both surface as +/// timeouts (a refusal is Bad but not a strike) or an allocation-budget breach. Both surface as /// so VirtualTagActor publishes Bad exactly as it does for a timeout. A quarantine lifts when the /// expression changes, on / (the deploy /// boundary), never by itself. diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs index 4074d0d9..ab2fde04 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineRunGuardTests.cs @@ -9,8 +9,8 @@ namespace ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests; /// /// runs every predicate through a keyed by /// alarm id. A runaway predicate used to leak one pool thread per upstream push (each re-evaluation -/// spawned another abandoned run); now the orphan is one, later pushes are refused and the alarm is -/// quarantined — frozen at its prior state and annotated Bad quality so the operator can see it — while a +/// spawned another abandoned run); now the orphan is one, later pushes are refused (and a +/// predicate that times out repeatedly is quarantined) — frozen at its prior state and annotated Bad quality so the operator can see it — while a /// sibling alarm on the same engine keeps evaluating. A reload lifts the quarantine. /// [Trait("Category", "Unit")] @@ -36,7 +36,7 @@ public sealed class ScriptedAlarmEngineRunGuardTests } [Fact] - public async Task Runaway_predicate_leaks_one_orphan_is_quarantined_annotated_Bad_and_a_sibling_keeps_evaluating() + public async Task Runaway_predicate_leaks_one_orphan_is_refused_annotated_Bad_once_and_a_sibling_keeps_evaluating() { var up = new FakeUpstream(); up.Set("Temp", 50); @@ -50,20 +50,27 @@ public sealed class ScriptedAlarmEngineRunGuardTests Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 100;""")], TestContext.Current.CancellationToken); - // Two more pushes: refused (orphan still running), then quarantined. - up.Push("Temp", 60); - await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= 1); - up.Push("Temp", 70); - await WaitForAsync(() => eng.RunGuard.IsQuarantined("runaway", out _)); + // N more pushes while the orphan runs: every one is refused (a refusal is not a strike — I1), and + // the Bad annotation is emitted ONCE for the whole streak, not once per push (I3). + const int pushes = 4; + for (var i = 0; i < pushes; i++) + { + up.Push("Temp", 60 + i); + var expected = i + 1; + await WaitForAsync(() => eng.RunGuard.RefusedWhileRunningCount >= expected); + } + await WaitForAsync(() => eng.RunGuard.StartedRunCount >= 1 + 1 + pushes); - eng.RunGuard.StartedRunCount.ShouldBe(1 + 3, "one dispatch for the runaway alarm, one per evaluation for the healthy one"); + eng.RunGuard.IsQuarantined("runaway", out _).ShouldBeFalse("refusals are not strikes"); + eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one dispatch for the runaway alarm, one per evaluation for the healthy one"); eng.RunGuard.InFlightCount.ShouldBe(1); - // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving. + // The frozen condition is annotated Bad out of band — the operator sees WHY it is not moving — + // exactly once across the whole run of non-completions. ScriptedAlarmEvent[] snapshot; lock (events) snapshot = events.ToArray(); var quality = snapshot.Where(e => e.AlarmId == "runaway" && e.Emission == EmissionKind.QualityChanged).ToArray(); - quality.ShouldNotBeEmpty(); + quality.Length.ShouldBe(1); quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); eng.GetState("runaway")!.Active.ShouldBe(AlarmActiveState.Inactive, "state is held, never invented"); @@ -72,6 +79,44 @@ public sealed class ScriptedAlarmEngineRunGuardTests await WaitForAsync(() => eng.GetState("healthy")!.Active == AlarmActiveState.Active); } + /// I3 — a QUARANTINED predicate is re-attempted on every push and refused before dispatch; the + /// Bad annotation must still be emitted once, not once per push. + [Fact] + public async Task A_quarantined_predicate_emits_QualityChanged_once_across_many_pushes() + { + var up = new FakeUpstream(); + up.Set("Temp", 50); + using var eng = Build(up, new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); + var events = new List(); + eng.OnEvent += (_, e) => { lock (events) events.Add(e); }; + + await eng.LoadAsync( + [Alarm("q", $"var t = ctx.GetTag(\"Temp\"); {SpinPredicate}"), + Alarm("healthy", """return (int)ctx.GetTag("Temp").Value > 1000;""")], + TestContext.Current.CancellationToken); + eng.RunGuard.IsQuarantined("q", out _).ShouldBeTrue(); + + // Each push re-evaluates BOTH alarms (same input). The healthy sibling's dispatch count is the + // positive control proving every push was actually processed — so the quarantined alarm was + // re-attempted N times, not coalesced into one. + const int pushes = 5; + for (var i = 0; i < pushes; i++) + { + var before = eng.RunGuard.StartedRunCount; + up.Push("Temp", 60 + i); + await WaitForAsync(() => eng.RunGuard.StartedRunCount > before); + } + eng.RunGuard.StartedRunCount.ShouldBe(1 + 1 + pushes, "one runaway dispatch, then healthy-only dispatches"); + await Task.Delay(300); // absence window for trailing emissions + + ScriptedAlarmEvent[] snapshot; + lock (events) snapshot = events.ToArray(); + var quality = snapshot.Where(e => e.AlarmId == "q" && e.Emission == EmissionKind.QualityChanged).ToArray(); + quality.Length.ShouldBe(1, "one Bad annotation for the quarantine, not one per push"); + quality[0].WorstInputStatusCode.ShouldBe(StatusBadInternalError); + snapshot.ShouldNotContain(e => e.AlarmId == "healthy" && e.Emission == EmissionKind.QualityChanged); + } + [Fact] public async Task A_reload_lifts_the_quarantine() { diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs index 974e6a7c..f60045b9 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs @@ -9,7 +9,7 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests; /// Calculation evaluator and the scripted-alarm engine. A timed-out script is abandoned, not stopped, so /// before the guard every re-trigger of a runaway script leaked one more thread-pool thread; the guard /// refuses to start a run while the previous one is still executing, quarantines a script after three -/// consecutive non-completions, and quarantines a script whose single run allocates past the budget. +/// consecutive timeouts (a refusal is not a strike), and quarantines a script whose single run allocates past the budget. /// [Trait("Category", "Unit")] public sealed class ScriptRunGuardTests @@ -28,6 +28,14 @@ public sealed class ScriptRunGuardTests private const string FastScript = "return 42;"; + /// Outlives the 60 ms budget but finishes within ~250 ms, so the slot frees up between attempts. + private const string BriefSpinScript = + """ + var end = System.DateTime.UtcNow.AddMilliseconds(250); + while (System.DateTime.UtcNow < end) { } + return 1; + """; + private static readonly TimeSpan ShortTimeout = TimeSpan.FromMilliseconds(60); private static ScriptEvaluator Compile(string source) => @@ -83,17 +91,21 @@ public sealed class ScriptRunGuardTests } [Fact] - public async Task Three_consecutive_non_completions_quarantine_the_script_and_a_sibling_keeps_evaluating() + public async Task Three_consecutive_timeouts_quarantine_the_script_and_a_sibling_keeps_evaluating() { var guard = new ScriptRunGuard(); - using var spin = Compile(SpinScript); + using var spin = Compile(BriefSpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; - // 1: times out, 2: refused (orphan still running), 3: refused → threshold reached → quarantined. - await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); - await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); - var quarantined = await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); + // Three timeouts in a row — each orphan is allowed to finish so the next attempt is a real run, + // not a refusal (a refusal is not a strike; see the next test). + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); + var quarantined = await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + await WaitForIdle(guard); quarantined.Message.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); quarantined.Message.ShouldContain("3 consecutive"); @@ -102,19 +114,46 @@ public sealed class ScriptRunGuardTests guard.QuarantinedCount.ShouldBe(1); // A quarantined script is never dispatched again — the 4th attempt refuses before Task.Run. - await Should.ThrowAsync(() => Run(guard, "bad", SpinScript, spin, ShortTimeout, ct)); - guard.StartedRunCount.ShouldBe(1); + await Should.ThrowAsync(() => Run(guard, "bad", BriefSpinScript, spin, ShortTimeout, ct)); + guard.StartedRunCount.ShouldBe(3); // Positive control: a healthy sibling on the SAME guard still runs — the quarantine is per key. (await Run(guard, "good", FastScript, fast, ct: ct)).ShouldBe(42); guard.IsQuarantined("good", out _).ShouldBeFalse(); - guard.StartedRunCount.ShouldBe(2); + guard.StartedRunCount.ShouldBe(4); + } + + /// + /// A refusal is NOT a strike (review fix round 1, I1): on a 1 Hz trigger with a 2 s budget a single + /// 4 s overrun is one timeout followed by two refusals, and counting those would quarantine a script + /// that merely ran long once. A refusal is a Bad publish plus its counter, and it leaves the timeout + /// streak untouched — neither adding to it nor resetting it. + /// + [Fact] + public async Task Refusals_neither_count_toward_nor_reset_the_timeout_streak() + { + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 2 }); + using var spin = Compile(BriefSpinScript); + var ct = TestContext.Current.CancellationToken; + + // timeout #1, then refusals while the orphan is still running — however many, no quarantine. + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + for (var i = 0; i < 5; i++) + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse("refusals are not strikes"); + guard.RefusedWhileRunningCount.ShouldBe(5); + await WaitForIdle(guard); + + // …and they did not RESET the streak either: the second real timeout reaches the threshold of 2. + await Should.ThrowAsync(() => Run(guard, "k", BriefSpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeTrue(); + await WaitForIdle(guard); } /// /// The thread-leak regression. Pre-guard, a while(true) script on a 1 Hz trigger leaked one /// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the - /// first trigger's orphan is the only one, every later trigger is refused or quarantined, and a healthy + /// first trigger's orphan is the only one, every later trigger is refused, and a healthy /// sibling keeps evaluating throughout (the positive control an absence assertion needs). /// /// @@ -145,7 +184,9 @@ public sealed class ScriptRunGuardTests guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing"); guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling"); - guard.IsQuarantined("runaway", out _).ShouldBeTrue(); + // One timeout + 29 refusals: not quarantined (a refusal is not a strike), but never re-dispatched. + guard.IsQuarantined("runaway", out _).ShouldBeFalse(); + guard.RefusedWhileRunningCount.ShouldBe(29); healthyResults.ShouldAllBe(v => v == 42); healthyResults.Count.ShouldBe(30); @@ -156,13 +197,11 @@ public sealed class ScriptRunGuardTests [Fact] public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot() { - var guard = new ScriptRunGuard(); + var guard = new ScriptRunGuard(new ScriptingOptions { QuarantineAfterConsecutiveTimeouts = 1 }); using var spin = Compile(SpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; - await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); - await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); await Should.ThrowAsync(() => Run(guard, "vt", SpinScript, spin, ShortTimeout, ct)); guard.IsQuarantined("vt", out _).ShouldBeTrue(); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs index 85943f16..679e7f71 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationEvaluatorRunGuardTests.cs @@ -10,8 +10,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests; /// runs every compiled script through a /// keyed by calc-tag RawPath. A timed-out run is abandoned on its pool thread; before the guard, every /// later trigger of the same tag stacked another orphan behind it (one leaked thread per trigger). Now -/// the next trigger is refused ("previous evaluation still running"), the third consecutive -/// non-completion quarantines the tag, and a healthy sibling on the same evaluator keeps computing. +/// every later trigger is refused ("previous evaluation still running" — Bad, but not a quarantine +/// strike; only consecutive timeouts quarantine), and a healthy sibling on the same evaluator keeps computing. /// public sealed class CalculationEvaluatorRunGuardTests { @@ -24,7 +24,7 @@ public sealed class CalculationEvaluatorRunGuardTests private static readonly Dictionary NoDeps = new(); [Fact] - public async Task A_trigger_during_the_previous_orphan_is_refused_then_the_tag_is_quarantined_and_a_sibling_keeps_computing() + public async Task Triggers_during_the_previous_orphan_are_refused_not_quarantined_and_a_sibling_keeps_computing() { using var sut = new CalculationEvaluator( NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); @@ -35,15 +35,16 @@ public sealed class CalculationEvaluatorRunGuardTests first.Reason.ShouldNotBeNull(); first.Reason.ShouldContain("timed out"); - var second = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); - second.Success.ShouldBeFalse(); - second.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); - - var third = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); - third.Success.ShouldBeFalse(); - third.Reason.ShouldNotBeNull(); - third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); - sut.QuarantinedScripts.ShouldBe(new[] { "calc/loop" }); + // Every later trigger while the orphan runs is a Bad "previous evaluation still running" — a + // refusal is not a strike, so the tag is NOT quarantined by them (fix round 1, I1). + for (var i = 0; i < 3; i++) + { + var refused = await sut.EvaluateAsync("calc/loop", SpinScript, Snap(NoDeps), ct); + refused.Success.ShouldBeFalse(); + refused.Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + } + sut.QuarantinedScripts.ShouldBeEmpty(); + sut.RunGuard.RefusedWhileRunningCount.ShouldBe(3); // Exactly one run was ever dispatched for the runaway tag — the leak is one orphan, not one per trigger. sut.RunGuard.StartedRunCount.ShouldBe(1); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs index f0de4999..cd18ae5e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/RoslynVirtualTagEvaluatorRunGuardTests.cs @@ -10,7 +10,7 @@ namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests; /// /// runs every compiled expression through a /// keyed by virtual-tag id: a runaway expression leaks one orphaned pool -/// thread, not one per trigger, the tag is quarantined after three consecutive non-completions (surfacing +/// thread, not one per trigger, later triggers are refused, the tag is quarantined after three consecutive timeouts (surfacing /// as a Failure the actor publishes Bad for), and the quarantine lifts at the deploy boundary. /// public sealed class RoslynVirtualTagEvaluatorRunGuardTests @@ -23,7 +23,7 @@ public sealed class RoslynVirtualTagEvaluatorRunGuardTests private static readonly Dictionary NoDeps = new(); [Fact] - public void Runaway_expression_is_refused_then_quarantined_and_a_sibling_keeps_evaluating() + public void Runaway_expression_is_refused_while_its_orphan_runs_and_a_sibling_keeps_evaluating() { using var sut = new RoslynVirtualTagEvaluator( NullLogger.Instance, NoOpScriptRoot(), TimeSpan.FromMilliseconds(60)); @@ -33,12 +33,10 @@ public sealed class RoslynVirtualTagEvaluatorRunGuardTests first.Reason.ShouldNotBeNull(); first.Reason.ShouldContain("timed out"); + // Refusals, not strikes (fix round 1, I1): the tag stays Bad but is not quarantined by them. sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); - - var third = sut.Evaluate("vt-loop", SpinScript, NoDeps); - third.Reason.ShouldNotBeNull(); - third.Reason.ShouldStartWith(ScriptRunGuard.QuarantinedReasonPrefix); - sut.QuarantinedScripts.ShouldBe(new[] { "vt-loop" }); + sut.Evaluate("vt-loop", SpinScript, NoDeps).Reason.ShouldBe(ScriptRunGuard.PreviousRunStillRunningReason); + sut.QuarantinedScripts.ShouldBeEmpty(); sut.RunGuard.StartedRunCount.ShouldBe(1, "one orphan, not one per trigger"); var sibling = sut.Evaluate("vt-ok", "return 42;", NoDeps); From ad6be56850f2c1551737014d0f6c6c224b864d94 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:31:41 -0400 Subject: [PATCH 15/51] fix(runtime): bound DependencyMux last-value retention to replay-opting refs (fix round 1 I2) DependencyMuxActor retained the last value of every ref ever registered, and never removed the entries. The continuous-historization recorder registers every historized ref, so on a historizing node the map grew with the historized address space and never shrank. The class remark claimed a bound that did not hold. - Retain only refs that at least one subscriber registered with ReplayRetained: true names (VT actors, the Calculation mux adapter, the scripted-alarm host). A non-opting subscriber is still delivered to but retains nothing. - When a ref's last replay-opting subscriber leaves, its value is kept for a grace (default 2 min, Props(retentionGrace)) so a restart (old adapter gone, new one not yet registered) still replays. A periodic sweep then drops it along with the entry. Entries with no subscriber and nothing retained are removed. - New SourceRemoved(DriverInstanceId) evicts the values a removed driver produced, so its last Good is never replayed. The mux records the publishing driver per retained ref. No production sender exists yet: DriverHostActor is outside this lane (follow-up). - Remark rewritten. Tests: retention bounded to replay-opting refs, release after grace (with an in-grace replay as the positive control), SourceRemoved eviction, and the restart-replay scenario still green. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../VirtualTags/DependencyMuxActor.cs | 251 ++++++++++++++---- .../VirtualTags/DependencyMuxReplayTests.cs | 95 ++++++- 2 files changed, 286 insertions(+), 60 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs index 3af36d6a..40e9bf19 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/DependencyMuxActor.cs @@ -28,53 +28,132 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; /// would over-emit if it spanned the cluster. /// /// -/// Last-value retention + replay (R07:H6). For every ref that has EVER been registered the -/// mux retains the last (value, status, timestamp) it forwarded — through unregisters too, -/// because the case that matters is a subscriber that went away and came back: a Calculation -/// driver Restart is a new driver instance + a new mux adapter, and before this every calc tag sat -/// BadWaitingForInitialData until each of its inputs happened to change again (a -/// scripted-alarm host restart has the same shape). A with -/// receives one batch of the retained values for its -/// refs, in the same shape a driver burst would arrive in. Opt-in per registration: the -/// continuous-historization recorder re-registers on every deploy and a replay there would -/// record a stale sample per tag per deploy. Memory is bounded by the number of refs scripts have -/// ever named on this node, not by the address space. +/// Last-value retention + replay (R07:H6, bounded in review fix round 1 / I2). The mux +/// retains the last (value, status, timestamp) of a ref only while at least one subscriber +/// registered with names it (the virtual-tag +/// actors, the Calculation mux adapter, the scripted-alarm host). A registration that asks receives +/// one batch of the retained values for its refs, in the shape a driver burst arrives in. The case +/// this serves is a subscriber that goes away and comes back: a Calculation driver Restart is a new +/// driver instance + a new mux adapter, and before this every calc tag sat +/// BadWaitingForInitialData until each of its inputs happened to change again. So when a +/// ref's last replay-opting subscriber leaves, its value is kept for +/// (default ; a returning replay registration inside it cancels +/// the release), then dropped together with the ref's entry. +/// +/// +/// A subscriber that does NOT opt in retains nothing: the continuous-historization recorder +/// registers every historized ref on the node (a replay there would record a stale sample per tag +/// per deploy), and retaining on its behalf made the map grow with the historized address space and +/// never shrink. Memory is therefore bounded by the refs replay-opting subscribers currently name, +/// plus those released within the last grace window. A ref with no subscriber and nothing retained +/// has no entry at all. +/// +/// +/// Source removal. The mux records which driver instance last published each retained ref; +/// evicts every value that driver produced, so a removed driver's last +/// Good is never replayed as current. A comms-loss fan (a Bad batch) already overwrites the retained +/// value with Bad. ⚠️ As of fix round 1 no production actor sends yet — +/// DriverHostActor (outside the scripting lane) must Tell it when it stops / removes a driver +/// child. Until then a removed driver's last value can still be replayed to a replay-opting +/// subscriber that re-registers for that ref while another replay-opting subscriber holds it, or +/// within the grace window. /// /// -public sealed class DependencyMuxActor : ReceiveActor +public sealed class DependencyMuxActor : ReceiveActor, IWithTimers { public const string ActorName = "dependency-mux"; + /// How long a ref's retained value outlives its last replay-opting subscriber. + public static readonly TimeSpan DefaultRetentionGrace = TimeSpan.FromMinutes(2); + /// Register a subscriber's interest in a set of tag refs. Idempotent on re-register — /// the second call replaces the prior interest set for that subscriber. /// The refs the subscriber wants. /// The subscriber. - /// When true, the mux immediately sends the subscriber one - /// holding the last retained value of every ref - /// in it has one for (nothing when it has none). + /// When true, the mux retains the last value of every ref in + /// while this subscriber is registered, and immediately sends the subscriber one + /// holding the last retained value of every ref it has + /// one for (nothing when it has none). public sealed record RegisterInterest(IReadOnlyList TagRefs, IActorRef Subscriber, bool ReplayRetained = false); /// Unregister every interest held by . Sent on PostStop by /// the subscriber, or by Terminated handling when the mux watches. public sealed record UnregisterInterest(IActorRef Subscriber); - // A ref stays in _byRef once registered — with an EMPTY set after its last subscriber leaves — so - // the hot path's single lookup per item also decides retention. _byRef.Count == 0 keeps meaning - // "nothing was ever registered", which the batch path early-outs on. - private readonly Dictionary> _byRef = new(StringComparer.Ordinal); - private readonly Dictionary> _bySubscriber = new(); - private readonly Dictionary _last = new(StringComparer.Ordinal); + /// A driver instance was removed / stopped: evict every retained value it produced so it is + /// never replayed. Subscriptions are untouched. + /// The removed driver instance. + public sealed record SourceRemoved(string DriverInstanceId); + + private sealed class SweepReleased + { + public static readonly SweepReleased Instance = new(); + } + + private const string SweepTimerKey = "sweep-released"; + + /// Per-ref state. One lookup on the hot path answers both "who wants it" and "retain it?". + private sealed class RefEntry + { + public readonly HashSet Subscribers = new(); + + /// Registered subscribers of this ref that opted into replay. + public int Replayers; + + /// When the last replay-opting subscriber left (retention in grace), or null. + public DateTime? ReleasedAtUtc; + + public VirtualTagActor.DependencyValueChanged? Last; + public string? LastSource; + + public bool Retains => Replayers > 0 || ReleasedAtUtc is not null; + public bool IsDead => Subscribers.Count == 0 && Replayers == 0 && Last is null; + + public void Forget() + { + Last = null; + LastSource = null; + ReleasedAtUtc = null; + } + } + + private sealed record Registration(HashSet Refs, bool Replay); + + private readonly Dictionary _byRef = new(StringComparer.Ordinal); + private readonly Dictionary _bySubscriber = new(); private readonly ILoggingAdapter _log = Context.GetLogger(); + /// How long a released ref's value is kept. + public TimeSpan RetentionGrace { get; } + + /// + public ITimerScheduler Timers { get; set; } = null!; + + /// Test hook — refs that currently hold a retained value. + internal int RetainedRefCount => _byRef.Values.Count(e => e.Last is not null); + + /// Test hook — refs the mux holds any state for (subscribers or a retained value). + internal int TrackedRefCount => _byRef.Count; + /// Creates props for the DependencyMuxActor. + /// How long a ref's value outlives its last replay-opting subscriber; + /// null ⇒ . /// The props for creating this actor. - public static Props Props() => Akka.Actor.Props.Create(); + public static Props Props(TimeSpan? retentionGrace = null) + { + var grace = retentionGrace ?? DefaultRetentionGrace; + return Akka.Actor.Props.Create(() => new DependencyMuxActor(grace)); + } /// Initializes a new instance of the class. - public DependencyMuxActor() + /// See ; non-positive ⇒ . + public DependencyMuxActor(TimeSpan retentionGrace) { + RetentionGrace = retentionGrace > TimeSpan.Zero ? retentionGrace : DefaultRetentionGrace; Receive(OnRegister); Receive(msg => RemoveSubscriber(msg.Subscriber)); + Receive(OnSourceRemoved); + Receive(_ => OnSweep()); Receive(OnAttributeValuePublished); Receive(OnAttributeValueBatchPublished); Receive(msg => RemoveSubscriber(msg.ActorRef)); @@ -82,31 +161,30 @@ public sealed class DependencyMuxActor : ReceiveActor private void OnRegister(RegisterInterest msg) { - // Replace any prior interest set so re-registers on actor restart don't leak old refs. - if (_bySubscriber.TryGetValue(msg.Subscriber, out var priorRefs)) - { - foreach (var r in priorRefs) - { - if (_byRef.TryGetValue(r, out var set)) set.Remove(msg.Subscriber); - } - } + // Replace any prior interest set so re-registers on actor restart don't leak old refs. Detaching first + // may put a ref into grace; the re-attach below takes it straight back out. + Detach(msg.Subscriber); var refs = new HashSet(msg.TagRefs, StringComparer.Ordinal); - _bySubscriber[msg.Subscriber] = refs; + _bySubscriber[msg.Subscriber] = new Registration(refs, msg.ReplayRetained); List? replay = null; foreach (var r in refs) { - if (!_byRef.TryGetValue(r, out var set)) + if (!_byRef.TryGetValue(r, out var entry)) { - set = new HashSet(); - _byRef[r] = set; + entry = new RefEntry(); + _byRef[r] = entry; } - set.Add(msg.Subscriber); - if (msg.ReplayRetained && _last.TryGetValue(r, out var retained)) + entry.Subscribers.Add(msg.Subscriber); + if (!msg.ReplayRetained) continue; + entry.Replayers++; + entry.ReleasedAtUtc = null; + if (entry.Last is { } retained) (replay ??= []).Add(retained); } Context.Watch(msg.Subscriber); - _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs", msg.Subscriber, refs.Count); + _log.Debug("DependencyMux: subscriber {Sub} registered for {Count} refs (replay {Replay})", + msg.Subscriber, refs.Count, msg.ReplayRetained); // One batch, after the registration is complete, in exactly the shape a driver burst arrives in — // so the subscriber's hot path handles it with no special case. @@ -119,29 +197,89 @@ public sealed class DependencyMuxActor : ReceiveActor private void RemoveSubscriber(IActorRef subscriber) { - if (!_bySubscriber.TryGetValue(subscriber, out var refs)) return; - foreach (var r in refs) - { - // The ref's entry stays (possibly empty) so its last value keeps being retained for a - // subscriber that comes back — see the class remarks. - if (_byRef.TryGetValue(r, out var set)) set.Remove(subscriber); - } - _bySubscriber.Remove(subscriber); + if (!Detach(subscriber)) return; Context.Unwatch(subscriber); _log.Debug("DependencyMux: subscriber {Sub} removed", subscriber); } + /// Drop 's registration. A ref losing its last replay-opting + /// subscriber goes into grace; a ref left with no state at all is removed. + private bool Detach(IActorRef subscriber) + { + if (!_bySubscriber.Remove(subscriber, out var registration)) return false; + var now = DateTime.UtcNow; + var released = false; + foreach (var r in registration.Refs) + { + if (!_byRef.TryGetValue(r, out var entry)) continue; + entry.Subscribers.Remove(subscriber); + if (registration.Replay && --entry.Replayers == 0 && entry.Last is not null) + { + entry.ReleasedAtUtc = now; + released = true; + } + if (entry.IsDead) _byRef.Remove(r); + } + if (released && !Timers.IsTimerActive(SweepTimerKey)) + Timers.StartPeriodicTimer(SweepTimerKey, SweepReleased.Instance, RetentionGrace); + return true; + } + + /// Drop retained values whose grace has run out — with a periodic sweep at the grace interval a + /// released value lives between one and two grace periods — and stop the sweep once nothing is in grace. + private void OnSweep() + { + var cutoff = DateTime.UtcNow - RetentionGrace; + var stillInGrace = false; + List? dead = null; + foreach (var (r, entry) in _byRef) + { + if (entry.ReleasedAtUtc is not { } releasedAt) continue; + if (releasedAt > cutoff) + { + stillInGrace = true; + continue; + } + entry.Forget(); + if (entry.IsDead) (dead ??= []).Add(r); + } + if (dead is not null) + foreach (var r in dead) _byRef.Remove(r); + if (!stillInGrace) Timers.Cancel(SweepTimerKey); + } + + private void OnSourceRemoved(SourceRemoved msg) + { + List? dead = null; + var evicted = 0; + foreach (var (r, entry) in _byRef) + { + if (entry.Last is null || !string.Equals(entry.LastSource, msg.DriverInstanceId, StringComparison.Ordinal)) + continue; + entry.Forget(); + evicted++; + if (entry.IsDead) (dead ??= []).Add(r); + } + if (dead is not null) + foreach (var r in dead) _byRef.Remove(r); + _log.Debug("DependencyMux: source {Driver} removed — evicted {Count} retained value(s)", msg.DriverInstanceId, evicted); + } + private void OnAttributeValuePublished(DriverInstanceActor.AttributeValuePublished msg) { - if (!_byRef.TryGetValue(msg.FullReference, out var subscribers)) + if (!_byRef.TryGetValue(msg.FullReference, out var entry)) { - // No virtual tag ever cared about this ref — drop. Common in normal operation; the address + // Nothing is registered for (or retaining) this ref — drop. Common in normal operation; the address // space carries thousands of tags and only a fraction feed virtual-tag expressions. return; } var dep = new VirtualTagActor.DependencyValueChanged(msg.FullReference, msg.Value, msg.TimestampUtc, msg.StatusCode); - _last[msg.FullReference] = dep; - foreach (var sub in subscribers) + if (entry.Retains) + { + entry.Last = dep; + entry.LastSource = msg.DriverInstanceId; + } + foreach (var sub in entry.Subscribers) { // A single-item batch, so the mux emits exactly ONE message shape and every subscriber needs // only the batch receive on its hot path. One list per subscriber: batch ownership transfers @@ -153,7 +291,7 @@ public sealed class DependencyMuxActor : ReceiveActor private void OnAttributeValueBatchPublished(DriverInstanceActor.AttributeValueBatchPublished msg) { - // Nobody has registered anything — skip the whole burst rather than probing an empty map per item. + // Nothing registered and nothing retained — skip the whole burst rather than probing an empty map per item. if (_byRef.Count == 0) return; // Allocated lazily on the first matched item: a burst in which nothing is registered for — still @@ -166,10 +304,15 @@ public sealed class DependencyMuxActor : ReceiveActor for (var i = 0; i < count; i++) { var item = msg.Items[i]; - if (!_byRef.TryGetValue(item.FullReference, out var subscribers)) continue; + if (!_byRef.TryGetValue(item.FullReference, out var entry)) continue; var dep = new VirtualTagActor.DependencyValueChanged(item.FullReference, item.Value, item.TimestampUtc, item.StatusCode); - _last[item.FullReference] = dep; - if (subscribers.Count == 0) continue; // retained for a returning subscriber; nobody to forward to now + if (entry.Retains) + { + entry.Last = dep; + entry.LastSource = msg.DriverInstanceId; + } + var subscribers = entry.Subscribers; + if (subscribers.Count == 0) continue; // retained (in grace) for a returning subscriber; nobody to forward to now perSubscriber ??= new Dictionary>(); foreach (var sub in subscribers) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs index ae42edb9..f761edd9 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/DependencyMuxReplayTests.cs @@ -12,10 +12,13 @@ using ZB.MOM.WW.OtOpcUa.Runtime.VirtualTags; namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.VirtualTags; /// -/// R07:H6 — the mux retains the last value per ever-registered ref and replays it to a registration -/// that asks (). A Calculation driver -/// Restart (new driver + new adapter) therefore republishes from current inputs instead of sitting -/// BadWaitingForInitialData until every input happens to change. +/// R07:H6 — the mux retains the last value of every ref a REPLAY-OPTING subscriber +/// () is registered for, keeps it for a +/// short grace after the last such subscriber leaves, and replays it to a registration that asks. A +/// Calculation driver Restart (new driver + new adapter) therefore republishes from current inputs instead +/// of sitting BadWaitingForInitialData until every input happens to change. Fix round 1 (I2): a +/// non-opting subscriber (the historization recorder) retains nothing, retention is released after the +/// grace, and a removed source's values are evicted so its last Good is never replayed. /// public sealed class DependencyMuxReplayTests : RuntimeActorTestBase { @@ -27,7 +30,7 @@ public sealed class DependencyMuxReplayTests : RuntimeActorTestBase { var mux = Sys.ActorOf(DependencyMuxActor.Props()); var first = CreateTestProbe(); - mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref)); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, first.Ref, ReplayRetained: true)); mux.Tell(Burst(("a", 1), ("b", 2), ("c", 3))); first.ExpectMsg(PresenceBudget).Items.Count.ShouldBe(2); @@ -38,7 +41,7 @@ public sealed class DependencyMuxReplayTests : RuntimeActorTestBase mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "c" }, second.Ref, ReplayRetained: true)); var replay = second.ExpectMsg(PresenceBudget); - replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by anyone, so it is not retained; 'b' is not in this subscriber's refs"); + replay.Items.Select(i => i.TagId).ShouldBe(new[] { "a" }, "'c' was never registered by a replay-opting subscriber, so it is not retained; 'b' is not in this subscriber's refs"); replay.Items[0].Value.ShouldBe(1); } @@ -60,6 +63,86 @@ public sealed class DependencyMuxReplayTests : RuntimeActorTestBase fresh.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); } + /// I2 — a subscriber that did NOT opt into replay (the continuous-historization recorder registers + /// every historized ref) retains nothing; retention is bounded to the refs a replay-opting subscriber named. + [Fact] + public void Only_refs_named_by_a_replay_opting_subscriber_are_retained() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props()); + var recorder = CreateTestProbe(); + var calc = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "h1", "h2", "h3", "shared" }, recorder.Ref)); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "r", "shared" }, calc.Ref, ReplayRetained: true)); + mux.Tell(Burst(("h1", 1), ("h2", 2), ("h3", 3), ("r", 4), ("shared", 5))); + recorder.ExpectMsg(PresenceBudget).Items.Count.ShouldBe(4, "the recorder is still DELIVERED to"); + calc.ExpectMsg(PresenceBudget); + + mux.UnderlyingActor.RetainedRefCount.ShouldBe(2, "only 'r' and 'shared' — the recorder-only refs retain nothing"); + + var late = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "h1", "r" }, late.Ref, ReplayRetained: true)); + late.ExpectMsg(PresenceBudget) + .Items.Select(i => i.TagId).ShouldBe(new[] { "r" }, "positive control: the opted-in ref replays; 'h1' was never retained"); + + // Unregistering everyone ends up with no entries at all once the grace is over — tested below; here + // the recorder leaving changes nothing about retention. + mux.Tell(new DependencyMuxActor.UnregisterInterest(recorder.Ref)); + mux.UnderlyingActor.RetainedRefCount.ShouldBe(2); + } + + /// I2 — when the last replay-opting subscriber of a ref leaves, its retained value survives a short + /// grace (the restart window: old adapter gone, new one not yet registered) and is then dropped. + [Fact] + public void Retention_is_released_after_the_grace_once_no_replay_subscriber_remains() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props(TimeSpan.FromMilliseconds(200))); + var gen1 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen1.Ref, ReplayRetained: true)); + mux.Tell(Burst(("a", 1))); + gen1.ExpectMsg(PresenceBudget); + mux.Tell(new DependencyMuxActor.UnregisterInterest(gen1.Ref)); + + // Positive control: inside the grace a returning subscriber still gets the value. + var gen2 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen2.Ref, ReplayRetained: true)); + gen2.ExpectMsg(PresenceBudget).Items.Single().Value.ShouldBe(1); + mux.Tell(new DependencyMuxActor.UnregisterInterest(gen2.Ref)); + + // Past the grace: nothing retained, no entry left behind, and a returning subscriber replays nothing. + AwaitAssert(() => + { + mux.UnderlyingActor.RetainedRefCount.ShouldBe(0); + mux.UnderlyingActor.TrackedRefCount.ShouldBe(0); + }, PresenceBudget); + var gen3 = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a" }, gen3.Ref, ReplayRetained: true)); + gen3.ExpectNoMsg(TimeSpan.FromMilliseconds(300)); + } + + /// I2 — a removed source's retained values are evicted, so its last Good is never replayed to a + /// subscriber that (re)registers after the removal. Values from other sources are untouched. + [Fact] + public void SourceRemoved_evicts_that_sources_retained_values_only() + { + var mux = ActorOfAsTestActorRef(DependencyMuxActor.Props()); + var sub = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, sub.Ref, ReplayRetained: true)); + mux.Tell(new DriverInstanceActor.AttributeValueBatchPublished("driver-1", + [new DriverInstanceActor.AttributeValueItem("a", 1, OpcUaStatus.Good, DateTime.UtcNow)])); + mux.Tell(new DriverInstanceActor.AttributeValueBatchPublished("driver-2", + [new DriverInstanceActor.AttributeValueItem("b", 2, OpcUaStatus.Good, DateTime.UtcNow)])); + sub.ExpectMsg(PresenceBudget); + sub.ExpectMsg(PresenceBudget); + + mux.Tell(new DependencyMuxActor.SourceRemoved("driver-1")); + mux.UnderlyingActor.RetainedRefCount.ShouldBe(1); + + var restarted = CreateTestProbe(); + mux.Tell(new DependencyMuxActor.RegisterInterest(new[] { "a", "b" }, restarted.Ref, ReplayRetained: true)); + restarted.ExpectMsg(PresenceBudget) + .Items.Select(i => i.TagId).ShouldBe(new[] { "b" }, "driver-1 was removed: its last Good must not be replayed"); + } + /// The user-facing scenario: a calc tag computes, the driver is restarted (new instance + new /// adapter) with NO input changing afterwards, and the new instance republishes the current value. [Fact] From 3ddb6876dbe65660c7b2f5e7a5fb3eba4a870151 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:35:54 -0400 Subject: [PATCH 16/51] fix(security): first-party JwtBearer with HS256 pinned; /auth/token mints for a cookie session only The hand-rolled bearer handler is replaced by AddJwtBearer (Microsoft.AspNetCore.Authentication.JwtBearer 10.0.7, pinned on the same servicing line as the other AspNetCore packages). Validation parameters come from JwtTokenService.BuildValidationParameters, which now pins ValidAlgorithms = HS256 for both the bearer scheme and TryValidate; MapInboundClaims is off so the minted Role/zb:username keys resolve. /auth/token requires the cookie scheme, so a bearer can no longer mint a fresh token for itself. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- Directory.Packages.props | 4 + docs/security.md | 13 ++- .../Auth/JwtBearerAuthenticationHandler.cs | 99 ------------------- .../Auth/OtOpcUaAuthSchemes.cs | 39 ++++++++ .../Endpoints/AuthEndpoints.cs | 16 ++- .../Jwt/JwtTokenService.cs | 28 ++---- .../ServiceCollectionExtensions.cs | 18 +++- .../ZB.MOM.WW.OtOpcUa.Security.csproj | 1 + .../AuthEndpointsIntegrationTests.cs | 93 +++++++++++++++++ .../JwtConfigurationGateTests.cs | 2 +- 10 files changed, 187 insertions(+), 126 deletions(-) delete mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs create mode 100644 src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 5b95a7e1..ed2dca67 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -35,6 +35,10 @@ + + diff --git a/docs/security.md b/docs/security.md index 6e44569a..fa8269a2 100644 --- a/docs/security.md +++ b/docs/security.md @@ -6,11 +6,14 @@ > auth + role-grant pages live in `OtOpcUa.AdminUI`), and Admin auth policies > register from `OtOpcUa.Host/Program.cs` via `AddOtOpcUaAuth` > (`src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs`) rather -> than in a separate Admin process. The Admin UI uses a **single Cookie -> authentication scheme** — there is no `AddJwtBearer` pipeline. The -> `Security:Jwt` section configures `JwtTokenService`, which mints a JWT at the -> `/auth/token` endpoint for **external** consumers (OPC UA clients / automation -> scripts); the cookie itself stores the `ClaimsPrincipal` directly. DataProtection +> than in a separate Admin process. The Admin UI session is a **Cookie** +> scheme; the cookie stores the `ClaimsPrincipal` directly. JWT is **opt-in by +> `Security:Jwt:SigningKey`** (validated at host start, ≥32 bytes): with a key, +> `/auth/token` mints an HS256 token for a **cookie session only** (a bearer +> cannot renew itself), and a first-party `AddJwtBearer` scheme (`Bearer`, +> `ValidAlgorithms = HS256`, parameters from `JwtTokenService.BuildValidationParameters`) +> accepts it on `/api/*` and every `RequireAuthorization` endpoint through the +> `OtOpcUa.CookieOrBearer` default policy scheme. Without a key neither exists. DataProtection > keys persist to the shared Config DB (`PersistKeysToDbContext`) > so cookies survive failover between admin-role nodes. > diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs deleted file mode 100644 index cc996758..00000000 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/JwtBearerAuthenticationHandler.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System.Text.Encodings.Web; -using Microsoft.AspNetCore.Authentication; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Primitives; -using ZB.MOM.WW.OtOpcUa.Security.Jwt; - -namespace ZB.MOM.WW.OtOpcUa.Security.Auth; - -/// The authentication scheme names OtOpcUa registers. -public static class OtOpcUaAuthSchemes -{ - /// - /// The bearer scheme that validates the HS256 tokens /auth/token mints, through - /// . Registered only when - /// . - /// - public const string Bearer = "Bearer"; - - /// - /// The default scheme: a policy scheme that forwards to when the request - /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one - /// RequireAuthorization() / policy accepts a browser session AND a machine token. - /// - public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; - - /// Whether the request presents a bearer token (the selector's whole rule). - /// The request. - /// when an Authorization: Bearer … header is present. - public static bool HasBearerToken(HttpContext context) => - TryGetBearerToken(context.Request.Headers.Authorization, out _); - - internal static bool TryGetBearerToken(StringValues authorization, out string token) - { - token = string.Empty; - var header = authorization.ToString(); - const string prefix = "Bearer "; - if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; - token = header[prefix.Length..].Trim(); - return token.Length > 0; - } -} - -/// -/// Validates an Authorization: Bearer token minted by /auth/token (R03:2.4). The JWT -/// feature used to be a dead end: tokens were issued and nothing in the product accepted them. This -/// handler is the consuming side, built on so the validation -/// parameters can never drift from the minting side ( -/// sets RoleClaimType/NameClaimType to the minted keys, so [Authorize(Roles)] and -/// IsInRole resolve). -/// -/// -/// Fail closed: no header ⇒ NoResult (the cookie scheme is not consulted by this handler — the -/// policy scheme selects one or the other); a malformed, expired, tampered or wrong-issuer token ⇒ -/// Fail, never a fall-through to the cookie. A direct AddJwtBearer needs the -/// Microsoft.AspNetCore.Authentication.JwtBearer package, which the central package list does -/// not carry; this handler is ~40 lines over the already-referenced System.IdentityModel.Tokens.Jwt -/// and is a drop-in to replace once that package is pinned. -/// -public sealed class JwtBearerAuthenticationHandler : AuthenticationHandler -{ - private readonly JwtTokenService _jwt; - - /// Initializes a new instance of the class. - /// The scheme options monitor. - /// The logger factory. - /// The URL encoder. - /// The token service whose validation parameters mirror its minting side. - public JwtBearerAuthenticationHandler( - IOptionsMonitor options, - ILoggerFactory logger, - UrlEncoder encoder, - JwtTokenService jwt) - : base(options, logger, encoder) - { - _jwt = jwt; - } - - /// - protected override Task HandleAuthenticateAsync() - { - if (!OtOpcUaAuthSchemes.TryGetBearerToken(Request.Headers.Authorization, out var token)) - return Task.FromResult(AuthenticateResult.NoResult()); - - if (!_jwt.TryValidate(token, out var principal) || principal is null) - return Task.FromResult(AuthenticateResult.Fail("Invalid bearer token.")); - - return Task.FromResult(AuthenticateResult.Success(new AuthenticationTicket(principal, Scheme.Name))); - } - - /// - protected override Task HandleChallengeAsync(AuthenticationProperties properties) - { - Response.StatusCode = StatusCodes.Status401Unauthorized; - Response.Headers.WWWAuthenticate = OtOpcUaAuthSchemes.Bearer; - return Task.CompletedTask; - } -} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs new file mode 100644 index 00000000..7c30915e --- /dev/null +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Auth/OtOpcUaAuthSchemes.cs @@ -0,0 +1,39 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Primitives; +using ZB.MOM.WW.OtOpcUa.Security.Jwt; + +namespace ZB.MOM.WW.OtOpcUa.Security.Auth; + +/// The authentication scheme names OtOpcUa registers. +public static class OtOpcUaAuthSchemes +{ + /// + /// The bearer scheme that validates the HS256 tokens /auth/token mints: the first-party + /// AddJwtBearer handler over . + /// Registered only when . + /// + public const string Bearer = "Bearer"; + + /// + /// The default scheme: a policy scheme that forwards to when the request + /// carries an Authorization: Bearer header and to the cookie scheme otherwise, so one + /// RequireAuthorization() / policy accepts a browser session AND a machine token. + /// + public const string CookieOrBearer = "OtOpcUa.CookieOrBearer"; + + /// Whether the request presents a bearer token (the selector's whole rule). + /// The request. + /// when an Authorization: Bearer … header is present. + public static bool HasBearerToken(HttpContext context) => + TryGetBearerToken(context.Request.Headers.Authorization, out _); + + internal static bool TryGetBearerToken(StringValues authorization, out string token) + { + token = string.Empty; + var header = authorization.ToString(); + const string prefix = "Bearer "; + if (!header.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return false; + token = header[prefix.Length..].Trim(); + return token.Length > 0; + } +} diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs index 6b0876d7..a3f661b6 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Endpoints/AuthEndpoints.cs @@ -24,6 +24,17 @@ public static class AuthEndpoints /// Response for a token issue request. public sealed record TokenResponse(string Token); + /// + /// The /auth/token policy: an authenticated principal from the COOKIE scheme. Naming the scheme + /// makes the authorization middleware authenticate that scheme alone, so an Authorization: Bearer + /// header is never consulted here (with login disabled the auto-login handler sits under the same + /// scheme name, so the dev rig is unchanged). + /// + internal static readonly AuthorizationPolicy CookieSessionOnly = + new AuthorizationPolicyBuilder(CookieAuthenticationDefaults.AuthenticationScheme) + .RequireAuthenticatedUser() + .Build(); + /// Maps OtOpcUa authentication endpoints to the application route builder. /// The endpoint route builder. /// The endpoint route builder for chaining. @@ -40,7 +51,10 @@ public static class AuthEndpoints // Mapped only when a signing key is configured (JwtOptions.IsConfigured): an unconfigured // endpoint used to answer HTTP 500 from the JwtTokenService constructor on every call. if (app.ServiceProvider.GetRequiredService>().Value.IsConfigured) - app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(); + // COOKIE scheme only (fix round 1 / I2): the default scheme also accepts a bearer, and a bearer + // that could mint a fresh token would renew itself forever — outliving its own expiry and the + // user's LDAP membership. A token is minted only for an interactive (cookie) session. + app.MapPost("/auth/token", (Delegate)IssueToken).RequireAuthorization(CookieSessionOnly); else app.ServiceProvider.GetService()? .CreateLogger("ZB.MOM.WW.OtOpcUa.Security.AuthEndpoints") diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs index 4dd38a2e..9dae8c14 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Jwt/JwtTokenService.cs @@ -25,24 +25,12 @@ public sealed class JwtTokenService /// /// Role claim type used in the JWT payload. /// - /// Issued-only / no internal JwtBearer scheme: OtOpcUa uses a single Cookie - /// authentication scheme; the JWT is minted by the /auth/token endpoint and - /// consumed externally (e.g. by OPC-UA clients or automation scripts). There is no - /// AddJwtBearer pipeline in OtOpcUa — the cookie stores the - /// directly. Because no internal - /// bearer validation path exists, the short "Role" key is intentionally used here rather - /// than the long URI; external consumers receive exactly the - /// key they expect. - /// - /// - /// If a JwtBearer scheme is ever added: the - /// passed to - /// AddJwtBearer MUST set RoleClaimType = JwtTokenService.RoleClaimType (and - /// NameClaimType = JwtTokenService.UsernameClaimType) so that - /// [Authorize(Roles=...)] and ClaimsPrincipal.IsInRole resolve correctly. - /// is already wired to do this and MUST be used - /// rather than constructing - /// ad hoc. + /// The short "Role" key is what mints and what external consumers of the token + /// expect. The internal bearer scheme (AddJwtBearer in AddOtOpcUaAuth) validates with + /// , which sets RoleClaimType = RoleClaimType and + /// NameClaimType = UsernameClaimType, and turns MapInboundClaims off, so + /// [Authorize(Roles=...)] and ClaimsPrincipal.IsInRole resolve against this key. Never + /// construct ad hoc. /// /// public const string RoleClaimType = "Role"; @@ -150,5 +138,9 @@ public sealed class JwtTokenService // if this is ever passed to AddJwtBearer. See RoleClaimType doc comment for rationale. RoleClaimType = RoleClaimType, NameClaimType = UsernameClaimType, + // Pinned to the one algorithm Issue signs with: an unsigned (alg:none) token or one signed with a + // different HMAC variant over the same key is refused on both the bearer and TryValidate paths. + ValidAlgorithms = [SecurityAlgorithms.HmacSha256], + RequireSignedTokens = true, }; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs index d58fbcf3..494145be 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; +using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; @@ -108,8 +109,21 @@ public static class ServiceCollectionExtensions }); if (jwtConfigured) { - authBuilder.AddScheme( - OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + // First-party JwtBearer (R03:2.4, fix round 1 / I1) — the earlier hand-rolled handler is gone. + // Validation parameters come from the SAME JwtTokenService that mints, so issuer/audience/key/ + // claim types can never drift; BuildValidationParameters pins ValidAlgorithms = HS256 (an + // alg:none or a different-HMAC token is refused). MapInboundClaims=false keeps the minted + // "Role"/"zb:username" claim keys verbatim, which RoleClaimType/NameClaimType name. A bad token + // is AuthenticateResult.Fail ⇒ 401 + WWW-Authenticate: Bearer, never a 500. + authBuilder.AddJwtBearer(OtOpcUaAuthSchemes.Bearer, "OtOpcUa bearer token", _ => { }); + services.AddOptions(OtOpcUaAuthSchemes.Bearer) + .Configure((o, jwt) => + { + o.TokenValidationParameters = jwt.BuildValidationParameters(); + o.MapInboundClaims = false; + o.SaveToken = false; + o.IncludeErrorDetails = false; + }); } if (disableLogin) { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj index d81e4eb6..507aca6c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj @@ -10,6 +10,7 @@ + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs index e5e3b620..599d0cf2 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/AuthEndpointsIntegrationTests.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; using Shouldly; using Xunit; using ZB.MOM.WW.Auth.AspNetCore; @@ -208,6 +209,98 @@ public sealed class AuthEndpointsIntegrationTests : IAsyncLifetime response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); } + /// + /// Fix round 1 / I1 — the first-party JwtBearer handler pins HS256: an unsigned (alg: none) + /// token carrying the right issuer/audience/roles, and a token signed with a different HMAC + /// variant, both fail CLOSED with a 401 — never a 500, never a pass. + /// + [Theory] + [InlineData("none")] + [InlineData("HS384")] + public async Task Bearer_with_a_non_HS256_algorithm_is_rejected_with_401(string alg) + { + var token = alg == "none" ? UnsignedToken() : SignedToken(SecurityAlgorithms.HmacSha384); + var client = NewClient(); + var req = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); + + var response = await client.SendAsync(req, Ct); + + response.StatusCode.ShouldBe(HttpStatusCode.Unauthorized); + response.Headers.WwwAuthenticate.ToString().ShouldContain("Bearer"); + + // Positive control: the same claims signed HS256 with the configured key ARE accepted, so the + // 401 above measured the algorithm and not a broken token shape. + var good = new HttpRequestMessage(HttpMethod.Get, "/api/viewer-only"); + good.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue( + "Bearer", SignedToken(SecurityAlgorithms.HmacSha256)); + (await client.SendAsync(good, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + } + + /// + /// Fix round 1 / I2 — /auth/token mints only for a COOKIE session. A bearer token used to + /// be able to mint a fresh token for itself, so a stolen token could renew itself forever and + /// outlive both its own expiry and the user's LDAP membership. Positive control: the cookie + /// session that minted the bearer still gets a token. + /// + [Fact] + public async Task Auth_token_refuses_a_bearer_only_caller_and_serves_a_cookie_session() + { + var client = NewClient(); + var loginResponse = await client.PostAsJsonAsync("/auth/login", + new AuthEndpoints.LoginRequest("alice", "valid-password"), Ct); + loginResponse.EnsureSuccessStatusCode(); + + var cookieReq = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + AttachCookies(cookieReq, loginResponse); + var cookieResp = await client.SendAsync(cookieReq, Ct); + cookieResp.StatusCode.ShouldBe(HttpStatusCode.OK); + var bearer = (await cookieResp.Content.ReadFromJsonAsync(Ct)).GetProperty("token").GetString()!; + + // The bearer authenticates elsewhere (positive control on the token itself) … + var root = new HttpRequestMessage(HttpMethod.Get, "/"); + root.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer); + (await client.SendAsync(root, Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + + // … but cannot renew itself. + var renew = new HttpRequestMessage(HttpMethod.Post, "/auth/token"); + renew.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", bearer); + renew.Headers.Add("X-Requested-With", "XMLHttpRequest"); + var renewResp = await client.SendAsync(renew, Ct); + renewResp.StatusCode.ShouldBeOneOf(HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden); + (await renewResp.Content.ReadAsStringAsync(Ct)).ShouldNotContain("token"); + } + + private const string TestSigningKey = "test-signing-key-with-at-least-32-bytes-of-utf8-content"; + + private static IEnumerable ViewerClaims() => + [ + new(JwtTokenService.UsernameClaimType, "mallory"), + new(JwtTokenService.RoleClaimType, "Viewer"), + ]; + + /// A token signed with using the configured key bytes. + private static string SignedToken(string algorithm) + { + var keyBytes = System.Text.Encoding.UTF8.GetBytes(TestSigningKey); + // The SAME key bytes for every algorithm, so a rejection measures the algorithm pin alone. + var token = new System.IdentityModel.Tokens.Jwt.JwtSecurityToken( + issuer: "otopcua-test", audience: "otopcua-test", claims: ViewerClaims(), + expires: DateTime.UtcNow.AddMinutes(5), + signingCredentials: new SigningCredentials(new SymmetricSecurityKey(keyBytes), algorithm)); + return new System.IdentityModel.Tokens.Jwt.JwtSecurityTokenHandler().WriteToken(token); + } + + /// An alg: none token with otherwise valid issuer/audience/lifetime/roles. + private static string UnsignedToken() + { + static string B64(string json) => Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(json); + var exp = DateTimeOffset.UtcNow.AddMinutes(5).ToUnixTimeSeconds(); + return B64("{\"alg\":\"none\",\"typ\":\"JWT\"}") + "." + + B64($"{{\"zb:username\":\"mallory\",\"Role\":\"Viewer\",\"iss\":\"otopcua-test\",\"aud\":\"otopcua-test\",\"exp\":{exp}}}") + + "."; + } + private async Task LoginAndIssueTokenAsync() { var client = NewClient(); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs index 96fc88ca..9d2cc7ef 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/JwtConfigurationGateTests.cs @@ -65,7 +65,7 @@ public sealed class JwtConfigurationGateTests var schemes = host.Services.GetRequiredService(); var bearer = await schemes.GetSchemeAsync(OtOpcUaAuthSchemes.Bearer); bearer.ShouldNotBeNull(); - bearer.HandlerType.ShouldBe(typeof(JwtBearerAuthenticationHandler)); + bearer.HandlerType.ShouldBe(typeof(Microsoft.AspNetCore.Authentication.JwtBearer.JwtBearerHandler)); await host.StopAsync(Ct); } From c0a0e7aa81626b5ff31e4a75284ac0aa5205334d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:36:51 -0400 Subject: [PATCH 17/51] fix(security): audit actor falls back to HttpContext when the server auth-state provider has no circuit ServerAuthenticationStateProvider throws InvalidOperationException from GetAuthenticationStateAsync in any scope where no circuit set its state (minimal-API requests, hosted-service scopes). The accessor now treats that as "no circuit identity" and falls back to the HTTP principal. Pinned with the real provider on both sides (unset -> HTTP fallback; set -> circuit principal wins). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AuthenticationStateAuditActorAccessor.cs | 32 +++++++++++++---- ...henticationStateAuditActorAccessorTests.cs | 36 +++++++++++++++++++ 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs index d689b85e..13ede448 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/Audit/AuthenticationStateAuditActorAccessor.cs @@ -47,17 +47,37 @@ public sealed class AuthenticationStateAuditActorAccessor : IAuditActorAccessor { if (_authenticationState is not null) { - var task = _authenticationState.GetAuthenticationStateAsync(); - if (task.IsCompletedSuccessfully) - { - var actor = ResolveActor(task.Result.User); - if (actor is not null) return actor; - } + var actor = TryResolveCircuitActor(_authenticationState); + if (actor is not null) return actor; } return ResolveActor(_httpContextAccessor.HttpContext?.User); } } + /// + /// The circuit's actor, or null when this scope has no circuit identity. The server-side provider + /// is registered for EVERY scope on an AdminUI node, but only a circuit calls + /// SetAuthenticationState; anywhere else (a minimal-API request, a hosted service's scope) + /// GetAuthenticationStateAsync throws synchronously + /// (fix round 1 / I5). That is "no circuit", never an audit failure — the caller falls back to + /// the HTTP context. + /// + /// The scoped provider. + /// The actor, or null. + private static string? TryResolveCircuitActor(AuthenticationStateProvider provider) + { + Task task; + try + { + task = provider.GetAuthenticationStateAsync(); + } + catch (InvalidOperationException) + { + return null; + } + return task.IsCompletedSuccessfully ? ResolveActor(task.Result.User) : null; + } + /// The claim-resolution rule shared with . /// The principal, or null. /// The actor string, or null when unauthenticated. diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs index 5dc1f391..6341ef6f 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Security.Tests/Audit/AuthenticationStateAuditActorAccessorTests.cs @@ -58,6 +58,42 @@ public sealed class AuthenticationStateAuditActorAccessorTests accessor.CurrentActor.ShouldBeNull(); } + /// + /// Fix round 1 / I5 — the REAL server-side provider, resolved in a non-circuit scope (a minimal-API + /// request, a background service), throws from + /// GetAuthenticationStateAsync because nothing called SetAuthenticationState. That used + /// to escape CurrentActor; it must fall back to the HTTP principal instead. + /// + [Fact] + public void Real_server_provider_outside_a_circuit_falls_back_to_the_http_principal() + { + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "dave")) }; + var accessor = new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(context), + new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider()); + + accessor.CurrentActor.ShouldBe("dave"); + + // No HTTP principal either ⇒ null, still no throw. + new AuthenticationStateAuditActorAccessor( + new HttpContextAccessorStub(null), + new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider()) + .CurrentActor.ShouldBeNull(); + } + + /// Positive control on the real provider: once a circuit has set its state, that principal wins. + [Fact] + public void Real_server_provider_inside_a_circuit_serves_the_circuit_principal() + { + var provider = new Microsoft.AspNetCore.Components.Server.ServerAuthenticationStateProvider(); + provider.SetAuthenticationState(Task.FromResult( + new AuthenticationState(Authenticated(new Claim(ZbClaimTypes.Username, "erin"))))); + var context = new DefaultHttpContext { User = Authenticated(new Claim(ZbClaimTypes.Username, "http-user")) }; + + new AuthenticationStateAuditActorAccessor(new HttpContextAccessorStub(context), provider) + .CurrentActor.ShouldBe("erin"); + } + private static ClaimsPrincipal Authenticated(params Claim[] claims) => new(new ClaimsIdentity(claims, "TestScheme", ZbClaimTypes.Name, ZbClaimTypes.Role)); From 88268ea3a6980a0c473d18b3acc8740add65d426 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:38:03 -0400 Subject: [PATCH 18/51] fix(host): OPC UA endpoint probe is Ready-only on a fused admin+driver node Coordinator ruling (I6): until ServiceLevel consumes IOpcUaEndpointState (F-C1), the probe must not sit on the Active tier of a fused node, because Traefik routes the AdminUI on /health/active. Driver-only nodes keep it on Ready + Active; admin-only nodes do not register it. Pinned per role. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Health/HealthEndpoints.cs | 21 ++++++++++++---- .../Health/OpcUaEndpointHealthCheck.cs | 6 +++-- .../Health/ActiveTierRegistrationTests.cs | 25 ++++++++++++------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs index 2f57158f..824223dd 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/HealthEndpoints.cs @@ -31,7 +31,8 @@ public static class HealthEndpoints /// /// /// Whether this node carries the driver role and therefore serves an OPC UA endpoint. When - /// true the joins the ready + active tiers (R03:2.1), reading + /// true the joins the ready tier, and the active tier on a + /// driver-ONLY node (R03:2.1; on a fused node the active tier routes the AdminUI — see the body), reading /// the the host registers alongside OtOpcUaServerHostedService. /// Defaults to false so admin-only callers and the existing harnesses register nothing new. /// @@ -40,16 +41,26 @@ public static class HealthEndpoints { var checks = services.AddHealthChecks(); - // A driver node's whole purpose is its OPC UA endpoint, so a failed SDK start must take it out of - // the ready AND active tiers: readiness because it cannot serve, active because a load balancer or - // the fleet must not route to it. Factory-registered so the state singleton is resolved per probe. + // A failed SDK start takes the node out of the READY tier on every driver node: it cannot serve. + // The ACTIVE tier depends on the roles (coordinator ruling, fix round 1 / I6): + // - driver-only: Active too — a site node's whole purpose is its OPC UA endpoint, so nothing may + // route to it; + // - fused admin+driver: NOT Active, until the ServiceLevel calculator consumes IOpcUaEndpointState + // (follow-up F-C1). Traefik routes the AdminUI on /health/active, so putting the probe there would + // take the operator's only console out of rotation precisely when they need it to diagnose the + // failed endpoint, while the OPC UA failover (ServiceLevel) still would not move. Once F-C1 lands + // the peer takes Primary and the active tier follows it on its own. + // Factory-registered so the state singleton is resolved per probe. if (hasDriver) { + var tags = hasAdmin + ? new[] { ZbHealthTags.Ready } + : new[] { ZbHealthTags.Ready, ZbHealthTags.Active }; checks.Add(new HealthCheckRegistration( OpcUaEndpointHealthCheck.Name, sp => new OpcUaEndpointHealthCheck(sp.GetRequiredService()), failureStatus: null, - tags: new[] { ZbHealthTags.Ready, ZbHealthTags.Active })); + tags: tags)); } // configdb probe is admin-only (per-cluster mesh Phase 4). A driver-only node gets its config diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs index 8caa69f5..e619d7b3 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Health/OpcUaEndpointHealthCheck.cs @@ -4,14 +4,16 @@ using ZB.MOM.WW.OtOpcUa.Host.OpcUa; namespace ZB.MOM.WW.OtOpcUa.Host.Health; /// -/// Ready + Active tier probe for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is +/// Ready-tier probe (plus Active on a driver-only node) for the node's OPC UA endpoint (R03:2.1): Healthy only while the SDK is /// listening. A driver node whose SDK failed to start is otherwise indistinguishable from a healthy /// one over HTTP — the configdb and akka probes are both green — while every OPC UA client gets /// connection-refused. /// /// /// Registered on driver-role nodes only (); an -/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. +/// admin-only node serves no OPC UA endpoint and must not be degraded by one it never starts. On a fused +/// admin+driver node it is Ready-only until ServiceLevel consumes the fact (F-C1), because the Active tier +/// is what Traefik routes the AdminUI on. /// public sealed class OpcUaEndpointHealthCheck : IHealthCheck { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs index d880e7b3..45ff262b 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Health/ActiveTierRegistrationTests.cs @@ -50,31 +50,38 @@ public sealed class ActiveTierRegistrationTests } /// - /// R03:2.1 — the OPC UA endpoint probe is on BOTH tiers of a driver node (it cannot serve, and must not - /// be routed to) and absent from an admin-only node, which starts no endpoint to be degraded by. + /// R03:2.1 + fix round 1 / I6 (coordinator ruling) — the OPC UA endpoint probe's tiers by role: + /// + /// driver-only: Ready + Active (it cannot serve and must not be routed to); + /// fused admin+driver: Ready ONLY, until the ServiceLevel calculator consumes the fact + /// (F-C1) — Traefik routes the AdminUI on /health/active, so a failed SDK on the active + /// central node must not take the AdminUI out of rotation; + /// admin-only: absent (it starts no endpoint to be degraded by). + /// /// [Theory] - [InlineData(true)] - [InlineData(false)] - public void Opcua_endpoint_probe_is_registered_on_ready_and_active_for_driver_nodes_only(bool hasDriver) + [InlineData(false, true, true, true)] + [InlineData(true, true, true, false)] + [InlineData(true, false, false, false)] + public void Opcua_endpoint_probe_tiers_follow_the_node_roles(bool hasAdmin, bool hasDriver, bool ready, bool active) { var registrations = new ServiceCollection() - .AddOtOpcUaHealth(hasAdmin: true, hasDriver: hasDriver) + .AddOtOpcUaHealth(hasAdmin: hasAdmin, hasDriver: hasDriver) .BuildServiceProvider() .GetRequiredService>() .Value.Registrations .Where(r => r.Name == OpcUaEndpointHealthCheck.Name) .ToList(); - if (!hasDriver) + if (!ready && !active) { registrations.ShouldBeEmpty(); return; } var registration = registrations.ShouldHaveSingleItem(); - registration.Tags.ShouldContain(ZbHealthTags.Ready); - registration.Tags.ShouldContain(ZbHealthTags.Active); + registration.Tags.Contains(ZbHealthTags.Ready).ShouldBe(ready); + registration.Tags.Contains(ZbHealthTags.Active).ShouldBe(active); } [Fact] From f1c52385edf42387e892a00dcad64bfb133a7838 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:39:34 -0400 Subject: [PATCH 19/51] fix(controlplane): retention keep window counts Sealed rows; the newest Sealed blob is never pruned The window used to count every terminal row, so 50+ failed deploys pushed the last good artifact out of it. Now the newest KeepSealed (>= 1) Sealed rows keep their blob, failed rows are prunable unless referenced in the ReferencedDays window. Pinned: 60 TimedOut + 1 old Sealed -> Sealed retained, 60 TimedOut emptied. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Retention/DeploymentRetentionActor.cs | 21 ++++-- .../Retention/DeploymentRetentionOptions.cs | 7 +- .../DeploymentRetentionTests.cs | 65 ++++++++++++++++++- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs index 55ffd7dc..67020803 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionActor.cs @@ -97,8 +97,9 @@ public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers } /// - /// The retention rule. Empties the blob of every terminal deployment that is (a) older than the - /// newest terminal rows, (b) not referenced by a + /// The retention rule. Empties the blob of every terminal deployment that is (a) not one of the + /// newest SEALED rows (at least one — the newest + /// Sealed blob is never pruned), (b) not referenced by a /// NodeDeploymentState row started or applied within /// , and (c) still carrying bytes — at most /// per call. @@ -125,11 +126,23 @@ public sealed class DeploymentRetentionActor : ReceiveActor, IWithTimers || d.Status == DeploymentStatus.PartiallyFailed || d.Status == DeploymentStatus.TimedOut) .OrderByDescending(d => d.CreatedAtUtc) - .Select(d => new { d.DeploymentId, HasBlob = d.ArtifactBlob.Length > 0 }) + .Select(d => new { d.DeploymentId, d.Status, HasBlob = d.ArtifactBlob.Length > 0 }) .ToListAsync(ct) .ConfigureAwait(false); - var aged = terminal.Skip(keep).Where(d => d.HasBlob).Select(d => d.DeploymentId).ToList(); + // The keep window counts SEALED rows only (fix round 1 / I3). Counting every terminal row let a burst + // of failed deploys (TimedOut / PartiallyFailed) fill the window by itself and push the fleet's last + // good artifact — the newest Sealed row, which boot, drift detection and the publish fallback all + // read — out of it. `keep` is clamped to ≥ 1, so the newest Sealed row is never a candidate. + var keptSealed = terminal + .Where(d => d.Status == DeploymentStatus.Sealed) + .Take(keep) + .Select(d => d.DeploymentId) + .ToHashSet(); + var aged = terminal + .Where(d => d.HasBlob && !keptSealed.Contains(d.DeploymentId)) + .Select(d => d.DeploymentId) + .ToList(); if (aged.Count == 0) return 0; var referenced = (await db.NodeDeploymentStates diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs index 6654ec1e..6bdc571f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Retention/DeploymentRetentionOptions.cs @@ -17,9 +17,10 @@ public sealed class DeploymentRetentionOptions public bool Enabled { get; set; } = true; /// - /// How many of the most recent terminal (Sealed / PartiallyFailed / TimedOut) deployments keep their - /// blob regardless of age. A node booting from central SQL reads the LATEST sealed row's blob, so this - /// must stay well above 1; 50 also keeps a comfortable re-deploy history. + /// How many of the most recent SEALED deployments keep their blob regardless of age (clamped to ≥ 1). + /// Failed deploys (PartiallyFailed / TimedOut) do not count toward it, so a burst of them can never push + /// the last good artifact out. A node booting from central SQL reads the LATEST sealed row's blob, so + /// this must stay well above 1; 50 also keeps a comfortable re-deploy history. /// public int KeepSealed { get; set; } = 50; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs index ecd397c4..bb607aaf 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/DeploymentRetentionTests.cs @@ -24,7 +24,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase public async Task Sweep_empties_blobs_beyond_the_keep_window_and_leaves_the_newest_intact() { var f = NewInMemoryDbFactory(); - var ids = Seed(f, sealedCount: 12); // ids[0] newest … ids[11] oldest + var ids = Seed(f, sealedCount: 12, mixInTimedOut: false); // ids[0] newest … ids[11] oldest var options = new DeploymentRetentionOptions { KeepSealed = 5, ReferencedDays = 30, MaxRowsPerSweep = 200 }; int pruned; @@ -91,6 +91,65 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase (await verify.Deployments.SingleAsync(d => d.DeploymentId == inFlight)).ArtifactBlob.ShouldNotBeEmpty("in flight"); } + /// + /// Fix round 1 / I3 — the keep window counts SEALED rows, not "terminal" ones. A burst of 60 failed + /// deploys used to fill a 50-row terminal window on its own and push the fleet's last good (Sealed) + /// artifact out of it — the one every boot / drift / fallback path reads. The newest Sealed blob is + /// never pruned; the unreferenced TimedOut rows are (positive control: the sweep did run). + /// + [Fact] + public async Task Sixty_timed_out_deploys_never_prune_the_newest_sealed_blob() + { + var f = NewInMemoryDbFactory(); + var sealedId = Guid.NewGuid(); + var timedOut = new List(); + await using (var db = f.CreateDbContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = sealedId, RevisionHash = new string('s', 64), Status = DeploymentStatus.Sealed, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddDays(-200), SealedAtUtc = SweepNow.AddDays(-200), + ArtifactBlob = new byte[] { 1, 2, 3 }, + }); + for (var i = 0; i < 60; i++) + { + var id = Guid.NewGuid(); + timedOut.Add(id); + db.Deployments.Add(new Deployment + { + DeploymentId = id, RevisionHash = new string('t', 64), Status = DeploymentStatus.TimedOut, + CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), ArtifactBlob = new byte[] { 9 }, + }); + } + await db.SaveChangesAsync(); + } + var options = new DeploymentRetentionOptions { KeepSealed = 50, ReferencedDays = 30, MaxRowsPerSweep = 200 }; + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, options, SweepNow)).ShouldBe(60); + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == sealedId)).ArtifactBlob + .ShouldNotBeEmpty("the newest Sealed artifact is never pruned"); + (await verify.Deployments.CountAsync(d => timedOut.Contains(d.DeploymentId) && d.ArtifactBlob.Length == 0)) + .ShouldBe(60); + } + + /// Even with KeepSealed misconfigured to 0, the newest Sealed row keeps its blob. + [Fact] + public async Task The_newest_sealed_blob_survives_a_zero_keep_window() + { + var f = NewInMemoryDbFactory(); + var ids = Seed(f, sealedCount: 3, mixInTimedOut: false); + + await using (var db = f.CreateDbContext()) + (await DeploymentRetentionActor.SweepAsync(db, new DeploymentRetentionOptions { KeepSealed = 0 }, SweepNow)) + .ShouldBe(2); + + await using var verify = f.CreateDbContext(); + (await verify.Deployments.SingleAsync(d => d.DeploymentId == ids[0])).ArtifactBlob.ShouldNotBeEmpty(); + } + [Fact] public async Task Sweep_is_bounded_per_call_and_drains_a_backlog_over_calls() { @@ -149,7 +208,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase } /// Seeds sealed deployments, newest first; each one hour older than the previous. - private static List Seed(IDbContextFactory f, int sealedCount) + private static List Seed(IDbContextFactory f, int sealedCount, bool mixInTimedOut = true) { var ids = new List(); using var db = f.CreateDbContext(); @@ -161,7 +220,7 @@ public sealed class DeploymentRetentionTests : ControlPlaneActorTestBase { DeploymentId = id, RevisionHash = new string('a', 64), - Status = i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, + Status = mixInTimedOut && i % 3 == 2 ? DeploymentStatus.TimedOut : DeploymentStatus.Sealed, CreatedBy = "test", CreatedAtUtc = SweepNow.AddHours(-i), SealedAtUtc = SweepNow.AddHours(-i), From ef2eebe6b6c46dcdc5bde3afad20cb44beeed08a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:41:15 -0400 Subject: [PATCH 20/51] fix(scripting): review minors: configured limit in summaries, dispose gate/CTS, validator, docs (fix round 1) - ScriptLogRateLimiter.LimitPerWindow. The ScriptLogTopicSink and VirtualTagActor summaries now print the configured limit, not DefaultLimitPerWindow. Tests assert "limit N/window" for a non-default N. - CalculationDriver.Dispose cancels, then takes _evalGate (bounded 2 s) and disposes the gate and _evalCts. The pump and timer ticks enter the gate through TryEnterEvalGateAsync, which stops cleanly once the driver is disposed. Test: Dispose with a 6 s runaway run in flight returns promptly, disposes both, and a late trigger is harmless. - ScriptedAlarmEngine captures _shutdownToken once and disposes _shutdownCts in Dispose when every drained task has completed. A task stuck past the grace leaves the CTS to the GC. Lifecycle tests assert the disposal. - DraftValidator: CalculationNoEffectiveTrigger is skipped for a change-triggered tag whose scriptId does not resolve. That case is already CalculationScriptMissing or CalculationScriptNotFound, and there are no reads to count. Both existing tests now assert the absence (red before the change). - docs/VirtualTags.md: System.Threading is denied as a whole namespace, with an allow-list; the type list is now accurate (Unsafe, RuntimeHelpers, and the ForbiddenMembers on System.Type); the resource limits section covers the run guard and the allocation budget. docs/ScriptEditor.md: accurate deny-list and a new "Runtime safety" section covering timeout, refusal, quarantine after 3 timeouts, allocation budget, lifting, and rate-limit summaries that appear only on the source's next entry after a burst then quiet. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/ScriptEditor.md | 42 +++++++++++++++- docs/VirtualTags.md | 8 +-- .../Validation/DraftValidator.cs | 4 ++ .../ScriptedAlarmEngine.cs | 28 +++++++++-- .../ScriptLogRateLimiter.cs | 3 ++ .../ScriptLogTopicSink.cs | 2 +- .../CalculationDriver.cs | 49 +++++++++++++++++-- .../VirtualTags/VirtualTagActor.cs | 2 +- .../DraftValidatorCalculationTests.cs | 10 +++- .../ScriptedAlarmEngineLifecycleTests.cs | 3 ++ .../ScriptLogRateLimiterTests.cs | 4 ++ .../CalculationDriverRemediationTests.cs | 23 +++++++++ ...VirtualTagActorFailureLogRateLimitTests.cs | 1 + 13 files changed, 161 insertions(+), 18 deletions(-) diff --git a/docs/ScriptEditor.md b/docs/ScriptEditor.md index 5321aca8..41421d34 100644 --- a/docs/ScriptEditor.md +++ b/docs/ScriptEditor.md @@ -162,11 +162,49 @@ user's fault. `ForbiddenTypeAnalyzer.Analyze(compilation)` walks the wrapped syntax tree and resolves every referenced symbol against the sandbox deny-list (namespace prefixes `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, -`System.Threading.Tasks`, etc., plus type-granular denials like -`System.Environment` and `System.Threading.Thread`). Each violation is mapped +`System.Threading` — the whole namespace, with only `CancellationToken`, +`Interlocked` and `Volatile` allowed back in — `System.Threading.Tasks`, +`System.Runtime.InteropServices`, `System.Runtime.Loader`, `Microsoft.Win32`; +type-granular denials `System.Environment`, `System.AppDomain`, `System.GC`, +`System.Activator`, `System.Runtime.CompilerServices.Unsafe` / +`RuntimeHelpers`; and member-granular denials on `System.Type`'s by-name +reflection entry points — `GetType(string)`, `GetMethod(s)`, `InvokeMember`, +and the `GetProperty`/`GetField`/`GetMember`/`GetConstructor`/`GetEvent` +families). The runtime publish gate uses the same analyzer, so a deny-list +change needs a `DiagnoseTests` case to keep editor ⇔ publish parity. Each violation is mapped via `tree.GetMappedLineSpan(span)` to user-source coordinates and emitted as an error marker with code `OTSCRIPT_FORBIDDEN`. +### Runtime safety (what happens after publish) + +The editor cannot catch a script that compiles cleanly but never returns or +allocates without bound, so every runtime seam (virtual tags, Calculation tags, +scripted-alarm predicates) runs scripts through the shared `ScriptRunGuard`: + +- **Timeout + one orphan.** A run that exceeds its budget is abandoned (it cannot + be pre-empted in-process) and the tag/alarm reads Bad. While that abandoned run + is still executing, every further trigger is **refused** — Bad + `"previous evaluation still running"`, counted on `otopcua.script.run_refused` — + so a `while(true)` costs one pool thread, not one per trigger. +- **Quarantine after 3 consecutive timeouts** (`Scripting:QuarantineAfterConsecutiveTimeouts`). + Only a timeout is a strike: a refusal neither adds to nor resets the streak, and + a run that returns (value or throw) resets it. A quarantined script is never + dispatched again; it reads Bad `"script quarantined: …"`, logs one Error and + increments `otopcua_script_quarantined`. An infinite loop therefore usually shows + as *refused* (one timeout, then refusals for as long as the orphan spins) rather + than *quarantined*. +- **Allocation budget.** A single run allocating more than + `Scripting:MaxAllocBytesPerRun` (64 MB) is quarantined. This is sampled after the + run returns, so a run that OOMs the process first is not stopped — that needs an + out-of-process runner. +- **Lifting.** A quarantine lifts when the script source changes or on redeploy, + never by itself. +- **Script-log rate limit.** The cluster-wide script-log fan-out admits 20 entries + per source per minute; the rest are counted and replaced by one "N suppressed" + summary. The summary is emitted on that source's **next** entry after the window + rolls, so a source that bursts and then goes quiet shows its summary only when + it logs again (the full stream is always in `scripts-*.log`). + ### 3. DependencyExtractor `DependencyExtractor.Extract(code).Rejections` enumerates every `ctx.GetTag` / diff --git a/docs/VirtualTags.md b/docs/VirtualTags.md index d58013ea..50ef578f 100644 --- a/docs/VirtualTags.md +++ b/docs/VirtualTags.md @@ -28,15 +28,15 @@ User scripts are compiled via `Microsoft.CodeAnalysis.CSharp` (regular compiler, `ScriptSandbox.Build` constructs the compile reference set in two parts. First, four pinned OtOpcUa assemblies are always included: `Core.Abstractions` (for `DataValueSnapshot` / `DriverDataType`), `Core.Scripting` (for `ScriptContext` + `Deadband`), `Serilog` (for `ILogger`), and the concrete context type's assembly. Second, the BCL subset is enumerated from the runtime's `TRUSTED_PLATFORM_ASSEMBLIES` list, restricted to filenames starting with `System.*` plus `netstandard.dll`, `mscorlib.dll`, and `Microsoft.Win32.Registry.dll` (the last needed so `ForbiddenTypeAnalyzer` can resolve and reject registry types). Pre-imported namespaces: `System`, `System.Linq`, `ZB.MOM.WW.OtOpcUa.Core.Abstractions`, `ZB.MOM.WW.OtOpcUa.Core.Scripting`. -`ForbiddenTypeAnalyzer.ForbiddenNamespacePrefixes` denies `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, `System.Threading.Tasks`, `System.Runtime.InteropServices`, `System.Runtime.Loader`, and `Microsoft.Win32`. Matching is by prefix against the resolved symbol's containing namespace, so `System.Net` catches `System.Net.Http.HttpClient` and every subnamespace. `System.Threading.Tasks` is denied because scripts are synchronous predicates with no legitimate need to start background tasks — a `Task.Run` fan-out would outlive the per-evaluation timeout entirely (Core.Scripting-003). `System.Runtime.Loader` is denied to block `AssemblyLoadContext` / `AssemblyDependencyResolver` — arbitrary DLL loads into the host process (Core.Scripting-012). +`ForbiddenTypeAnalyzer.ForbiddenNamespacePrefixes` denies `System.IO`, `System.Net`, `System.Diagnostics`, `System.Reflection`, `System.Threading` (the whole namespace — `ForbiddenTypeAnalyzer.AllowedTypesInForbiddenNamespaces` lets only `CancellationToken`, `Interlocked` and `Volatile` back in), `System.Threading.Tasks`, `System.Runtime.InteropServices`, `System.Runtime.Loader`, and `Microsoft.Win32`. Matching is by prefix against the resolved symbol's containing namespace, so `System.Net` catches `System.Net.Http.HttpClient` and every subnamespace. `System.Threading.Tasks` is denied because scripts are synchronous predicates with no legitimate need to start background tasks — a `Task.Run` fan-out would outlive the per-evaluation timeout entirely (Core.Scripting-003). `System.Runtime.Loader` is denied to block `AssemblyLoadContext` / `AssemblyDependencyResolver` — arbitrary DLL loads into the host process (Core.Scripting-012). -`ForbiddenTypeAnalyzer.ForbiddenFullTypeNames` denies type-granularly: `System.Environment`, `System.AppDomain`, `System.GC`, `System.Activator`, `System.Threading.Thread`, `System.Threading.ThreadPool`, and `System.Threading.Timer`. These types require granular denial rather than namespace-prefix denial for different reasons: `Environment` / `AppDomain` / `GC` / `Activator` live directly in the `System` namespace (which is otherwise allowed for primitives), so a namespace-prefix rule cannot reach them without blocking `int` / `string` / `Math`; `Thread` / `ThreadPool` / `Timer` live in `System.Threading` (shared with allowed types like `CancellationToken` and `SemaphoreSlim`), so a prefix on `System.Threading` would block those too. `Environment.Exit` / `FailFast` terminate the host process outright (Core.Scripting-001); `Thread` and `ThreadPool` reintroduce background-fanout vectors that `System.Threading.Tasks` denial closed (Core.Scripting-010 / -012). +`ForbiddenTypeAnalyzer.ForbiddenFullTypeNames` denies type-granularly: `System.Environment`, `System.AppDomain`, `System.GC`, `System.Activator`, `System.Runtime.CompilerServices.Unsafe`, and `System.Runtime.CompilerServices.RuntimeHelpers`. These live in namespaces that are otherwise allowed — `System` (primitives, `Math`) and `System.Runtime.CompilerServices` (compile-time attributes such as `CallerMemberNameAttribute`) — so a namespace-prefix rule cannot reach them. `Environment.Exit` / `FailFast` terminate the host process outright (Core.Scripting-001); `Unsafe.As` bypasses the type system; `RuntimeHelpers.GetUninitializedObject` builds an instance of any type without a constructor. `Thread` / `ThreadPool` / `Timer` / `SemaphoreSlim` and the rest of `System.Threading` are now caught by the namespace prefix above instead of being listed one by one — a script is a synchronous expression with no use for background work or blocking primitives (Core.Scripting-010 / -012, review R05:10). `ForbiddenTypeAnalyzer.ForbiddenMembers` additionally denies `System.Type`'s by-name reflection entry points (`GetType(string)`, `GetMethod(s)`, `InvokeMember`, and the `GetProperty`/`GetField`/`GetMember`/`GetConstructor`/`GetEvent` families), which would otherwise reach a forbidden type without naming it syntactically. #### Known resource limits (accepted trade-offs) -The sandbox cannot prevent a script from **allocating unbounded memory**. A script calling `new byte[int.MaxValue]` repeatedly, or accumulating a large LINQ enumeration, can drive the server process to `OutOfMemoryException` before the 250 ms timeout fires. Script authoring is gated behind the Admin permission as the primary control; the test-harness preview (Stream F.4) allows operators to exercise a script before publishing. Out-of-process script execution is a v3 concern. +The sandbox cannot pre-empt a script. **Memory:** each run's own-thread allocation is sampled (`GC.GetAllocatedBytesForCurrentThread`) and a run over `Scripting:MaxAllocBytesPerRun` (default 64 MB) quarantines the script — but the sample is taken after the run returns, so a script that allocates fast enough (`new byte[int.MaxValue]` in a loop) can still OOM the process first. **CPU:** a tight loop keeps its pool thread after the timeout fires; `ScriptRunGuard` bounds that to one orphan per script by refusing further runs while it spins (Bad `"previous evaluation still running"`) and quarantines after three consecutive timeouts (a refusal is not a strike). Both are mitigations; pre-emptive CPU/memory limits need an out-of-process runner. Script authoring is gated behind the Admin permission as the primary control. See `docs/ScriptEditor.md` §"Runtime safety". -Similarly, **`System.Threading.Tasks` is now denied** (Core.Scripting-003), which prevents `Task.Run` / `Parallel` fan-out that would spawn background work outliving the timeout. However, a tight CPU-bound loop still runs on its thread-pool thread after `WaitAsync` returns — see the `TimedScriptEvaluator` remarks for detail. The orphaned thread is reclaimed when the Roslyn runtime eventually returns; in practice the operator fixes the script once the structured timeout warning appears in `scripts-*.log`. +`System.Threading.Tasks` is denied (Core.Scripting-003), which prevents `Task.Run` / `Parallel` fan-out that would spawn background work outliving the timeout. ### Compile cache (`CompiledScriptCache`) diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs index 3b8d0bb1..96a25da5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftValidator.cs @@ -463,6 +463,10 @@ public static class DraftValidator var (changeTriggered, timerIntervalMs) = ParseTriggers(t.TagConfig); if (timerIntervalMs is > 0) continue; var source = scriptId is not null && scriptSourceById.TryGetValue(scriptId, out var s) ? s : null; + // An unresolved scriptId is already CalculationScriptMissing / CalculationScriptNotFound; with no + // source there are no reads to count, so a change-triggered tag would get a second, misleading + // "no literal ctx.GetTag read" error. (changeTriggered=false + no timer is source-independent.) + if (changeTriggered && source is null) continue; var depCount = EquipmentScriptPaths.ExtractDependencyRefs(source).Count; if (changeTriggered && depCount > 0) continue; errors.Add(new("CalculationNoEffectiveTrigger", diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index ae6352f5..c96c8c87 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -188,6 +188,15 @@ public sealed class ScriptedAlarmEngine : IDisposable /// (DrainDirtyAsync, ShelvingCheckAsync) runs on its token, so a store call that honours /// cancellation returns instead of outliving the engine. private readonly CancellationTokenSource _shutdownCts = new(); + + /// 's token, captured once: background work reads this, never the CTS, + /// so can dispose the CTS without racing a late reader. + private readonly CancellationToken _shutdownToken; + + private volatile bool _shutdownCtsDisposed; + + /// Test hook — whether disposed . + internal bool ShutdownCtsDisposedForTest => _shutdownCtsDisposed; private readonly Dictionary> _alarmsReferencing = new(StringComparer.Ordinal); // tag path -> alarm ids @@ -247,6 +256,7 @@ public sealed class ScriptedAlarmEngine : IDisposable _clock = clock ?? (() => DateTime.UtcNow); _scriptTimeout = scriptTimeout ?? TimedScriptEvaluator.DefaultTimeout; _guard = new ScriptRunGuard(scripting, _engineLogger); + _shutdownToken = _shutdownCts.Token; } /// The run guard every predicate evaluation goes through (diagnostics / tests). @@ -611,7 +621,7 @@ public sealed class ScriptedAlarmEngine : IDisposable // Started outside the lock: DrainDirtyAsync runs synchronously up to its // first real await, and nothing it does should happen under _dirtyLock. - if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownCts.Token)); + if (startDrain) TrackBackgroundTask(() => DrainDirtyAsync(_shutdownToken)); } /// @@ -754,7 +764,7 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } - catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + catch (OperationCanceledException) when (_shutdownToken.IsCancellationRequested) { // Dispose cut the drain short after its bound — expected, not a failure. return; @@ -1080,7 +1090,7 @@ public sealed class ScriptedAlarmEngine : IDisposable { if (_disposed) return; var ids = _alarms.Keys.ToArray(); - TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownCts.Token)); + TrackBackgroundTask(() => ShelvingCheckAsync(ids, _shutdownToken)); } /// @@ -1219,7 +1229,7 @@ public sealed class ScriptedAlarmEngine : IDisposable } finally { _evalGate.Release(); } } - catch (OperationCanceledException) when (_shutdownCts.IsCancellationRequested) + catch (OperationCanceledException) when (_shutdownToken.IsCancellationRequested) { return; // Dispose cut the check short after its bound } @@ -1304,6 +1314,16 @@ public sealed class ScriptedAlarmEngine : IDisposable _alarmsReferencing.Clear(); _scratchByAlarmId.Clear(); _lastQualityBucketByAlarmId.Clear(); + + // Release the shutdown CTS once nothing can still be observing it through a registration: every + // drained task has completed. A task still stuck past the grace keeps its registration, so the CTS is + // left to the GC in that case (logged above). + if (toAwait.All(t => t.IsCompleted)) + { + _shutdownCts.Dispose(); + _shutdownCtsDisposed = true; + } + // Dispose every compiled-predicate ALC so the engine's shutdown actually // releases the emitted assemblies. The drain above ensures no evaluator is // mid-call; CompiledScriptCache.Dispose internally guards against use-after- diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs index 17a6f2f6..5cce5fcd 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogRateLimiter.cs @@ -43,6 +43,9 @@ public sealed class ScriptLogRateLimiter _clock = clock ?? (static () => DateTime.UtcNow); } + /// Entries admitted per source per window, as configured (after clamping to at least 1). + public int LimitPerWindow => _limit; + /// The window length. public TimeSpan Window => _window; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs index a7b8f03e..88bdd973 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptLogTopicSink.cs @@ -83,7 +83,7 @@ public sealed class ScriptLogTopicSink : ILogEventSink Level: nameof(LogEventLevel.Warning), Message: $"{admission.SuppressedInPreviousWindow} script-log entr{(admission.SuppressedInPreviousWindow == 1 ? "y" : "ies")} " + $"from this source were suppressed in the previous {_limiter.Window.TotalSeconds:F0} s " + - $"(limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); see scripts-*.log for the full stream", + $"(limit {_limiter.LimitPerWindow}/window); see scripts-*.log for the full stream", TimestampUtc: logEvent.Timestamp.UtcDateTime, VirtualTagId: virtualTagId, AlarmId: alarmId, diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs index 905fff13..9f8ef41b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationDriver.cs @@ -111,7 +111,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba private CalculationTimerScheduler? _timers; private volatile bool _running; private DateTime? _lastComputeUtc; - private bool _disposed; + private volatile bool _disposed; /// Occurs when a calc tag's computed value changes (or transitions Good↔Bad). public event EventHandler? OnDataChange; @@ -538,7 +538,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba // _evalGate (never _lock) is held across the run, so the pump and the timer ticks still // evaluate one at a time while no host thread can block behind a script. - await _evalGate.WaitAsync().ConfigureAwait(false); + if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed try { await RunEvaluationAsync(next).ConfigureAwait(false); @@ -566,7 +566,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba PendingEvaluation? prepared; lock (_lock) prepared = PrepareEvaluation(rawPath, DateTime.UtcNow); if (prepared is null) return; - await _evalGate.WaitAsync().ConfigureAwait(false); + if (!await TryEnterEvalGateAsync().ConfigureAwait(false)) return; // disposed try { await RunEvaluationAsync(prepared).ConfigureAwait(false); @@ -577,6 +577,25 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba } } + /// Acquire ; false once has disposed it (a pump + /// iteration or timer tick that raced the teardown simply stops). + private async Task TryEnterEvalGateAsync() + { + if (_disposed) return false; + try + { + await _evalGate.WaitAsync().ConfigureAwait(false); + } + catch (ObjectDisposedException) + { + return false; + } + if (!_disposed) return true; + // Dispose landed while we waited (it gave up the gate without disposing it): back out. + _evalGate.Release(); + return false; + } + /// Run one prepared evaluation and publish the outcome. MUST be called under /// and NOT under : the script run is the whole point of the /// split, and the script log + fan-out both invoke subscriber code. @@ -784,11 +803,33 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba StopTimers(); lock (_lock) _pending.Clear(); // Cancel an awaited run first (the guard lets the caller's cancel win over the timeout), then drop - // the compiled scripts. The CTS is not disposed: a pump mid-await may still read its token. + // the compiled scripts. try { _evalCts.Cancel(); } catch (ObjectDisposedException) { } _evaluator.Dispose(); + + // Then take the gate: once Dispose holds it no evaluation is mid-run (the token — read only under + // the gate — has been observed), so the gate and the CTS can be disposed. A late pump iteration or + // timer tick sees _disposed / an ObjectDisposedException in TryEnterEvalGateAsync and stops. Bounded: + // a run that ignores the cancel cannot hold Dispose; in that case both are left to the GC. + if (_evalGate.Wait(DisposeGateTimeout)) + { + _evalCts.Dispose(); + _evalGate.Dispose(); + _evalResourcesDisposed = true; + } + else + { + _logger.LogWarning("Calculation {Driver}: an evaluation did not release the gate within {Timeout} of Dispose; leaving it to the GC", + _driverInstanceId, DisposeGateTimeout); + } } + private static readonly TimeSpan DisposeGateTimeout = TimeSpan.FromSeconds(2); + private volatile bool _evalResourcesDisposed; + + /// Test hook — whether disposed the evaluation gate and its CTS. + internal bool EvalResourcesDisposedForTest => _evalResourcesDisposed; + /// Per-tag published state (guarded by ). private sealed class TagState { diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs index 63ea8736..8c7b4f41 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/VirtualTags/VirtualTagActor.cs @@ -377,7 +377,7 @@ public sealed class VirtualTagActor : ReceiveActor { PublishLog("Warning", $"{admission.SuppressedInPreviousWindow} further evaluation failure(s) were suppressed from the script log " + - $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {ScriptLogRateLimiter.DefaultLimitPerWindow}/window); last: {reason}"); + $"in the previous {_failureLogLimiter.Window.TotalSeconds:F0} s (limit {_failureLogLimiter.LimitPerWindow}/window); last: {reason}"); } if (admission.Admitted) PublishLog(level, reason); diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs index b542871f..b68fb1cd 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Configuration.Tests/DraftValidatorCalculationTests.cs @@ -55,7 +55,10 @@ public sealed class DraftValidatorCalculationTests Tags = [CalcTagRow("t-1", "dev-1", "a", "{\"changeTriggered\":true}")], }; - DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationScriptMissing" && e.Context == "t-1"); + var errors = DraftValidator.Validate(draft); + errors.ShouldContain(e => e.Code == "CalculationScriptMissing" && e.Context == "t-1"); + errors.ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger", + "an unresolved script has no reads to count — the missing script is the one error to report"); } [Fact] @@ -70,7 +73,10 @@ public sealed class DraftValidatorCalculationTests Scripts = [], }; - DraftValidator.Validate(draft).ShouldContain(e => e.Code == "CalculationScriptNotFound" && e.Context == "t-1"); + var errors = DraftValidator.Validate(draft); + errors.ShouldContain(e => e.Code == "CalculationScriptNotFound" && e.Context == "t-1"); + errors.ShouldNotContain(e => e.Code == "CalculationNoEffectiveTrigger", + "an unresolved script has no reads to count — the missing script is the one error to report"); } [Fact] diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs index 7afea07a..2e9b7df6 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms.Tests/ScriptedAlarmEngineLifecycleTests.cs @@ -82,6 +82,7 @@ public sealed class ScriptedAlarmEngineLifecycleTests sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the drain is bounded; a hung store cannot hang shutdown"); await WaitForAsync(() => store.ObservedCancellation, timeoutMs: 2000); store.ObservedCancellation.ShouldBeTrue("the in-flight save was cancelled through the engine's shutdown token"); + eng.ShutdownCtsDisposedForTest.ShouldBeTrue("the cancelled save completed inside the grace, so the CTS is released"); } [Fact] @@ -105,6 +106,8 @@ public sealed class ScriptedAlarmEngineLifecycleTests store.SaveInProgress.ShouldBeFalse(); store.ObservedCancellation.ShouldBeFalse("a save that completes within the bound is never cancelled"); store.Saved.ShouldContain("A"); + eng.ShutdownCtsDisposedForTest.ShouldBeTrue("every background task completed, so the shutdown CTS is disposed"); + Should.NotThrow(() => eng.OnUpstreamChange("Temp", new DataValueSnapshot(1, 0u, DateTime.UtcNow, DateTime.UtcNow))); } private static async Task WaitForAsync(Func cond, int timeoutMs = 5000) diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs index 3f560e40..70674942 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptLogRateLimiterTests.cs @@ -59,6 +59,9 @@ public sealed class ScriptLogRateLimiterTests ScriptLogRateLimiter.DefaultLimitPerWindow.ShouldBe(20); ScriptLogRateLimiter.DefaultWindow.ShouldBe(TimeSpan.FromMinutes(1)); var limiter = new ScriptLogRateLimiter(); + limiter.LimitPerWindow.ShouldBe(20); + new ScriptLogRateLimiter(limitPerWindow: 3).LimitPerWindow.ShouldBe(3); + new ScriptLogRateLimiter(limitPerWindow: 0).LimitPerWindow.ShouldBe(1, "values below 1 are treated as 1"); for (var i = 0; i < 20; i++) limiter.Admit("s").Admitted.ShouldBeTrue(); limiter.Admit("s").Admitted.ShouldBeFalse(); } @@ -101,6 +104,7 @@ public sealed class ScriptLogRateLimiterTests var summary = s1[2]; summary.Message.ShouldContain("3"); summary.Message.ShouldContain("suppressed"); + summary.Message.ShouldContain("limit 2/window", Case.Sensitive, "the summary names the CONFIGURED limit, not the default"); summary.VirtualTagId.ShouldBe("V1", "the summary carries the source's identity so the page attributes it"); summary.Level.ShouldBe("Warning"); s1[3].Message.ShouldBe("chatty again"); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs index 88015ffc..0372ea15 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation.Tests/CalculationDriverRemediationTests.cs @@ -50,6 +50,29 @@ public sealed class CalculationDriverRemediationTests driver.IsEvaluationIdle.ShouldBeTrue("the calc driver's evaluation pump should have drained"); } + /// Fix round 1 minor: Dispose releases the evaluation gate and its cancellation source — even + /// with a runaway run in flight (the guard lets the caller's cancel win, so the awaited run returns + /// promptly) — and a trigger arriving after Dispose is harmless. + [Fact] + public async Task Dispose_with_a_run_in_flight_disposes_the_eval_gate_and_a_late_trigger_is_harmless() + { + var (driver, _, _, _) = Build( + CalcTag("calc/e/spin", "var v = ctx.GetTag(\"src/a\").Value; var end = System.DateTime.UtcNow.AddSeconds(6); while (System.DateTime.UtcNow < end) { } return 1;")); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + driver.OnDependencyValue("src/a", 1, StatusGood, DateTime.UtcNow); + var deadline = DateTime.UtcNow.AddSeconds(10); + while (driver.IsEvaluationIdle && DateTime.UtcNow < deadline) await Task.Delay(5); + driver.IsEvaluationIdle.ShouldBeFalse("positive control: a run is in flight when Dispose lands"); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + driver.Dispose(); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(5), "the caller's cancel releases the awaited run; Dispose is not held for the 6 s spin"); + driver.EvalResourcesDisposedForTest.ShouldBeTrue(); + + Should.NotThrow(() => driver.OnDependencyValue("src/a", 2, StatusGood, DateTime.UtcNow)); + Should.NotThrow(() => driver.Dispose()); + } + [Fact] public async Task A_compile_failure_is_remembered_and_logged_once_across_repeated_triggers() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs index 74ee2e32..c1a6253e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/VirtualTags/VirtualTagActorFailureLogRateLimitTests.cs @@ -43,6 +43,7 @@ public sealed class VirtualTagActorFailureLogRateLimitTests : RuntimeActorTestBa var entries = capture.ToArray(); entries[3].Message.ShouldContain("7"); entries[3].Message.ShouldContain("suppressed"); + entries[3].Message.ShouldContain("limit 3/window", Case.Sensitive, "the summary names the CONFIGURED limit, not the default"); entries[3].VirtualTagId.ShouldBe("vt-chatty"); entries[4].Message.ShouldBe("boom"); } From f7e7ba0263045b1ce0ae1bbba3a60b3ead7bc12a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:42:29 -0400 Subject: [PATCH 21/51] fix(controlplane): audit writer drops only already-persisted EventIds, never the whole SQL Server batch SQL Server's DbUpdateException for one duplicate names every entry in the statement batch, so detaching ex.Entries dropped the whole flush. On a row-level rejection the writer now asks the database which of the batch's EventIds it already holds, drops exactly those, re-saves the rest, and falls back to row-by-row only if a non-duplicate row is still refused. Fresh context + entities per attempt. Pinned against the real central SQL Server (new SqlServerTestDatabase harness) and a faithful whole-batch interceptor. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Audit/AuditWriterActor.cs | 134 +++++++++++------- .../AuditWriterActorRejectedRowTests.cs | 109 ++++++++++++-- .../Harness/SqlServerTestDatabase.cs | 77 ++++++++++ 3 files changed, 255 insertions(+), 65 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs index 0e20da2f..3f0aa914 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Audit/AuditWriterActor.cs @@ -17,8 +17,8 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; /// Dedup is two-layer: in-buffer (the below collapses /// duplicate EventIds before flush) and at the database via the filtered unique index /// UX_ConfigAuditLog_EventId (cross-restart safety — a retry of an already-flushed -/// batch hits the constraint; detaches the rejected row -/// and re-saves, so the duplicate is dropped without losing the rest of the batch). +/// batch hits the constraint; drops exactly the already-persisted EventIds +/// and re-saves the rest, so the duplicate is dropped without losing the rest of the batch). /// /// Implements the shared seam: is a /// best-effort, never-throwing entry point that simply Tells this actor and returns @@ -92,22 +92,7 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter try { - using var db = _dbFactory.CreateDbContext(); - foreach (var evt in snapshot) - { - db.ConfigAuditLogs.Add(new ConfigAuditLog - { - Timestamp = evt.OccurredAtUtc.UtcDateTime, - Principal = evt.Actor, - EventType = $"{evt.Category}:{evt.Action}", - NodeId = evt.SourceNode, - DetailsJson = evt.DetailsJson, - EventId = evt.EventId, - CorrelationId = evt.CorrelationId, - Outcome = evt.Outcome.ToString(), - }); - } - var dropped = SaveDetachingRejectedRows(db); + var dropped = SaveBatch(snapshot); _log.Debug("AuditWriter flushed {Count} events ({Dropped} rejected rows dropped)", snapshot.Count - dropped, dropped); } catch (Exception ex) @@ -116,46 +101,93 @@ public sealed class AuditWriterActor : ReceiveActor, IWithTimers, IAuditWriter } } + private static ConfigAuditLog ToRow(AuditEvent evt) => new() + { + Timestamp = evt.OccurredAtUtc.UtcDateTime, + Principal = evt.Actor, + EventType = $"{evt.Category}:{evt.Action}", + NodeId = evt.SourceNode, + DetailsJson = evt.DetailsJson, + EventId = evt.EventId, + CorrelationId = evt.CorrelationId, + Outcome = evt.Outcome.ToString(), + }; + /// - /// Saves the batch, detaching and retrying past any row the database rejects (R03:2.6). One - /// statement batch used to carry the whole flush: a single duplicate EventId (any at-least-once - /// producer — the audit seam is Tell-based through a singleton proxy that re-delivers across a - /// hand-over) threw for all 500 rows and the class remarks' per-row dedup claim was not what ran. + /// Persists one flush, dropping ONLY the rows the database will not take (R03:2.6, fix round 1 / I4). /// - /// The context holding the batch. + /// The flush. /// How many rows were dropped. /// - /// Only a that NAMES its entries is handled here — that is the shape a - /// constraint violation takes (SQL 2601/2627 for the unique index). A failure with no entries (the - /// connection died, a deadlock) is the whole-batch failure it always was and propagates to the - /// caller's catch. Bounded by the batch size so a store that rejects every row still terminates. + /// + /// Why not detach ex.Entries. SQL Server inserts the batch as one statement batch, so + /// the raised for ONE duplicate EventId names EVERY entry in + /// the batch. Detaching what it names dropped the whole flush — the exact loss the fix was for + /// (pinned against a real SQL Server by On_sql_server_a_duplicate_event_id_costs_only_that_row). + /// + /// + /// So, on a rejection that names entries (a constraint violation; 2601/2627 for the unique index): + /// (1) ask the database which of the batch's EventIds it already holds and drop exactly those — + /// the at-least-once re-delivery the dedup contract expects; (2) re-save the rest as a batch; (3) if + /// that is still rejected (some OTHER per-row constraint), fall back to row-by-row and drop only the + /// rows that fail on their own. Every attempt uses a fresh context and fresh entities, so no state + /// from a failed save leaks into the next one. A failure that names no entry (connection loss, + /// deadlock) propagates — the whole-batch failure it always was. Bounded: at most 2 + N saves. + /// /// - private int SaveDetachingRejectedRows(OtOpcUaConfigDbContext db) + private int SaveBatch(IReadOnlyList batch) { - var dropped = 0; - var attempts = 0; - var maxAttempts = db.ChangeTracker.Entries().Count() + 1; - while (true) + if (TrySave(batch, out var first)) return 0; + _log.Debug(first, "AuditWriter: a batch of {Count} was rejected; dropping only already-persisted EventIds", batch.Count); + + HashSet existing; + using (var db = _dbFactory.CreateDbContext()) { - try - { - db.SaveChanges(); - return dropped; - } - catch (DbUpdateException ex) when (ex.Entries.Count > 0 && ++attempts < maxAttempts) - { - var duplicate = IsUniqueViolation(ex); - foreach (var entry in ex.Entries) - { - var eventId = (entry.Entity as ConfigAuditLog)?.EventId; - if (duplicate) - _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", eventId); - else - _log.Warning(ex, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", eventId); - entry.State = EntityState.Detached; - dropped++; - } - } + var ids = batch.Select(e => (Guid?)e.EventId).ToList(); + existing = db.ConfigAuditLogs + .AsNoTracking() + .Where(r => ids.Contains(r.EventId)) + .Select(r => r.EventId!.Value) + .ToHashSet(); + } + foreach (var id in existing) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", id); + + var remaining = batch.Where(e => !existing.Contains(e.EventId)).ToList(); + var dropped = batch.Count - remaining.Count; + if (remaining.Count == 0 || TrySave(remaining, out _)) return dropped; + + // Still rejected with no duplicate left to blame: isolate the offending row(s). + foreach (var evt in remaining) + { + if (TrySave([evt], out var rowError)) continue; + dropped++; + if (IsUniqueViolation(rowError!)) + _log.Debug("AuditWriter: EventId {EventId} already persisted (unique index); dropping the duplicate", evt.EventId); + else + _log.Warning(rowError, "AuditWriter: the database rejected EventId {EventId}; dropping that row and keeping the batch", evt.EventId); + } + return dropped; + } + + /// Saves on a fresh context; a rejection that names entries is reported, anything else throws. + /// The rows to save. + /// The row-level rejection, when this returns false. + /// when saved. + private bool TrySave(IReadOnlyList events, out DbUpdateException? rejection) + { + using var db = _dbFactory.CreateDbContext(); + foreach (var evt in events) db.ConfigAuditLogs.Add(ToRow(evt)); + try + { + db.SaveChanges(); + rejection = null; + return true; + } + catch (DbUpdateException ex) when (ex.Entries.Count > 0 || IsUniqueViolation(ex)) + { + rejection = ex; + return false; } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs index bd64161a..aba37648 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/AuditWriterActorRejectedRowTests.cs @@ -26,7 +26,7 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase var dbName = Guid.NewGuid().ToString("N"); var root = new InMemoryDatabaseRoot(); var duplicate = Guid.NewGuid(); - var rejecting = new RejectingDbFactory(dbName, root, new RejectOnce(duplicate)); + var rejecting = new RejectingDbFactory(dbName, root, new AlwaysRejectBatchContaining(duplicate)); var actor = Sys.ActorOf(AuditWriterActor.Props(rejecting)); var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); @@ -46,6 +46,68 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase persisted.ShouldContain(ids[8]); } + /// + /// Fix round 1 / I4 — a FAITHFUL SQL Server shape: the provider sends the batch as one statement batch, + /// so the for one duplicate names EVERY entry in it. Detaching + /// ex.Entries therefore dropped the whole flush. The writer must drop only the rows whose + /// EventId is already persisted and keep the rest. + /// + [Fact] + public void A_whole_batch_rejection_drops_only_the_rows_already_persisted() + { + var dbName = Guid.NewGuid().ToString("N"); + var root = new InMemoryDatabaseRoot(); + var factory = new RejectingDbFactory(dbName, root, new RejectWholeBatchOnDuplicate()); + var duplicate = Guid.NewGuid(); + using (var seed = factory.CreateDbContext()) + { + seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate }); + seed.SaveChanges(); + } + var actor = Sys.ActorOf(AuditWriterActor.Props(factory)); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor); + + using var db = factory.CreateDbContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(21, "the 20 new rows land beside the one that was already there"); + persisted.Count(id => id == duplicate).ShouldBe(1); + db.ConfigAuditLogs.Single(r => r.EventId == duplicate).Principal.ShouldBe("earlier"); + } + + /// The same contract against the REAL provider and the real filtered unique index. + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_a_duplicate_event_id_costs_only_that_row() + { + using var sql = new SqlServerTestDatabase(); + var duplicate = Guid.NewGuid(); + using (var seed = sql.NewContext()) + { + seed.ConfigAuditLogs.Add(new ConfigAuditLog { Principal = "earlier", EventType = "Config:Edit", EventId = duplicate }); + seed.SaveChanges(); + } + var actor = Sys.ActorOf(AuditWriterActor.Props(sql.Factory())); + + var ids = Enumerable.Range(0, 20).Select(_ => Guid.NewGuid()).ToList(); + ids.Insert(7, duplicate); + foreach (var id in ids) actor.Tell(NewEvent(id)); + Watch(actor); + actor.Tell(PoisonPill.Instance); + ExpectTerminated(actor, PresenceBudget); + + using var db = sql.NewContext(); + var persisted = db.ConfigAuditLogs.Select(r => r.EventId).ToList(); + persisted.Count.ShouldBe(21); + persisted.Count(id => id == duplicate).ShouldBe(1); + foreach (var id in ids) persisted.ShouldContain(id); + } + /// A failure that names no entry is a whole-batch failure, exactly as before. [Fact] public void A_connection_level_failure_still_drops_the_batch_without_looping() @@ -71,24 +133,43 @@ public sealed class AuditWriterActorRejectedRowTests : ControlPlaneActorTestBase DetailsJson = "{}", SourceNode = "node-a", CorrelationId = Guid.NewGuid(), }; - /// Throws a naming the entry for one EventId, the first time it is saved. - private sealed class RejectOnce(Guid eventId) : ISaveChangesInterceptor + /// + /// A row the database refuses on its own terms (not a duplicate — e.g. a per-row constraint), every + /// time it is in the save. Faithful to SQL Server's batch shape: the exception names EVERY added + /// entry, so only a row-by-row retry can isolate the offender. + /// + private sealed class AlwaysRejectBatchContaining(Guid eventId) : ISaveChangesInterceptor { - private int _fired; - public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) { - if (_fired == 1 || eventData.Context is null) return result; - var entry = eventData.Context.ChangeTracker.Entries() - .FirstOrDefault(e => e.Entity.EventId == eventId); - if (entry is null) return result; - _fired = 1; - // GetInfrastructure() hands back the pubternal InternalEntityEntry, which is the IUpdateEntry the - // real provider puts on the exception; the test only needs that one cast. + if (eventData.Context is null) return result; + var added = eventData.Context.ChangeTracker.Entries() + .Where(e => e.State == EntityState.Added).ToList(); + if (!added.Any(e => e.Entity.EventId == eventId)) return result; #pragma warning disable EF1001 - var updateEntry = (IUpdateEntry)entry.GetInfrastructure(); + var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList(); #pragma warning restore EF1001 - throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601)", new[] { updateEntry }); + throw new DbUpdateException("The statement conflicted with a CHECK constraint (simulated, whole batch)", entries); + } + } + + /// + /// Models SQL Server's batched insert: when ANY added row's EventId is already stored, the whole save + /// fails with an exception naming EVERY added entry (the InMemory provider enforces no unique index, + /// so this interceptor is the constraint). + /// + private sealed class RejectWholeBatchOnDuplicate : ISaveChangesInterceptor + { + public InterceptionResult SavingChanges(DbContextEventData eventData, InterceptionResult result) + { + if (eventData.Context is not OtOpcUaConfigDbContext ctx) return result; + var added = ctx.ChangeTracker.Entries().Where(e => e.State == EntityState.Added).ToList(); + var ids = added.Select(e => e.Entity.EventId).Where(id => id != null).ToList(); + if (ids.Count == 0 || !ctx.ConfigAuditLogs.AsNoTracking().Any(r => ids.Contains(r.EventId))) return result; +#pragma warning disable EF1001 + var entries = added.Select(e => (IUpdateEntry)e.GetInfrastructure()).ToList(); +#pragma warning restore EF1001 + throw new DbUpdateException("Cannot insert duplicate key row (simulated 2601, whole batch)", entries); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs new file mode 100644 index 00000000..3415bb99 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Harness/SqlServerTestDatabase.cs @@ -0,0 +1,77 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.OtOpcUa.Configuration; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +/// +/// A migrated, throw-away config database on the central SQL Server, for the few ControlPlane behaviours +/// the InMemory provider cannot model (a SQL Server DbUpdateException names the WHOLE statement +/// batch; collation-ordered reads). Same server / credentials / env overrides as +/// Configuration.Tests.SchemaComplianceFixture (OTOPCUA_CONFIG_TEST_SERVER, +/// OTOPCUA_CONFIG_TEST_SA_PASSWORD; default the shared docker host's dev SA), and the same database +/// name shape, so that suite's 24 h orphan reaper also reclaims a database a killed run leaks here. +/// +public sealed class SqlServerTestDatabase : IDisposable +{ + private const string DefaultServer = "10.100.0.35,14330"; + private const string DefaultSaPassword = "OtOpcUaDev_2026!"; + + /// Gets the connection string of the migrated database. + public string ConnectionString { get; } + + private readonly string _databaseName; + + /// Creates and migrates the database. + public SqlServerTestDatabase() + { + var server = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SERVER") ?? DefaultServer; + var saPassword = Environment.GetEnvironmentVariable("OTOPCUA_CONFIG_TEST_SA_PASSWORD") ?? DefaultSaPassword; + _databaseName = $"OtOpcUaConfig_Test_{DateTime.UtcNow:yyyyMMddHHmmss}_{Guid.NewGuid():N}"; + ConnectionString = + $"Server={server};Database={_databaseName};User Id=sa;Password={saPassword};TrustServerCertificate=True;Encrypt=False;"; + using var ctx = NewContext(); + ctx.Database.Migrate(); + } + + /// A new context on the database. + /// The context. + public OtOpcUaConfigDbContext NewContext() => + new(new DbContextOptionsBuilder().UseSqlServer(ConnectionString).Options); + + /// A factory over . + /// The factory. + public IDbContextFactory Factory() => new ContextFactory(this); + + /// Drops the database (best effort; a leak is reaped by the Configuration suite). + public void Dispose() + { + try + { + SqlConnection.ClearAllPools(); + var master = new SqlConnectionStringBuilder(ConnectionString) { InitialCatalog = "master" }.ConnectionString; + using var conn = new SqlConnection(master); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = @" +DECLARE @sql nvarchar(max); +IF DB_ID(@name) IS NOT NULL +BEGIN + SET @sql = N'ALTER DATABASE ' + QUOTENAME(@name) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; ' + + N'DROP DATABASE ' + QUOTENAME(@name) + N';'; + EXEC sp_executesql @sql; +END"; + cmd.Parameters.Add(new SqlParameter("@name", _databaseName)); + cmd.ExecuteNonQuery(); + } + catch (Exception) + { + // Best effort — see the class remarks. + } + } + + private sealed class ContextFactory(SqlServerTestDatabase owner) : IDbContextFactory + { + public OtOpcUaConfigDbContext CreateDbContext() => owner.NewContext(); + } +} From 9466b8b242f80d36f3ef6c481a6058e9eee46722 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:46:54 -0400 Subject: [PATCH 22/51] fix(controlplane): close a lost dispatch from the in-flight check; guard terminal rows; blob-free status writes I7: a Dispatching row committed after the coordinator's PreStart whose Tell never arrived had no owner and refused every later deploy. The in-flight check (now fully off the admin singleton's thread, serialised by the actor-local compose flag alone) closes a Dispatching row with zero NodeDeploymentState rows that is past the apply deadline as TimedOut (optimistic on RowVersion) and proceeds. HandleDispatch ignores a dispatch for a Sealed/PartiallyFailed/TimedOut row, so a late or duplicated dispatch cannot flip it back. Coordinator status/reason writes attach a key+RowVersion stub instead of loading the multi-MB ArtifactBlob; proven on the real SQL Server (blob untouched, seal lands). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdminOperations/AdminOperationsActor.cs | 172 +++++++++++++----- .../Coordinators/ConfigPublishCoordinator.cs | 84 +++++++-- .../ConfigPublishCoordinatorRecoveryTests.cs | 135 ++++++++++++++ 3 files changed, 332 insertions(+), 59 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index e78c926e..81f09dae 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -11,6 +11,7 @@ using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Configuration.Validation; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; @@ -28,6 +29,12 @@ public sealed class AdminOperationsActor : ReceiveActor private readonly IReadOnlyDictionary _probesByType; private readonly TagConfigValidationMode _tagConfigValidationMode; + /// + /// The coordinator's apply deadline. A Dispatching row with no expected-ack rows that is older + /// than this is a lost dispatch, closed by the in-flight check (fix round 1 / I7). + /// + private readonly TimeSpan _applyDeadline; + /// /// Where central→node commands are handed off (per-cluster mesh), or /// to publish on the mediator directly. @@ -48,15 +55,17 @@ public sealed class AdminOperationsActor : ReceiveActor /// Central comm actor commands are handed to, or to publish on the /// DistributedPubSub mediator directly (the behaviour before the per-cluster mesh). /// + /// The coordinator's apply deadline; defaults to . /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) => + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) => Akka.Actor.Props.Create(() => - new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter)); + new AdminOperationsActor(dbFactory, coordinator, probes, tagConfigValidationMode, meshRouter, applyDeadline)); /// Initializes a new instance of the AdminOperationsActor. /// Factory for creating config database contexts. @@ -64,13 +73,16 @@ public sealed class AdminOperationsActor : ReceiveActor /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). /// Central comm actor, or for the mediator. + /// The coordinator's apply deadline; defaults to . public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, IEnumerable probes, TagConfigValidationMode tagConfigValidationMode = TagConfigValidationMode.Warn, - IActorRef? meshRouter = null) + IActorRef? meshRouter = null, + TimeSpan? applyDeadline = null) { + _applyDeadline = applyDeadline ?? ConfigPublishCoordinator.DefaultApplyDeadline; _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); @@ -201,16 +213,17 @@ public sealed class AdminOperationsActor : ReceiveActor } /// - /// Serialised part of StartDeployment: the in-flight check. Everything after it — the - /// snapshot read, validation, gzip and the row write — runs on the thread pool and comes back as a - /// / message, so a large compose no - /// longer suspends the mailbox (R03:3.1) and alarm acks keep flowing while it runs. + /// StartDeployment. The only serialised part is the actor-local + /// flag; the DB in-flight check, the snapshot read, validation, gzip and the row write all run on the + /// thread pool and come back as a / + /// message, so a slow or blocked ConfigDb never suspends the mailbox (R03:3.1; fix round 1 moved the + /// in-flight query off the actor thread too) and alarm acks keep flowing while it runs. /// /// - /// is what keeps the check correct once the compose is off-actor: the - /// Deployment row is written at the END of the compose, so a second request arriving mid-compose - /// would pass a DB-only check and two artifacts would race for one coordinator. The flag is cleared - /// when the outcome message lands, on both branches. + /// is what keeps the check correct once it is off-actor: it is set + /// BEFORE the off-actor work starts and cleared when the outcome message lands (both branches), so a + /// second request is refused in memory for the whole check-plus-compose and two artifacts can never + /// race for one coordinator. /// private void HandleStartDeployment(StartDeployment msg) { @@ -226,49 +239,118 @@ public sealed class AdminOperationsActor : ReceiveActor return; } - try - { - // Refuse if any deployment is already in flight — keeps the coordinator's state - // unambiguous. The UI is expected to wait for the in-flight one to seal or fail. One indexed - // projection (IX_Deployment_Status), synchronous on purpose: it IS the serialised part. - using var db = _dbFactory.CreateDbContext(); - var inflight = db.Deployments - .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) - .Select(d => d.DeploymentId) - .FirstOrDefault(); - if (inflight != Guid.Empty) - { - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.AnotherDeploymentInFlight, - DeploymentId: new DeploymentId(inflight), - RevisionHash: null, - Message: $"Deployment {inflight:N} is still in flight.", - msg.CorrelationId)); - return; - } - } - catch (Exception ex) - { - _log.Error(ex, "StartDeployment failed for {CreatedBy} (correlation {CorrelationId})", msg.CreatedBy, msg.CorrelationId); - replyTo.Tell(new StartDeploymentResult( - StartDeploymentOutcome.Rejected, - DeploymentId: null, - RevisionHash: null, - Message: ServerFault("StartDeployment", msg.CorrelationId), - msg.CorrelationId)); - return; - } - _composeInFlight = msg; // Captured here: Context/Sender are not available on the thread-pool thread. var sourceNode = Akka.Cluster.Cluster.Get(Context.System).SelfAddress.Host ?? "unknown"; var self = Self; - Task.Run(() => ComposeDeploymentAsync(msg, sourceNode)) + Task.Run(async () => + { + var refusal = await CheckInFlightAsync(msg).ConfigureAwait(false); + return refusal ?? await ComposeDeploymentAsync(msg, sourceNode).ConfigureAwait(false); + }) .PipeTo(self, success: composed => new DeployComposed(msg, replyTo, composed.Result, composed.Dispatch), failure: ex => new DeployComposeFailed(msg, replyTo, ex)); } + /// + /// The DB half of the in-flight check, off the actor thread. Refuses while a deployment is in flight — + /// keeping the coordinator's state unambiguous — EXCEPT a lost dispatch, which it closes first. + /// + /// The request. + /// The refusal, or null to proceed. + /// + /// + /// Lost dispatch (fix round 1 / I7). A Dispatching row with ZERO + /// NodeDeploymentState rows was never taken by the coordinator (its HandleDispatch + /// seeds the expected set and flips the status in one SaveChanges). The coordinator's PreStart + /// recovers such a row — but only one that already existed when it started. A row committed AFTER + /// PreStart whose Tell never arrived (this actor died between the compose's commit and + /// ) has no owner at all, and refused every later deploy forever. + /// Once it is past the apply deadline nothing can still be legitimately dispatching it, so it is + /// closed TimedOut with a reason and the new deploy proceeds. A late DispatchDeployment + /// for it is ignored by the coordinator's terminal-state guard. + /// + /// + /// One indexed projection (IX_Deployment_Status) plus, only for a Dispatching row past the + /// deadline, a count and a tracked update. + /// + /// + private async Task CheckInFlightAsync(StartDeployment msg) + { + await using var db = await _dbFactory.CreateDbContextAsync().ConfigureAwait(false); + var inflight = await db.Deployments + .AsNoTracking() + .Where(d => d.Status == DeploymentStatus.Dispatching || d.Status == DeploymentStatus.AwaitingApplyAcks) + .OrderByDescending(d => d.CreatedAtUtc) + .Select(d => new { d.DeploymentId, d.Status, d.CreatedAtUtc }) + .ToListAsync() + .ConfigureAwait(false); + + foreach (var row in inflight) + { + if (row.Status == DeploymentStatus.Dispatching + && DateTime.UtcNow - row.CreatedAtUtc > _applyDeadline + && !await db.NodeDeploymentStates.AnyAsync(s => s.DeploymentId == row.DeploymentId).ConfigureAwait(false) + && await CloseLostDispatchAsync(db, row.DeploymentId).ConfigureAwait(false)) + { + continue; + } + + return new ComposeOutcome( + new StartDeploymentResult( + StartDeploymentOutcome.AnotherDeploymentInFlight, + DeploymentId: new DeploymentId(row.DeploymentId), + RevisionHash: null, + Message: $"Deployment {row.DeploymentId:N} is still in flight.", + msg.CorrelationId), + Dispatch: null); + } + return null; + } + + /// Closes a lost dispatch as TimedOut, unless it moved on meanwhile (optimistic, on its RowVersion). + /// The context. + /// The row. + /// when the row is now closed. + private async Task CloseLostDispatchAsync(OtOpcUaConfigDbContext db, Guid deploymentId) + { + var stamp = await db.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == deploymentId && d.Status == DeploymentStatus.Dispatching) + .Select(d => new { d.RowVersion }) + .FirstOrDefaultAsync() + .ConfigureAwait(false); + if (stamp is null) return true; // no longer Dispatching — not in flight as far as this check goes + + // Stub update: the row's ArtifactBlob is never read just to change its status. + var stub = new Deployment + { + DeploymentId = deploymentId, RevisionHash = string.Empty, CreatedBy = string.Empty, + RowVersion = stamp.RowVersion, + Status = DeploymentStatus.TimedOut, + FailureReason = + "Dispatch was lost before any node was expected to ack (the coordinator was never told, or never " + + "took it) and the apply deadline has passed; closed by the next StartDeployment. Redeploy.", + }; + var entry = db.Deployments.Attach(stub); + entry.Property(d => d.Status).IsModified = true; + entry.Property(d => d.FailureReason).IsModified = true; + try + { + await db.SaveChangesAsync().ConfigureAwait(false); + } + catch (DbUpdateConcurrencyException) + { + // The coordinator took it after all (status flipped, RowVersion moved): it is genuinely in flight. + return false; + } + _log.Warning( + "StartDeployment: closed lost dispatch {DeploymentId} as TimedOut — Dispatching with no expected-ack rows " + + "past the {Deadline} apply deadline (the coordinator never took it)", deploymentId, _applyDeadline); + return true; + } + private void HandleDeployComposed(DeployComposed msg) { _composeInFlight = null; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 3c99e998..49b54df4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -234,6 +234,26 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers private void HandleDispatch(DispatchDeployment msg) { + // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already + // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message + // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. + using (var check = _dbFactory.CreateDbContext()) + { + var status = check.Deployments + .AsNoTracking() + .Where(d => d.DeploymentId == msg.DeploymentId.Value) + .Select(d => (DeploymentStatus?)d.Status) + .FirstOrDefault(); + if (status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) + { + _log.Warning( + "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)", + msg.DeploymentId, status); + if (_current == msg.DeploymentId) ResetForNext(); + return; + } + } + _current = msg.DeploymentId; _acks.Clear(); _degraded.Clear(); @@ -371,11 +391,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // same as clean, though: the deployment's reason names the degraded nodes (per-node detail is on // each NodeDeploymentState row) and the seal logs at Warning. var nodes = _degraded.Select(n => n.Value).Order(StringComparer.OrdinalIgnoreCase).ToList(); - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + - "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it."); + SetFailureReason(db, Truncate( + $"Sealed with a degraded address-space apply on {nodes.Count} node(s): {string.Join(", ", nodes)}. " + + "See each node's NodeDeploymentState.FailureReason; re-deploying the same revision re-applies it.")); db.SaveChanges(); _log.Warning( "Deployment {Id} sealed DEGRADED (acks={Count}) — the address-space apply was degraded on: {Nodes}", @@ -420,11 +438,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers UpdateDeploymentStatus(db, _current.Value, DeploymentStatus.TimedOut); if (_expectedAcks.Count == 0) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == _current.Value.Value); - if (d is not null) - d.FailureReason = Truncate( - "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + - "hand-over mid-dispatch) and the apply deadline has passed. Redeploy."); + SetFailureReason(db, Truncate( + "Dispatch was lost before any node was expected to ack (coordinator restart or singleton " + + "hand-over mid-dispatch) and the apply deadline has passed. Redeploy.")); db.SaveChanges(); _log.Warning( "Deployment {Id} timed out with NO expected-ack rows: the dispatch was lost before any node was " + @@ -459,13 +475,53 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers public sealed record DeadlineElapsed(DeploymentId DeploymentId); + /// + /// Stages a status change WITHOUT materialising the row: the row carries the whole ArtifactBlob + /// (varbinary(max), multi-MB on a real fleet), which no status transition reads (fix round 1). A stub + /// carrying only the key and the row's current RowVersion is attached and the changed columns marked + /// modified, so the UPDATE writes those columns and keeps the optimistic-concurrency check. The staged + /// stub stays tracked so can add to the same UPDATE. + /// private static void UpdateDeploymentStatus( OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false) { - var d = db.Deployments.FirstOrDefault(x => x.DeploymentId == id.Value); - if (d is null) return; - d.Status = status; - if (sealNow) d.SealedAtUtc = DateTime.UtcNow; + var entry = TrackStub(db, id); + if (entry is null) return; + entry.Entity.Status = status; + entry.Property(x => x.Status).IsModified = true; + if (sealNow) + { + entry.Entity.SealedAtUtc = DateTime.UtcNow; + entry.Property(x => x.SealedAtUtc).IsModified = true; + } + } + + /// Stages the current deployment's failure reason onto the stub staged. + private void SetFailureReason(OtOpcUaConfigDbContext db, string reason) + { + if (_current is null) return; + var entry = TrackStub(db, _current.Value); + if (entry is null) return; + entry.Entity.FailureReason = reason; + entry.Property(x => x.FailureReason).IsModified = true; + } + + /// The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone. + private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( + OtOpcUaConfigDbContext db, DeploymentId id) + { + var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); + if (tracked is not null) return tracked; + var stamp = db.Deployments + .AsNoTracking() + .Where(x => x.DeploymentId == id.Value) + .Select(x => new { x.RowVersion }) + .FirstOrDefault(); + if (stamp is null) return null; + return db.Deployments.Attach(new Deployment + { + DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = stamp.RowVersion, + }); } /// diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs index 86f6efa7..0added3d 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -176,6 +176,141 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes }, duration: PresenceBudget); } + /// + /// Fix round 1 / I7 — the window PreStart cannot see: the Dispatching row is committed AFTER the + /// coordinator's PreStart ran, and the coordinator is never told (the admin singleton died between + /// its commit and the Tell). Nothing would ever close it. The in-flight check now closes a + /// Dispatching row with NO expected-ack rows once it is past the apply deadline, so the next + /// StartDeployment is Accepted (and seals — positive control on the pipeline). + /// + [Fact] + public async Task Lost_dispatch_committed_after_PreStart_is_closed_by_the_next_StartDeployment() + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart saw nothing + + var stale = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddMinutes(-10)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, result.Message); + + using (var db = dbFactory.CreateDbContext()) + { + var row = db.Deployments.Single(d => d.DeploymentId == stale.Value); + row.Status.ShouldBe(DeploymentStatus.TimedOut); + row.FailureReason.ShouldNotBeNull(); + row.FailureReason.ShouldContain("lost"); + } + + coordinator.Tell(new ApplyAck(result.DeploymentId!.Value, NodeId.Parse("site-a-1:4053"), + ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == result.DeploymentId.Value.Value).Status + .ShouldBe(DeploymentStatus.Sealed); + }, duration: PresenceBudget); + } + + /// Control for the closure: the same row INSIDE the deadline is still in flight and still refuses. + [Fact] + public void Dispatching_row_inside_the_deadline_still_refuses_a_new_deployment() + { + var dbFactory = NewInMemoryDbFactory(); + var coordinator = CreateTestProbe(); + var pending = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow.AddSeconds(-5)); + var admin = Sys.ActorOf(AdminOperationsActor.Props( + dbFactory, coordinator.Ref, Enumerable.Empty(), applyDeadline: NoTimeout)); + + admin.Tell(new StartDeployment("joe", CorrelationId.NewId())); + var result = ExpectMsg(PresenceBudget); + result.Outcome.ShouldBe(StartDeploymentOutcome.AnotherDeploymentInFlight); + result.DeploymentId.ShouldBe(pending); + + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single().Status.ShouldBe(DeploymentStatus.Dispatching); + } + + /// + /// Fix round 1 minor — a late or duplicated DispatchDeployment for a row that is already + /// terminal (it raced the lost-dispatch closure above, or a re-delivery after a seal) must not flip + /// it back to AwaitingApplyAcks or seed an expected set. Positive control: a live row dispatched right + /// after it IS seeded (so the first message was processed, not merely not-yet-processed). + /// + [Theory] + [InlineData(DeploymentStatus.Sealed)] + [InlineData(DeploymentStatus.TimedOut)] + [InlineData(DeploymentStatus.PartiallyFailed)] + public void Dispatch_of_a_terminal_row_is_ignored(DeploymentStatus terminal) + { + var dbFactory = NewInMemoryDbFactory(); + SeedNodes(dbFactory, "site-a-1:4053"); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(dbFactory, NoTimeout)); + + var done = SeedDeployment(dbFactory, terminal, DateTime.UtcNow.AddMinutes(-1)); + coordinator.Tell(new DispatchDeployment(done, TestRevision, CorrelationId.NewId())); + var live = SeedDeployment(dbFactory, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = dbFactory.CreateDbContext(); + db.Deployments.Single(d => d.DeploymentId == live.Value).Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = dbFactory.CreateDbContext(); + verify.Deployments.Single(d => d.DeploymentId == done.Value).Status.ShouldBe(terminal); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == done.Value).ShouldBe(0); + } + + /// + /// Fix round 1 minor — status transitions update a blob-free stub (key + RowVersion) instead of + /// materialising the multi-MB row. Proven on the REAL provider, where the stub's RowVersion is the + /// optimistic-concurrency predicate: dispatch → ack → Sealed lands, and the artifact bytes are + /// untouched by the partial UPDATEs. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public void On_sql_server_status_transitions_do_not_touch_the_artifact_blob() + { + using var sql = new SqlServerTestDatabase(); + var factory = sql.Factory(); + SeedNodes(factory, "site-a-1:4053"); + var blob = Enumerable.Range(0, 4096).Select(i => (byte)i).ToArray(); + var id = DeploymentId.NewId(); + using (var db = sql.NewContext()) + { + db.Deployments.Add(new Deployment + { + DeploymentId = id.Value, RevisionHash = TestRevision.Value, Status = DeploymentStatus.Dispatching, + CreatedBy = "test", ArtifactBlob = blob, + }); + db.SaveChanges(); + } + + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(factory, NoTimeout)); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single().Status.ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); // PreStart re-drives the un-dispatched row + coordinator.Tell(new ApplyAck(id, NodeId.Parse("site-a-1:4053"), ApplyAckOutcome.Applied, null, CorrelationId.NewId())); + + AwaitAssert(() => + { + using var db = sql.NewContext(); + var row = db.Deployments.AsNoTracking().Single(); + row.Status.ShouldBe(DeploymentStatus.Sealed); + row.SealedAtUtc.ShouldNotBeNull(); + row.ArtifactBlob.ShouldBe(blob); + }, duration: PresenceBudget); + } + private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) { From 2adeacc65582d4704cbc254c4c6e4194058c7d4a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:48:40 -0400 Subject: [PATCH 23/51] fix(controlplane): keep the revision hash byte-identical to the pre-refactor composer The one-snapshot refactor re-sorted every collection in memory with an ordinal comparer, while the old composer's SQL ORDER BY used the database's case-insensitive collation; mixed-case keys therefore re-hashed an unchanged configuration (spurious drift fleet-wide, a "new" revision on a no-op redeploy). The snapshot factory now reads every composed table ORDER BY its natural key server-side and Compose serialises in the order given. Pinned byte-for-byte against a verbatim replica of the old composer on the real SQL Server. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Validation/DraftSnapshotFactory.cs | 40 +++++---- .../AdminOperations/ConfigComposer.cs | 51 ++++++----- ...nfigComposerSqlServerHashStabilityTests.cs | 87 +++++++++++++++++++ 3 files changed, 139 insertions(+), 39 deletions(-) create mode 100644 tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs index 154d664c..30873aca 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Validation/DraftSnapshotFactory.cs @@ -56,6 +56,12 @@ public static class DraftSnapshotFactory } } + // Every composed table is read ORDERED BY ITS NATURAL KEY, server-side. That order IS the deployment + // artifact's order (ConfigComposer.Compose serialises the snapshot as given): it is exactly the order the + // pre-R03:3.2 composer's own queries produced — the database collation's, case-insensitive on SQL Server — + // so the revision hash of an unchanged configuration is byte-identical across the refactor (fix round 1; + // pinned by ConfigComposerSqlServerHashStabilityTests). Re-sorting in memory with any other comparer would + // re-hash every mixed-case configuration and make drift detection report the whole fleet drifted. private static async Task ReadAsync(OtOpcUaConfigDbContext db, CancellationToken ct) => new DraftSnapshot { @@ -64,26 +70,26 @@ public static class DraftSnapshotFactory // EquipmentUuid checks run in separate validator passes — no rule here reads these fields. GenerationId = 0, // generation model dropped; placeholder (no rule reads it) ClusterId = string.Empty, // global snapshot; rules compare entity ClusterId fields, not this - ServerClusters = await db.ServerClusters.AsNoTracking().ToListAsync(ct), - ClusterNodes = await db.ClusterNodes.AsNoTracking().ToListAsync(ct), + ServerClusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), + ClusterNodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), // v3: Namespace entity retired — no namespace list to materialize. The Raw tree // (RawFolders/TagGroups) + UnsTagReferences feed the v3 rules (charset, tagname length, // effective-leaf uniqueness). - RawFolders = await db.RawFolders.AsNoTracking().ToListAsync(ct), - DriverInstances = await db.DriverInstances.AsNoTracking().ToListAsync(ct), - Devices = await db.Devices.AsNoTracking().ToListAsync(ct), - TagGroups = await db.TagGroups.AsNoTracking().ToListAsync(ct), - UnsAreas = await db.UnsAreas.AsNoTracking().ToListAsync(ct), - UnsLines = await db.UnsLines.AsNoTracking().ToListAsync(ct), - Equipment = await db.Equipment.AsNoTracking().ToListAsync(ct), - Tags = await db.Tags.AsNoTracking().ToListAsync(ct), - UnsTagReferences = await db.UnsTagReferences.AsNoTracking().ToListAsync(ct), - VirtualTags = await db.VirtualTags.AsNoTracking().ToListAsync(ct), - ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().ToListAsync(ct), - Scripts = await db.Scripts.AsNoTracking().ToListAsync(ct), - PollGroups = await db.PollGroups.AsNoTracking().ToListAsync(ct), - NodeAcls = await db.NodeAcls.AsNoTracking().ToListAsync(ct), - ApiKeys = await db.ApiKeys.AsNoTracking().ToListAsync(ct), + RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), + DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), + Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), + TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), + UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), + UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), + Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), + Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), + UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), + VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), + ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), + PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), + NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), PriorEquipment = [], // intentional: no prior-generation table to diff against at this level ActiveReservations = await db.ExternalIdReservations .AsNoTracking() diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs index 6f332143..39735ddb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/ConfigComposer.cs @@ -9,8 +9,9 @@ namespace ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; /// /// Pure snapshot composer: reads the current live-edit state from /// and serialises it into a deterministic, gzip-compressed byte[] artifact + SHA-256 hex revision hash. -/// Determinism comes from sorting every collection by its natural key before serialising, so two snapshots -/// over the same DB state always produce the same hash regardless of EF row ordering; compression wraps the +/// Determinism comes from reading every collection ordered by its natural key on the server +/// (), so two snapshots over the same DB state always produce the same hash +/// — and the same hash as the pre-R03:3.2 composer, which used those same queries; compression wraps the /// final bytes and reorders nothing. The hash is taken over the COMPRESSED bytes — see . /// public static class ConfigComposer @@ -41,6 +42,11 @@ public static class ConfigComposer /// Serialises an already-read into the deterministic artifact. Pure over /// its input: the same snapshot always yields the same bytes and hash, whatever the DB holds by now. /// + /// + /// Serialises the collections IN THE ORDER GIVEN. Determinism comes from + /// , which reads every table ordered by its natural key + /// on the server; a snapshot built any other way must be ordered the same way or its hash will differ. + /// /// The rows to compose — the SAME rows the validator was run over. /// The compressed artifact blob and its revision hash. public static ConfigArtifact Compose(DraftSnapshot snapshot) @@ -52,34 +58,35 @@ public static class ConfigComposer // projection (UnsAreas/UnsLines/Equipment/UnsTagReferences) and the retained VirtualTags/ScriptedAlarms/ // PollGroups/NodeAcls/Scripts. The Namespaces snapshot is RETIRED (the two OPC UA namespaces are implicit). // RevisionHash inputs follow the blob automatically — adding/removing a table shifts the SHA-256. - // Every collection is sorted by its natural key with an ORDINAL comparer so the hash is independent of - // both EF row order and the process culture (a culture-aware sort could order "a-1"/"A-1" differently - // on two central nodes and make the same config hash twice). + // NO re-sort here (fix round 1): the snapshot's collections arrive ordered by their natural key from + // DraftSnapshotFactory's server-side ORDER BY — the same order, and so the same bytes and RevisionHash, + // as the pre-R03:3.2 composer's own queries. An in-memory ordinal re-sort orders mixed-case keys + // differently from the SQL collation and silently re-hashed every unchanged configuration. var ordered = new { - Clusters = snapshot.ServerClusters.OrderBy(x => x.ClusterId, StringComparer.Ordinal).ToList(), - Nodes = snapshot.ClusterNodes.OrderBy(x => x.NodeId, StringComparer.Ordinal).ToList(), - RawFolders = snapshot.RawFolders.OrderBy(x => x.RawFolderId, StringComparer.Ordinal).ToList(), - DriverInstances = snapshot.DriverInstances.OrderBy(x => x.DriverInstanceId, StringComparer.Ordinal).ToList(), - Devices = snapshot.Devices.OrderBy(x => x.DeviceId, StringComparer.Ordinal).ToList(), - TagGroups = snapshot.TagGroups.OrderBy(x => x.TagGroupId, StringComparer.Ordinal).ToList(), - Equipment = snapshot.Equipment.OrderBy(x => x.EquipmentId, StringComparer.Ordinal).ToList(), - Tags = snapshot.Tags.OrderBy(x => x.TagId, StringComparer.Ordinal).ToList(), - UnsTagReferences = snapshot.UnsTagReferences.OrderBy(x => x.UnsTagReferenceId, StringComparer.Ordinal).ToList(), - PollGroups = snapshot.PollGroups.OrderBy(x => x.PollGroupId, StringComparer.Ordinal).ToList(), - UnsAreas = snapshot.UnsAreas.OrderBy(x => x.UnsAreaId, StringComparer.Ordinal).ToList(), - UnsLines = snapshot.UnsLines.OrderBy(x => x.UnsLineId, StringComparer.Ordinal).ToList(), - NodeAcls = snapshot.NodeAcls.OrderBy(x => x.NodeAclId, StringComparer.Ordinal).ToList(), + Clusters = snapshot.ServerClusters, + Nodes = snapshot.ClusterNodes, + RawFolders = snapshot.RawFolders, + DriverInstances = snapshot.DriverInstances, + Devices = snapshot.Devices, + TagGroups = snapshot.TagGroups, + Equipment = snapshot.Equipment, + Tags = snapshot.Tags, + UnsTagReferences = snapshot.UnsTagReferences, + PollGroups = snapshot.PollGroups, + UnsAreas = snapshot.UnsAreas, + UnsLines = snapshot.UnsLines, + NodeAcls = snapshot.NodeAcls, // the data-plane credentials travel with the config, because a driver-only node // has no ConfigDb to read them from (per-cluster mesh Phase 4) and would otherwise deny every // UserName token. Only the peppered HMAC of each secret ships — never the secret — and the // pepper is node configuration that is deliberately NOT in the artifact, so a stolen blob // still cannot be turned into a working credential. This is also why revoking a key reaches a // driver-only node only on the next deployment; see the ApiKey entity's remarks. - ApiKeys = snapshot.ApiKeys.OrderBy(x => x.ApiKeyId, StringComparer.Ordinal).ToList(), - Scripts = snapshot.Scripts.OrderBy(x => x.ScriptId, StringComparer.Ordinal).ToList(), - VirtualTags = snapshot.VirtualTags.OrderBy(x => x.VirtualTagId, StringComparer.Ordinal).ToList(), - ScriptedAlarms = snapshot.ScriptedAlarms.OrderBy(x => x.ScriptedAlarmId, StringComparer.Ordinal).ToList(), + ApiKeys = snapshot.ApiKeys, + Scripts = snapshot.Scripts, + VirtualTags = snapshot.VirtualTags, + ScriptedAlarms = snapshot.ScriptedAlarms, }; return Pack(JsonSerializer.SerializeToUtf8Bytes(ordered, JsonOptions)); diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs new file mode 100644 index 00000000..67ac26bd --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigComposerSqlServerHashStabilityTests.cs @@ -0,0 +1,87 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Configuration; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Validation; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness; + +namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests; + +/// +/// Fix round 1 minor — revision-hash stability across the R03:3.2 refactor. The pre-change composer read +/// every table with a SQL ORDER BY (the database's case-INSENSITIVE collation); the refactor re-sorted +/// in memory with an ORDINAL comparer, which orders mixed-case keys differently ("B-1" before "a-2"). The +/// same configuration would then hash differently after the upgrade: drift detection reports every node +/// drifted, and a re-deploy of an unchanged config shows a "new" revision. This pins the composer's output +/// BYTE-identical to a verbatim replica of the pre-change composer on the real provider. +/// +[Trait("Category", "SchemaCompliance")] +public sealed class ConfigComposerSqlServerHashStabilityTests +{ + [Fact] + public async Task Composer_output_is_byte_identical_to_the_pre_change_composer_for_mixed_case_keys() + { + using var sql = new SqlServerTestDatabase(); + await using (var seed = sql.NewContext()) + { + // Collation order a-2, B-1, c-3; ordinal order B-1, a-2, c-3. + foreach (var id in new[] { "c-3", "B-1", "a-2" }) + seed.Scripts.Add(new Script { ScriptId = id, Name = id, SourceCode = "return 1;", SourceHash = id }); + await seed.SaveChangesAsync(); + } + + await using var legacyDb = sql.NewContext(); + var legacy = await LegacySnapshotAndFlattenAsync(legacyDb); + + await using var viaDb = sql.NewContext(); + var current = await ConfigComposer.SnapshotAndFlattenAsync(viaDb); + current.Blob.ShouldBe(legacy.Blob); + current.RevisionHash.ShouldBe(legacy.RevisionHash); + + // The deploy path's form (validate + compose from ONE snapshot) must agree too. + await using var deployDb = sql.NewContext(); + var viaSnapshot = ConfigComposer.Compose(await DraftSnapshotFactory.FromConfigDbAsync(deployDb)); + viaSnapshot.RevisionHash.ShouldBe(legacy.RevisionHash); + } + + /// Verbatim replica of ConfigComposer.SnapshotAndFlattenAsync before R03:3.2 (ad2b5c40). + private static async Task<(byte[] Blob, string RevisionHash)> LegacySnapshotAndFlattenAsync(OtOpcUaConfigDbContext db) + { + var ct = CancellationToken.None; + var snapshot = new + { + Clusters = await db.ServerClusters.AsNoTracking().OrderBy(x => x.ClusterId).ToListAsync(ct), + Nodes = await db.ClusterNodes.AsNoTracking().OrderBy(x => x.NodeId).ToListAsync(ct), + RawFolders = await db.RawFolders.AsNoTracking().OrderBy(x => x.RawFolderId).ToListAsync(ct), + DriverInstances = await db.DriverInstances.AsNoTracking().OrderBy(x => x.DriverInstanceId).ToListAsync(ct), + Devices = await db.Devices.AsNoTracking().OrderBy(x => x.DeviceId).ToListAsync(ct), + TagGroups = await db.TagGroups.AsNoTracking().OrderBy(x => x.TagGroupId).ToListAsync(ct), + Equipment = await db.Equipment.AsNoTracking().OrderBy(x => x.EquipmentId).ToListAsync(ct), + Tags = await db.Tags.AsNoTracking().OrderBy(x => x.TagId).ToListAsync(ct), + UnsTagReferences = await db.UnsTagReferences.AsNoTracking().OrderBy(x => x.UnsTagReferenceId).ToListAsync(ct), + PollGroups = await db.PollGroups.AsNoTracking().OrderBy(x => x.PollGroupId).ToListAsync(ct), + UnsAreas = await db.UnsAreas.AsNoTracking().OrderBy(x => x.UnsAreaId).ToListAsync(ct), + UnsLines = await db.UnsLines.AsNoTracking().OrderBy(x => x.UnsLineId).ToListAsync(ct), + NodeAcls = await db.NodeAcls.AsNoTracking().OrderBy(x => x.NodeAclId).ToListAsync(ct), + ApiKeys = await db.ApiKeys.AsNoTracking().OrderBy(x => x.ApiKeyId).ToListAsync(ct), + Scripts = await db.Scripts.AsNoTracking().OrderBy(x => x.ScriptId).ToListAsync(ct), + VirtualTags = await db.VirtualTags.AsNoTracking().OrderBy(x => x.VirtualTagId).ToListAsync(ct), + ScriptedAlarms = await db.ScriptedAlarms.AsNoTracking().OrderBy(x => x.ScriptedAlarmId).ToListAsync(ct), + }; + var json = JsonSerializer.SerializeToUtf8Bytes(snapshot, new JsonSerializerOptions + { + WriteIndented = false, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.Never, + }); + using var output = new MemoryStream(json.Length / 8); + using (var gzip = new GZipStream(output, CompressionLevel.Fastest, leaveOpen: true)) + gzip.Write(json, 0, json.Length); + var blob = output.ToArray(); + return (blob, Convert.ToHexStringLower(SHA256.HashData(blob))); + } +} From 896aaa2b02258e4f829f7d5fdff10c7efb57e0b0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:49:56 -0400 Subject: [PATCH 24/51] fix(controlplane): a PSK secret deleted from the store revokes the key; bound refresh concurrency RefreshAsync now tells "the store answered: no such secret" (revocation - drop the refreshed key AND the lazily cached one) apart from "the read failed" (keep the previous key). Store reads fan out through Parallel.ForEachAsync capped at MaxConcurrentRefreshReads = 8 instead of one read per cluster at once. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Communication/IMeshPskProvider.cs | 90 +++++++++++++------ .../MeshPskProviderRefreshTests.cs | 71 +++++++++++++++ 2 files changed, 136 insertions(+), 25 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs index bdab7ba7..bad1956c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Communication/IMeshPskProvider.cs @@ -110,7 +110,8 @@ public interface IMeshPskProvider /// /// Re-reads the keys for from the store OFF the request path and /// swaps them into the cache atomically (R03:3.4). Until the refresh completes, lookups keep - /// serving the previous key; a cluster whose read fails keeps its previous key too. Replaces + /// serving the previous key; a cluster whose read FAILS keeps its previous key too, while a cluster whose + /// secret the store reports ABSENT is revoked (no key from then on). Replaces /// "invalidate on every refresh", which made the first inbound ack per cluster per /// ContactRefreshSeconds block a gRPC thread on a synchronous, 3 s-bounded store read. /// @@ -331,45 +332,76 @@ public sealed class MeshPskProvider : IMeshPskProvider _refreshed = new Dictionary(StringComparer.OrdinalIgnoreCase); } + /// + /// Ceiling on concurrent store reads during one . A fleet of N clusters used to + /// fan out N simultaneous reads every contact refresh; the store (and anything behind it) sees at most + /// this many at once now. + /// + public const int MaxConcurrentRefreshReads = 8; + /// public async Task RefreshAsync(IReadOnlyCollection clusterIds, CancellationToken ct) { ArgumentNullException.ThrowIfNull(clusterIds); var previous = _refreshed; - var next = new Dictionary(StringComparer.OrdinalIgnoreCase); var ids = clusterIds.Where(id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + var results = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - // Resolve every cluster concurrently — each read bounded like the synchronous path — and carry a - // failed cluster's PREVIOUS refreshed key forward: a transient store fault must not turn into a - // window in which that cluster has no key at all. - var reads = ids.Select(async id => + // Bounded fan-out; each read bounded like the synchronous path. + await Parallel.ForEachAsync( + ids, + new ParallelOptions { MaxDegreeOfParallelism = MaxConcurrentRefreshReads, CancellationToken = ct }, + async (id, token) => results[id] = await ResolveFromStoreAsync(id, token).ConfigureAwait(false)) + .ConfigureAwait(false); + + // Three outcomes per cluster (fix round 1 / I8): + // - Found → the new key; + // - NotFound → the store answered that the secret does not exist: a REVOCATION — the cluster gets no + // key (and its lazily cached one is evicted below), so a deleted secret stops working + // within one refresh instead of living on for the life of the process; + // - Failed → the store could not be read (threw / timed out / unnameable): keep the PREVIOUS key — + // a transient fault must not open a window in which the cluster has no key at all. + var next = new Dictionary(StringComparer.OrdinalIgnoreCase); + var answered = new List(); + foreach (var id in ids) { - var value = await ResolveFromStoreAsync(id, ct).ConfigureAwait(false); - return (Id: id, Value: value); - }).ToList(); - foreach (var (id, value) in await Task.WhenAll(reads).ConfigureAwait(false)) - { - if (!string.IsNullOrWhiteSpace(value)) next[id] = value; - else if (previous.TryGetValue(id, out var kept)) next[id] = kept; + var read = results[id]; + switch (read.Outcome) + { + case StoreReadOutcome.Found: + next[id] = read.Value!; + answered.Add(id); + break; + case StoreReadOutcome.NotFound: + answered.Add(id); + break; + default: + if (previous.TryGetValue(id, out var kept)) next[id] = kept; + break; + } } - // The swap. Then drop the lazy entries for the clusters now covered, so a key rotated between - // refreshes is served from the new snapshot rather than from a stale lazy resolution; entries - // for clusters NOT covered (their read failed and they had no previous value) are left alone. + // The swap. Then drop the lazy entries for every cluster the store ANSWERED for — found (serve the new + // snapshot, not a stale lazy resolution) or not found (revoked: a lazily cached key must die too). + // Entries for clusters whose read failed are left alone. _refreshed = next; - foreach (var id in next.Keys) _cache.TryRemove(id, out _); + foreach (var id in answered) _cache.TryRemove(id, out _); } + private enum StoreReadOutcome { Found, NotFound, Failed } + + private readonly record struct StoreRead(StoreReadOutcome Outcome, string? Value); + /// - /// The asynchronous twin of : same name derivation, same bound, same - /// "anything but a usable key is null" contract — awaited instead of blocked on, because this one - /// runs on a refresh rather than inside a gRPC interceptor. + /// The asynchronous twin of : same name derivation, same bound — awaited + /// instead of blocked on, because this one runs on a refresh rather than inside a gRPC interceptor — + /// but it keeps "the secret does not exist" (a blank answer) apart from "the store could not be read". /// /// The application Cluster id. /// The caller's token, linked under the store-read bound. - /// The stored key, or null. - private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) + /// The outcome and, when found, the key. + private async Task ResolveFromStoreAsync(string clusterId, CancellationToken ct) { SecretName name; try @@ -384,14 +416,22 @@ public sealed class MeshPskProvider : IMeshPskProvider + "refreshed from the store. Supply it through MeshTransport:ClusterPsks instead, or " + "rename the cluster to [a-z0-9._/-].", clusterId); - return null; + return new StoreRead(StoreReadOutcome.Failed, null); } using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); cts.CancelAfter(StoreReadTimeout); try { - return await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + var value = await _secretResolver.GetAsync(name, cts.Token).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(value)) return new StoreRead(StoreReadOutcome.Found, value); + + _logger.LogWarning( + "Mesh PSK secret {SecretName} for cluster {ClusterId} is absent (or blank) in the store; the key is " + + "REVOKED — commands to that cluster and acks from it are refused until it is restored.", + name.Value, + clusterId); + return new StoreRead(StoreReadOutcome.NotFound, null); } catch (Exception ex) { @@ -401,7 +441,7 @@ public sealed class MeshPskProvider : IMeshPskProvider + "previously resolved key (if any) stays in service until the next refresh.", clusterId, name.Value); - return null; + return new StoreRead(StoreReadOutcome.Failed, null); } } diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs index e727c153..34129235 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/Communication/MeshPskProviderRefreshTests.cs @@ -78,6 +78,77 @@ public sealed class MeshPskProviderRefreshTests psk.ShouldBe("k-1"); } + /// + /// Fix round 1 / I8 — "the store says there is no such secret" is a REVOCATION, not a read failure: + /// the previous key must stop being served (from the refreshed snapshot AND the lazy cache). A thrown + /// read keeps the previous key (the test above); a definitive not-found drops it. + /// + [Fact] + public async Task A_secret_deleted_from_the_store_revokes_the_key_on_the_next_refresh() + { + var resolver = new GatedResolver { Value = "k-1" }; + var provider = Provider(resolver); + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + provider.Matches("SITE-A", "k-1").ShouldBeTrue("positive control: the key was in service"); + + resolver.Value = null; // the secret was deleted + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + provider.Matches("SITE-A", "k-1").ShouldBeFalse(); + } + + /// The same revocation reaches a key that was first served from the LAZY cache, not a refresh. + [Fact] + public async Task A_deleted_secret_also_evicts_a_lazily_cached_key() + { + var resolver = new GatedResolver { Value = "k-lazy" }; + var provider = Provider(resolver); + provider.TryGetPsk("SITE-A", out var lazy).ShouldBeTrue(); + lazy.ShouldBe("k-lazy"); + + resolver.Value = null; + await provider.RefreshAsync(new[] { "SITE-A" }, CancellationToken.None); + + provider.TryGetPsk("SITE-A", out _).ShouldBeFalse(); + } + + /// Fix round 1 minor — the refresh fans out to the store with BOUNDED concurrency, not one read per cluster at once. + [Fact] + public async Task Refresh_store_reads_are_concurrency_bounded() + { + var resolver = new ConcurrencyTrackingResolver(); + var provider = Provider(resolver); + var clusters = Enumerable.Range(0, 40).Select(i => $"SITE-{i}").ToList(); + + await provider.RefreshAsync(clusters, CancellationToken.None); + + resolver.Calls.ShouldBe(40); + resolver.MaxConcurrent.ShouldBeLessThanOrEqualTo(MeshPskProvider.MaxConcurrentRefreshReads); + resolver.MaxConcurrent.ShouldBeGreaterThan(1, "still concurrent, just bounded"); + foreach (var c in clusters) provider.TryGetPsk(c, out _).ShouldBeTrue(); + } + + private sealed class ConcurrencyTrackingResolver : ISecretResolver + { + private int _inFlight; + private int _max; + private int _calls; + public int MaxConcurrent => Volatile.Read(ref _max); + public int Calls => Volatile.Read(ref _calls); + + public async Task GetAsync(SecretName name, CancellationToken ct) + { + Interlocked.Increment(ref _calls); + var now = Interlocked.Increment(ref _inFlight); + int seen; + while (now > (seen = Volatile.Read(ref _max)) && Interlocked.CompareExchange(ref _max, now, seen) != seen) { } + await Task.Delay(20, ct); + Interlocked.Decrement(ref _inFlight); + return "k-" + name.Value; + } + } + /// A cluster that was never refreshed still resolves lazily on the request path (unchanged behaviour). [Fact] public async Task A_cluster_outside_the_refresh_set_still_resolves_lazily() From dc3323fcca2d6a213fc63d5b1c95049015bd17ff Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:51:28 -0400 Subject: [PATCH 25/51] fix(host): publish the known-id set and its stamp atomically; never write the shared guard options BoundedApiKeyStore kept the set (volatile) and its DateTimeOffset stamp (non-volatile, 16 bytes) in separate fields, so a reader could pair a new set with an old stamp or read a torn stamp; they are now one immutable snapshot behind a volatile reference. An inner store that cannot list ids now switches the set off with a per-instance flag instead of writing KnownIdSet = false into the shared ApiKeyStoreGuardOptions. Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../OpcUa/BoundedApiKeyStore.cs | 35 ++++++++++++------- .../OpcUa/BoundedApiKeyStoreKnownIdTests.cs | 30 ++++++++++++++++ 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs index bb3154ed..6ebb7626 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/BoundedApiKeyStore.cs @@ -61,10 +61,22 @@ public sealed class BoundedApiKeyStore : IApiKeyStore // R03:3.3 — the known-id set: every ApiKeyId the inner store holds, refreshed every StoreCacheSeconds // (single-flight; a refresh that fails or times out keeps the previous set). Null until the first // successful load, or when the inner store cannot list ids — then the per-id path runs as before. - private volatile IReadOnlySet? _knownIds; - private DateTimeOffset _knownIdsStampedUtc = DateTimeOffset.MinValue; + // The set and the instant it was loaded are published TOGETHER as one immutable snapshot behind a volatile + // reference (fix round 1): a reader on another thread can never pair a new set with an old stamp, or read + // a torn DateTimeOffset (a 16-byte struct, not atomically readable), as the separate fields allowed. + private volatile KnownIdSnapshot? _knownIds; private readonly SemaphoreSlim _knownIdsRefresh = new(1, 1); + // Set when the inner store reports it cannot list ids. Per-instance on purpose: the guard options object is + // shared configuration and is never written (it used to be — `_guard.KnownIdSet = false` — which leaked + // one store's capability into every other consumer of the same options). + private volatile bool _listingUnsupported; + + /// The known-id set and the instant it was loaded, published as one unit. + /// The ids the inner store holds. + /// When they were read. + private sealed record KnownIdSnapshot(IReadOnlySet Ids, DateTimeOffset StampedUtc); + // R03:3.3 — the miss budget: a token bucket on lookups that REACH the inner store and miss. Guards // the path the known-id set cannot (set unavailable), and bounds a flood of unknown ids to the // configured rate instead of one SQL round trip per attempt under the SessionManager lock. @@ -104,7 +116,7 @@ public sealed class BoundedApiKeyStore : IApiKeyStore } /// The known-id set currently in force, or null when none has loaded. Exposed for tests and wiring guards. - public IReadOnlySet? KnownKeyIds => _knownIds; + public IReadOnlySet? KnownKeyIds => _knownIds?.Ids; /// The store being bounded. /// @@ -173,26 +185,25 @@ public sealed class BoundedApiKeyStore : IApiKeyStore /// The set, or null. private async Task?> KnownIdsAsync(TimeSpan ttl, CancellationToken ct) { - if (!_guard.KnownIdSet || ttl <= TimeSpan.Zero) return null; + if (!_guard.KnownIdSet || _listingUnsupported || ttl <= TimeSpan.Zero) return null; var now = _time.GetUtcNow(); var current = _knownIds; - if (current is not null && now - _knownIdsStampedUtc < ttl) return current; + if (current is not null && now - current.StampedUtc < ttl) return current.Ids; - if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current; + if (!await _knownIdsRefresh.WaitAsync(0, ct).ConfigureAwait(false)) return current?.Ids; try { // Re-check under the gate: another caller may have refreshed while this one waited. - if (_knownIds is { } refreshed && now - _knownIdsStampedUtc < ttl) return refreshed; + if (_knownIds is { } refreshed && now - refreshed.StampedUtc < ttl) return refreshed.Ids; var ids = await BoundAsync(token => _inner.ListKeyIdsAsync(token), ct).ConfigureAwait(false); if (ids is null) { - // Unsupported by the inner store: say so once and stop asking. - _guard.KnownIdSet = false; + // Unsupported by the inner store: stop asking — for this decorator only. + _listingUnsupported = true; return null; } - _knownIds = ids; - _knownIdsStampedUtc = _time.GetUtcNow(); + _knownIds = new KnownIdSnapshot(ids, _time.GetUtcNow()); return ids; } catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) @@ -200,7 +211,7 @@ public sealed class BoundedApiKeyStore : IApiKeyStore _log.LogWarning(ex, "ApiKeyStore: refreshing the known key-id set failed; {Mode}", current is null ? "unknown ids reach the store until it loads" : "the previous set stays in force"); - return current; + return current?.Ids; } finally { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs index 7051b151..8d08df5e 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/BoundedApiKeyStoreKnownIdTests.cs @@ -120,6 +120,36 @@ public sealed class BoundedApiKeyStoreKnownIdTests inner.ListCalls.ShouldBe(0); } + /// + /// Fix round 1 minor — an inner store that cannot list ids switches the known-id set off for THIS + /// decorator only. It used to do so by writing KnownIdSet = false into the guard options it was + /// handed, a shared options object, so one store's capability leaked into every other consumer. + /// Positive control: a second decorator over a listing store on the same options instance still uses + /// the set. + /// + [Fact] + public async Task An_unlisting_store_does_not_mutate_the_shared_guard_options() + { + var guard = new ApiKeyStoreGuardOptions { MissBurst = 100, MissRefillPerSecond = 20 }; + var options = Options.Create(new ApiKeyOptions + { + Username = "apikey", Pepper = "unit-test-pepper-xxxx", StoreTimeoutMs = 0, StoreCacheSeconds = 5, + }); + var clock = new FakeClock(T0); + var unlisting = new ListingStore("k-real") { SupportsListing = false }; + var first = new BoundedApiKeyStore(unlisting, options, NullLogger.Instance, clock, guard); + + (await first.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + unlisting.FindCalls.ShouldBe(1, "no set: the per-id path ran"); + guard.KnownIdSet.ShouldBeTrue("the shared options object is never written"); + + var listing = new ListingStore("k-real"); + var second = new BoundedApiKeyStore(listing, options, NullLogger.Instance, clock, guard); + (await second.FindAsync("k-guess", T0.UtcDateTime, default)).ShouldBeNull(); + listing.FindCalls.ShouldBe(0, "the second decorator still refuses unknown ids in memory"); + listing.ListCalls.ShouldBe(1); + } + private static BoundedApiKeyStore Bounded( IApiKeyStore inner, TimeProvider clock, int cacheSeconds, int missBurst = 100, double refillPerSecond = 20) => new( inner, From de21a54949422aa568948a9c0af17f469e9e5d51 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:57:01 -0400 Subject: [PATCH 26/51] fix(twincat): publish BadNodeIdUnknown for unresolvable native-subscribe references (U12) The native subscribe path silently skipped a reference it could not resolve (unknown RawPath, or a tag routed to an unconfigured device), leaving the node on whatever status the server seeded. It now publishes one BadNodeIdUnknown snapshot, matching the poll path. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../TwinCATDriver.cs | 16 ++++- .../TwinCATNativeIsolationTests.cs | 59 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeIsolationTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index 66663ef8..1dda4ad7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -666,8 +666,16 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { foreach (var reference in fullReferences) { - if (!_resolver.TryResolve(reference, out var def)) continue; - if (!_devices.TryGetValue(def.DeviceHostAddress, out var device)) continue; + // An unresolvable reference (unknown RawPath, or a tag routed to an unconfigured device) + // publishes BadNodeIdUnknown once — the poll path's status — instead of leaving the node on + // whatever status the server seeded it with. + if (!_resolver.TryResolve(reference, out var def) + || !_devices.TryGetValue(def.DeviceHostAddress, out var device)) + { + RaiseDataChange(handle, reference, new DataValueSnapshot( + null, TwinCATStatusMapper.BadNodeIdUnknown, null, DateTime.UtcNow)); + continue; + } var parsed = TwinCATSymbolPath.TryParse(def.SymbolPath); var symbolName = parsed?.ToAdsSymbolName() ?? def.SymbolPath; @@ -724,6 +732,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery return handle; } + /// Raises one data change for a native subscription. + private void RaiseDataChange(ISubscriptionHandle handle, string reference, DataValueSnapshot snapshot) => + OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, reference, snapshot)); + /// /// Registers one native notification and records its replayable intent on the device — both /// under the device so the register can't race a diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeIsolationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeIsolationTests.cs new file mode 100644 index 00000000..0d04eaf6 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeIsolationTests.cs @@ -0,0 +1,59 @@ +using System.Collections.Concurrent; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// Native-subscribe isolation (#739): an unresolvable reference publishes BadNodeIdUnknown +/// (U12) instead of keeping whatever status the server seeded, and one failing registration does not +/// take down the rest of the subscription (H11). +/// +[Trait("Category", "Unit")] +public sealed class TwinCATNativeIsolationTests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + + private static TwinCATDriver NewNativeDriver(FakeTwinCATClientFactory factory, params TwinCATTagDefinition[] tags) => + new(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = TwinCATRawTags.Entries(tags), + Probe = new TwinCATProbeOptions { Enabled = false }, + UseNativeNotifications = true, + }, "drv-1", factory); + + /// + /// U12: an unknown reference and a tag routed to an unconfigured device each publish one + /// BadNodeIdUnknown snapshot; the resolvable tag still registers and delivers Good (positive control). + /// + [Fact] + public async Task Native_subscribe_publishes_BadNodeIdUnknown_for_unresolvable_references() + { + var factory = new FakeTwinCATClientFactory(); + var drv = NewNativeDriver(factory, + new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt), + new TwinCATTagDefinition("Orphan", "ads://9.9.9.9.1.1:851", "MAIN.Orphan", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + + _ = await drv.SubscribeAsync(["Ghost", "Speed", "Orphan"], TimeSpan.FromMilliseconds(100), + TestContext.Current.CancellationToken); + + var ghost = events.Single(e => e.FullReference == "Ghost"); + ghost.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + ghost.Snapshot.Value.ShouldBeNull(); + events.Single(e => e.FullReference == "Orphan").Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + + // Positive control: the resolvable tag registered and its push arrives Good. + factory.Clients.ShouldHaveSingleItem().Notifications.ShouldHaveSingleItem().SymbolPath.ShouldBe("MAIN.Speed"); + factory.Clients[0].FireNotification("MAIN.Speed", 42); + var good = events.Single(e => e.FullReference == "Speed"); + good.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good); + good.Snapshot.Value.ShouldBe(42); + } +} From a85d1c069a477abb308051303c1fb0a9ccf62b4b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 13:59:10 -0400 Subject: [PATCH 27/51] fix(scripting): quarantine a refusal behind a stuck orphan (fix round 2) Ruling refinement on I1. Only timeouts are strikes, so a never-returning while(true), which times out once and is then only refused, was never quarantined, and that defeated E1. A refusal now quarantines immediately when the orphan it was refused behind has been executing for longer than ScriptingOptions.StuckOrphanQuarantineFactor x the script's timeout (default 10, so 20 s at the 2 s default; 0 or less disables it). It uses the same quarantine path: one Error, otopcua.script.quarantined with reason=stuck, Bad, lifted on source change or redeploy. A young orphan (a short overrun) is still a plain refusal. The guard stamps the run's start (Stopwatch timestamp) when the pool picks it up and clears it when the run really finishes. Tests: the 30-trigger while(true) regression (paced) now ends quarantined with reason "stuck", still with exactly one orphan and a healthy sibling. Positive control: a 2x overrun is refused, not quarantined, and evaluates again once the orphan completes. A factor of 0 disables the rule. Docs (ScriptEditor "Runtime safety", VirtualTags) and the seam remarks are updated. Refs #734 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/ScriptEditor.md | 11 ++- docs/VirtualTags.md | 2 +- .../ScriptedAlarmEngine.cs | 2 +- .../ScriptRunGuard.cs | 36 +++++++- .../ScriptingOptions.cs | 13 +++ .../CalculationEvaluator.cs | 2 +- .../Engines/RoslynVirtualTagEvaluator.cs | 2 +- .../ScriptRunGuardTests.cs | 88 ++++++++++++++++--- 8 files changed, 133 insertions(+), 23 deletions(-) diff --git a/docs/ScriptEditor.md b/docs/ScriptEditor.md index 41421d34..eb6985e2 100644 --- a/docs/ScriptEditor.md +++ b/docs/ScriptEditor.md @@ -190,9 +190,14 @@ scripted-alarm predicates) runs scripts through the shared `ScriptRunGuard`: Only a timeout is a strike: a refusal neither adds to nor resets the streak, and a run that returns (value or throw) resets it. A quarantined script is never dispatched again; it reads Bad `"script quarantined: …"`, logs one Error and - increments `otopcua_script_quarantined`. An infinite loop therefore usually shows - as *refused* (one timeout, then refusals for as long as the orphan spins) rather - than *quarantined*. + increments `otopcua_script_quarantined`. +- **Stuck-orphan quarantine.** A never-returning script (`while(true)`) times out + once and is then only refused, so the timeout streak alone never reaches 3. + Instead, a refusal behind an orphan that has been running longer than + `Scripting:StuckOrphanQuarantineFactor` × the timeout (default 10 × — 20 s at the + 2 s default) quarantines immediately, through the same path (Error log, metric + `reason=stuck`, Bad). A short overrun (a young orphan) stays a plain refusal and + the script resumes once the orphan completes. - **Allocation budget.** A single run allocating more than `Scripting:MaxAllocBytesPerRun` (64 MB) is quarantined. This is sampled after the run returns, so a run that OOMs the process first is not stopped — that needs an diff --git a/docs/VirtualTags.md b/docs/VirtualTags.md index 50ef578f..183fcef0 100644 --- a/docs/VirtualTags.md +++ b/docs/VirtualTags.md @@ -34,7 +34,7 @@ User scripts are compiled via `Microsoft.CodeAnalysis.CSharp` (regular compiler, #### Known resource limits (accepted trade-offs) -The sandbox cannot pre-empt a script. **Memory:** each run's own-thread allocation is sampled (`GC.GetAllocatedBytesForCurrentThread`) and a run over `Scripting:MaxAllocBytesPerRun` (default 64 MB) quarantines the script — but the sample is taken after the run returns, so a script that allocates fast enough (`new byte[int.MaxValue]` in a loop) can still OOM the process first. **CPU:** a tight loop keeps its pool thread after the timeout fires; `ScriptRunGuard` bounds that to one orphan per script by refusing further runs while it spins (Bad `"previous evaluation still running"`) and quarantines after three consecutive timeouts (a refusal is not a strike). Both are mitigations; pre-emptive CPU/memory limits need an out-of-process runner. Script authoring is gated behind the Admin permission as the primary control. See `docs/ScriptEditor.md` §"Runtime safety". +The sandbox cannot pre-empt a script. **Memory:** each run's own-thread allocation is sampled (`GC.GetAllocatedBytesForCurrentThread`) and a run over `Scripting:MaxAllocBytesPerRun` (default 64 MB) quarantines the script — but the sample is taken after the run returns, so a script that allocates fast enough (`new byte[int.MaxValue]` in a loop) can still OOM the process first. **CPU:** a tight loop keeps its pool thread after the timeout fires; `ScriptRunGuard` bounds that to one orphan per script by refusing further runs while it spins (Bad `"previous evaluation still running"`) and quarantines after three consecutive timeouts (a refusal is not a strike) — or immediately on a refusal behind an orphan that has run longer than `Scripting:StuckOrphanQuarantineFactor` (default 10) × the timeout, which is what quarantines a never-returning `while(true)`. Both are mitigations; pre-emptive CPU/memory limits need an out-of-process runner. Script authoring is gated behind the Admin permission as the primary control. See `docs/ScriptEditor.md` §"Runtime safety". `System.Threading.Tasks` is denied (Core.Scripting-003), which prevents `Task.Run` / `Parallel` fan-out that would spawn background work outliving the timeout. diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs index c96c8c87..17b513ac 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.ScriptedAlarms/ScriptedAlarmEngine.cs @@ -76,7 +76,7 @@ public sealed class ScriptedAlarmEngine : IDisposable /// /// Per-alarm run quarantine (keyed by alarm id, source identity = predicate source). Refuses a /// predicate run while the alarm's previous (timed-out, abandoned) run is still executing and - /// quarantines after three consecutive timeouts (a refusal is not a strike) or an allocation-budget breach — one orphan + /// quarantines after three consecutive timeouts (a refusal is not a strike unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — one orphan /// per alarm instead of one per upstream push. Reset on every (a redeploy). /// private readonly ScriptRunGuard _guard; diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs index dc308ce6..de1afee2 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptRunGuard.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Diagnostics; using System.Diagnostics.Metrics; using Serilog; using ZB.MOM.WW.OtOpcUa.Commons.Observability; @@ -27,7 +28,10 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Scripting; /// consecutive timeouts (only a timeout is a strike; a refusal is a Bad publish plus its counter /// and leaves the streak untouched — on a 1 Hz trigger with a 2 s budget one 4 s overrun is a /// timeout plus two refusals and must not quarantine; a run that returns — a value or a throw — -/// resets the streak): no further dispatch, every attempt throws +/// resets the streak), or on a refusal behind a stuck orphan — one that has been executing for +/// longer than × the timeout (20 s at the +/// 2 s default), which is how a never-returning while(true) (one timeout, then only refusals) +/// gets quarantined: no further dispatch, every attempt throws /// , one Error log, one otopcua.script.quarantined /// increment, one event. Lifted by a source change (the operator fixed the /// script — a new source identity under the same key starts clean and the old orphan no longer blocks @@ -63,11 +67,11 @@ public sealed class ScriptRunGuard /// /// Counter otopcua.script.quarantined (Prometheus: otopcua_script_quarantined): one - /// increment per script quarantined, tagged reason = timeouts | allocation. + /// increment per script quarantined, tagged reason = timeouts | stuck | allocation. /// public static readonly Counter QuarantinedCounter = OtOpcUaTelemetry.Meter.CreateCounter("otopcua.script.quarantined", unit: "{script}", - description: "Scripts quarantined by the run guard (reason: timeouts | allocation)."); + description: "Scripts quarantined by the run guard (reason: timeouts | stuck | allocation)."); /// /// Counter otopcua.script.run_refused: runs refused because the script's previous run was @@ -80,6 +84,7 @@ public sealed class ScriptRunGuard private readonly ConcurrentDictionary _keys = new(StringComparer.Ordinal); private readonly long _maxAllocBytesPerRun; private readonly int _quarantineThreshold; + private readonly double _stuckFactor; private readonly ILogger _logger; private int _inFlight; private long _started; @@ -94,6 +99,7 @@ public sealed class ScriptRunGuard options ??= new ScriptingOptions(); _maxAllocBytesPerRun = options.MaxAllocBytesPerRun; _quarantineThreshold = Math.Max(1, options.QuarantineAfterConsecutiveTimeouts); + _stuckFactor = options.StuckOrphanQuarantineFactor; _logger = logger ?? Log.Logger; } @@ -185,6 +191,24 @@ public sealed class ScriptRunGuard // ran long once. The refusal also leaves the timeout streak untouched. Interlocked.Increment(ref _refused); RefusedCounter.Add(1, new KeyValuePair("script", key)); + // …unless the orphan is STUCK: a never-returning script times out once and is then only ever + // refused, so the timeout streak alone would never quarantine it. Past the stuck threshold the + // refusal itself quarantines (same path: Error log, metric, Bad, lifted on redeploy/source change). + if (_stuckFactor > 0 && Volatile.Read(ref state.RunStartedTimestamp) is var startedAt and not 0) + { + var running = Stopwatch.GetElapsedTime(startedAt); + var threshold = timeout * _stuckFactor; + if (running > threshold) + { + Quarantine(key, state, + $"its previous evaluation has been running for {running.TotalSeconds:F1} s, over " + + $"{_stuckFactor:0.##}x its {timeout.TotalMilliseconds:F0} ms budget " + + "(Scripting:StuckOrphanQuarantineFactor) — it is stuck; the script is not evaluated again " + + "until its source changes or it is redeployed", + "stuck"); + throw new ScriptQuarantinedException(key, state.QuarantineReason!); + } + } throw new ScriptRunRefusedException(key); } @@ -200,6 +224,7 @@ public sealed class ScriptRunGuard // never complete, and the slot would never be released. The evaluator observes the token instead. var runTask = Task.Run(async () => { + Volatile.Write(ref state.RunStartedTimestamp, Stopwatch.GetTimestamp()); started.TrySetResult(); var before = GC.GetAllocatedBytesForCurrentThread(); try @@ -223,6 +248,7 @@ public sealed class ScriptRunGuard // still quarantines its key here. _ = runTask.ContinueWith(t => { + Volatile.Write(ref state.RunStartedTimestamp, 0); Interlocked.Exchange(ref state.InFlight, 0); Interlocked.Decrement(ref _inFlight); runCts.Dispose(); @@ -339,6 +365,8 @@ public sealed class ScriptRunGuard public string? SourceIdentity = sourceIdentity; public int InFlight; public int ConsecutiveTimeouts; + /// when the pool started the in-flight run; 0 when idle. + public long RunStartedTimestamp; public string? QuarantineReason; } @@ -373,7 +401,7 @@ public sealed class ScriptRunRefusedException : Exception /// /// Thrown by for a quarantined script: repeated -/// timeouts or an allocation-budget breach. Nothing is dispatched while quarantined; the seam +/// timeouts, a stuck orphan or an allocation-budget breach. Nothing is dispatched while quarantined; the seam /// publishes Bad carrying the reason, which an operator clears by fixing the script (source change) or /// redeploying. /// diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs index f101a0e1..296d862c 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting/ScriptingOptions.cs @@ -25,6 +25,9 @@ public sealed class ScriptingOptions /// Default : 3. public const int DefaultQuarantineAfterConsecutiveTimeouts = 3; + /// Default : 10. + public const double DefaultStuckOrphanQuarantineFactor = 10; + /// /// Managed bytes one script run may allocate on its own thread (sampled with /// around the run) before the script is @@ -40,4 +43,14 @@ public sealed class ScriptingOptions /// throw) resets it. Default 3; values below 1 are treated as 1. /// public int QuarantineAfterConsecutiveTimeouts { get; set; } = DefaultQuarantineAfterConsecutiveTimeouts; + + /// + /// A refusal quarantines the script immediately when the abandoned run it was refused behind has been + /// executing for longer than this many times the script's timeout (20 s at the 2 s default). This is + /// what quarantines a permanently stuck script (while(true)): it times out once and is then + /// only ever refused, so the timeout streak alone would never reach + /// . A young orphan (a short overrun) is still just a + /// refusal. Zero or negative disables the rule. Default 10. + /// + public double StuckOrphanQuarantineFactor { get; set; } = DefaultStuckOrphanQuarantineFactor; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs index e47cc745..dd7085e3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Calculation/CalculationEvaluator.cs @@ -42,7 +42,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Calculation; /// Run quarantine. Every compiled run goes through one keyed by /// calc-tag RawPath with the script source as the identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive -/// timeouts (a refusal is Bad but not a strike) or an allocation-budget breach — both surface as a +/// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach — both surface as a /// the driver maps to Bad. The driver's evaluation pump awaits rather than /// blocking a pool thread on the sync , /// and the guard starts the budget clock only once the pool has picked the script up, so pool starvation diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs index f0bbd150..8a9347d9 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Engines/RoslynVirtualTagEvaluator.cs @@ -35,7 +35,7 @@ namespace ZB.MOM.WW.OtOpcUa.Host.Engines; /// Run quarantine. Every compiled run goes through one keyed by /// virtual-tag id with the expression as the source identity: a run is refused while the tag's previous /// (timed-out, abandoned) run is still executing, and the tag is quarantined after three consecutive -/// timeouts (a refusal is Bad but not a strike) or an allocation-budget breach. Both surface as +/// timeouts (a refusal is Bad but not a strike, unless its orphan is stuck past StuckOrphanQuarantineFactor × the budget) or an allocation-budget breach. Both surface as /// so VirtualTagActor publishes Bad exactly as it does for a timeout. A quarantine lifts when the /// expression changes, on / (the deploy /// boundary), never by itself. diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs index f60045b9..b4efe8af 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Scripting.Tests/ScriptRunGuardTests.cs @@ -151,10 +151,13 @@ public sealed class ScriptRunGuardTests } /// - /// The thread-leak regression. Pre-guard, a while(true) script on a 1 Hz trigger leaked one - /// pool thread per trigger — 30 orphans after 30 triggers, trending toward pool starvation. Now the - /// first trigger's orphan is the only one, every later trigger is refused, and a healthy - /// sibling keeps evaluating throughout (the positive control an absence assertion needs). + /// The thread-leak regression, and the stuck-orphan quarantine (fix round 2). Pre-guard, a + /// while(true) script on a 1 Hz trigger leaked one pool thread per trigger — 30 orphans after 30 + /// triggers, trending toward pool starvation. Now the first trigger's orphan is the only one; later + /// triggers are refused while it is young, and once it has run longer than + /// × the timeout (10 × 60 ms here — the + /// trigger cadence is scaled down from 1 Hz to keep the test short) the next refusal quarantines it. + /// A healthy sibling keeps evaluating throughout (the positive control an absence assertion needs). /// /// /// The load-bearing assertions are on the guard's own bookkeeping ( @@ -163,30 +166,37 @@ public sealed class ScriptRunGuardTests /// it must stay far below the 30 the leak would have produced. /// [Fact] - public async Task Runaway_script_leaves_at_most_one_orphan_after_30_triggers_while_a_sibling_keeps_evaluating() + public async Task Runaway_script_leaves_one_orphan_is_quarantined_once_stuck_and_a_sibling_keeps_evaluating() { var guard = new ScriptRunGuard(); using var spin = Compile(SpinScript); using var fast = Compile(FastScript); var ct = TestContext.Current.CancellationToken; var threadsBefore = ThreadPool.ThreadCount; + var stuckAfter = ShortTimeout * ScriptingOptions.DefaultStuckOrphanQuarantineFactor; // 600 ms var healthyResults = new List(); + var outcomes = new List(); + var sw = System.Diagnostics.Stopwatch.StartNew(); for (var trigger = 0; trigger < 30; trigger++) { - try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); } - catch (ScriptTimeoutException) { } - catch (ScriptRunRefusedException) { } - catch (ScriptQuarantinedException) { } + try { await Run(guard, "runaway", SpinScript, spin, ShortTimeout, ct); outcomes.Add("value"); } + catch (ScriptTimeoutException) { outcomes.Add("timeout"); } + catch (ScriptRunRefusedException) { outcomes.Add(sw.Elapsed < stuckAfter ? "refused-young" : "refused"); } + catch (ScriptQuarantinedException) { outcomes.Add("quarantined"); } healthyResults.Add(await Run(guard, "healthy", FastScript, fast, ct: ct)); + await Task.Delay(50, ct); } guard.InFlightCount.ShouldBe(1, "exactly one orphaned run — the first trigger's — may be executing"); guard.StartedRunCount.ShouldBe(1 + 30, "one dispatch for the runaway script, thirty for the healthy sibling"); - // One timeout + 29 refusals: not quarantined (a refusal is not a strike), but never re-dispatched. - guard.IsQuarantined("runaway", out _).ShouldBeFalse(); - guard.RefusedWhileRunningCount.ShouldBe(29); + outcomes[0].ShouldBe("timeout"); + outcomes.ShouldContain("refused-young", "a young orphan is only refused — a short overrun is not quarantined"); + guard.IsQuarantined("runaway", out var reason).ShouldBeTrue("the orphan outlived 10x its budget: it is stuck"); + reason!.ShouldContain("stuck"); + outcomes[^1].ShouldBe("quarantined"); + guard.QuarantinedCount.ShouldBe(1); healthyResults.ShouldAllBe(v => v == 42); healthyResults.Count.ShouldBe(30); @@ -194,6 +204,60 @@ public sealed class ScriptRunGuardTests threadDelta.ShouldBeLessThan(15, "thirty leaked orphans would have grown the pool by ~30 threads"); } + /// + /// Positive control for the stuck rule: a 2x overrun (one timeout, refusals while the orphan is young) + /// is NOT quarantined, and once the orphan completes the script evaluates normally again. + /// + [Fact] + public async Task A_2x_overrun_is_not_quarantined_and_resumes_evaluating_once_the_orphan_completes() + { + var guard = new ScriptRunGuard(); + const string source = + """ + var ms = (int)(ctx.GetTag("spinMs").Value ?? 0); + var end = System.DateTime.UtcNow.AddMilliseconds(ms); + while (System.DateTime.UtcNow < end) { } + return 7; + """; + using var eval = Compile(source); + var ct = TestContext.Current.CancellationToken; + var timeout = TimeSpan.FromMilliseconds(100); + + FakeScriptContext Ctx(int spinMs) + { + var c = new FakeScriptContext(); + c.Tags["spinMs"] = new Core.Abstractions.DataValueSnapshot(spinMs, 0u, c.Now, c.Now); + return c; + } + + // Warm the JIT so the first measured run's overrun is the script's, not the compiler's. + (await guard.RunAsync("k", source, eval, Ctx(0), TimeSpan.FromSeconds(5), ct)).ShouldBe(7); + + // 2x overrun: 200 ms against a 100 ms budget — times out, then is refused while it finishes. + await Should.ThrowAsync(() => guard.RunAsync("k", source, eval, Ctx(200), timeout, ct)); + await Should.ThrowAsync(() => guard.RunAsync("k", source, eval, Ctx(0), timeout, ct)); + await WaitForIdle(guard); + + guard.IsQuarantined("k", out _).ShouldBeFalse("a 2x overrun is far below the 10x stuck threshold"); + (await guard.RunAsync("k", source, eval, Ctx(0), timeout, ct)).ShouldBe(7, "the script resumes evaluating"); + guard.QuarantinedCount.ShouldBe(0); + } + + /// The stuck rule is a knob: with ≤ 0 a + /// stuck orphan is only ever refused. + [Fact] + public async Task StuckOrphanQuarantineFactor_zero_disables_the_stuck_rule() + { + var guard = new ScriptRunGuard(new ScriptingOptions { StuckOrphanQuarantineFactor = 0 }); + using var spin = Compile(SpinScript); + var ct = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct)); + await Task.Delay(ShortTimeout * 12, ct); + await Should.ThrowAsync(() => Run(guard, "k", SpinScript, spin, ShortTimeout, ct)); + guard.IsQuarantined("k", out _).ShouldBeFalse(); + } + [Fact] public async Task A_source_change_clears_the_quarantine_and_abandons_the_old_orphan_slot() { From 17f4bf6d33e9ba30845cfdd61607e9ab77fa44d5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:00:38 -0400 Subject: [PATCH 28/51] fix(twincat): isolate native registrations per tag and per device (H11) Native subscribe was all-or-nothing: any registration failure other than a type refusal disposed every registration and rethrew, so one typo'd symbol or one unreachable PLC in a multi-device driver served zero data, churned register/delete every reconcile tick, and left health Healthy. Now each registration fails on its own: a refusal (the device answered: symbol not found, PLC not ready, type mismatch) publishes its status with no health write; any other per-tag failure publishes BadCommunicationError and degrades; a connect failure marks the device unreachable for the rest of the call and stores its tags as dead intents. Every failed tag is kept as a dead intent that the probe-tick retry and the reconnect replay re-register. The ADS client now classifies a failed symbol resolve / AddDeviceNotification by status instead of throwing InvalidOperationException for all of them. Cancellation still tears the partial subscription down. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdsTwinCATClient.cs | 21 ++- .../TwinCATDriver.cs | 105 ++++++++--- .../TwinCATNativeIsolationTests.cs | 166 ++++++++++++++++++ .../TwinCATNativeNotificationTests.cs | 24 ++- 4 files changed, 278 insertions(+), 38 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs index c7ab8850..53481b5e 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs @@ -523,8 +523,7 @@ internal sealed class AdsTwinCATClient : ITwinCATClient var (parent, _, _, parentStatus) = await DescribeSymbolAsync(registerPath, cancellationToken) .ConfigureAwait(false); if (parentStatus != TwinCATStatusMapper.Good) - throw new InvalidOperationException( - $"Resolving {registerPath} for a notification failed with status 0x{parentStatus:X8}"); + throw RegistrationFailure(registerPath, "resolving the parent word", parentStatus); if (TwinCATBitAccess.CheckParent(bit, parent) is { } bitReason) throw new TwinCATTypeMismatchException(symbolPath, bitReason); clrType = typeof(byte[]); @@ -538,8 +537,7 @@ internal sealed class AdsTwinCATClient : ITwinCATClient var (shape, array, _, symbolStatus) = await DescribeSymbolAsync(symbolPath, cancellationToken) .ConfigureAwait(false); if (symbolStatus != TwinCATStatusMapper.Good) - throw new InvalidOperationException( - $"Resolving {symbolPath} for a notification failed with status 0x{symbolStatus:X8}"); + throw RegistrationFailure(symbolPath, "resolving the symbol", symbolStatus); if (arrayCount is int count) { @@ -582,8 +580,8 @@ internal sealed class AdsTwinCATClient : ITwinCATClient registerPath, settings, userData: null, clrType, anyTypeArgs, cancellationToken) .ConfigureAwait(false); if (result.ErrorCode != AdsErrorCode.NoError) - throw new InvalidOperationException( - $"AddDeviceNotificationExAsync failed with ADS error {result.ErrorCode} for {registerPath}"); + throw RegistrationFailure(registerPath, $"AddDeviceNotificationEx (ADS error {result.ErrorCode})", + MapAndSignal((uint)result.ErrorCode)); var reg = new NotificationRegistration(symbolPath, type, bitIndex, onChange, this, result.Handle) { @@ -594,6 +592,17 @@ internal sealed class AdsTwinCATClient : ITwinCATClient return reg; } + /// + /// Classifies a failed notification registration. A status proving the device ANSWERED (symbol not + /// found, invalid access, PLC not ready, …) is a per-tag refusal the driver isolates without a health + /// write; a comms-class status (BadCommunicationError / BadTimeout) stays an ordinary + /// exception — a comms fault the driver degrades on and retries. + /// + internal static Exception RegistrationFailure(string symbolPath, string step, uint status) => + status is TwinCATStatusMapper.BadCommunicationError or TwinCATStatusMapper.BadTimeout + ? new InvalidOperationException($"{step} for {symbolPath} failed with status 0x{status:X8}") + : new TwinCATNotificationRefusedException(symbolPath, $"{step} failed with status 0x{status:X8}", status); + /// /// Runs on the AMS router thread. Does the cheap decode (bit /// extraction, string, array) then enqueues — no driver logic, no consumer callbacks — so a slow consumer diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index 1dda4ad7..021c4e32 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -661,6 +661,9 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery var id = Interlocked.Increment(ref _nextNativeSubId); var handle = new NativeSubscriptionHandle(id); var registrations = new List(fullReferences.Count); + // Devices whose connect failed during THIS call: the rest of their tags are stored as dead + // intents straight away rather than each paying (and logging) its own failed connect. + HashSet? unreachable = null; try { @@ -686,44 +689,69 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery var reg = new NativeRegistration( Interlocked.Increment(ref _nextRegId), device, reference, symbolName, def.DataType, bitIndex, def.ArrayLength, publishingInterval, - onChange: (_, value) => OnDataChange?.Invoke(this, - new DataChangeEventArgs(handle, reference, new DataValueSnapshot( - value, TwinCATStatusMapper.Good, DateTime.UtcNow, DateTime.UtcNow))), + onChange: (_, value) => RaiseDataChange(handle, reference, new DataValueSnapshot( + value, TwinCATStatusMapper.Good, DateTime.UtcNow, DateTime.UtcNow)), // When the handle goes dead (a failed replay) push a Bad snapshot so // subscribers see the tag go Bad instead of a frozen-Good-stale value. - onUnavailable: () => OnDataChange?.Invoke(this, - new DataChangeEventArgs(handle, reference, new DataValueSnapshot( - null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)))); + onUnavailable: () => RaiseDataChange(handle, reference, new DataValueSnapshot( + null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow))); + registrations.Add(reg); + + if (unreachable?.Contains(device) == true) + { + // Same device already failed to connect in this call: keep the intent dead for the + // probe-tick retry / reconnect replay and surface Bad now. + device.NativeRegistrations[reg.Id] = reg; + RaiseDataChange(handle, reference, new DataValueSnapshot( + null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)); + continue; + } try { await RegisterNotificationAsync(device, reg, cancellationToken).ConfigureAwait(false); } + catch (OperationCanceledException) { throw; } catch (TwinCATNotificationRefusedException refused) { - // The declared type / array length cannot decode this PLC symbol (ignitionoee audit), or the array is over the native-notification size cap: isolate - // THIS tag to Bad — one WARN, one snapshot carrying the refusal's status - // (BadTypeMismatch / BadOutOfRange) — and keep the rest of the subscription. Its - // intent stays on the device with no live handle, so the probe-tick retry re-checks - // it after a PLC program change. + // The device answered and refused THIS tag (declared type cannot decode the symbol, + // array over the notification cap, symbol not found): isolate it — one WARN, one + // snapshot carrying the refusal's status, no health write (a refusal is not a comms + // fault). Its intent stays on the device with no live handle, so the probe-tick retry + // re-checks it after a PLC program change. _logger.LogWarning( "TwinCAT driver '{DriverInstanceId}' refused native notification for '{Reference}': {Reason}", _driverInstanceId, reference, refused.Message); - OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, reference, new DataValueSnapshot( - null, refused.StatusCode, null, DateTime.UtcNow))); + RaiseDataChange(handle, reference, new DataValueSnapshot( + null, refused.StatusCode, null, DateTime.UtcNow)); + } + catch (DeviceUnavailableException unavailable) + { + // The device could not be connected (offline, router down, backoff window open). + // Every tag of this device goes Bad + dead for the retry; other devices carry on. + (unreachable ??= []).Add(device); + MarkCommsFault(unavailable.InnerException?.Message ?? unavailable.Message); + RaiseDataChange(handle, reference, new DataValueSnapshot( + null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)); + } + catch (Exception ex) + { + // A per-tag registration failure that is not a refusal (the AddDeviceNotification + // round-trip failed): this tag goes Bad + dead for the retry; the rest keep going. + _logger.LogWarning(ex, + "TwinCAT driver '{DriverInstanceId}' native-notification registration failed for " + + "'{Reference}'; it will be retried on the next healthy probe tick", + _driverInstanceId, reference); + MarkCommsFault(ex.Message); + RaiseDataChange(handle, reference, new DataValueSnapshot( + null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)); } - registrations.Add(reg); } } - catch (Exception ex) + catch (OperationCanceledException) { - // On any registration failure, tear down everything we got so far + rethrow. Leaves - // the subscription in a clean "never existed" state rather than a half-registered - // state the caller has to clean up. - _logger.LogWarning(ex, - "TwinCAT driver '{DriverInstanceId}' native-notification registration failed; " + - "tearing down {Count} partial registrations", - _driverInstanceId, registrations.Count); + // Cancelled mid-subscribe: leave the subscription in a clean "never existed" state rather + // than a half-registered one the caller has to clean up. foreach (var r in registrations) { try { r.Dispose(); } catch { } } throw; } @@ -732,6 +760,21 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery return handle; } + /// The connect step of a native registration failed — the whole device is unreachable. + private sealed class DeviceUnavailableException(Exception inner) + : Exception("TwinCAT device unreachable: " + inner.Message, inner); + + /// + /// Records a comms fault: preserving LastSuccessfulRead. + /// Never downgrades . + /// + private void MarkCommsFault(string message) + { + var current = _health; + if (current.State != DriverState.Faulted) + _health = new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, message); + } + /// Raises one data change for a native subscription. private void RaiseDataChange(ISubscriptionHandle handle, string reference, DataValueSnapshot snapshot) => OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, reference, snapshot)); @@ -748,7 +791,20 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery await device.ConnectGate.WaitAsync(ct).ConfigureAwait(false); try { - var client = await EnsureConnectedUnderGateAsync(device, ct, timeoutOverride: null).ConfigureAwait(false); + ITwinCATClient client; + try + { + client = await EnsureConnectedUnderGateAsync(device, ct, timeoutOverride: null).ConfigureAwait(false); + } + catch (OperationCanceledException) { throw; } + catch (Exception ex) + { + // Keep the intent dead (no live handle): the replay after the next successful connect + // re-registers it. The caller short-circuits the rest of this device's tags. + device.NativeRegistrations[reg.Id] = reg; + throw new DeviceUnavailableException(ex); + } + ITwinCATNotificationHandle newHandle; try { @@ -756,7 +812,8 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery reg.SymbolName, reg.DataType, reg.BitIndex, reg.ArrayLength, reg.Interval, _options.NotificationMaxDelayMs, reg.OnChange, ct).ConfigureAwait(false); } - catch (TwinCATNotificationRefusedException) + catch (OperationCanceledException) { throw; } + catch { // Keep the intent with no live handle (a "dead" registration) so the probe-tick // retry re-checks it and UnsubscribeAsync still removes it; the caller isolates it. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeIsolationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeIsolationTests.cs index 0d04eaf6..144c7570 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeIsolationTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeIsolationTests.cs @@ -56,4 +56,170 @@ public sealed class TwinCATNativeIsolationTests good.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good); good.Snapshot.Value.ShouldBe(42); } + + /// + /// H11: one tag whose AddDeviceNotification round-trip fails (not a refusal) goes Bad on its own and + /// the subscribe call SUCCEEDS — before, the exception disposed every registration and rethrew, so one + /// bad tag served zero data for the whole driver. The good tags keep delivering (positive control). + /// + [Fact] + public async Task One_failing_registration_does_not_tear_down_the_subscription() + { + var fake = new FakeTwinCATClient(); + fake.AddNotificationErrors["MAIN.B"] = new InvalidOperationException("AddDeviceNotificationEx blew up"); + var factory = new FakeTwinCATClientFactory { Customise = () => fake }; + var drv = NewNativeDriver(factory, + new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt), + new TwinCATTagDefinition("B", Host, "MAIN.B", TwinCATDataType.DInt), + new TwinCATTagDefinition("C", Host, "MAIN.C", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + + var handle = await drv.SubscribeAsync(["A", "B", "C"], TimeSpan.FromMilliseconds(100), + TestContext.Current.CancellationToken); + + fake.Notifications.Select(n => n.SymbolPath).ShouldBe(["MAIN.A", "MAIN.C"], ignoreOrder: true); + events.ShouldHaveSingleItem().FullReference.ShouldBe("B"); + events.Single().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError); + + fake.FireNotification("MAIN.C", 3); + events.Last().FullReference.ShouldBe("C"); + events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good); + + // The failed tag is kept as a dead intent and unsubscribe still removes everything. + drv.GetDeviceState(Host)!.NativeRegistrations.Count.ShouldBe(3); + await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken); + drv.GetDeviceState(Host)!.NativeRegistrations.ShouldBeEmpty(); + fake.Notifications.ShouldBeEmpty(); + } + + /// + /// H11: a refusal (symbol not found — the device answered) isolates the tag with the refusal's own + /// status and leaves driver health alone; the good tag delivers. + /// + [Fact] + public async Task A_refused_symbol_is_isolated_without_a_health_write() + { + var fake = new FakeTwinCATClient(); + fake.AddNotificationErrors["MAIN.Typo"] = new TwinCATNotificationRefusedException( + "MAIN.Typo", "resolving the symbol failed", TwinCATStatusMapper.BadNodeIdUnknown); + var factory = new FakeTwinCATClientFactory { Customise = () => fake }; + var drv = NewNativeDriver(factory, + new TwinCATTagDefinition("Typo", Host, "MAIN.Typo", TwinCATDataType.DInt), + new TwinCATTagDefinition("Good", Host, "MAIN.Good", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + + _ = await drv.SubscribeAsync(["Typo", "Good"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + + events.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + fake.FireNotification("MAIN.Good", 1); + events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good); + } + + /// + /// H11: in a two-device driver an unreachable device only blacks out ITS tags (Bad + dead intents, + /// health Degraded); the reachable device's tags register and deliver. + /// + [Fact] + public async Task An_unreachable_device_blacks_out_only_its_own_tags() + { + const string hostB = "ads://5.23.91.24.1.1:851"; + var build = 0; + var factory = new FakeTwinCATClientFactory + { + // Device A's client is created first and works; device B's connect throws. + Customise = () => new FakeTwinCATClient { ThrowOnConnect = ++build == 2 }, + }; + var drv = new TwinCATDriver(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host), new TwinCATDeviceOptions(hostB)], + RawTags = TwinCATRawTags.Entries( + [ + new TwinCATTagDefinition("A1", Host, "MAIN.A1", TwinCATDataType.DInt), + new TwinCATTagDefinition("B1", hostB, "MAIN.B1", TwinCATDataType.DInt), + new TwinCATTagDefinition("B2", hostB, "MAIN.B2", TwinCATDataType.DInt), + new TwinCATTagDefinition("A2", Host, "MAIN.A2", TwinCATDataType.DInt), + ]), + Probe = new TwinCATProbeOptions { Enabled = false }, + UseNativeNotifications = true, + }, "drv-1", factory); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + + _ = await drv.SubscribeAsync(["A1", "B1", "B2", "A2"], TimeSpan.FromMilliseconds(100), + TestContext.Current.CancellationToken); + + factory.Clients.Count.ShouldBe(2); // B2 did not attempt a second connect + factory.Clients[0].Notifications.Select(n => n.SymbolPath).ShouldBe(["MAIN.A1", "MAIN.A2"], ignoreOrder: true); + events.Select(e => e.FullReference).ShouldBe(["B1", "B2"], ignoreOrder: true); + events.ShouldAllBe(e => e.Snapshot.StatusCode == TwinCATStatusMapper.BadCommunicationError); + drv.GetDeviceState(hostB)!.NativeRegistrations.Count.ShouldBe(2); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + + factory.Clients[0].FireNotification("MAIN.A2", 22); + events.Last().FullReference.ShouldBe("A2"); + events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good); + } + + /// + /// H11: the dead intent left by a failed registration is re-registered by the probe-tick retry, and + /// its first push then arrives Good — the subscription heals without a re-subscribe. + /// + [Fact] + public async Task A_failed_registration_is_recovered_by_the_probe_tick_retry() + { + var fake = new FakeTwinCATClient(); + fake.AddNotificationErrors["MAIN.B"] = new InvalidOperationException("transient"); + var factory = new FakeTwinCATClientFactory { Customise = () => fake }; + var drv = new TwinCATDriver(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = TwinCATRawTags.Entries([new TwinCATTagDefinition("B", Host, "MAIN.B", TwinCATDataType.DInt)]), + Probe = new TwinCATProbeOptions + { + Enabled = true, + Interval = TimeSpan.FromMilliseconds(40), + Timeout = TimeSpan.FromMilliseconds(100), + }, + UseNativeNotifications = true, + }, "drv-1", factory); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + + _ = await drv.SubscribeAsync(["B"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + fake.AddNotificationErrors.Clear(); + + await WaitForAsync(() => fake.Notifications.Count == 1, TimeSpan.FromSeconds(3)); + fake.Notifications.Count.ShouldBe(1); + fake.FireNotification("MAIN.B", 5); + events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + /// A comms-class registration status stays an exception; any other status is a per-tag refusal. + [Fact] + public void Registration_failures_are_classified_by_status() + { + AdsTwinCATClient.RegistrationFailure("MAIN.X", "step", TwinCATStatusMapper.BadNodeIdUnknown) + .ShouldBeOfType().StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + AdsTwinCATClient.RegistrationFailure("MAIN.X", "step", TwinCATStatusMapper.BadOutOfService) + .ShouldBeOfType(); + AdsTwinCATClient.RegistrationFailure("MAIN.X", "step", TwinCATStatusMapper.BadCommunicationError) + .ShouldBeOfType(); + AdsTwinCATClient.RegistrationFailure("MAIN.X", "step", TwinCATStatusMapper.BadTimeout) + .ShouldBeOfType(); + } + + private static async Task WaitForAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(20); + } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeNotificationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeNotificationTests.cs index 7b40cbc0..3fdceb13 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeNotificationTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATNativeNotificationTests.cs @@ -98,13 +98,14 @@ public sealed class TwinCATNativeNotificationTests events.Count.ShouldBe(afterUnsub); } - /// Verifies that native subscribe failure mid-registration cleans up partial state. + /// + /// A registration failing mid-subscribe isolates that one tag (#739 H11): the first registration + /// stays live, the second is kept as a dead intent and goes Bad, and the call does not throw. + /// [Fact] - public async Task Native_subscribe_failure_mid_registration_cleans_up_partial_state() + public async Task Native_subscribe_failure_mid_registration_isolates_the_failed_tag() { // Fail-on-second-call fake — first AddNotificationAsync succeeds, second throws. - // Subscribe's catch block must tear the first one down before rethrowing so no zombie - // notification lingers. var fake = new FailAfterNAddsFake(new AbTagParamsIrrelevant(), succeedBefore: 1); var factory = new FakeTwinCATClientFactory { Customise = () => fake }; var drv = new TwinCATDriver(new TwinCATDriverOptions @@ -120,12 +121,19 @@ public sealed class TwinCATNativeNotificationTests }, "drv-1", factory); await drv.InitializeAsync("{}", CancellationToken.None); - await Should.ThrowAsync(() => - drv.SubscribeAsync(["A", "B"], TimeSpan.FromMilliseconds(100), CancellationToken.None)); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + + var handle = await drv.SubscribeAsync(["A", "B"], TimeSpan.FromMilliseconds(100), CancellationToken.None); - // First registration succeeded then got torn down by the catch; second threw. fake.AddCallCount.ShouldBe(2); - fake.Notifications.Count.ShouldBe(0); // partial handle cleaned up + fake.Notifications.ShouldHaveSingleItem().SymbolPath.ShouldBe("MAIN.A"); + var bad = events.ShouldHaveSingleItem(); + bad.FullReference.ShouldBe("B"); + bad.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError); + + await drv.UnsubscribeAsync(handle, CancellationToken.None); + fake.Notifications.ShouldBeEmpty(); } /// From a32894e93b5eaa1339be6afa9f4976b4bf072b8d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:03:08 -0400 Subject: [PATCH 29/51] fix(focas): build-and-swap the device table; make the lifecycle overlap-safe (R07:H13) _devices was a plain Dictionary cleared and refilled in place by InitializeAsync / ShutdownAsync while the retained poll engine, the fixed-tree and alarm loops and the actor's health tick enumerated it ("Collection was modified"). It is now built whole and published by reference swap, and a release swaps it out to empty before tearing it down. Lane A's runtime overlaps lifecycle calls, so the driver now tolerates both shapes: - a lifecycle generation is bumped by every Initialize/Shutdown; an initialise that finds itself superseded (abandoned by the runtime, then a shutdown ran) publishes nothing, starts no loops, writes no health and throws OperationCanceledException; - a shutdown overlapping another (dispose beside a still-running shutdown) waits for the first's teardown via a release tail instead of returning on an empty table; - released devices are Retired: EnsureConnectedAsync refuses to reconnect them, and a connect that completes after retirement disposes what it opened. Clients are disposed under the device ConnectGate (bounded). Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../FocasDriver.cs | 334 ++++++++++++------ .../FakeFocasClient.cs | 2 +- .../FocasDeviceTableLifecycleTests.cs | 200 +++++++++++ 3 files changed, 424 insertions(+), 112 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasDeviceTableLifecycleTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs index cecc6299..f0c0226f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs @@ -43,7 +43,40 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, private readonly Func? _rebind; private readonly PollGroupEngine _poll; private readonly ILogger _logger; - private readonly Dictionary _devices = new(StringComparer.OrdinalIgnoreCase); + /// + /// Host address → device state. Built whole in and SWAPPED in (and swapped + /// out whole by a release), never cleared or refilled in place: the retained poll engine, the fixed-tree + /// and alarm loops and the actor's health tick all read it concurrently with a re-initialise, and a + /// mutated under an enumerator throws "Collection was modified". + /// Readers take one Volatile.Read snapshot and use it for the whole operation. + /// + private IReadOnlyDictionary _devices = EmptyDevices; + + private static readonly IReadOnlyDictionary EmptyDevices = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + /// The live device table (one consistent snapshot). + private IReadOnlyDictionary Devices => Volatile.Read(ref _devices); + + /// + /// Guards the publish/retire of the device table against an overlapping lifecycle call. Held only for + /// synchronous swaps — never across an await, and nothing a loop runs ever takes it. + /// + private readonly object _lifecycleLock = new(); + + /// + /// Bumped by every and . An initialise that finds + /// the counter moved on by the time it would publish has been superseded (the runtime abandoned it and + /// tore the driver down) and must start nothing. + /// + private int _lifecycleGeneration; + + /// + /// Completes when every device table released so far has finished its teardown. A shutdown that overlaps + /// another (Lane A's dispose beside a still-running shutdown) awaits it, so it never returns while the + /// first one is still joining loops and disposing clients. + /// + private Task _releaseTail = Task.CompletedTask; // v3: RawPath → definition, case-sensitive ordinal. Built in InitializeAsync from _options.RawTags // via the pure FocasTagDefinitionFactory mapper (the reference is always a RawPath). // Built whole in InitializeAsync and SWAPPED in: the retained poll engine keeps calling @@ -196,9 +229,14 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // release the previous device states' loops + clients BEFORE they are overwritten (a live driver can be // re-initialised without a teardown first). Not ShutdownAsync: that also drops _poll // and subscriptions, which are owned by the shutdown path (the actor re-subscribes after every successful init). + // + // The generation is taken FIRST: an initialise the runtime abandons (it timed out and moved on to a + // shutdown) is superseded by that shutdown's bump, and checks for it before publishing anything. + var generation = Interlocked.Increment(ref _lifecycleGeneration); await ReleaseDeviceStatesAsync().ConfigureAwait(false); ResetAlarmHealth(); - Volatile.Write(ref _health, new DriverHealth(DriverState.Initializing, null, null)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Initializing, null, null)); + var devices = new Dictionary(StringComparer.OrdinalIgnoreCase); try { // adopt a changed config. Options and the backend client factory are re-derived by ONE @@ -222,7 +260,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, var addr = FocasHostAddress.TryParse(device.HostAddress) ?? throw new InvalidOperationException( $"FOCAS device has invalid HostAddress '{device.HostAddress}' — expected 'focas://{{ip}}[:{{port}}]'."); - _devices[device.HostAddress] = new DeviceState(addr, device); + devices[device.HostAddress] = new DeviceState(addr, device); } // Build the RawPath → definition table from the authored raw tags. Each entry's TagConfig // blob is mapped by the pure factory; the entry's WriteIdempotent flag + DeviceName are @@ -256,7 +294,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, throw new InvalidOperationException( $"FOCAS tag '{entry.RawPath}' references device '{entry.DeviceName}' " + $"which is not in the Devices list. Check for a typo (e.g. wrong port or hostname)."); - if (!_devices.TryGetValue(deviceHost, out var device)) + if (!devices.TryGetValue(deviceHost, out var device)) throw new InvalidOperationException( $"FOCAS tag '{entry.RawPath}' references device '{entry.DeviceName}' " + $"which is not in the Devices list. Check for a typo (e.g. wrong port or hostname)."); @@ -309,75 +347,135 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // above; reusing the parsed record avoids per-tick allocs on subscription pollers. parsedAddresses[entry.RawPath] = parsed; } - // Publish the skip tally only once the whole table is built. An unknown-device throw above - // leaves the previous snapshot standing rather than a truncated one that would read as - // "we now skip fewer tags". - _skippedTags.Commit(); - Volatile.Write(ref _parsedAddressesByTagName, parsedAddresses); - Volatile.Write(ref _tagsByRawPath, tagsByRawPath); - Volatile.Write(ref _alarmTags, alarmTags); - - if (_options.Probe.Enabled) + // Publish everything at once, and only if no later lifecycle call has superseded this one. The check + // and the publish share the lock with the shutdown's swap-out, so either the shutdown sees (and + // releases) this table, or this initialise sees the shutdown and publishes nothing. + lock (_lifecycleLock) { - foreach (var state in _devices.Values) - { - state.ProbeCts = new CancellationTokenSource(); - var ct = state.ProbeCts.Token; - state.ProbeTask = Task.Run(() => ProbeLoopAsync(state, ct), ct); - } - } + if (generation != Volatile.Read(ref _lifecycleGeneration)) + throw new OperationCanceledException( + $"FOCAS initialise of {_driverInstanceId} was superseded by a later shutdown/initialise; nothing was started."); - if (_options.HandleRecycle.Enabled) - { - foreach (var state in _devices.Values) - { - state.RecycleCts = new CancellationTokenSource(); - var ct = state.RecycleCts.Token; - state.RecycleTask = Task.Run(() => RecycleLoopAsync(state, ct), ct); - } - } - - // See _fixedTreeActive: an authored FIXED: tag arms the loop on its own, because the - // config flag defaults off and the operator who committed those tags has already said - // what they want. - _fixedTreeActive = _options.FixedTree.Enabled - || _parsedAddressesByTagName.Values.Any(a => a.Kind == FocasAreaKind.Fixed); - if (_fixedTreeActive) - { - foreach (var state in _devices.Values) - { - state.FixedTreeCts = new CancellationTokenSource(); - var ct = state.FixedTreeCts.Token; - state.FixedTreeTask = Task.Run(() => FixedTreeLoopAsync(state, ct), ct); - } + // Publish the skip tally only once the whole table is built. An unknown-device throw above + // leaves the previous snapshot standing rather than a truncated one that would read as + // "we now skip fewer tags". + _skippedTags.Commit(); + Volatile.Write(ref _parsedAddressesByTagName, parsedAddresses); + Volatile.Write(ref _tagsByRawPath, tagsByRawPath); + Volatile.Write(ref _alarmTags, alarmTags); + // See _fixedTreeActive: an authored FIXED: tag arms the loop on its own, because the + // config flag defaults off and the operator who committed those tags has already said + // what they want. + _fixedTreeActive = _options.FixedTree.Enabled + || parsedAddresses.Values.Any(a => a.Kind == FocasAreaKind.Fixed); + Volatile.Write(ref _devices, devices); + StartDeviceLoops(devices.Values); } // Restart the alarm poll a re-initialise paused, against the tables just rebuilt. A // driver with no subscription yet starts nothing — the actor subscribes once it is Connected. await _alarmProjection.ResumeAsync().ConfigureAwait(false); + // A shutdown that landed while the resume ran owns the projection now: put it back the way it left it. + if (generation != Volatile.Read(ref _lifecycleGeneration)) + { + await _alarmProjection.PauseAsync().ConfigureAwait(false); + throw new OperationCanceledException( + $"FOCAS initialise of {_driverInstanceId} was superseded while resuming the alarm poll."); + } - Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null)); + } + catch (OperationCanceledException) when (generation != Volatile.Read(ref _lifecycleGeneration)) + { + // Superseded: whatever this attempt built but never published is released here (a published table + // is the superseding call's to release); health belongs to that call too. + if (!ReferenceEquals(Devices, devices)) await ReleaseTableAsync(devices.Values).ConfigureAwait(false); + throw; } catch (Exception ex) { - Volatile.Write(ref _health, new DriverHealth(DriverState.Faulted, null, ex.Message)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Faulted, null, ex.Message)); throw; } } + /// Writes unless a later lifecycle call has taken over. + private void WriteHealthIfCurrent(int generation, DriverHealth health) + { + lock (_lifecycleLock) + { + if (generation == Volatile.Read(ref _lifecycleGeneration)) Volatile.Write(ref _health, health); + } + } + + /// Starts the probe / recycle / fixed-tree loops for a freshly built table. Caller holds the lifecycle lock. + private void StartDeviceLoops(IEnumerable states) + { + foreach (var state in states) + { + if (_options.Probe.Enabled) + { + state.ProbeCts = new CancellationTokenSource(); + var ct = state.ProbeCts.Token; + state.ProbeTask = Task.Run(() => ProbeLoopAsync(state, ct), ct); + } + if (_options.HandleRecycle.Enabled) + { + state.RecycleCts = new CancellationTokenSource(); + var ct = state.RecycleCts.Token; + state.RecycleTask = Task.Run(() => RecycleLoopAsync(state, ct), ct); + } + if (_fixedTreeActive) + { + state.FixedTreeCts = new CancellationTokenSource(); + var ct = state.FixedTreeCts.Token; + state.FixedTreeTask = Task.Run(() => FixedTreeLoopAsync(state, ct), ct); + } + } + } + /// - /// Cancels, bounded-joins and disposes every existing device state (probe / recycle / fixed-tree - /// loops + client) and the alarm projection, then clears the table. Device part of - /// only (no poll / subscription teardown). + /// Swaps the live device table out (to empty) and releases it: cancels, bounded-joins and disposes every + /// device state's loops and client, after pausing the alarm projection. Device part of + /// only (no poll / subscription teardown). Also waits for any release an + /// overlapping lifecycle call started earlier, so it never returns while that one is still running. /// private async Task ReleaseDeviceStatesAsync() { // Pause, not dispose: the subscription and what the server was told survive the rebuild. await _alarmProjection.PauseAsync().ConfigureAwait(false); - if (_devices.Count == 0) return; - var loops = new List(); - foreach (var state in _devices.Values) + IReadOnlyDictionary old; + Task prior; + var done = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + lock (_lifecycleLock) { + old = _devices; + Volatile.Write(ref _devices, EmptyDevices); + prior = _releaseTail; + _releaseTail = done.Task; + } + try + { + if (old.Count > 0) await ReleaseTableAsync(old.Values).ConfigureAwait(false); + await prior.ConfigureAwait(false); + } + finally { done.TrySetResult(); } + } + + /// + /// Retires and tears down one swapped-out table: retire every device first (so a connect still in flight + /// disposes what it opens), cancel every loop, join them (bounded), then dispose the CTSs and — under each + /// device's connect gate — its client. + /// + private async Task ReleaseTableAsync(IEnumerable table) + { + var states = table.ToList(); + var loops = new List(); + foreach (var state in states) + { + state.Retire(); + // Cancel-then-dispose can race in tight shutdown loops; swallowing is intentional + // but we log the cause so a noisy shutdown leaves a Debug trace. try { state.ProbeCts?.Cancel(); } catch (Exception ex) { _logger.LogDebug(ex, "Cancelling probe CTS for {Host} failed", state.Options.HostAddress); } try { state.RecycleCts?.Cancel(); } @@ -388,15 +486,18 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, if (state.RecycleTask is { } rt) loops.Add(rt); if (state.FixedTreeTask is { } ft) loops.Add(ft); } + // Cancel every loop first, JOIN them (bounded), and only then dispose the CTSs and the + // client. Fire-and-forget loops used to outlive shutdown: one parked in EnsureConnectedAsync could + // finish its connect AFTER DisposeClient and leave a live socket on a shut-down driver. if (loops.Count > 0) { var all = Task.WhenAll(loops); var finished = await Task.WhenAny(all, Task.Delay(LoopJoinTimeout, CancellationToken.None)).ConfigureAwait(false); if (finished != all) - _logger.LogWarning("FOCAS re-initialise: {Count} background loop(s) did not stop within {Timeout}; proceeding", + _logger.LogWarning("FOCAS teardown: {Count} background loop(s) did not stop within {Timeout}; proceeding", loops.Count, LoopJoinTimeout); } - foreach (var state in _devices.Values) + foreach (var state in states) { state.ProbeCts?.Dispose(); state.ProbeCts = null; @@ -407,9 +508,22 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, state.FixedTreeCts?.Dispose(); state.FixedTreeCts = null; state.FixedTreeTask = null; - state.DisposeClient(); + await DisposeClientGatedAsync(state).ConfigureAwait(false); } - _devices.Clear(); + } + + /// + /// Disposes a device's client under its , so it can neither null a + /// client a concurrent just installed nor race that connect. The wait + /// is bounded: a connect wedged past may not wedge teardown too (the device + /// is retired, so that connect disposes its own client when it finally returns). + /// + private async Task DisposeClientGatedAsync(DeviceState state) + { + var entered = await state.ConnectGate.WaitAsync(LoopJoinTimeout).ConfigureAwait(false); + try { state.DisposeClient(); } + catch (Exception ex) { _logger.LogDebug(ex, "FOCAS client dispose failed for {Host}", state.Options.HostAddress); } + finally { if (entered) state.ConnectGate.Release(); } } /// @@ -423,58 +537,28 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, internal static readonly TimeSpan LoopJoinTimeout = TimeSpan.FromSeconds(5); /// + // Safe to overlap with itself (Lane A disposes beside a still-running shutdown — the second call waits for + // the first's teardown via _releaseTail) and with an abandoned InitializeAsync (the generation bump below + // supersedes it, so it publishes nothing). public async Task ShutdownAsync(CancellationToken cancellationToken) { + var generation = Interlocked.Increment(ref _lifecycleGeneration); await _poll.DisposeAsync().ConfigureAwait(false); - // Pause, not dispose — see _alarmProjection. A later InitializeAsync resumes it. - await _alarmProjection.PauseAsync().ConfigureAwait(false); - // cancel every loop first, JOIN them (bounded), and only then dispose the CTSs and the - // client. Fire-and-forget loops used to outlive shutdown: one parked in EnsureConnectedAsync could - // finish its connect AFTER DisposeClient and leave a live socket on a shut-down driver. - var loops = new List(); - foreach (var state in _devices.Values) + // Pauses (not disposes) the alarm projection — see _alarmProjection; a later InitializeAsync resumes it. + await ReleaseDeviceStatesAsync().ConfigureAwait(false); + lock (_lifecycleLock) { - // Cancel-then-dispose can race in tight shutdown loops; swallowing is intentional - // but we now log the cause so a noisy shutdown leaves a Debug trace. - try { state.ProbeCts?.Cancel(); } - catch (Exception ex) { _logger.LogDebug(ex, "Cancelling probe CTS for {Host} failed", state.Options.HostAddress); } - try { state.RecycleCts?.Cancel(); } - catch (Exception ex) { _logger.LogDebug(ex, "Cancelling recycle CTS for {Host} failed", state.Options.HostAddress); } - try { state.FixedTreeCts?.Cancel(); } - catch (Exception ex) { _logger.LogDebug(ex, "Cancelling fixed-tree CTS for {Host} failed", state.Options.HostAddress); } - if (state.ProbeTask is { } pt) loops.Add(pt); - if (state.RecycleTask is { } rt) loops.Add(rt); - if (state.FixedTreeTask is { } ft) loops.Add(ft); + // A later initialise owns the tables (and health) now — leave them alone. + if (generation != Volatile.Read(ref _lifecycleGeneration)) return; + Volatile.Write(ref _tagsByRawPath, new Dictionary(StringComparer.Ordinal)); + Volatile.Write(ref _alarmTags, new Dictionary(StringComparer.Ordinal)); + Volatile.Write(ref _parsedAddressesByTagName, + new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase)); + _resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry + _fixedTreeActive = false; + ResetAlarmHealth(); + Volatile.Write(ref _health, new DriverHealth(DriverState.Unknown, Volatile.Read(ref _health).LastSuccessfulRead, null)); } - if (loops.Count > 0) - { - var all = Task.WhenAll(loops); - var finished = await Task.WhenAny(all, Task.Delay(LoopJoinTimeout, CancellationToken.None)).ConfigureAwait(false); - if (finished != all) - _logger.LogWarning("FOCAS shutdown: {Count} background loop(s) did not stop within {Timeout}; proceeding with teardown", - loops.Count, LoopJoinTimeout); - } - foreach (var state in _devices.Values) - { - state.ProbeCts?.Dispose(); - state.ProbeCts = null; - state.ProbeTask = null; - state.RecycleCts?.Dispose(); - state.RecycleCts = null; - state.RecycleTask = null; - state.FixedTreeCts?.Dispose(); - state.FixedTreeCts = null; - state.FixedTreeTask = null; - state.DisposeClient(); - } - _devices.Clear(); - Volatile.Write(ref _tagsByRawPath, new Dictionary(StringComparer.Ordinal)); - Volatile.Write(ref _alarmTags, new Dictionary(StringComparer.Ordinal)); - _resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry - _parsedAddressesByTagName.Clear(); - _fixedTreeActive = false; - ResetAlarmHealth(); - Volatile.Write(ref _health, new DriverHealth(DriverState.Unknown, Volatile.Read(ref _health).LastSuccessfulRead, null)); } /// @@ -499,12 +583,14 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, } /// Gets the number of configured devices. - internal int DeviceCount => _devices.Count; + internal int DeviceCount => Devices.Count; + /// Test seam — the live device table snapshot (swapped whole, never mutated in place). + internal IReadOnlyDictionary DeviceTable => Devices; /// Gets the state of a device by host address. /// The host address of the device. /// The device state if found; otherwise null. internal DeviceState? GetDeviceState(string hostAddress) => - _devices.TryGetValue(hostAddress, out var s) ? s : null; + Devices.TryGetValue(hostAddress, out var s) ? s : null; /// Test seam — the resolved options the factory built this driver from. internal FocasDriverOptions Options => _options; @@ -592,7 +678,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadNodeIdUnknown, null, now); continue; } - if (!_devices.TryGetValue(def.DeviceHostAddress, out var device)) + if (!Devices.TryGetValue(def.DeviceHostAddress, out var device)) { results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadNodeIdUnknown, null, now); continue; @@ -680,7 +766,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, var reference = references[i]; if (_fixedTreeActive && TryReadFixedTree(reference, now) is not null) continue; if (!_resolver.TryResolve(reference, out var def)) continue; - if (!_devices.TryGetValue(def.DeviceHostAddress, out var device)) continue; + if (!Devices.TryGetValue(def.DeviceHostAddress, out var device)) continue; var parsed = ResolveParsedAddress(def); if (parsed.Kind != FocasAreaKind.Pmc || parsed.PmcLetter is null) continue; (byDevice ??= []).TryAdd(device, []); @@ -802,7 +888,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, results[i] = new WriteResult(FocasStatusMapper.BadNotWritable); continue; } - if (!_devices.TryGetValue(def.DeviceHostAddress, out var device)) + if (!Devices.TryGetValue(def.DeviceHostAddress, out var device)) { results[i] = new WriteResult(FocasStatusMapper.BadNodeIdUnknown); continue; @@ -890,7 +976,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // Fixed-tree T1 — Identity + Axes subtrees, populated once per session // from cnc_sysinfo + cnc_rdaxisname at init time and kept in DeviceState. if (_fixedTreeActive - && _devices.TryGetValue(device.HostAddress, out var state) + && Devices.TryGetValue(device.HostAddress, out var state) && state.FixedTreeCache is { } cache) { var identity = deviceFolder.Folder("Identity", "Identity"); @@ -1142,7 +1228,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// public IReadOnlyList GetHostStatuses() => - [.. _devices.Values.Select(s => new HostConnectivityStatus(s.Options.HostAddress, s.HostState, s.HostStateChangedUtc))]; + [.. Devices.Values.Select(s => new HostConnectivityStatus(s.Options.HostAddress, s.HostState, s.HostStateChangedUtc))]; private async Task ProbeLoopAsync(DeviceState state, CancellationToken ct) { @@ -1510,7 +1596,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// private DataValueSnapshot? TryReadFixedTree(string reference, DateTime now) { - foreach (var state in _devices.Values) + foreach (var state in Devices.Values) { if (!reference.StartsWith(state.Options.HostAddress + "/", StringComparison.OrdinalIgnoreCase)) continue; if (TryReadFixedTreeOnDevice(state, reference[(state.Options.HostAddress.Length + 1)..], now) is { } hit) @@ -1776,7 +1862,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// The device's active alarms. internal async Task> ReadDeviceAlarmsAsync(string hostAddress, CancellationToken ct) { - if (!_devices.TryGetValue(hostAddress, out var state)) + if (!Devices.TryGetValue(hostAddress, out var state)) throw new InvalidOperationException($"FOCAS device '{hostAddress}' is not configured."); var client = await EnsureConnectedAsync(state, ct).ConfigureAwait(false); return await client.ReadAlarmsAsync(ct).ConfigureAwait(false); @@ -1822,6 +1908,12 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, await device.ConnectGate.WaitAsync(ct).ConfigureAwait(false); try { + // A released device (re-initialise / shutdown) is never reconnected: whoever still holds it read + // the table before the swap, and a client opened now would be orphaned on a state nobody tears down. + if (device.IsRetired) + throw new ObjectDisposedException(nameof(DeviceState), + $"FOCAS device {device.Options.HostAddress} was released by a re-initialise or shutdown."); + // Another caller may have connected while this one waited. if (device.Client is { IsConnected: true } connected) return connected; @@ -1864,6 +1956,15 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, throw; } device.Backoff.RecordSuccess(); // recovery resets the window immediately + // Retired WHILE this connect ran (the release waits on this gate only up to its bound): the connect + // is ours to undo, or it leaves a live session on a shut-down driver. + if (device.IsRetired) + { + client.Dispose(); + if (ReferenceEquals(device.Client, client)) device.Client = null; + throw new ObjectDisposedException(nameof(DeviceState), + $"FOCAS device {device.Options.HostAddress} was released while connecting."); + } return client; } finally { device.ConnectGate.Release(); } @@ -2020,6 +2121,17 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// public ConcurrentDictionary LastSpindleMotorSpeeds { get; } = new(); + private int _retired; + + /// + /// True once a re-initialise / shutdown has released this device. A retired device never connects + /// again, and a connect that was already running when it retired disposes what it opened. + /// + public bool IsRetired => Volatile.Read(ref _retired) != 0; + + /// Marks the device released. Idempotent. + public void Retire() => Volatile.Write(ref _retired, 1); + /// Disposes the FOCAS client instance. public void DisposeClient() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FakeFocasClient.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FakeFocasClient.cs index 00cf61d5..370bff15 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FakeFocasClient.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FakeFocasClient.cs @@ -259,7 +259,7 @@ internal sealed class FakeFocasClientFactory : IFocasClientFactory public IFocasClient Create() { var c = Customise?.Invoke() ?? new FakeFocasClient(); - Clients.Add(c); + lock (Clients) Clients.Add(c); return c; } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasDeviceTableLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasDeviceTableLifecycleTests.cs new file mode 100644 index 00000000..056b7ca8 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasDeviceTableLifecycleTests.cs @@ -0,0 +1,200 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; + +/// +/// #740 / R07:H13 — the device table is read by the retained poll engine, the fixed-tree loop, the alarm +/// loop and the actor's health tick, so a re-initialise must build a fresh table and swap it in rather than +/// clearing and refilling the live one. Also pins the lifecycle overlaps the runtime produces since Lane A: +/// a dispose running beside a still-running shutdown, and an abandoned initialise overlapping the next +/// attempt's shutdown. +/// +[Trait("Category", "Unit")] +public sealed class FocasDeviceTableLifecycleTests +{ + private const string Host = "focas://10.0.0.5:8193"; + + private static FocasDriverOptions Options(bool probe = false, bool fixedTree = false) => new() + { + Devices = [new FocasDeviceOptions(Host)], + RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/R", Host, "R100", FocasDataType.Byte))], + Probe = new FocasProbeOptions { Enabled = probe, Interval = TimeSpan.FromMilliseconds(20) }, + FixedTree = new FocasFixedTreeOptions { Enabled = fixedTree, PollInterval = TimeSpan.FromMilliseconds(10) }, + }; + + /// A re-initialise publishes a NEW table; a reader holding the old one never sees it emptied. + [Fact] + public async Task Reinitialize_swaps_in_a_new_table_and_never_mutates_the_one_a_reader_holds() + { + var drv = new FocasDriver(Options(), "focas-swap", new FakeFocasClientFactory()); + await drv.InitializeAsync("{}", CancellationToken.None); + var before = drv.DeviceTable; + before.Count.ShouldBe(1); + + await drv.InitializeAsync("{}", CancellationToken.None); + + drv.DeviceTable.ShouldNotBeSameAs(before); + drv.DeviceTable.Count.ShouldBe(1); + before.Count.ShouldBe(1, "the table a concurrent reader enumerates must not be cleared under it"); + await drv.ShutdownAsync(CancellationToken.None); + before.Count.ShouldBe(1); + drv.DeviceTable.Count.ShouldBe(0); + } + + /// Hammering every table reader during repeated re-initialises throws nothing — and reads still succeed. + [Fact] + public async Task Readers_survive_repeated_reinitialise() + { + var factory = new FakeFocasClientFactory(); + var drv = new FocasDriver(Options(fixedTree: true), "focas-hammer", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + using var stop = new CancellationTokenSource(); + var good = 0; + var errors = new List(); + var reader = Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + try + { + _ = drv.GetHostStatuses(); + var snaps = await drv.ReadAsync(["raw/R", $"{Host}/Identity/SeriesNumber"], CancellationToken.None); + if (snaps[0].StatusCode == FocasStatusMapper.Good) Interlocked.Increment(ref good); + } + catch (Exception ex) { lock (errors) errors.Add(ex); } + } + }); + + for (var i = 0; i < 100; i++) + await drv.InitializeAsync("{}", CancellationToken.None); + await Task.Delay(100); + stop.Cancel(); + await reader; + await drv.ShutdownAsync(CancellationToken.None); + + errors.ShouldBeEmpty(); + good.ShouldBeGreaterThan(0, "positive control: the reader really did read through the table"); + } + + /// A client whose connect parks until released and ignores cancellation. + private sealed class ParkedConnectClient(TaskCompletionSource entered, Task release) : FakeFocasClient + { + public override async Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken ct) + { + entered.TrySetResult(); + await release.ConfigureAwait(false); + await base.ConnectAsync(address, timeout, CancellationToken.None).ConfigureAwait(false); + } + } + + /// + /// A data-path read parked in connect when shutdown runs must not leave a live client on the released + /// device: the device is retired, so the connect that completes afterwards disposes what it opened. + /// + [Fact] + public async Task Connect_completing_after_shutdown_does_not_leave_a_live_client() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var factory = new FakeFocasClientFactory { Customise = () => new ParkedConnectClient(entered, release.Task) }; + var drv = new FocasDriver(Options(), "focas-retire", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + var read = drv.ReadAsync(["raw/R"], CancellationToken.None); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + var shutdown = drv.ShutdownAsync(CancellationToken.None); + await Task.Delay(50); + release.SetResult(); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10)); + var snaps = await read.WaitAsync(TimeSpan.FromSeconds(5)); + + factory.Clients.Count.ShouldBe(1); + factory.Clients[0].IsConnected.ShouldBeFalse("a connect finishing on a retired device must dispose its client"); + factory.Clients[0].DisposeCount.ShouldBeGreaterThanOrEqualTo(1); + snaps[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError); + } + + /// + /// An initialise abandoned mid-way (the runtime timed it out and moved on to a shutdown) must not + /// resurrect the driver when it finally completes: no device table, no loops, no connection. + /// + [Fact] + public async Task Abandoned_initialise_finishing_after_shutdown_starts_nothing() + { + var factory = new FakeFocasClientFactory(); + var options = Options(probe: true); + using var gate = new ManualResetEventSlim(false); + var inRebind = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var drv = new FocasDriver(options, "focas-abandoned", factory, rebind: _ => + { + inRebind.TrySetResult(); + gate.Wait(TimeSpan.FromSeconds(10)); + return new FocasDriverBinding(options, factory); + }); + + var init = Task.Run(() => drv.InitializeAsync("{\"devices\":[]}", CancellationToken.None)); + await inRebind.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await drv.ShutdownAsync(CancellationToken.None); + gate.Set(); + await Should.ThrowAsync(() => init.WaitAsync(TimeSpan.FromSeconds(10))); + + await Task.Delay(200); // several probe intervals — a started loop would have connected by now + drv.DeviceCount.ShouldBe(0); + factory.Clients.ShouldBeEmpty(); + drv.GetHealth().State.ShouldNotBe(DriverState.Healthy); + + // Positive control: the same driver, initialised without being superseded, does start its probe. + await drv.InitializeAsync("{\"devices\":[]}", CancellationToken.None); + await WaitUntil(() => factory.Clients.Count > 0); + drv.DeviceCount.ShouldBe(1); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + await drv.DisposeAsync(); + } + + /// A loop wedged in a read that ignores cancellation for a while. + private sealed class SlowProbeClient : FakeFocasClient + { + public override async Task ProbeAsync(CancellationToken ct) + { + await Task.Delay(400, CancellationToken.None).ConfigureAwait(false); + return true; + } + } + + /// + /// Lane A disposes while its own shutdown may still be running. The dispose must not return before the + /// teardown it overlaps has released the client — it waits for it rather than finding an empty table. + /// + [Fact] + public async Task Dispose_concurrent_with_shutdown_waits_for_the_teardown() + { + var factory = new FakeFocasClientFactory { Customise = () => new SlowProbeClient() }; + var drv = new FocasDriver(Options(probe: true), "focas-dispose", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + await WaitUntil(() => factory.Clients.Count > 0 && factory.Clients[0].IsConnected); + + var shutdown = drv.ShutdownAsync(CancellationToken.None); + var dispose = drv.DisposeAsync().AsTask(); + await dispose.WaitAsync(TimeSpan.FromSeconds(10)); + + factory.Clients.ShouldAllBe(c => !c.IsConnected); + factory.Clients.ShouldAllBe(c => c.DisposeCount >= 1); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10)); + drv.DeviceCount.ShouldBe(0); + } + + private static async Task WaitUntil(Func condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached"); + await Task.Delay(10); + } + } +} From ecdd6dbecb53e860dec659b4abc92cf26010bc52 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:03:28 -0400 Subject: [PATCH 30/51] fix(twincat): native/probe evidence advances LastSuccessfulRead; probe failures degrade (H12) In native-notification mode LastSuccessfulRead never moved past the init stamp, so one Degraded write on a connection older than 90 s latched the comms-loss watchdog into a full reconnect (every notification on every device deleted and re-registered). Conversely an unreachable AMS router was swallowed by the probe and never left Healthy. - a native push and a successful probe now stamp LastSuccessfulRead (throttled to one health record per 250 ms) and restore Healthy unless a device is still probe-unreachable; - 3 consecutive failed probe ticks write Degraded with the stamp preserved; - a NotSupportedException write is BadNotSupported with no health write (#611); - a reconnect replay the PLC refuses republishes the refusal's status without degrading (it used to report BadCommunicationError + Degraded on every reconnect for a correctly-isolated type-mismatch tag). Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../TwinCATDriver.cs | 106 ++++++++++-- .../TwinCATHealthContractTests.cs | 161 ++++++++++++++++++ 2 files changed, 252 insertions(+), 15 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATHealthContractTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index 021c4e32..f1c87ebb 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -346,7 +346,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { if (status == TwinCATStatusMapper.Good) { - _health = new DriverHealth(DriverState.Healthy, now, null); + RecordDeviceAnswered(now, force: true); return new DataValueSnapshot(value, status, now, now); } @@ -524,10 +524,11 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery results[i] = new WriteResult(status); } catch (OperationCanceledException) { throw; } - catch (NotSupportedException nse) + catch (NotSupportedException) { + // A write the client cannot express is a refusal, not a comms fault (#611): no health + // write, or the comms-loss watchdog would reconnect a healthy device over a config issue. results[i] = new WriteResult(TwinCATStatusMapper.BadNotSupported); - _health = new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead, nse.Message); } catch (Exception ex) when (ex is FormatException or InvalidCastException) { @@ -540,7 +541,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery catch (Exception ex) { results[i] = new WriteResult(TwinCATStatusMapper.BadCommunicationError); - _health = new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead, ex.Message); + MarkCommsFault(ex.Message); } } @@ -689,12 +690,18 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery var reg = new NativeRegistration( Interlocked.Increment(ref _nextRegId), device, reference, symbolName, def.DataType, bitIndex, def.ArrayLength, publishingInterval, - onChange: (_, value) => RaiseDataChange(handle, reference, new DataValueSnapshot( - value, TwinCATStatusMapper.Good, DateTime.UtcNow, DateTime.UtcNow)), + onChange: (_, value) => + { + // A push is the device answering: stamp LastSuccessfulRead so a notification-only + // driver does not look silent to the comms-loss watchdog. + var now = DateTime.UtcNow; + RecordDeviceAnswered(now); + RaiseDataChange(handle, reference, new DataValueSnapshot(value, TwinCATStatusMapper.Good, now, now)); + }, // When the handle goes dead (a failed replay) push a Bad snapshot so // subscribers see the tag go Bad instead of a frozen-Good-stale value. - onUnavailable: () => RaiseDataChange(handle, reference, new DataValueSnapshot( - null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow))); + onUnavailable: status => RaiseDataChange(handle, reference, new DataValueSnapshot( + null, status, null, DateTime.UtcNow))); registrations.Add(reg); if (unreachable?.Contains(device) == true) @@ -775,6 +782,43 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery _health = new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, message); } + /// + /// Consecutive failed probe ticks (connect failure, failed ReadState) after which a device counts as + /// unreachable and the driver goes with LastSuccessfulRead + /// preserved — so an unreachable AMS router reaches the comms-loss watchdog instead of leaving the + /// driver Healthy with stale-Good values. Three ticks ride out a single dropped ReadState. + /// + internal const int ProbeFailuresBeforeDegraded = 3; + + /// Minimum spacing between LastSuccessfulRead stamps from the notification path. + private static readonly TimeSpan AnsweredStampInterval = TimeSpan.FromMilliseconds(250); + + /// + /// Records that a device answered (a native push, a successful probe, a Good read): advances + /// LastSuccessfulRead and restores unless a device is still + /// probe-unreachable (then the state stays Degraded, the stamp still advances). Throttled so a + /// notification burst does not allocate a health record per value; never touches Faulted. + /// + /// The time the device answered. + /// Write even inside the throttle window (an explicit read / probe outcome). + private void RecordDeviceAnswered(DateTime now, bool force = false) + { + var current = _health; + if (current.State == DriverState.Faulted) return; + // Hot path (a notification on a healthy driver inside the window): no device scan, no allocation. + if (!force && current.State == DriverState.Healthy && current.LastSuccessfulRead is { } recent + && now - recent < AnsweredStampInterval) + return; + var unreachable = _devices.Values.FirstOrDefault(d => d.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded); + var state = unreachable is null ? DriverState.Healthy : DriverState.Degraded; + if (!force && current.State == state && current.LastSuccessfulRead is { } last + && now - last < AnsweredStampInterval) + return; + _health = new DriverHealth(state, now, unreachable is null + ? null + : $"TwinCAT device {unreachable.Options.HostAddress} unreachable ({unreachable.ConsecutiveProbeFailures} consecutive failed probes)"); + } + /// Raises one data change for a native subscription. private void RaiseDataChange(ISubscriptionHandle handle, string reference, DataValueSnapshot snapshot) => OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, reference, snapshot)); @@ -888,7 +932,14 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// the shape has no quality channel. Closes over the same /// (handle, reference) pair as . /// - public Action OnUnavailable { get; } + public Action OnUnavailableStatus { get; } + + /// Publishes BadCommunicationError for a handle that went dead on a comms failure. + public void OnUnavailable() => OnUnavailableStatus(TwinCATStatusMapper.BadCommunicationError); + + /// Publishes the status of a refusal (the PLC answered and refused this tag). + /// The refusal's OPC UA status. + public void OnRefused(uint status) => OnUnavailableStatus(status); private ITwinCATNotificationHandle? _handle; @@ -908,10 +959,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// Authored array length for an array tag; null for a scalar. /// Requested notification interval. /// Value-change callback (closes over the owning subscription handle + reference). - /// Bad-quality callback fired when the handle goes dead. + /// Bad-quality callback (given the status to publish) fired when the handle goes dead. public NativeRegistration(long id, DeviceState device, string reference, string symbolName, TwinCATDataType dataType, int? bitIndex, int? arrayLength, TimeSpan interval, Action onChange, - Action onUnavailable) + Action onUnavailable) { Id = id; Device = device; @@ -922,7 +973,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery ArrayLength = arrayLength; Interval = interval; OnChange = onChange; - OnUnavailable = onUnavailable; + OnUnavailableStatus = onUnavailable; } /// Installs a freshly-registered handle and returns the previous one (null on first set). @@ -996,6 +1047,17 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery catch (OperationCanceledException) when (ct.IsCancellationRequested) { break; } } + if (success) + { + state.ConsecutiveProbeFailures = 0; + RecordDeviceAnswered(DateTime.UtcNow, force: true); + } + else if (++state.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded) + { + MarkCommsFault($"TwinCAT device {state.Options.HostAddress} unreachable " + + $"({state.ConsecutiveProbeFailures} consecutive failed probes)"); + } + TransitionDeviceState(state, success ? HostState.Running : HostState.Stopped); // Opt-in PLC state / task-cycle sampling for /hosts. After the host-state transition so a @@ -1360,6 +1422,19 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } replayed++; } + catch (TwinCATNotificationRefusedException refused) + { + // The PLC answered and refused this tag again (it was already isolated at subscribe, or a + // program change made it undecodable): republish the refusal's status, keep it dead for the + // probe-tick retry, and leave health alone — a refusal is not a comms fault. + _logger.LogDebug( + "TwinCAT driver '{DriverInstanceId}' native notification '{Symbol}' on device '{HostAddress}' " + + "still refused after reconnect: {Reason}", + _driverInstanceId, reg.SymbolName, device.Options.HostAddress, refused.Message); + var dead = reg.MarkHandleDead(); + try { dead?.Dispose(); } catch { /* already-dead handle from the disposed client */ } + try { reg.OnRefused(refused.StatusCode); } catch { /* never abort recovery of the rest */ } + } catch (Exception ex) { _logger.LogWarning(ex, @@ -1373,9 +1448,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery var dead = reg.MarkHandleDead(); try { dead?.Dispose(); } catch { /* already-dead handle from the disposed client */ } try { reg.OnUnavailable(); } catch { /* never let a subscriber callback abort recovery of the rest */ } - if (_health.State != DriverState.Faulted) - _health = new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead, - $"native-notification replay failed for '{reg.SymbolName}' on {device.Options.HostAddress}"); + MarkCommsFault($"native-notification replay failed for '{reg.SymbolName}' on {device.Options.HostAddress}"); } } @@ -1586,6 +1659,9 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// True once an unexpected sum-read chunk failure was logged for this device. public volatile bool SumChunkFailureWarned; + /// Consecutive failed probe ticks; reset by a successful probe. Written only by this device's probe loop. + public volatile int ConsecutiveProbeFailures; + /// Latest probe-sampled PLC diagnostics; null before the first sample. public TwinCATDeviceDiagnostics? LastDiagnostics { get; set; } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATHealthContractTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATHealthContractTests.cs new file mode 100644 index 00000000..f1c06bfd --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATHealthContractTests.cs @@ -0,0 +1,161 @@ +using System.Collections.Concurrent; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// The driver health contract (#739 H12): data proving the device answered — a native notification or +/// a successful probe — advances LastSuccessfulRead; sustained probe/connect failure degrades with +/// it preserved; a refusal (a write the client cannot express, a replay the PLC refuses) is no comms fault. +/// +[Trait("Category", "Unit")] +public sealed class TwinCATHealthContractTests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + + private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, TwinCATProbeOptions probe, bool native = true) => + new(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = TwinCATRawTags.Entries([new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)]), + Probe = probe, + UseNativeNotifications = native, + }, "drv-1", factory); + + private static TwinCATProbeOptions FastProbe => new() + { + Enabled = true, + Interval = TimeSpan.FromMilliseconds(30), + Timeout = TimeSpan.FromMilliseconds(100), + }; + + /// A native push advances LastSuccessfulRead and restores Healthy after a comms-fault write. + [Fact] + public async Task A_native_notification_advances_LastSuccessfulRead_and_restores_Healthy() + { + var fake = new FakeTwinCATClient(); + var factory = new FakeTwinCATClientFactory { Customise = () => fake }; + var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + var initStamp = drv.GetHealth().LastSuccessfulRead; + initStamp.ShouldNotBeNull(); + + // A write that fails on the wire degrades (comms fault) with the stamp preserved. + fake.ThrowOnWrite = true; + _ = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + drv.GetHealth().LastSuccessfulRead.ShouldBe(initStamp); + + await Task.Delay(300, TestContext.Current.CancellationToken); + fake.FireNotification("MAIN.Speed", 7); + + var after = drv.GetHealth(); + after.State.ShouldBe(DriverState.Healthy); + after.LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp!.Value); + } + + /// A successful probe advances LastSuccessfulRead in a driver that serves only notifications. + [Fact] + public async Task A_successful_probe_advances_LastSuccessfulRead() + { + var factory = new FakeTwinCATClientFactory(); + var drv = NewDriver(factory, FastProbe); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var initStamp = drv.GetHealth().LastSuccessfulRead!.Value; + + await WaitForAsync(() => drv.GetHealth().LastSuccessfulRead > initStamp.AddMilliseconds(250), TimeSpan.FromSeconds(3)); + + drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp.AddMilliseconds(250)); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + /// + /// An unreachable AMS router (every probe connect throws) degrades the driver after + /// consecutive failures, preserving the stamp — + /// before, the probe caught it and health stayed Healthy forever. + /// + [Fact] + public async Task Consecutive_probe_failures_degrade_preserving_LastSuccessfulRead() + { + var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { ThrowOnConnect = true } }; + var drv = NewDriver(factory, FastProbe); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var initStamp = drv.GetHealth().LastSuccessfulRead; + + await WaitForAsync(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(3)); + + var health = drv.GetHealth(); + health.State.ShouldBe(DriverState.Degraded); + health.LastSuccessfulRead.ShouldBe(initStamp); + factory.Clients.Count.ShouldBeGreaterThanOrEqualTo(TwinCATDriver.ProbeFailuresBeforeDegraded); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + /// + /// A write the client refuses with is BadNotSupported with no + /// health write (#611: a refusal is not a comms fault). Positive control: a wire failure does degrade. + /// + [Fact] + public async Task A_NotSupported_write_leaves_health_alone() + { + var fake = new FakeTwinCATClient { ThrowOnWrite = true, Exception = new NotSupportedException("no") }; + var factory = new FakeTwinCATClientFactory { Customise = () => fake }; + var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }, native: false); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var result = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken); + result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadNotSupported); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + + fake.Exception = new InvalidOperationException("wire"); + result = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken); + result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + } + + /// + /// A reconnect replay the PLC REFUSES (an isolated type-mismatch tag re-attempted on every reconnect) + /// republishes the refusal's status and does not degrade. Positive control: a replay that fails on the + /// wire degrades (covered for the same flow in TwinCATReconnectReplayTests). + /// + [Fact] + public async Task A_refused_replay_republishes_the_refusal_status_without_degrading() + { + var build = 0; + var factory = new FakeTwinCATClientFactory + { + Customise = () => + { + var c = new FakeTwinCATClient(); + if (++build >= 2) + c.AddNotificationErrors["MAIN.Speed"] = new TwinCATTypeMismatchException("MAIN.Speed", "now a STRING"); + return c; + }, + }; + var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + _ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + + factory.Clients[0].SimulateWireDrop(); + // A write reconnects (and replays) without itself touching health on success. + _ = await drv.WriteAsync([new WriteRequest("Speed", 1)], TestContext.Current.CancellationToken); + + factory.Clients.Count.ShouldBe(2); + events.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadTypeMismatch); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + private static async Task WaitForAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(20); + } +} From 4083b4aada5b99ae9b6b48baecad2cc14486a641 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:07:03 -0400 Subject: [PATCH 31/51] fix(twincat): warn once per bad tag; a device refusal is not a comms fault (P10) CompleteRead logged a Warning for every non-Good status on every poll and latched Degraded for all of them. Now a bad tag warns once per bad episode (re-armed when it reads Good, and on re-initialise), a comms-class status (BadCommunicationError / BadTimeout) degrades with LastSuccessfulRead preserved, and any other status - the device answered and refused this tag - advances LastSuccessfulRead with no state change. Read-path exception handlers now go through MarkCommsFault, so they never downgrade Faulted. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../TwinCATDriver.cs | 59 +++++++--- .../TwinCATCapturingLogger.cs | 26 +++++ .../TwinCATReadWarningTests.cs | 103 ++++++++++++++++++ 3 files changed, 175 insertions(+), 13 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATCapturingLogger.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWarningTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index f1c87ebb..7c95a8f0 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -42,6 +42,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // so concurrent bit-writers to the same word must serialise or they lose each other's // updates. Keyed by device + parent symbol. (Cannot guard against the PLC program writing // the word between our read and write — inherent to RMW.) + // References that already logged their bad-read Warning in the current bad episode (#739 P10). + // Removed when the tag reads Good; cleared on (re-)initialise and shutdown. + private readonly ConcurrentDictionary _warnedRefs = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _bitRmwLocks = new(StringComparer.OrdinalIgnoreCase); @@ -117,6 +121,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // allows it, and a reconnect retry can overlap a running attempt). Release the previous device states' probe loops + clients before // the overwrite below. Deliberately NOT ShutdownAsync: that also drops _poll and the subscriptions. await ReleaseDeviceStatesAsync().ConfigureAwait(false); + _warnedRefs.Clear(); _health = new DriverHealth(DriverState.Initializing, null, null); try { @@ -335,7 +340,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery catch (OperationCanceledException) { throw; } catch (Exception ex) { - _health = new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead, ex.Message); + MarkCommsFault(ex.Message); return new DataValueSnapshot(null, TwinCATStatusMapper.BadCommunicationError, null, now); } } @@ -346,25 +351,53 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { if (status == TwinCATStatusMapper.Good) { + _warnedRefs.TryRemove(reference, out _); // a later failure is a new episode and warns again RecordDeviceAnswered(now, force: true); return new DataValueSnapshot(value, status, now, now); } - if (status == TwinCATStatusMapper.BadTypeMismatch) - _logger.LogWarning( - "TwinCAT driver '{DriverInstanceId}' refused '{Reference}': declared {DataType} " + - "cannot decode the PLC symbol '{Symbol}' (string/numeric kind mismatch, array element type " + - "or length mismatch, or a reference/container)", - _driverInstanceId, reference, def.DataType, symbolName); + // Once per tag per bad episode: a Warning every poll for every bad tag (50 typo'd symbols on a + // 500 ms poll = 100 Warnings/s) buried everything else in the log. + if (_warnedRefs.TryAdd(reference, 0)) + { + if (status == TwinCATStatusMapper.BadTypeMismatch) + _logger.LogWarning( + "TwinCAT driver '{DriverInstanceId}' refused '{Reference}': declared {DataType} " + + "cannot decode the PLC symbol '{Symbol}' (string/numeric kind mismatch, array element type " + + "or length mismatch, or a reference/container); logged once until the tag reads Good", + _driverInstanceId, reference, def.DataType, symbolName); + else + _logger.LogWarning( + "TwinCAT driver '{DriverInstanceId}' ADS read error 0x{Status:X8} for '{Reference}'; " + + "logged once until the tag reads Good", + _driverInstanceId, status, reference); + } + + if (IsCommsStatus(status)) + MarkCommsFault($"ADS status {status:X8} reading {reference}"); else - _logger.LogWarning( - "TwinCAT driver '{DriverInstanceId}' ADS read error 0x{Status:X8} for '{Reference}'", - _driverInstanceId, status, reference); - _health = new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead, - $"ADS status {status:X8} reading {reference}"); + RecordDeviceAnsweredKeepState(now); // a refusal: the device answered — no state change return new DataValueSnapshot(value, status, now, now); } + /// + /// True for a status that means the request did not get a device answer (router / transport / + /// timeout). Every other non-Good status is the device refusing this one tag (symbol not found, type + /// mismatch, PLC not running, access denied): per-tag Bad with no health write. + /// + /// The mapped OPC UA status. + /// True for a comms-class status. + internal static bool IsCommsStatus(uint status) => + status is TwinCATStatusMapper.BadCommunicationError or TwinCATStatusMapper.BadTimeout; + + /// Advances LastSuccessfulRead without changing the state (the device answered with a refusal). + private void RecordDeviceAnsweredKeepState(DateTime now) + { + var current = _health; + if (current.State is DriverState.Faulted or DriverState.Unknown or DriverState.Initializing) return; + _health = current with { LastSuccessfulRead = now }; + } + /// /// Reads one device's gathered tags as ADS sum-reads of at most SumReadMaxItems symbols each. /// A target that refuses the sum command flips and the @@ -430,7 +463,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { foreach (var e in chunk) results[e.Index] = new DataValueSnapshot(null, TwinCATStatusMapper.BadCommunicationError, null, now); - _health = new DriverHealth(DriverState.Degraded, _health.LastSuccessfulRead, ex.Message); + MarkCommsFault(ex.Message); // Once per device: the health message is overwritten by the next good read in the same poll, so // without this a sum-read path failing on every poll left no trace (hw2 live gate, 2026-10-08). if (connected && !device.SumChunkFailureWarned) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATCapturingLogger.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATCapturingLogger.cs new file mode 100644 index 00000000..9df5d1b8 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATCapturingLogger.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// Collects the driver's Warning-level messages (thread-safe) for once-per-X assertions. +internal sealed class TwinCATCapturingLogger : ILogger +{ + private readonly List _warnings = []; + + /// Gets a snapshot of the formatted Warning messages logged so far. + public IReadOnlyList Warnings { get { lock (_warnings) return [.. _warnings]; } } + + /// + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + /// + public bool IsEnabled(LogLevel logLevel) => true; + + /// + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) + { + if (logLevel == LogLevel.Warning) lock (_warnings) _warnings.Add(formatter(state, exception)); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWarningTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWarningTests.cs new file mode 100644 index 00000000..dd669164 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWarningTests.cs @@ -0,0 +1,103 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// #739 P10: a bad tag warns ONCE (not once per poll — 50 typo'd symbols on a 500 ms poll used to be 100 +/// Warnings/s forever), re-arms when it reads Good, and a device refusal leaves driver health alone while a +/// comms-class status degrades. +/// +[Trait("Category", "Unit")] +public sealed class TwinCATReadWarningTests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + + private static (TwinCATDriver drv, FakeTwinCATClient fake, TwinCATCapturingLogger log) NewDriver() + { + var fake = new FakeTwinCATClient(); + var log = new TwinCATCapturingLogger(); + var drv = new TwinCATDriver(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = TwinCATRawTags.Entries( + [ + new TwinCATTagDefinition("Typo", Host, "MAIN.Typo", TwinCATDataType.DInt), + new TwinCATTagDefinition("Other", Host, "MAIN.Other", TwinCATDataType.DInt), + ]), + Probe = new TwinCATProbeOptions { Enabled = false }, + }, "drv-1", new FakeTwinCATClientFactory { Customise = () => fake }, log); + return (drv, fake, log); + } + + private static int WarningsFor(TwinCATCapturingLogger log, string reference) => + log.Warnings.Count(w => w.Contains($"'{reference}'", StringComparison.Ordinal)); + + /// Repeated bad reads of one tag log one Warning; a second bad tag gets its own (positive control). + [Fact] + public async Task A_bad_tag_warns_once_across_polls_and_rearms_after_Good() + { + var (drv, fake, log) = NewDriver(); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown; + + for (var i = 0; i < 5; i++) + (await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken)) + .Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + WarningsFor(log, "Typo").ShouldBe(1); + + fake.ReadStatuses["MAIN.Other"] = TwinCATStatusMapper.BadNodeIdUnknown; + _ = await drv.ReadAsync(["Other"], TestContext.Current.CancellationToken); + WarningsFor(log, "Other").ShouldBe(1); + + // Fixed, then broken again: a new episode warns again. + fake.ReadStatuses.Remove("MAIN.Typo"); + _ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken); + fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown; + _ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken); + _ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken); + WarningsFor(log, "Typo").ShouldBe(2); + } + + /// A re-initialise re-arms the warning. + [Fact] + public async Task A_reinitialise_rearms_the_warning() + { + var (drv, fake, log) = NewDriver(); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown; + _ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken); + + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken); + + WarningsFor(log, "Typo").ShouldBe(2); + } + + /// + /// A status proving the device answered (symbol not found) keeps the driver Healthy and advances the + /// stamp; a comms-class status (timeout) degrades with the stamp preserved. + /// + [Fact] + public async Task A_refusal_status_does_not_degrade_but_a_comms_status_does() + { + var (drv, fake, _) = NewDriver(); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var initStamp = drv.GetHealth().LastSuccessfulRead!.Value; + fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown; + await Task.Delay(300, TestContext.Current.CancellationToken); + + _ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken); + var afterRefusal = drv.GetHealth(); + afterRefusal.State.ShouldBe(DriverState.Healthy); + afterRefusal.LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp); + + fake.ReadStatuses["MAIN.Other"] = TwinCATStatusMapper.BadTimeout; + _ = await drv.ReadAsync(["Other"], TestContext.Current.CancellationToken); + var afterTimeout = drv.GetHealth(); + afterTimeout.State.ShouldBe(DriverState.Degraded); + afterTimeout.LastSuccessfulRead.ShouldBe(afterRefusal.LastSuccessfulRead); + } +} From 12b981dc1bd522e27b9b4c4eb7858226a37e9491 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:07:42 -0400 Subject: [PATCH 32/51] fix(focas): carry macro/parameter numbers to the CNC unnarrowed (R07:H14) The capability matrix admits #0..99999 on 30i/31i/32i (and anything on Unknown), and the FOCAS/2 request block has always carried 32-bit arguments, but WireFocasClient cast every number to short: MACRO:99999 asked the CNC for #-31073 and published that variable at Good under the authored tag. FocasWireClient's macro/parameter reads, WireMacro.Number and WireParameter.DataNumber are now int; the request blocks come from MacroRequest / ParameterRequest builders (byte-identical to the live 31i-B capture for #500). Same class on PMC: a range whose last byte passes 65535 (Unknown series admits it) is now refused BadOutOfRange instead of wrapping onto a different 16-bit address. The wire client is in-process (the other end is the CNC itself), so this is not a cross-repo contract change. Whether a real 30i honours cnc_rdmacro Arg1 > 32767 is the hardware userGate. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Wire/FocasWireClient.cs | 54 ++++++++---- .../Wire/FocasWireModels.cs | 4 +- .../Wire/WireFocasClient.cs | 39 +++++++-- .../Series/WireBackendCoverageTests.cs | 69 +++++++++++++++ .../FocasWideNumberTests.cs | 84 +++++++++++++++++++ 5 files changed, 225 insertions(+), 25 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasWideNumberTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireClient.cs index db407d18..53a23d2c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireClient.cs @@ -467,18 +467,41 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// Optional path ID override; defaults to . /// The raw parameter payload bytes, wrapped in a . public async Task> ReadParameterBytesAsync( - short dataNumber, + int dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) { using var callTimeout = CreateCallTimeout(cancellationToken, timeout); - var secondArg = axis == 0 ? dataNumber : axis; - var block = await SendSingleRequestAsync(callTimeout.Token, new RequestBlock(0x000e, dataNumber, secondArg, PathId: EffectivePathId(pathId))).ConfigureAwait(false); + var block = await SendSingleRequestAsync(callTimeout.Token, ParameterRequest(dataNumber, axis, EffectivePathId(pathId))).ConfigureAwait(false); return ToResult(block, payload => payload); } + /// + /// The cnc_rdparam (0x000e) request block. The data number travels in the 32-bit + /// Arg1 UNNARROWED — the request block has always carried arguments, and + /// a short cast in the adapter once asked the CNC for a different (wrapped) number than the + /// one authored. A global parameter repeats the number in Arg2; an axis parameter carries + /// the axis there. + /// + /// The parameter data number. + /// Axis selector; zero means a global parameter. + /// The CNC path. + /// The request block. + internal static RequestBlock ParameterRequest(int dataNumber, short axis, ushort pathId) => + new(0x000e, dataNumber, axis == 0 ? dataNumber : axis, PathId: pathId); + + /// + /// The cnc_rdmacro (0x0015) request block: the variable number, unnarrowed, in both 32-bit + /// arguments (start / end of a one-variable range) — byte-identical to the live 31i-B capture for #500. + /// + /// The macro variable number. + /// The CNC path. + /// The request block. + internal static RequestBlock MacroRequest(int number, ushort pathId) => + new(0x0015, number, number, PathId: pathId); + /// Typed Int32 parameter read — convenience over . /// The parameter data number. /// The FOCAS parameter type code. @@ -487,7 +510,7 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// Optional path ID override; defaults to . /// The typed parameter value, wrapped in a . public async Task> ReadParameterAsync( - short dataNumber, + int dataNumber, short type = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, @@ -507,7 +530,7 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// Optional per-call timeout override. /// Optional path ID override; defaults to . /// The typed 8-bit parameter value, wrapped in a . - public async Task> ReadParameterByteAsync(short dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) + public async Task> ReadParameterByteAsync(int dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) { var result = await ReadParameterBytesAsync(dataNumber, axis, cancellationToken, timeout, pathId).ConfigureAwait(false); return !result.IsOk || result.Value is null @@ -522,7 +545,7 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// Optional per-call timeout override. /// Optional path ID override; defaults to . /// The typed 16-bit parameter value, wrapped in a . - public async Task> ReadParameterInt16Async(short dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) + public async Task> ReadParameterInt16Async(int dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) { var result = await ReadParameterBytesAsync(dataNumber, axis, cancellationToken, timeout, pathId).ConfigureAwait(false); return !result.IsOk || result.Value is null @@ -537,7 +560,7 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// Optional per-call timeout override. /// Optional path ID override; defaults to . /// The typed 32-bit parameter value, wrapped in a . - public async Task> ReadParameterInt32Async(short dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) + public async Task> ReadParameterInt32Async(int dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) { var result = await ReadParameterBytesAsync(dataNumber, axis, cancellationToken, timeout, pathId).ConfigureAwait(false); return !result.IsOk || result.Value is null @@ -552,7 +575,7 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// Optional per-call timeout override. /// Optional path ID override; defaults to . /// The typed single-precision parameter value, wrapped in a . - public async Task> ReadParameterFloat32Async(short dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) + public async Task> ReadParameterFloat32Async(int dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) { var result = await ReadParameterBytesAsync(dataNumber, axis, cancellationToken, timeout, pathId).ConfigureAwait(false); return !result.IsOk || result.Value is null || result.Value.Length < 4 @@ -567,7 +590,7 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// Optional per-call timeout override. /// Optional path ID override; defaults to . /// The typed double-precision parameter value, wrapped in a . - public async Task> ReadParameterFloat64Async(short dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) + public async Task> ReadParameterFloat64Async(int dataNumber, short axis = 0, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) { var result = await ReadParameterBytesAsync(dataNumber, axis, cancellationToken, timeout, pathId).ConfigureAwait(false); return !result.IsOk || result.Value is null || result.Value.Length < 8 @@ -582,14 +605,17 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// Optional path ID override; defaults to . /// The macro variable value, wrapped in a . public Task> ReadMacroAsync( - short number, + int number, CancellationToken cancellationToken = default, TimeSpan? timeout = null, ushort? pathId = null) - => ReadSingleWithTimeoutAsync( - 0x0015, + { + var request = MacroRequest(number, EffectivePathId(pathId)); + return ReadSingleWithTimeoutAsync( + request.Command, payload => ParseMacro(number, payload), - cancellationToken, timeout, EffectivePathId(pathId), number, number); + cancellationToken, timeout, request.PathId, request.Arg1, request.Arg2); + } /// /// Decode a cnc_rdmacro (0x0015) payload. The on-wire ODBM record is 8 @@ -609,7 +635,7 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// The response-block payload. /// The decoded macro. /// The payload is shorter than 8 bytes. - internal static WireMacro ParseMacro(short number, byte[] payload) + internal static WireMacro ParseMacro(int number, byte[] payload) { RequireLength(payload, 8, "cnc_rdmacro"); return new WireMacro(number, ReadInt32(payload, 0), ReadInt16(payload, 6)); diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireModels.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireModels.cs index ac6f32ee..70ca55f6 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireModels.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireModels.cs @@ -170,7 +170,7 @@ public readonly record struct WireSpindleRecord(short Index, string Name); /// Parameter value returned by cnc_rdparam, interpreted as a scalar Int32. public sealed record WireParameter( - short DataNumber, + int DataNumber, short Type, int Value); @@ -184,7 +184,7 @@ public sealed record WireParameter( /// The 9-significant-digit mantissa. /// Implied decimal places; negative (0xFFFF on the wire) = vacant. public sealed record WireMacro( - short Number, + int Number, int Value, short Decimal) { diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/WireFocasClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/WireFocasClient.cs index d6d5a3a4..ec44e6e3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/WireFocasClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/WireFocasClient.cs @@ -80,6 +80,7 @@ public sealed class WireFocasClient : IFocasClient var area = FocasPmcAreaLookup.FromLetter(letter); if (area is null) return (null, FocasStatusMapper.BadNodeIdUnknown); var dataType = FocasPmcDataTypeLookup.FromFocasDataType(slotType); + if (start < 0 || end > ushort.MaxValue || end < start) return (null, FocasStatusMapper.BadOutOfRange); try { var result = await _wire.ReadPmcRangeAsync(area.Value, dataType, (ushort)start, (ushort)end, cancellationToken) @@ -334,18 +335,38 @@ public sealed class WireFocasClient : IFocasClient // ---- PMC / Parameter / Macro read paths ------------------------------------------ + /// + /// The 16-bit byte range pmc_rdpmcrng addresses for a slot of bytes at + /// . False when any byte of it falls outside 0..65535 — the range is refused + /// rather than wrapped onto a different PMC address (an Unknown-series device admits any number). + /// + /// The authored PMC byte address. + /// The slot width in bytes (≥ 1). + /// The first byte, when representable. + /// The last byte, when representable. + /// True when the whole slot is addressable. + internal static bool TryPmcRange(int number, int width, out ushort start, out ushort end) + { + start = end = 0; + var last = (long)number + Math.Max(width, 1) - 1; + if (number < 0 || last > ushort.MaxValue) return false; + start = (ushort)number; + end = (ushort)last; + return true; + } + private async Task<(object? value, uint status)> ReadPmcAsync( FocasAddress address, FocasDataType type, CancellationToken cancellationToken) { var area = FocasPmcAreaLookup.FromLetter(address.PmcLetter ?? string.Empty); if (area is null) return (null, FocasStatusMapper.BadNodeIdUnknown); var dataType = FocasPmcDataTypeLookup.FromFocasDataType(type); - var start = (ushort)address.Number; // pmc_rdpmcrng returns (end-start+1) BYTES. A multi-byte slot (Word/Long/Real/Double) needs // the range widened to its byte width, else the value parser gets too few bytes → 0 values // → spurious BadOutOfRange (live-confirmed on the 31i-B: a Word read with end=start returned // a single byte). Bit/Byte stay width 1 so end==start. 2026-06-25. - var end = (ushort)(start + FocasWireClient.PmcByteWidth((short)dataType) - 1); + if (!TryPmcRange(address.Number, FocasWireClient.PmcByteWidth((short)dataType), out var start, out var end)) + return (null, FocasStatusMapper.BadOutOfRange); try { @@ -373,23 +394,23 @@ public sealed class WireFocasClient : IFocasClient switch (type) { case FocasDataType.Byte: - var b = await _wire.ReadParameterByteAsync((short)address.Number, 0, cancellationToken).ConfigureAwait(false); + var b = await _wire.ReadParameterByteAsync(address.Number, 0, cancellationToken).ConfigureAwait(false); return b.IsOk ? ((object)(sbyte)b.Value, FocasStatusMapper.Good) : (null, FocasStatusMapper.MapFocasReturn(b.Rc)); case FocasDataType.Int16: - var s = await _wire.ReadParameterInt16Async((short)address.Number, 0, cancellationToken).ConfigureAwait(false); + var s = await _wire.ReadParameterInt16Async(address.Number, 0, cancellationToken).ConfigureAwait(false); return s.IsOk ? ((object)s.Value, FocasStatusMapper.Good) : (null, FocasStatusMapper.MapFocasReturn(s.Rc)); case FocasDataType.Float32: - var f = await _wire.ReadParameterFloat32Async((short)address.Number, 0, cancellationToken).ConfigureAwait(false); + var f = await _wire.ReadParameterFloat32Async(address.Number, 0, cancellationToken).ConfigureAwait(false); return f.IsOk ? ((object)f.Value, FocasStatusMapper.Good) : (null, FocasStatusMapper.MapFocasReturn(f.Rc)); case FocasDataType.Float64: - var d = await _wire.ReadParameterFloat64Async((short)address.Number, 0, cancellationToken).ConfigureAwait(false); + var d = await _wire.ReadParameterFloat64Async(address.Number, 0, cancellationToken).ConfigureAwait(false); return d.IsOk ? ((object)d.Value, FocasStatusMapper.Good) : (null, FocasStatusMapper.MapFocasReturn(d.Rc)); case FocasDataType.Bit when address.BitIndex is int bit: - var bi = await _wire.ReadParameterInt32Async((short)address.Number, 0, cancellationToken).ConfigureAwait(false); + var bi = await _wire.ReadParameterInt32Async(address.Number, 0, cancellationToken).ConfigureAwait(false); if (!bi.IsOk) return (null, FocasStatusMapper.MapFocasReturn(bi.Rc)); return ((object)((bi.Value >> bit & 1) != 0), FocasStatusMapper.Good); default: - var i = await _wire.ReadParameterInt32Async((short)address.Number, 0, cancellationToken).ConfigureAwait(false); + var i = await _wire.ReadParameterInt32Async(address.Number, 0, cancellationToken).ConfigureAwait(false); return i.IsOk ? ((object)i.Value, FocasStatusMapper.Good) : (null, FocasStatusMapper.MapFocasReturn(i.Rc)); } } @@ -404,7 +425,7 @@ public sealed class WireFocasClient : IFocasClient { try { - var result = await _wire.ReadMacroAsync((short)address.Number, cancellationToken).ConfigureAwait(false); + var result = await _wire.ReadMacroAsync(address.Number, cancellationToken).ConfigureAwait(false); if (!result.IsOk || result.Value is null) return (null, FocasStatusMapper.MapFocasReturn(result.Rc)); return MapMacro(result.Value, type, address.BitIndex); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendCoverageTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendCoverageTests.cs index 314fda86..4cbf460e 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendCoverageTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendCoverageTests.cs @@ -72,6 +72,75 @@ public sealed class WireBackendCoverageTests } } + /// + /// #740 / R07:H14 — a macro / parameter number above 32767 reaches the CNC unnarrowed. The mock decodes + /// the request's 32-bit Arg1 as the number, so a decoy is seeded at the address the old + /// (short) cast produced (99999 → -31073, 40000 → -25536, as the unsigned key the mock sees): + /// reading the decoy's value means the number was wrapped. #500 / #6711 are the positive control that + /// ordinary numbers still read. + /// + [Fact] + public async Task Macro_and_parameter_numbers_above_short_range_read_the_authored_variable() + { + if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason); + var ct = TestContext.Current.CancellationToken; + + static string Wrapped(int number) => ((uint)(int)(short)number).ToString(System.Globalization.CultureInfo.InvariantCulture); + + await _fx.LoadProfileAsync("FWLIB64", ct); + await _fx.PatchStateAsync(new + { + parameters = new Dictionary + { + ["6711"] = new { type = "long", value = 1234, @decimal = 0 }, + ["40000"] = new { type = "long", value = 4242, @decimal = 0 }, + [Wrapped(40000)] = new { type = "long", value = -1, @decimal = 0 }, + }, + macros = new Dictionary + { + ["500"] = new { value = 42000, @decimal = 3 }, + ["99999"] = new { value = 7, @decimal = 0 }, + [Wrapped(99999)] = new { value = 666, @decimal = 0 }, + }, + }, ct); + + var drv = new FocasDriver(new FocasDriverOptions + { + // ThirtyOne_i: the capability matrix admits #99999 (its documented ceiling); the parameter is read on + // an Unknown-series device, which admits any number. + Devices = [new FocasDeviceOptions(DeviceHost, Series: FocasCncSeries.ThirtyOne_i)], + RawTags = FocasRawTags.Entries( + new FocasTagDefinition("Macro500", DeviceHost, "MACRO:500", FocasDataType.Float64, Writable: false), + new FocasTagDefinition("Macro99999", DeviceHost, "MACRO:99999", FocasDataType.Int32, Writable: false), + new FocasTagDefinition("Param6711", DeviceHost, "PARAM:6711", FocasDataType.Int32, Writable: false)), + Probe = new FocasProbeOptions { Enabled = false }, + }, driverInstanceId: "wire-wide-numbers", clientFactory: new WireFocasClientFactory()); + + var unknownSeries = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(DeviceHost)], + RawTags = FocasRawTags.Entries( + new FocasTagDefinition("Param40000", DeviceHost, "PARAM:40000", FocasDataType.Int32, Writable: false)), + Probe = new FocasProbeOptions { Enabled = false }, + }, driverInstanceId: "wire-wide-param", clientFactory: new WireFocasClientFactory()); + + await using (drv) + await using (unknownSeries) + { + await drv.InitializeAsync("{}", ct); + var snaps = await drv.ReadAsync(["Macro500", "Macro99999", "Param6711"], ct); + snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.Good); + Convert.ToDouble(snaps[0].Value).ShouldBe(42.0, tolerance: 0.001); + snaps[1].Value.ShouldBe(7, "MACRO:99999 must read #99999, not the wrapped #-31073 decoy (666)"); + Convert.ToInt32(snaps[2].Value).ShouldBe(1234); + + await unknownSeries.InitializeAsync("{}", ct); + var param = await unknownSeries.ReadAsync(["Param40000"], ct); + param[0].StatusCode.ShouldBe(FocasStatusMapper.Good); + Convert.ToInt32(param[0].Value).ShouldBe(4242, "PARAM:40000 must not read the wrapped decoy (-1)"); + } + } + /// Verifies that discover emits device folder and tag variables. [Fact] public async Task Discover_emits_device_folder_and_tag_variables() diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasWideNumberTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasWideNumberTests.cs new file mode 100644 index 00000000..7bf63a40 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasWideNumberTests.cs @@ -0,0 +1,84 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; + +/// +/// #740 / R07:H14 — the capability matrix admits macro numbers up to 99999 (30i/31i/32i) and parameters up +/// to 29999 (Unknown: anything), and the FOCAS/2 request block carries 32-bit arguments, but the adapter +/// cast every number to short: MACRO:99999 asked the CNC for #-31073 and published +/// that variable Good under the authored tag. Numbers now travel as int end to end. +/// +[Trait("Category", "Unit")] +public sealed class FocasWideNumberTests +{ + /// The macro request carries the full number in both 32-bit argument slots. + [Theory] + [InlineData(500)] + [InlineData(32767)] + [InlineData(32768)] + [InlineData(99999)] + public void Macro_request_carries_the_unnarrowed_number(int number) + { + var block = FocasWireClient.MacroRequest(number, pathId: 1); + block.Command.ShouldBe((ushort)0x0015); + block.Arg1.ShouldBe(number); + block.Arg2.ShouldBe(number); + } + + /// + /// Positive control on the wire encoding: the request for #500 is byte-identical to the one the real + /// 31i-B answered in the ref-macro-500-0x0015 capture, so widening changed nothing below 32768. + /// + [Fact] + public void Macro_request_for_500_matches_the_live_capture_bytes() + { + var body = FocasWireProtocol.BuildRequestBody([FocasWireClient.MacroRequest(500, pathId: 1)]); + Convert.ToHexString(body).ShouldBe("0001001C0001000100150000 01F4000001F4000000000000000000000000".Replace(" ", "")); + } + + /// A global parameter puts its number in both arguments; an axis parameter puts the axis second. + [Theory] + [InlineData(6711, (short)0, 6711)] + [InlineData(40000, (short)0, 40000)] + [InlineData(40000, (short)2, 2)] + public void Parameter_request_carries_the_unnarrowed_number(int number, short axis, int expectedArg2) + { + var block = FocasWireClient.ParameterRequest(number, axis, pathId: 1); + block.Command.ShouldBe((ushort)0x000e); + block.Arg1.ShouldBe(number); + block.Arg2.ShouldBe(expectedArg2); + } + + /// The decoded macro reports the number that was asked for, not a wrapped one. + [Fact] + public void Parsed_macro_keeps_a_number_above_short_range() + { + var payload = new byte[] { 0x05, 0xF5, 0xE1, 0x00, 0x00, 0x0A, 0x00, 0x08 }; + var m = FocasWireClient.ParseMacro(99999, payload); + m.Number.ShouldBe(99999); + m.Scaled.ShouldBe(1.0); + } + + /// + /// pmc_rdpmcrng addresses are 16-bit on the wire; a PMC number whose slot runs past 65535 is refused + /// rather than wrapped onto a different address (Unknown series admits any number). + /// + [Theory] + [InlineData(100, 1, true)] + [InlineData(65535, 1, true)] + [InlineData(65534, 2, true)] + [InlineData(65535, 2, false)] + [InlineData(70000, 1, false)] + public void Pmc_range_outside_16_bits_is_refused_not_wrapped(int number, int width, bool ok) + { + WireFocasClient.TryPmcRange(number, width, out var start, out var end).ShouldBe(ok); + if (ok) + { + ((int)start).ShouldBe(number); + ((int)end).ShouldBe(number + width - 1); + } + } +} From 5097444d56f9b7cad8ae979eba5c17fd35ed2c41 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:08:42 -0400 Subject: [PATCH 33/51] fix(controlplane): condition the dispatch flip on the guard's RowVersion; one deadline source; align IdentityModel pins Re-review 1: HandleDispatch's terminal guard and its AwaitingApplyAcks flip read the row separately, so a lost-dispatch closure committed between them was silently reopened. The flip now carries the RowVersion the guard read; a change in between is a concurrency conflict (nothing seeded, nothing published, coordinator reset). Pinned on real SQL Server with a closure injected between the two reads (fails without the fix). AdminOps' apply deadline and the coordinator's now both read DeployTimings.ApplyDeadline, passed explicitly to both singletons. Microsoft.IdentityModel.Protocols(.OpenIdConnect) pinned to 8.11.0 and referenced directly by the Security project, so the whole IdentityModel family resolves 8.11.0 (fresh restore clean of NU190x/NU1605). Refs #733 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- Directory.Packages.props | 8 ++- .../AdminOperations/AdminOperationsActor.cs | 9 +-- .../Coordinators/ConfigPublishCoordinator.cs | 63 +++++++++++----- .../ServiceCollectionExtensions.cs | 8 ++- .../ZB.MOM.WW.OtOpcUa.Security.csproj | 3 + .../ConfigPublishCoordinatorRecoveryTests.cs | 71 +++++++++++++++++++ 6 files changed, 138 insertions(+), 24 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index ed2dca67..d63e819e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -36,8 +36,10 @@ + line as the other Microsoft.AspNetCore.* pins. It depends on Microsoft.IdentityModel.Protocols(.OpenIdConnect) + 8.0.1; those two are pinned to 8.11.0 below (and referenced directly by the Security project, the JwtBearer + entry point, so the pin applies without transitive pinning) to keep the whole IdentityModel family on one + version with Microsoft.IdentityModel.Tokens / System.IdentityModel.Tokens.Jwt. --> @@ -86,6 +88,8 @@ + + diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs index 81f09dae..69e878db 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/AdminOperations/AdminOperationsActor.cs @@ -11,7 +11,6 @@ using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Configuration.Entities; using ZB.MOM.WW.OtOpcUa.Configuration.Enums; using ZB.MOM.WW.OtOpcUa.Configuration.Validation; -using ZB.MOM.WW.OtOpcUa.ControlPlane.Coordinators; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.Scripting; @@ -55,7 +54,7 @@ public sealed class AdminOperationsActor : ReceiveActor /// Central comm actor commands are handed to, or to publish on the /// DistributedPubSub mediator directly (the behaviour before the per-cluster mesh). /// - /// The coordinator's apply deadline; defaults to . + /// The coordinator's apply deadline; defaults to (the coordinator's default source). /// Props configured to create an AdminOperationsActor. public static Props Props( IDbContextFactory dbFactory, @@ -73,7 +72,7 @@ public sealed class AdminOperationsActor : ReceiveActor /// Driver probes registered in DI; keyed by DriverType (case-insensitive). /// Deploy-gate TagConfig strictness mode (Warn default | Error opt-in). /// Central comm actor, or for the mediator. - /// The coordinator's apply deadline; defaults to . + /// The coordinator's apply deadline; defaults to (the coordinator's default source). public AdminOperationsActor( IDbContextFactory dbFactory, IActorRef coordinator, @@ -82,7 +81,9 @@ public sealed class AdminOperationsActor : ReceiveActor IActorRef? meshRouter = null, TimeSpan? applyDeadline = null) { - _applyDeadline = applyDeadline ?? ConfigPublishCoordinator.DefaultApplyDeadline; + // The same source the coordinator's default reads (ConfigPublishCoordinator.DefaultApplyDeadline IS + // DeployTimings.ApplyDeadline), so the lost-dispatch closure and the coordinator's own deadline agree. + _applyDeadline = applyDeadline ?? DeployTimings.ApplyDeadline; _dbFactory = dbFactory; _coordinator = coordinator; _probesByType = probes.ToDictionary(p => p.DriverType, StringComparer.OrdinalIgnoreCase); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs index 49b54df4..d408343c 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/Coordinators/ConfigPublishCoordinator.cs @@ -237,21 +237,28 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers // Terminal-state guard (fix round 1): a late or re-delivered dispatch for a row that is already // Sealed / PartiallyFailed / TimedOut — e.g. one AdminOps closed as a lost dispatch while this message // was still queued — must never flip it back to AwaitingApplyAcks or seed an expected set for it. + // + // The RowVersion read HERE is the one the AwaitingApplyAcks flip below is conditioned on (re-review 1): + // reading a fresh one at flip time left a window in which AdminOps' lost-dispatch closure could commit + // TimedOut between the guard and the flip, and the flip would silently reopen the row. With the guard's + // RowVersion, any change since the guard read makes the flip a concurrency conflict instead. + byte[]? guardRowVersion; using (var check = _dbFactory.CreateDbContext()) { - var status = check.Deployments + var guard = check.Deployments .AsNoTracking() .Where(d => d.DeploymentId == msg.DeploymentId.Value) - .Select(d => (DeploymentStatus?)d.Status) + .Select(d => new { d.Status, d.RowVersion }) .FirstOrDefault(); - if (status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) + if (guard?.Status is DeploymentStatus.Sealed or DeploymentStatus.PartiallyFailed or DeploymentStatus.TimedOut) { _log.Warning( "Ignoring DispatchDeployment {Id}: the deployment is already {Status} (a late or duplicated dispatch)", - msg.DeploymentId, status); + msg.DeploymentId, guard.Status); if (_current == msg.DeploymentId) ResetForNext(); return; } + guardRowVersion = guard?.RowVersion; } _current = msg.DeploymentId; @@ -290,8 +297,22 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers }); } } - UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks); - db.SaveChanges(); + UpdateDeploymentStatus(db, msg.DeploymentId, DeploymentStatus.AwaitingApplyAcks, rowVersion: guardRowVersion); + try + { + db.SaveChanges(); + } + catch (DbUpdateConcurrencyException) + { + // The row moved after the guard read (on SQL Server: AdminOps closed it as a lost dispatch). The + // expected-set rows and the flip were one SaveChanges, so nothing was written; nothing is + // published and the coordinator is free for the next deployment. + _log.Warning( + "DispatchDeployment {Id} abandoned: the deployment changed state while it was being dispatched " + + "(closed as a lost dispatch?) — not reopening it", msg.DeploymentId); + ResetForNext(); + return; + } } // Per-cluster mesh Phase 2: hand off to the transport router rather than publishing @@ -483,9 +504,9 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers /// stub stays tracked so can add to the same UPDATE. /// private static void UpdateDeploymentStatus( - OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false) + OtOpcUaConfigDbContext db, DeploymentId id, DeploymentStatus status, bool sealNow = false, byte[]? rowVersion = null) { - var entry = TrackStub(db, id); + var entry = TrackStub(db, id, rowVersion); if (entry is null) return; entry.Entity.Status = status; entry.Property(x => x.Status).IsModified = true; @@ -507,20 +528,30 @@ public sealed class ConfigPublishCoordinator : ReceiveActor, IWithTimers } /// The tracked blob-free stub for a deployment, attaching one on first use; null when the row is gone. + /// The context. + /// The deployment. + /// + /// The RowVersion the update must be conditioned on, when the caller already read one (the dispatch + /// guard); reads the current one. + /// private static Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry? TrackStub( - OtOpcUaConfigDbContext db, DeploymentId id) + OtOpcUaConfigDbContext db, DeploymentId id, byte[]? rowVersion = null) { var tracked = db.ChangeTracker.Entries().FirstOrDefault(e => e.Entity.DeploymentId == id.Value); if (tracked is not null) return tracked; - var stamp = db.Deployments - .AsNoTracking() - .Where(x => x.DeploymentId == id.Value) - .Select(x => new { x.RowVersion }) - .FirstOrDefault(); - if (stamp is null) return null; + if (rowVersion is null) + { + var stamp = db.Deployments + .AsNoTracking() + .Where(x => x.DeploymentId == id.Value) + .Select(x => new { x.RowVersion }) + .FirstOrDefault(); + if (stamp is null) return null; + rowVersion = stamp.RowVersion; + } return db.Deployments.Attach(new Deployment { - DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = stamp.RowVersion, + DeploymentId = id.Value, RevisionHash = string.Empty, CreatedBy = string.Empty, RowVersion = rowVersion, }); } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs index f838a004..9a9f0742 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.ControlPlane/ServiceCollectionExtensions.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; using ZB.MOM.WW.OtOpcUa.Configuration; using ZB.MOM.WW.OtOpcUa.Cluster; +using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; using ZB.MOM.WW.OtOpcUa.ControlPlane.Audit; @@ -111,7 +112,7 @@ public static class ServiceCollectionExtensions // coordinator. Deliberately NOT an ActorSelection.ResolveOne here: that blocks // inside a props factory and races the spawn it is waiting for. var meshRouter = registry.Get(); - return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: null, meshRouter: meshRouter); + return ConfigPublishCoordinator.Props(dbFactory, applyDeadline: DeployTimings.ApplyDeadline, meshRouter: meshRouter); }, singletonOptions); @@ -128,7 +129,10 @@ public static class ServiceCollectionExtensions resolver.GetService()?["Deployment:TagConfigValidationMode"], ignoreCase: true, out var m) ? m : TagConfigValidationMode.Warn; var meshRouter = registry.Get(); - return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter); + // The SAME deadline source as the coordinator above: AdminOps closes a lost dispatch only once it + // is past the coordinator's deadline, so the two must never drift. + return AdminOperationsActor.Props(dbFactory, coordinator, probes, mode, meshRouter, + applyDeadline: DeployTimings.ApplyDeadline); }, singletonOptions); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj index 507aca6c..8ee03814 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Security/ZB.MOM.WW.OtOpcUa.Security.csproj @@ -11,6 +11,9 @@ + + + diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs index 0added3d..8c93fb1c 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.ControlPlane.Tests/ConfigPublishCoordinatorRecoveryTests.cs @@ -311,6 +311,77 @@ public sealed class ConfigPublishCoordinatorRecoveryTests : ControlPlaneActorTes }, duration: PresenceBudget); } + /// + /// Re-review 1 — the reopen race. AdminOps' lost-dispatch closure commits TimedOut AFTER the coordinator's + /// terminal guard read the row as Dispatching but BEFORE the AwaitingApplyAcks flip. The flip is + /// conditioned on the guard's RowVersion, so on SQL Server it becomes a concurrency conflict and the row + /// stays TimedOut with no expected set. Positive control: the coordinator is not wedged — the next live + /// dispatch is seeded and flipped. + /// + [Fact] + [Trait("Category", "SchemaCompliance")] + public async Task On_sql_server_a_closure_between_guard_and_flip_is_not_reopened() + { + using var sql = new SqlServerTestDatabase(); + var inner = sql.Factory(); + SeedNodes(inner, "site-a-1:4053"); + var interposer = new InterposingDbFactory(inner); + var coordinator = Sys.ActorOf(ConfigPublishCoordinator.Props(interposer, NoTimeout)); + await Sys.ActorSelection(coordinator.Path).ResolveOne(PresenceBudget); // PreStart ran over an empty table + + // The race proper: a new Dispatching row whose dispatch is interleaved with an AdminOps-style closure. + var victim = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow); + interposer.OnSecondCreate(() => + { + using var db = sql.NewContext(); + db.Database.ExecuteSqlRaw( + "UPDATE Deployment SET Status = {0}, FailureReason = N'closed as lost' WHERE DeploymentId = {1}", + (int)DeploymentStatus.TimedOut, victim.Value); + }); + coordinator.Tell(new DispatchDeployment(victim, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => interposer.Fired.ShouldBeTrue(), duration: PresenceBudget); + + // Positive control + ordering: a live row dispatched after it IS flipped, so the victim's dispatch was processed. + var live = SeedDeployment(inner, DeploymentStatus.Dispatching, DateTime.UtcNow); + coordinator.Tell(new DispatchDeployment(live, TestRevision, CorrelationId.NewId())); + AwaitAssert(() => + { + using var db = sql.NewContext(); + db.Deployments.AsNoTracking().Single(d => d.DeploymentId == live.Value).Status + .ShouldBe(DeploymentStatus.AwaitingApplyAcks); + }, duration: PresenceBudget); + + using var verify = sql.NewContext(); + verify.Deployments.AsNoTracking().Single(d => d.DeploymentId == victim.Value).Status + .ShouldBe(DeploymentStatus.TimedOut, "the closure must not be reopened"); + verify.NodeDeploymentStates.Count(s => s.DeploymentId == victim.Value).ShouldBe(0); + } + + /// Runs an action just before handing out the SECOND context after arming (HandleDispatch: guard, then flip). + private sealed class InterposingDbFactory(IDbContextFactory inner) + : IDbContextFactory + { + private Action? _action; + private int _countdown; + public volatile bool Fired; + + public void OnSecondCreate(Action action) + { + _action = action; + Volatile.Write(ref _countdown, 2); + } + + public OtOpcUaConfigDbContext CreateDbContext() + { + if (Volatile.Read(ref _countdown) > 0 && Interlocked.Decrement(ref _countdown) == 0) + { + _action?.Invoke(); + Fired = true; + } + return inner.CreateDbContext(); + } + } + private static DeploymentId SeedDeployment( IDbContextFactory dbFactory, DeploymentStatus status, DateTime createdAtUtc) { From 40c74938298fc732f2de2e29b3ba133817f2f476 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:10:12 -0400 Subject: [PATCH 34/51] fix(twincat): BOOL-within-word writes RMW the parent at its own width (H18) The bit write read and wrote the parent word as a 32-bit UDInt with no parent check: a bit beyond a 16-bit parent was written into neighbouring bytes, and a signed parent with its sign bit set read back >= 2^31, so the checked conversion turned any bit write into BadOutOfRange. The write now goes through ITwinCATClient.WriteBitAsync: the ADS client resolves the parent symbol, checks the bit against it (TwinCATBitAccess. CheckParent, as the notification path already did), reads exactly parent.ByteSize raw bytes, flips one bit (TwinCATBitAccess.ApplyBitWrite) and writes the bytes back by index group/offset or variable handle. A bit beyond the parent is BadOutOfRange, a non-integer parent BadTypeMismatch, a short parent read BadCommunicationError (never written back), all without a health write. The per-parent RMW lock stays in the driver. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdsTwinCATClient.cs | 55 +++++++++++++++++ .../ITwinCATClient.cs | 16 +++++ .../TwinCATBitAccess.cs | 30 ++++++++++ .../TwinCATDriver.cs | 31 +++------- .../FakeTwinCATClient.cs | 34 +++++++++++ .../TwinCATBitAccessTests.cs | 46 +++++++++++++++ .../TwinCATReadWriteTests.cs | 59 ++++++++++++------- 7 files changed, 227 insertions(+), 44 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs index 53481b5e..a143d018 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs @@ -482,6 +482,61 @@ internal sealed class AdsTwinCATClient : ITwinCATClient } } + /// + public async Task WriteBitAsync(string parentPath, int bit, bool value, CancellationToken cancellationToken) + { + try + { + // Resolve the parent so the RMW happens at its OWN width (the notification path's model): + // the old 32-bit UDInt round-trip wrote a 16-bit parent's neighbours and overflowed on a + // signed parent with its sign bit set. + var (parent, _, symbol, status) = await DescribeSymbolAsync(parentPath, cancellationToken) + .ConfigureAwait(false); + if (status != TwinCATStatusMapper.Good) return status; + if (TwinCATBitAccess.CheckParent(bit, parent) is not null) + return TwinCATBitAccess.ApplyBitWrite(parent, ReadOnlySpan.Empty, bit, value, out _); + + var (bytes, readStatus) = await ReadRawAsync(parentPath, symbol!, parent.ByteSize, cancellationToken) + .ConfigureAwait(false); + if (readStatus != TwinCATStatusMapper.Good) return readStatus; + var modify = TwinCATBitAccess.ApplyBitWrite(parent, bytes, bit, value, out var updated); + if (modify != TwinCATStatusMapper.Good) return modify; + return await WriteRawAsync(parentPath, symbol!, updated, cancellationToken).ConfigureAwait(false); + } + catch (AdsErrorException ex) + { + return MapAndSignal((uint)ex.ErrorCode); + } + } + + /// + /// Writes raw bytes to a resolved symbol — the mirror of : by its index + /// group/offset, or through a fresh variable handle for a handle-addressed symbol (0xF005). + /// + private async Task WriteRawAsync( + string symbolPath, IAdsSymbol symbol, ReadOnlyMemory data, CancellationToken cancellationToken) + { + if (symbol.IndexGroup != SymbolValueByHandle) + { + var direct = await _client.WriteAsync(symbol.IndexGroup, symbol.IndexOffset, data, cancellationToken) + .ConfigureAwait(false); + return direct.ErrorCode == AdsErrorCode.NoError ? TwinCATStatusMapper.Good : MapAndSignal((uint)direct.ErrorCode); + } + + var handle = await _client.CreateVariableHandleAsync(symbolPath, cancellationToken).ConfigureAwait(false); + if (handle.ErrorCode != AdsErrorCode.NoError) return MapAndSignal((uint)handle.ErrorCode); + try + { + var write = await _client.WriteAsync(handle.Handle, data, cancellationToken).ConfigureAwait(false); + return write.ErrorCode == AdsErrorCode.NoError ? TwinCATStatusMapper.Good : MapAndSignal((uint)write.ErrorCode); + } + finally + { + try { await _client.DeleteVariableHandleAsync(handle.Handle, CancellationToken.None).ConfigureAwait(false); } + catch { /* best-effort release */ } + } + } + /// public async Task ProbeAsync(CancellationToken cancellationToken) { diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs index 3849edd0..1ffeb6fa 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs @@ -119,6 +119,22 @@ public interface ITwinCATClient : IDisposable object? value, CancellationToken cancellationToken); + /// + /// Sets or clears one bit of an integer parent word — the write half of a BOOL-within-word tag + /// (MAIN.Flags.3; ADS has no symbol for WORD.N). A read-modify-write of the parent's raw + /// bytes at the parent's OWN resolved width (), so a 16-bit + /// parent is never written as 32 bits and a signed parent's sign bit is just a bit. The caller + /// serialises concurrent writers to the same parent; the PLC program writing the word between the + /// read and the write is inherent to RMW. + /// + /// The parent word's ADS symbol path (the bit path without its .N). + /// The bit index (0 = least significant bit of the first byte). + /// The bit value to write. + /// Cancellation token for the read and the write. + /// The mapped OPC UA status: Good; the parent read's status; BadOutOfRange for a bit + /// beyond the parent; BadTypeMismatch for a parent that is not an integer word. + Task WriteBitAsync(string parentPath, int bit, bool value, CancellationToken cancellationToken); + /// /// Cheap health probe — returns true when the target's AMS state is reachable. /// Used by 's probe loop. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATBitAccess.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATBitAccess.cs index 186e1cee..277bd317 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATBitAccess.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATBitAccess.cs @@ -32,6 +32,36 @@ internal static class TwinCATBitAccess : $"bit {bit} is outside the {parent.ByteSize * 8}-bit parent {parent.TypeName}"; } + /// + /// The modify step of a bit write's read-modify-write: checks the bit against the parent + /// () and returns the parent's bytes — exactly parent.ByteSize of them — + /// with one bit set or cleared. Works on raw bytes, so the parent's width and signedness play no part. + /// + /// The parent symbol's resolved shape. + /// The parent's current bytes as read from the PLC. + /// The bit index. + /// The bit value. + /// The bytes to write back when the result is Good; otherwise empty. + /// Good; BadOutOfRange for a bit beyond the parent; BadTypeMismatch for a parent + /// that is not an integer word; BadCommunicationError for a read shorter than the parent + /// (writing it back would zero the missing bytes). + public static uint ApplyBitWrite(TwinCATPlcSymbolShape parent, ReadOnlySpan current, int bit, bool value, + out byte[] updated) + { + updated = []; + if (CheckParent(bit, parent) is not null) + return parent.Kind != TwinCATPlcKind.Scalar || parent.IsFloat + ? TwinCATStatusMapper.BadTypeMismatch + : TwinCATStatusMapper.BadOutOfRange; + if (current.Length < parent.ByteSize) return TwinCATStatusMapper.BadCommunicationError; + + var bytes = current[..parent.ByteSize].ToArray(); + var mask = (byte)(1 << (bit % 8)); + bytes[bit / 8] = value ? (byte)(bytes[bit / 8] | mask) : (byte)(bytes[bit / 8] & ~mask); + updated = bytes; + return TwinCATStatusMapper.Good; + } + /// Reads bit of a little-endian parent word. /// The parent word's raw bytes. /// The bit index (0 = least significant bit of the first byte). diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index 7c95a8f0..2a5d4d94 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -511,38 +511,21 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery var client = await EnsureConnectedAsync(device, cancellationToken).ConfigureAwait(false); var parsed = TwinCATSymbolPath.TryParse(def.SymbolPath); - // BOOL-within-word write — read-modify-write of the parent word. Mirrors the bit-read - // (AdsTwinCATClient.ReadValueAsync) which reads the parent as uint: read the parent as - // UDInt (-> uint), flip the bit, write it back, all under a per-parent lock. + // BOOL-within-word write — a read-modify-write of the parent word at the parent's OWN + // width and on raw bytes (the client resolves the parent and checks the bit against it, the + // notification path's model), under a per-parent lock so concurrent bit writers to the same + // word don't lose each other's updates. if (def.DataType == TwinCATDataType.Bool && parsed?.BitIndex is int bit) { var parentPath = (parsed with { BitIndex = null }).ToAdsSymbolName(); + var bitValue = Convert.ToBoolean(w.Value); var gate = _bitRmwLocks.GetOrAdd( $"{def.DeviceHostAddress}|{parentPath}", static _ => new SemaphoreSlim(1, 1)); await gate.WaitAsync(cancellationToken).ConfigureAwait(false); try { - var (parentValue, readStatus) = await client.ReadValueAsync( - parentPath, TwinCATDataType.UDInt, null, null, cancellationToken).ConfigureAwait(false); - if (readStatus != TwinCATStatusMapper.Good) - { - results[i] = new WriteResult(readStatus); - } - else if (parentValue is null) - { - // Good status but no value — treating null as 0 would write a zeroed - // word and clear every bit set on the device. Surface a Bad status and - // write nothing rather than corrupt the parent word. - results[i] = new WriteResult(TwinCATStatusMapper.BadCommunicationError); - } - else - { - var word = Convert.ToUInt32(parentValue); - var updated = Convert.ToBoolean(w.Value) ? word | (1u << bit) : word & ~(1u << bit); - var writeStatus = await client.WriteValueAsync( - parentPath, TwinCATDataType.UDInt, null, updated, cancellationToken).ConfigureAwait(false); - results[i] = new WriteResult(writeStatus); - } + results[i] = new WriteResult(await client.WriteBitAsync( + parentPath, bit, bitValue, cancellationToken).ConfigureAwait(false)); } finally { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs index 2ad33e42..e31340a5 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs @@ -136,6 +136,40 @@ internal class FakeTwinCATClient : ITwinCATClient return Task.FromResult(status); } + /// Raw bytes of bit-write parent words, by parent path (#739 H18). The byte count is the parent's width. + public Dictionary ParentWords { get; } = new(StringComparer.OrdinalIgnoreCase); + /// Optional parent shapes; absent = an unsigned integer word as wide as its entry. + public Dictionary ParentShapes { get; } = new(StringComparer.OrdinalIgnoreCase); + /// Gets the log of every call. + public List<(string parent, int bit, bool value)> BitWriteLog { get; } = new(); + /// When set, yields between its read and its write so a missing + /// per-parent lock in the driver loses updates. + public bool YieldInBitWrite { get; set; } + + /// Simulates the parent-width bit RMW with the driver's own . + /// The parent word's symbol path. + /// The bit index. + /// The bit value. + /// The cancellation token. + /// The write status. + public virtual async Task WriteBitAsync(string parentPath, int bit, bool value, CancellationToken ct) + { + if (ThrowOnWrite) throw Exception ?? new InvalidOperationException(); + lock (BitWriteLog) BitWriteLog.Add((parentPath, bit, value)); + if (ReadStatuses.TryGetValue(parentPath, out var readStatus) && readStatus != TwinCATStatusMapper.Good) + return readStatus; + byte[] current; + lock (ParentWords) current = ParentWords.TryGetValue(parentPath, out var w) ? [.. w] : []; + var shape = ParentShapes.TryGetValue(parentPath, out var sh) + ? sh + : new TwinCATPlcSymbolShape(TwinCATPlcKind.Scalar, current.Length == 0 ? 4 : current.Length, false, "UDINT"); + if (YieldInBitWrite) await Task.Yield(); + var status = TwinCATBitAccess.ApplyBitWrite(shape, current, bit, value, out var updated); + if (status != TwinCATStatusMapper.Good) return status; + lock (ParentWords) ParentWords[parentPath] = updated; + return WriteStatuses.TryGetValue(parentPath, out var s) ? s : TwinCATStatusMapper.Good; + } + /// Simulates probing the connection status. /// The cancellation token. /// A task that returns the probe result. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATBitAccessTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATBitAccessTests.cs index 3dc22507..a5b339c8 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATBitAccessTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATBitAccessTests.cs @@ -55,4 +55,50 @@ public sealed class TwinCATBitAccessTests TwinCATBitAccess.CheckParent(0, TwinCATPlcSymbolShape.Classify("ST_X", DataTypeCategory.Struct, 8)).ShouldNotBeNull(); TwinCATBitAccess.CheckParent(31, TwinCATPlcSymbolShape.Classify("DWORD", DataTypeCategory.Primitive, 4)).ShouldBeNull(); } + + // ---- #739 H18: the bit WRITE is a read-modify-write at the parent's own width ---- + + /// A bit write flips exactly one bit of the parent's raw bytes at the parent's width. + [Fact] + public void ApplyBitWrite_flips_one_bit_at_the_parent_width() + { + var word = TwinCATPlcSymbolShape.Classify("WORD", DataTypeCategory.Primitive, 2); + TwinCATBitAccess.ApplyBitWrite(word, [0x01, 0x00], 9, true, out var set).ShouldBe(TwinCATStatusMapper.Good); + set.ShouldBe(new byte[] { 0x01, 0x02 }); + TwinCATBitAccess.ApplyBitWrite(word, [0xFF, 0xFF], 3, false, out var cleared).ShouldBe(TwinCATStatusMapper.Good); + cleared.ShouldBe(new byte[] { 0xF7, 0xFF }); + } + + /// + /// A signed parent with its sign bit set is plain bytes — the old UDInt round-trip read it back as + /// a value ≥ 2³¹ and a checked conversion turned ANY bit write into BadOutOfRange. + /// + [Fact] + public void ApplyBitWrite_on_a_negative_signed_parent_preserves_the_sign_bit() + { + var intWord = TwinCATPlcSymbolShape.Classify("INT", DataTypeCategory.Primitive, 2); + TwinCATBitAccess.ApplyBitWrite(intWord, [0x00, 0x80], 0, true, out var updated).ShouldBe(TwinCATStatusMapper.Good); + updated.ShouldBe(new byte[] { 0x01, 0x80 }); + } + + /// A bit beyond the parent is BadOutOfRange; a non-integer parent BadTypeMismatch; nothing is produced. + [Fact] + public void ApplyBitWrite_refuses_a_bit_the_parent_does_not_have() + { + var word = TwinCATPlcSymbolShape.Classify("UINT", DataTypeCategory.Primitive, 2); + TwinCATBitAccess.ApplyBitWrite(word, [0, 0], 16, true, out var none).ShouldBe(TwinCATStatusMapper.BadOutOfRange); + none.ShouldBeEmpty(); + var real = TwinCATPlcSymbolShape.Classify("REAL", DataTypeCategory.Primitive, 4); + TwinCATBitAccess.ApplyBitWrite(real, [0, 0, 0, 0], 1, true, out _).ShouldBe(TwinCATStatusMapper.BadTypeMismatch); + } + + /// A parent read shorter than the parent's width is never written back (it would zero bytes). + [Fact] + public void ApplyBitWrite_refuses_a_short_parent_buffer() + { + var dword = TwinCATPlcSymbolShape.Classify("DWORD", DataTypeCategory.Primitive, 4); + TwinCATBitAccess.ApplyBitWrite(dword, [0xFF, 0xFF], 1, true, out var none) + .ShouldBe(TwinCATStatusMapper.BadCommunicationError); + none.ShouldBeEmpty(); + } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWriteTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWriteTests.cs index c81e7b88..d7dbcc0d 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWriteTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWriteTests.cs @@ -296,23 +296,23 @@ public sealed class TwinCATReadWriteTests factory.Clients[0].DisposeCount.ShouldBe(1); } - // ---- BOOL-within-word RMW writes ---- + // ---- BOOL-within-word RMW writes (#739 H18: at the parent's own width, on raw bytes) ---- - /// A BOOL-within-word set reads the parent word, ORs the bit, writes it back as UDInt. + /// A BOOL-within-word set read-modify-writes the parent word's raw bytes through WriteBitAsync. [Fact] - public async Task Bit_set_RMWs_parent_word_as_UDInt() + public async Task Bit_set_RMWs_the_parent_word_bytes() { var (drv, factory) = NewDriver( new TwinCATTagDefinition("Flag", "ads://5.23.91.23.1.1:851", "MAIN.Flags.3", TwinCATDataType.Bool)); - factory.Customise = () => new FakeTwinCATClient { Values = { ["MAIN.Flags"] = 0b0001u } }; + factory.Customise = () => new FakeTwinCATClient { ParentWords = { ["MAIN.Flags"] = [0b0001, 0, 0, 0] } }; await drv.InitializeAsync("{}", CancellationToken.None); var results = await drv.WriteAsync([new WriteRequest("Flag", true)], CancellationToken.None); results.Single().StatusCode.ShouldBe(TwinCATStatusMapper.Good); - factory.Clients[0].Values["MAIN.Flags"].ShouldBe(0b1001u); - factory.Clients[0].WriteLog.ShouldContain(e => - e.symbol == "MAIN.Flags" && e.type == TwinCATDataType.UDInt && e.bit == null); + factory.Clients[0].ParentWords["MAIN.Flags"].ShouldBe(new byte[] { 0b1001, 0, 0, 0 }); + factory.Clients[0].BitWriteLog.ShouldBe([("MAIN.Flags", 3, true)]); + factory.Clients[0].WriteLog.ShouldBeEmpty(); // no 32-bit UDInt round-trip any more } /// A BOOL-within-word clear preserves the other bits in the parent word. @@ -321,12 +321,12 @@ public sealed class TwinCATReadWriteTests { var (drv, factory) = NewDriver( new TwinCATTagDefinition("Flag", "ads://5.23.91.23.1.1:851", "MAIN.Flags.3", TwinCATDataType.Bool)); - factory.Customise = () => new FakeTwinCATClient { Values = { ["MAIN.Flags"] = 0xFFFFu } }; + factory.Customise = () => new FakeTwinCATClient { ParentWords = { ["MAIN.Flags"] = [0xFF, 0xFF] } }; await drv.InitializeAsync("{}", CancellationToken.None); await drv.WriteAsync([new WriteRequest("Flag", false)], CancellationToken.None); - factory.Clients[0].Values["MAIN.Flags"].ShouldBe(0xFFF7u); + factory.Clients[0].ParentWords["MAIN.Flags"].ShouldBe(new byte[] { 0xF7, 0xFF }); } /// RMW works on a DWORD parent (bit 20 set above the 16-bit boundary). @@ -335,12 +335,31 @@ public sealed class TwinCATReadWriteTests { var (drv, factory) = NewDriver( new TwinCATTagDefinition("Hi", "ads://5.23.91.23.1.1:851", "GVL.Status.20", TwinCATDataType.Bool)); - factory.Customise = () => new FakeTwinCATClient { Values = { ["GVL.Status"] = 0u } }; + factory.Customise = () => new FakeTwinCATClient { ParentWords = { ["GVL.Status"] = [0, 0, 0, 0] } }; await drv.InitializeAsync("{}", CancellationToken.None); await drv.WriteAsync([new WriteRequest("Hi", true)], CancellationToken.None); - factory.Clients[0].Values["GVL.Status"].ShouldBe(1u << 20); + BitConverter.ToUInt32(factory.Clients[0].ParentWords["GVL.Status"]).ShouldBe(1u << 20); + } + + /// + /// A bit beyond a 16-bit parent is refused BadOutOfRange and nothing is written — before, the hardcoded + /// 32-bit RMW wrote into the neighbouring bytes. No health write: a refusal is not a comms fault. + /// + [Fact] + public async Task Bit_beyond_a_16_bit_parent_is_refused_without_writing() + { + var (drv, factory) = NewDriver( + new TwinCATTagDefinition("Hi", "ads://5.23.91.23.1.1:851", "MAIN.Word.20", TwinCATDataType.Bool)); + factory.Customise = () => new FakeTwinCATClient { ParentWords = { ["MAIN.Word"] = [0, 0] } }; + await drv.InitializeAsync("{}", CancellationToken.None); + + var results = await drv.WriteAsync([new WriteRequest("Hi", true)], CancellationToken.None); + + results.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange); + factory.Clients[0].ParentWords["MAIN.Word"].ShouldBe(new byte[] { 0, 0 }); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); } /// A failed parent read short-circuits the RMW and surfaces the read status. @@ -351,6 +370,7 @@ public sealed class TwinCATReadWriteTests new TwinCATTagDefinition("Flag", "ads://5.23.91.23.1.1:851", "MAIN.Flags.3", TwinCATDataType.Bool)); factory.Customise = () => new FakeTwinCATClient { + ParentWords = { ["MAIN.Flags"] = [0xAA, 0] }, ReadStatuses = { ["MAIN.Flags"] = TwinCATStatusMapper.BadNodeIdUnknown }, }; await drv.InitializeAsync("{}", CancellationToken.None); @@ -358,25 +378,24 @@ public sealed class TwinCATReadWriteTests var results = await drv.WriteAsync([new WriteRequest("Flag", true)], CancellationToken.None); results.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); - factory.Clients[0].WriteLog.ShouldBeEmpty(); + factory.Clients[0].ParentWords["MAIN.Flags"].ShouldBe(new byte[] { 0xAA, 0 }); } - /// A Good parent read that yields a null value must NOT zero the word — it surfaces - /// a Bad status and writes nothing (treating null as 0 would clear every set bit). + /// A Good parent read that yields no bytes must NOT zero the word — it surfaces a Bad status + /// and writes nothing (treating it as 0 would clear every set bit). [Fact] - public async Task Bit_write_with_null_parent_value_does_not_zero_the_word() + public async Task Bit_write_with_no_parent_bytes_does_not_zero_the_word() { var (drv, factory) = NewDriver( new TwinCATTagDefinition("Flag", "ads://5.23.91.23.1.1:851", "MAIN.Flags.3", TwinCATDataType.Bool)); - // Do NOT seed Values["MAIN.Flags"] — the fake returns (null, Good) for the parent read. + // Do NOT seed ParentWords["MAIN.Flags"] — the fake answers the parent read Good with no bytes. factory.Customise = () => new FakeTwinCATClient(); await drv.InitializeAsync("{}", CancellationToken.None); var results = await drv.WriteAsync([new WriteRequest("Flag", true)], CancellationToken.None); results.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError); - // No parent write attempted — the word is left untouched on the device. - factory.Clients[0].WriteLog.ShouldBeEmpty(); + factory.Clients[0].ParentWords.ShouldNotContainKey("MAIN.Flags"); } /// Concurrent bit writes to the same word compose correctly (per-parent lock). @@ -387,12 +406,12 @@ public sealed class TwinCATReadWriteTests .Select(b => new TwinCATTagDefinition($"Bit{b}", "ads://5.23.91.23.1.1:851", $"MAIN.Flags.{b}", TwinCATDataType.Bool)) .ToArray(); var (drv, factory) = NewDriver(tags); - factory.Customise = () => new FakeTwinCATClient { Values = { ["MAIN.Flags"] = 0u } }; + factory.Customise = () => new FakeTwinCATClient { YieldInBitWrite = true, ParentWords = { ["MAIN.Flags"] = [0, 0] } }; await drv.InitializeAsync("{}", CancellationToken.None); await Task.WhenAll(Enumerable.Range(0, 8).Select(b => drv.WriteAsync([new WriteRequest($"Bit{b}", true)], CancellationToken.None))); - Convert.ToUInt32(factory.Clients[0].Values["MAIN.Flags"]).ShouldBe(0xFFu); + factory.Clients[0].ParentWords["MAIN.Flags"].ShouldBe(new byte[] { 0xFF, 0 }); } } From 6423ed272fafb7dde5f349bcbdd0319d00b64e47 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:11:25 -0400 Subject: [PATCH 35/51] fix(focas): the fixed-tree loop reports device silence to driver health (R07:H15) The fixed-tree loop swallowed every failure at Debug and never touched health, so a driver serving only FIXED: tags (Identity/Axes/Program/Status, the typical FOCAS shape) read Healthy on /hosts with every node Bad and the #527 watchdog could never fire. The fixed tree is now a third health path beside the value and alarm paths, combined in GetHealth (worst state, latest LastSuccessfulRead): - an answered bootstrap / axis pass clears the device's failure and advances LastSuccessfulRead; - a transport failure degrades the driver once the device has been silent past the same staleness budget that turns its values Bad (from its last answer, or from loop start when it never answered), with LastSuccessfulRead frozen; - a CNC refusal (non-OK rc) is not a comms loss: it clears the failure and credits nothing. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../FocasDriver.cs | 86 +++++++++-- .../FocasFixedTreeHealthTests.cs | 133 ++++++++++++++++++ 2 files changed, 207 insertions(+), 12 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs index f0c0226f..c8dad722 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs @@ -562,24 +562,27 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, } /// - // The worst of the value path (_health) and the alarm path (_alarmTransportFailures), - // computed together. LastSuccessfulRead is the later of the two paths' last successes: while either - // path still reads, the comms-loss watchdog stays quiet; when both have stopped it is frozen. - // Faulted / Unknown / Initializing come from the value (lifecycle) path unchanged. + // The worst of the value path (_health), the alarm path (_alarmTransportFailures) and the fixed-tree path + // (_fixedTreeFailures), computed together. LastSuccessfulRead is the latest of the three paths' last + // successes: while any path still reads, the comms-loss watchdog stays quiet; when all have stopped it is + // frozen. Faulted / Unknown / Initializing come from the value (lifecycle) path unchanged. public DriverHealth GetHealth() { var value = Volatile.Read(ref _health); - var ticks = Interlocked.Read(ref _alarmLastSuccessTicks); - DateTime? alarmLast = ticks == 0 ? null : new DateTime(ticks, DateTimeKind.Utc); - var last = value.LastSuccessfulRead is { } v && (alarmLast is null || v >= alarmLast) ? v : alarmLast; + var last = Latest(Latest(value.LastSuccessfulRead, Stamp(Interlocked.Read(ref _alarmLastSuccessTicks))), + Stamp(Interlocked.Read(ref _fixedTreeLastSuccessTicks))); if (value.State is not (DriverState.Healthy or DriverState.Degraded)) return value; - if (value.State == DriverState.Healthy && !_alarmTransportFailures.IsEmpty) - { - var reason = _alarmTransportFailures.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase).First().Value; + if (value.State == DriverState.Healthy + && (FirstFailure(_alarmTransportFailures) ?? FirstFailure(_fixedTreeFailures)) is { } reason) return new DriverHealth(DriverState.Degraded, last, reason); - } return value.LastSuccessfulRead == last ? value : value with { LastSuccessfulRead = last }; + + static DateTime? Stamp(long ticks) => ticks == 0 ? null : new DateTime(ticks, DateTimeKind.Utc); + static DateTime? Latest(DateTime? a, DateTime? b) => a is null ? b : b is null || a >= b ? a : b; + static string? FirstFailure(ConcurrentDictionary failures) => failures.IsEmpty + ? null + : failures.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase).Select(kv => kv.Value).FirstOrDefault(); } /// Gets the number of configured devices. @@ -1290,6 +1293,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // Each optional call is attempted once; failures (EW_FUNC / EW_NOOPT / EW_VERSION) // record the capability as unsupported and suppress the corresponding nodes // in DiscoverAsync + the poll loop. + state.FixedTreeStartedUtc = DateTime.UtcNow; while (!ct.IsCancellationRequested && state.FixedTreeCache is null) { try @@ -1320,6 +1324,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // so stamp liveness here as well as per tick, or a CNC whose axis poll never // succeeds would serve a perfectly valid sysinfo as stale. state.LastFixedPollUtc = DateTime.UtcNow; + NoteFixedTreeAnswered(state); // Published LAST (release write): LastStatus is the field callers watch for "the first // answer is in", and an authored FIXED:Status/* read is only servable once // FixedTreeCache and LastFixedPollUtc are set (TryReadFixedTreeOnDevice). Publishing it @@ -1331,6 +1336,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, { _logger.LogDebug(ex, "FOCAS fixed-tree bootstrap failed for {Host} — retrying", state.Options.HostAddress); + NoteFixedTreeFailed(state, ex); try { await Task.Delay(TimeSpan.FromSeconds(2), ct).ConfigureAwait(false); } catch (OperationCanceledException) { return; } } @@ -1379,6 +1385,8 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // of the loop: EnsureConnectedAsync can hand back a cached handle without any wire // traffic, so it alone does not prove the CNC is still there. state.LastFixedPollUtc = DateTime.UtcNow; + // …and it is the driver's evidence of life too: a fixed-only driver has no value read to credit. + NoteFixedTreeAnswered(state); // Servo loads + spindle loads — both return bulk arrays, so folding // into the axis cadence is cheap. Each is gated by the bootstrap @@ -1474,6 +1482,8 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /* next tick retries — transient blips are expected */ _logger.LogDebug(ex, "FOCAS fixed-tree poll tick failed for {Host}", state.Options.HostAddress); + // …but a sustained silence is a comms loss, and nothing else will report it for a fixed-only driver. + NoteFixedTreeFailed(state, ex); } try { await Task.Delay(_options.FixedTree.PollInterval, ct).ConfigureAwait(false); } @@ -1635,7 +1645,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, { var polledUtc = state.LastFixedPollUtc; if (polledUtc is null) return null; - var status = now - polledUtc.Value > FixedTreeStaleAfter(_options.FixedTree.PollInterval) + var status = now - polledUtc.Value > FixedTreeStaleBudget ? FocasStatusMapper.BadCommunicationError : FocasStatusMapper.Good; var sourceUtc = polledUtc.Value; @@ -1850,6 +1860,55 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, { _alarmTransportFailures.Clear(); Interlocked.Exchange(ref _alarmLastSuccessTicks, 0); + _fixedTreeFailures.Clear(); + Interlocked.Exchange(ref _fixedTreeLastSuccessTicks, 0); + } + + /// + /// The FIXED-TREE half of driver health: device host → why its fixed-tree loop has had no answer for longer + /// than the staleness budget. Kept apart from like the alarm path, and for the same + /// reason — a driver serving only FIXED: tags has no value read to degrade it, so before this it + /// read Healthy on /hosts with every node Bad and the #527 watchdog could never fire. + /// + private readonly ConcurrentDictionary _fixedTreeFailures = new(StringComparer.OrdinalIgnoreCase); + + /// UTC ticks of the last answered fixed-tree poll on any device (0 = none). + private long _fixedTreeLastSuccessTicks; + + /// Test seam: replaces the fixed-tree staleness budget (production derives it from the poll interval). + internal TimeSpan? FixedTreeStaleBudgetOverride { get; init; } + + /// + /// How long a device's fixed tree may go unanswered before its values read Bad AND the driver degrades — + /// one budget for both, so the node quality and the health surface cannot disagree. + /// + private TimeSpan FixedTreeStaleBudget => + FixedTreeStaleBudgetOverride ?? FixedTreeStaleAfter(_options.FixedTree.PollInterval); + + /// An answered fixed-tree pass of : clears its failure, advances LastSuccessfulRead. + private void NoteFixedTreeAnswered(DeviceState state) + { + _fixedTreeFailures.TryRemove(state.Options.HostAddress, out _); + Interlocked.Exchange(ref _fixedTreeLastSuccessTicks, DateTime.UtcNow.Ticks); + } + + /// + /// A failed fixed-tree tick. A CNC REFUSAL (non-OK rc, transport fine) is not a comms loss and clears any + /// failure without crediting a read; a transport failure degrades the driver only once the device has been + /// silent past (measured from its last answer, or from loop start when + /// it never answered), so one dropped tick on a slow control does not flap /hosts. + /// + private void NoteFixedTreeFailed(DeviceState state, Exception error) + { + if (FocasAlarmProjection.IsCncRefusal(error)) + { + _fixedTreeFailures.TryRemove(state.Options.HostAddress, out _); + return; + } + var since = state.LastFixedPollUtc ?? state.FixedTreeStartedUtc; + if (DateTime.UtcNow - since > FixedTreeStaleBudget) + _fixedTreeFailures[state.Options.HostAddress] = + $"FOCAS fixed-tree poll of {state.Options.HostAddress} has had no answer since {since:O}: {error.Message}"; } /// @@ -2082,6 +2141,9 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// it falls outside . /// public DateTime? LastFixedPollUtc { get; set; } + + /// When this device's fixed-tree loop started — the silence origin for a device that never answered. + public DateTime FixedTreeStartedUtc { get; set; } = DateTime.UtcNow; /// /// Gets the last fixed-tree snapshots by field name. Double-typed so axis /// positions can carry the engineering-unit scale applied at diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs new file mode 100644 index 00000000..b83898e8 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs @@ -0,0 +1,133 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; + +/// +/// #740 / R07:H15 — the fixed-tree loop swallowed every failure at Debug and never touched driver health, so a +/// driver serving only FIXED: tags (Identity / Axes / Program / Status — the typical FOCAS shape) read +/// Healthy on /hosts with every node Bad, and the #527 comms-loss watchdog could never fire. The loop +/// now credits each answered axis pass (Healthy + LastSuccessfulRead) and, once a device has been silent past +/// the fixed-tree staleness budget, degrades the driver with LastSuccessfulRead frozen. +/// +[Trait("Category", "Unit")] +public sealed class FocasFixedTreeHealthTests +{ + private const string Host = "focas://10.0.0.5:8193"; + private static readonly TimeSpan Stale = TimeSpan.FromMilliseconds(300); + + /// A CNC that can be switched off (transport failure) or made to refuse the dynamic read. + private sealed class CncSwitch + { + public volatile bool Down; + public volatile bool Refuse; + } + + private sealed class SwitchableCnc(CncSwitch sw) : FakeFocasClient + { + private bool Down => sw.Down; + private bool Refuse => sw.Refuse; + + public override Task GetSysInfoAsync(CancellationToken ct) => + Down ? throw new IOException("socket closed") : base.GetSysInfoAsync(ct); + + public override Task ReadDynamicAsync(int axisIndex, CancellationToken ct) + { + if (Down) throw new IOException("socket closed"); + if (Refuse) throw new FocasWireException("cnc_rddynamic2 refused", rc: 6); + return base.ReadDynamicAsync(axisIndex, ct); + } + } + + private static FocasDriver FixedOnlyDriver(CncSwitch sw) => new(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(Host)], + RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/X", Host, "FIXED:Axes/X/AbsolutePosition", FocasDataType.Float64))], + Probe = new FocasProbeOptions { Enabled = false }, + FixedTree = new FocasFixedTreeOptions + { + PollInterval = TimeSpan.FromMilliseconds(20), + ProgramPollInterval = TimeSpan.Zero, + TimerPollInterval = TimeSpan.Zero, + }, + }, "focas-fixed-health", new FakeFocasClientFactory { Customise = () => new SwitchableCnc(sw) }) + { + FixedTreeStaleBudgetOverride = Stale, + }; + + /// + /// Positive control first, then the outage: an answering CNC keeps the driver Healthy with a moving + /// LastSuccessfulRead; a CNC that stops answering degrades it once the budget passes, with + /// LastSuccessfulRead frozen (the watchdog's conjunction); and its return restores Healthy. + /// + [Fact] + public async Task Fixed_only_driver_degrades_when_its_cnc_disappears_and_recovers_when_it_returns() + { + var sw = new CncSwitch(); + await using var drv = FixedOnlyDriver(sw); + await drv.InitializeAsync("{}", CancellationToken.None); + await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null); + + var first = drv.GetHealth().LastSuccessfulRead; + await Task.Delay(100); + var healthy = drv.GetHealth(); + healthy.State.ShouldBe(DriverState.Healthy); + healthy.LastSuccessfulRead!.Value.ShouldBeGreaterThan(first!.Value, "an answered axis pass is a successful read"); + + sw.Down = true; + await Task.Delay(100); + drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade"); + await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded); + var frozen = drv.GetHealth().LastSuccessfulRead; + await Task.Delay(150); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "LastSuccessfulRead must stay frozen while the CNC is silent"); + + sw.Down = false; + await WaitUntil(() => drv.GetHealth().State == DriverState.Healthy); + drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value); + } + + /// A CNC that is down from the start (bootstrap never succeeds) degrades once the budget passes. + [Fact] + public async Task Bootstrap_that_never_succeeds_degrades_the_driver() + { + var sw = new CncSwitch { Down = true }; + await using var drv = FixedOnlyDriver(sw); + await drv.InitializeAsync("{}", CancellationToken.None); + // The bootstrap retries every 2 s, so the first failure is what has to trip it. + await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(6)); + sw.Down = false; + } + + /// + /// A CNC that ANSWERS with a refusal is not a comms loss: the transport is fine, so the driver is not + /// degraded (else the watchdog would reconnect-loop a production control). + /// + [Fact] + public async Task Cnc_refusal_of_the_dynamic_read_does_not_degrade() + { + var sw = new CncSwitch(); + await using var drv = FixedOnlyDriver(sw); + await drv.InitializeAsync("{}", CancellationToken.None); + await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null); + + sw.Refuse = true; + await Task.Delay(Stale * 3); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + sw.Refuse = false; + } + + private static async Task WaitUntil(Func condition, TimeSpan? timeout = null) + { + var deadline = DateTime.UtcNow + (timeout ?? TimeSpan.FromSeconds(5)); + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached"); + await Task.Delay(10); + } + } +} From f17ba8e2585e0036bfe83f036c37f23bc141d3e6 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:15:29 -0400 Subject: [PATCH 36/51] feat(twincat): array and STRING writes through the codecs (U9) Array tags were advertised writable but every write threw InvalidCastException in ConvertForWrite (reported BadTypeMismatch); STRING writes went through the Beckhoff marshaller with no length check against STRING(n) and the host's ANSI code page, which the read side deliberately avoids. - Array write: TwinCATArrayCodec.ToWriteElements converts exactly ArrayLength elements to the declared type (CLR lists convert - ScadaBridge writes int[] to Int16 nodes; a length mismatch or out-of-range element is BadOutOfRange, a scalar value BadTypeMismatch) before any wire traffic; the ADS client plans the layout from the resolved PLC array (TwinCATTypeGuard.CheckArray, as the read does), encodes with TwinCATArrayCodec.Encode - the exact inverse of Decode - and writes the raw bytes (new ITwinCATClient.WriteArrayAsync). - STRING / WSTRING write: encoded at the PLC's declared width by TwinCATStringCodec.EncodeString/EncodeWString (Windows-1252 or UTF-8 per target, NUL-padded, strict: an over-long value or unrepresentable character is OverflowException -> BadOutOfRange) and written as raw bytes. - TwinCAT CLI read/write gain --array-length (+ --separator) for the hardware gate. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Commands/ReadCommand.cs | 11 +- .../Commands/WriteCommand.cs | 38 +++- .../AdsTwinCATClient.cs | 39 ++++ .../ITwinCATClient.cs | 17 ++ .../TwinCATArrayCodec.cs | 78 +++++++ .../TwinCATDriver.cs | 26 ++- .../TwinCATStringCodec.cs | 51 +++++ .../WriteCommandParseValueTests.cs | 20 ++ .../FakeTwinCATClient.cs | 18 ++ .../TwinCATArrayStringWriteTests.cs | 199 ++++++++++++++++++ 10 files changed, 490 insertions(+), 7 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATArrayStringWriteTests.cs diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli/Commands/ReadCommand.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli/Commands/ReadCommand.cs index 96fe1d87..4603825d 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli/Commands/ReadCommand.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli/Commands/ReadCommand.cs @@ -25,6 +25,11 @@ public sealed class ReadCommand : TwinCATTagCommandBase "String / WString / Time / Date / DateTime / TimeOfDay (default DInt).")] public TwinCATDataType DataType { get; init; } = TwinCATDataType.DInt; + /// Gets the element count for an array read; null reads a scalar. + [CommandOption("array-length", Description = + "Read the symbol as a 1-D array of this many --type elements (the leading sub-range when shorter).")] + public int? ArrayLength { get; init; } + /// public override async ValueTask ExecuteAsync(IConsole console) { @@ -38,7 +43,8 @@ public sealed class ReadCommand : TwinCATTagCommandBase DeviceHostAddress: Gateway, SymbolPath: SymbolPath, DataType: DataType, - Writable: false); + Writable: false, + ArrayLength: ArrayLength); var options = BuildOptions([tag]); await using var driver = new TwinCATDriver(options, DriverInstanceId); @@ -47,6 +53,9 @@ public sealed class ReadCommand : TwinCATTagCommandBase await driver.InitializeAsync("{}", ct); var snapshot = await driver.ReadAsync([tagName], ct); await console.Output.WriteLineAsync(SnapshotFormatter.Format(SymbolPath, snapshot[0])); + if (snapshot[0].Value is Array elements) + await console.Output.WriteLineAsync("Elements: [" + string.Join(", ", + elements.Cast().Select(e => Convert.ToString(e, System.Globalization.CultureInfo.InvariantCulture))) + "]"); } finally { diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli/Commands/WriteCommand.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli/Commands/WriteCommand.cs index f72adea4..a1f882ef 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli/Commands/WriteCommand.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli/Commands/WriteCommand.cs @@ -30,6 +30,16 @@ public sealed class WriteCommand : TwinCATTagCommandBase IsRequired = true)] public string Value { get; init; } = default!; + /// Gets the element count for an array write; null writes a scalar. + [CommandOption("array-length", Description = + "Write the symbol as a 1-D array of this many --type elements. --value then carries the elements " + + "separated by --separator (e.g. -v 1,2,3 --array-length 3); the count must match.")] + public int? ArrayLength { get; init; } + + /// Gets the element separator for an array write. + [CommandOption("separator", Description = "Element separator for --array-length writes (default ',').")] + public string Separator { get; init; } = ","; + /// public override async ValueTask ExecuteAsync(IConsole console) { @@ -48,10 +58,13 @@ public sealed class WriteCommand : TwinCATTagCommandBase DeviceHostAddress: Gateway, SymbolPath: SymbolPath, DataType: DataType, - Writable: true); + Writable: true, + ArrayLength: ArrayLength); var options = BuildOptions([tag]); - var parsed = ParseValue(Value, DataType); + var parsed = ArrayLength is int count + ? ParseArrayValue(Value, DataType, count, Separator) + : ParseValue(Value, DataType); await using var driver = new TwinCATDriver(options, DriverInstanceId); try @@ -113,6 +126,27 @@ public sealed class WriteCommand : TwinCATTagCommandBase } } + /// + /// Parse an array --value: -separated elements, each parsed by + /// . The element count must equal — the driver + /// refuses a mismatch with BadOutOfRange, but the CLI says so before touching the PLC. + /// + /// The raw separated element list. + /// The element type. + /// The expected element count (--array-length). + /// The element separator. + /// The parsed elements. + internal static object[] ParseArrayValue(string raw, TwinCATDataType type, int count, string separator) + { + if (count < 1) + throw new CliFx.Exceptions.CommandException("--array-length must be >= 1."); + var parts = raw.Split(string.IsNullOrEmpty(separator) ? "," : separator); + if (parts.Length != count) + throw new CliFx.Exceptions.CommandException( + $"--value has {parts.Length} element(s) but --array-length is {count}."); + return [.. parts.Select(p => ParseValue(type is TwinCATDataType.String or TwinCATDataType.WString ? p : p.Trim(), type))]; + } + private static bool ParseBool(string raw) => raw.Trim().ToLowerInvariant() switch { "1" or "true" or "on" or "yes" => true, diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs index a143d018..1781bedf 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs @@ -469,6 +469,23 @@ internal sealed class AdsTwinCATClient : ITwinCATClient { try { + if (type is TwinCATDataType.String or TwinCATDataType.WString) + { + // A STRING / WSTRING is written as raw bytes at the PLC's own declared width, encoded by the + // same codec the read decodes with: Windows-1252 (or UTF-8 when the target says so) rather + // than the host's ANSI code page, NUL-padded so no stale tail survives, and refused + // (OverflowException → BadOutOfRange) when it does not fit STRING(n). + var (shape, _, symbol, symbolStatus) = await DescribeSymbolAsync(symbolPath, cancellationToken) + .ConfigureAwait(false); + if (symbolStatus != TwinCATStatusMapper.Good) return symbolStatus; + if (TwinCATTypeGuard.CheckRead(type, shape) is not null) return TwinCATStatusMapper.BadTypeMismatch; + var text = Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + var encoded = type == TwinCATDataType.WString + ? TwinCATStringCodec.EncodeWString(text, shape.ByteSize) + : TwinCATStringCodec.EncodeString(text, shape.ByteSize, StringsAreUtf8()); + return await WriteRawAsync(symbolPath, symbol!, encoded, cancellationToken).ConfigureAwait(false); + } + var converted = ConvertForWrite(type, value); var result = await _client.WriteValueAsync(symbolPath, converted, cancellationToken) .ConfigureAwait(false); @@ -482,6 +499,28 @@ internal sealed class AdsTwinCATClient : ITwinCATClient } } + /// + public async Task WriteArrayAsync(string symbolPath, TwinCATDataType type, IReadOnlyList elements, + CancellationToken cancellationToken) + { + try + { + // Plan the layout from the RESOLVED PLC array exactly as the array read does, so the element + // stride (incl. a STRING(n) element's n+1) and the strict element-type rule are shared. + var (shape, array, symbol, symbolStatus) = await DescribeSymbolAsync(symbolPath, cancellationToken) + .ConfigureAwait(false); + if (symbolStatus != TwinCATStatusMapper.Good) return symbolStatus; + if (TwinCATTypeGuard.CheckArray(type, elements.Count, shape, array, out var layout) is not null) + return TwinCATStatusMapper.BadTypeMismatch; + var bytes = TwinCATArrayCodec.Encode(type, elements, layout, StringsAreUtf8()); + return await WriteRawAsync(symbolPath, symbol!, bytes, cancellationToken).ConfigureAwait(false); + } + catch (AdsErrorException ex) + { + return MapAndSignal((uint)ex.ErrorCode); + } + } + /// public async Task WriteBitAsync(string parentPath, int bit, bool value, CancellationToken cancellationToken) { diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs index 1ffeb6fa..b44f3038 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs @@ -119,6 +119,23 @@ public interface ITwinCATClient : IDisposable object? value, CancellationToken cancellationToken); + /// + /// Writes an array tag: (already converted to the declared element type by + /// ) as raw bytes at the layout planned from the RESOLVED + /// PLC array () and encoded by + /// — the exact inverse of the array read, so an array tag writes + /// what it reads. Fewer elements than the PLC array writes its leading sub-range. + /// + /// The ADS symbol path of the array. + /// The declared element type. + /// The converted elements (the authored ArrayLength of them). + /// Cancellation token for the write. + /// The mapped OPC UA status: Good; BadTypeMismatch when the declared element type / + /// length cannot address the PLC array; the ADS write's status otherwise. + /// A string element does not fit the PLC element's width. + Task WriteArrayAsync(string symbolPath, TwinCATDataType type, IReadOnlyList elements, + CancellationToken cancellationToken); + /// /// Sets or clears one bit of an integer parent word — the write half of a BOOL-within-word tag /// (MAIN.Flags.3; ADS has no symbol for WORD.N). A read-modify-write of the parent's raw diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATArrayCodec.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATArrayCodec.cs index d008a714..d8fa56c1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATArrayCodec.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATArrayCodec.cs @@ -19,6 +19,84 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; /// internal static class TwinCATArrayCodec { + /// + /// Converts a client-supplied array value into exactly elements of the + /// declared type, each converted the way a scalar write converts ( + /// — so TIME/DATE elements become their encoded uint). Any CLR collection is accepted and the + /// element type is CONVERTED, not exact-matched: ScadaBridge writes an int[] to an Int16 node. + /// + /// The tag's declared element type. + /// The written value — an array or list. + /// The tag's authored ArrayLength. + /// The converted elements, in order. + /// The value is not a collection (a scalar, a string, null), or + /// an element cannot convert. + /// The element count differs from , or an + /// element is outside the declared type's range. + public static object[] ToWriteElements(TwinCATDataType declared, object? value, int count) + { + if (value is null or string || value is not System.Collections.IEnumerable items) + throw new InvalidCastException( + $"an array tag of {count} {declared} takes an array value, not {value?.GetType().Name ?? "null"}"); + var elements = new List(count); + foreach (var item in items) elements.Add(AdsTwinCATClient.ConvertForWrite(declared, item)); + if (elements.Count != count) + throw new OverflowException($"wrote {elements.Count} elements to an array tag of arrayLength {count}"); + return [.. elements]; + } + + /// + /// Encodes converted elements () into the PLC's raw bytes at the planned + /// layout — the exact inverse of : little-endian at the element stride, a + /// STRING / WSTRING element encoded by within its fixed stride. + /// + /// The tag's declared element type. + /// The converted elements; exactly . + /// The layout planned from the PLC array (). + /// True when the target reports UTF-8 STRING data (else Windows-1252). + /// Exactly bytes. + /// The element count differs from the layout. + /// A string element does not fit its stride. + public static byte[] Encode(TwinCATDataType declared, IReadOnlyList elements, TwinCATArrayLayout layout, bool utf8) + { + if (layout.Stride <= 0 || elements.Count != layout.ElementCount) + throw new ArgumentException( + $"{elements.Count} elements do not match a layout of {layout.ElementCount} × {layout.Stride} B", nameof(elements)); + + var bytes = new byte[layout.ByteLength]; + var stride = layout.Stride; + for (var i = 0; i < elements.Count; i++) + { + var slot = bytes.AsSpan(i * stride, stride); + var e = elements[i]; + switch (declared) + { + case TwinCATDataType.Bool: slot[0] = Convert.ToBoolean(e) ? (byte)1 : (byte)0; break; + case TwinCATDataType.SInt: slot[0] = unchecked((byte)Convert.ToSByte(e)); break; + case TwinCATDataType.USInt: slot[0] = Convert.ToByte(e); break; + case TwinCATDataType.Int: BinaryPrimitives.WriteInt16LittleEndian(slot, Convert.ToInt16(e)); break; + case TwinCATDataType.UInt: BinaryPrimitives.WriteUInt16LittleEndian(slot, Convert.ToUInt16(e)); break; + case TwinCATDataType.DInt: BinaryPrimitives.WriteInt32LittleEndian(slot, Convert.ToInt32(e)); break; + case TwinCATDataType.UDInt or TwinCATDataType.Time or TwinCATDataType.TimeOfDay + or TwinCATDataType.Date or TwinCATDataType.DateTime: + BinaryPrimitives.WriteUInt32LittleEndian(slot, Convert.ToUInt32(e)); break; + case TwinCATDataType.LInt: BinaryPrimitives.WriteInt64LittleEndian(slot, Convert.ToInt64(e)); break; + case TwinCATDataType.ULInt: BinaryPrimitives.WriteUInt64LittleEndian(slot, Convert.ToUInt64(e)); break; + case TwinCATDataType.Real: BinaryPrimitives.WriteSingleLittleEndian(slot, Convert.ToSingle(e)); break; + case TwinCATDataType.LReal: BinaryPrimitives.WriteDoubleLittleEndian(slot, Convert.ToDouble(e)); break; + case TwinCATDataType.String: + TwinCATStringCodec.EncodeString(Convert.ToString(e) ?? string.Empty, stride, utf8).CopyTo(slot); + break; + case TwinCATDataType.WString: + TwinCATStringCodec.EncodeWString(Convert.ToString(e) ?? string.Empty, stride).CopyTo(slot); + break; + default: + throw new NotSupportedException($"TwinCATDataType {declared} has no array element encoding"); + } + } + return bytes; + } + /// Decodes .ElementCount elements of . /// The tag's declared element type. /// The PLC bytes from the array's first element (at least ). diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index 2a5d4d94..30d9f488 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -508,17 +508,26 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery try { - var client = await EnsureConnectedAsync(device, cancellationToken).ConfigureAwait(false); var parsed = TwinCATSymbolPath.TryParse(def.SymbolPath); + var bitIndex = def.DataType == TwinCATDataType.Bool ? parsed?.BitIndex : null; + // Shape checks run BEFORE any wire traffic (connect included): an array write needs exactly + // ArrayLength elements, each converted to the declared element type (a CLR int[] to an Int16 + // array converts; it is not exact-matched) — a length mismatch / out-of-range element is + // OverflowException (BadOutOfRange), a scalar value InvalidCastException (BadTypeMismatch). + var bitValue = bitIndex is not null ? Convert.ToBoolean(w.Value) : false; + var elements = bitIndex is null && def.ArrayLength is int arrayLength + ? TwinCATArrayCodec.ToWriteElements(def.DataType, w.Value, arrayLength) + : null; + + var client = await EnsureConnectedAsync(device, cancellationToken).ConfigureAwait(false); // BOOL-within-word write — a read-modify-write of the parent word at the parent's OWN // width and on raw bytes (the client resolves the parent and checks the bit against it, the // notification path's model), under a per-parent lock so concurrent bit writers to the same // word don't lose each other's updates. - if (def.DataType == TwinCATDataType.Bool && parsed?.BitIndex is int bit) + if (bitIndex is int bit) { - var parentPath = (parsed with { BitIndex = null }).ToAdsSymbolName(); - var bitValue = Convert.ToBoolean(w.Value); + var parentPath = (parsed! with { BitIndex = null }).ToAdsSymbolName(); var gate = _bitRmwLocks.GetOrAdd( $"{def.DeviceHostAddress}|{parentPath}", static _ => new SemaphoreSlim(1, 1)); await gate.WaitAsync(cancellationToken).ConfigureAwait(false); @@ -535,6 +544,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } var symbolName = parsed?.ToAdsSymbolName() ?? def.SymbolPath; + if (elements is not null) + { + // Array tag: raw bytes encoded by TwinCATArrayCodec at the layout planned from the PLC + // array — the inverse of the array read. + results[i] = new WriteResult(await client.WriteArrayAsync( + symbolName, def.DataType, elements, cancellationToken).ConfigureAwait(false)); + continue; + } + var status = await client.WriteValueAsync( symbolName, def.DataType, parsed?.BitIndex, w.Value, cancellationToken).ConfigureAwait(false); results[i] = new WriteResult(status); diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATStringCodec.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATStringCodec.cs index d3a90c1f..bdaff2ae 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATStringCodec.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATStringCodec.cs @@ -50,6 +50,57 @@ internal static class TwinCATStringCodec return (utf8 ? Encoding.UTF8 : Windows1252).GetString(bytes); } + // Strict encoders for the WRITE side: a character the target encoding cannot hold is refused rather + // than silently replaced with '?' (the replacement would be written to the PLC as if it were the value). + private static readonly Encoding StrictWindows1252 = CreateStrict(Windows1252.CodePage); + private static readonly Encoding StrictUtf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + private static readonly Encoding StrictUtf16 = new UnicodeEncoding(bigEndian: false, byteOrderMark: false, throwOnInvalidBytes: true); + + private static Encoding CreateStrict(int codePage) => + Encoding.GetEncoding(codePage, EncoderFallback.ExceptionFallback, DecoderFallback.ReplacementFallback); + + /// + /// Encodes a value into an IEC STRING(n) buffer of the PLC's declared width (n+1 bytes): the + /// inverse of — Windows-1252 unless , NUL-terminated + /// and NUL-padded so no stale tail of a longer earlier value survives. + /// + /// The value to write. + /// The PLC symbol's byte width (n+1 for a STRING(n)). + /// True when the target reports UTF-8 string data. + /// Exactly bytes. + /// The value does not fit STRING(n), or holds a character the + /// target encoding cannot represent. + public static byte[] EncodeString(string value, int byteWidth, bool utf8) => + Encode(value, byteWidth, utf8 ? StrictUtf8 : StrictWindows1252, terminatorBytes: 1, "STRING"); + + /// + /// Encodes a value into an IEC WSTRING(n) buffer (UTF-16LE, 2(n+1) bytes): the inverse of + /// , terminated by an aligned U+0000 and zero-padded. + /// + /// The value to write. + /// The PLC symbol's byte width. + /// Exactly bytes. + /// The value does not fit WSTRING(n). + public static byte[] EncodeWString(string value, int byteWidth) => + Encode(value, byteWidth, StrictUtf16, terminatorBytes: 2, "WSTRING"); + + private static byte[] Encode(string value, int byteWidth, Encoding encoding, int terminatorBytes, string kind) + { + ArgumentNullException.ThrowIfNull(value); + byte[] encoded; + try { encoded = encoding.GetBytes(value); } + catch (ArgumentException ex) // EncoderFallbackException: a character the encoding cannot hold + { + throw new OverflowException($"'{value}' cannot be represented in the PLC's {kind} encoding: {ex.Message}", ex); + } + if (encoded.Length + terminatorBytes > byteWidth) + throw new OverflowException( + $"'{value}' is {encoded.Length} B but the PLC {kind} holds {Math.Max(0, byteWidth - terminatorBytes)} B"); + var buffer = new byte[byteWidth]; + encoded.CopyTo(buffer, 0); + return buffer; + } + /// Decodes an IEC WSTRING(n) buffer (UTF-16LE): ends at the first aligned U+0000. /// The raw symbol bytes (2(n+1) for a WSTRING(n)). /// The decoded value. diff --git a/tests/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli.Tests/WriteCommandParseValueTests.cs b/tests/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli.Tests/WriteCommandParseValueTests.cs index 89ed5c22..a06662a7 100644 --- a/tests/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli.Tests/WriteCommandParseValueTests.cs +++ b/tests/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Cli.Tests/WriteCommandParseValueTests.cs @@ -184,4 +184,24 @@ public sealed class WriteCommandParseValueTests { ReadCommand.SynthesiseTagName(symbol, type).ShouldBe(expected); } + + /// An array value splits on the separator and parses each element per --type (#739 U9 gate tooling). + [Fact] + public void ParseArrayValue_parses_each_element() + { + WriteCommand.ParseArrayValue("1, -2,3", TwinCATDataType.Int, 3, ",") + .ShouldBe(new object[] { (short)1, (short)-2, (short)3 }); + WriteCommand.ParseArrayValue("a b|c", TwinCATDataType.String, 2, "|") + .ShouldBe(new object[] { "a b", "c" }); + } + + /// A count that differs from --array-length is refused before the PLC is touched. + [Fact] + public void ParseArrayValue_refuses_a_count_mismatch() + { + Should.Throw( + () => WriteCommand.ParseArrayValue("1,2", TwinCATDataType.Int, 3, ",")); + Should.Throw( + () => WriteCommand.ParseArrayValue("1", TwinCATDataType.Int, 0, ",")); + } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs index e31340a5..39f795be 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs @@ -136,6 +136,24 @@ internal class FakeTwinCATClient : ITwinCATClient return Task.FromResult(status); } + /// Gets the log of every call. + public List<(string symbol, TwinCATDataType type, IReadOnlyList elements)> ArrayWriteLog { get; } = new(); + + /// Simulates an array write: records it, stores the elements and answers from . + /// The array symbol path. + /// The declared element type. + /// The converted elements. + /// The cancellation token. + /// The write status. + public virtual Task WriteArrayAsync(string symbolPath, TwinCATDataType type, IReadOnlyList elements, + CancellationToken ct) + { + if (ThrowOnWrite) throw Exception ?? new InvalidOperationException(); + ArrayWriteLog.Add((symbolPath, type, elements)); + Values[symbolPath] = elements; + return Task.FromResult(WriteStatuses.TryGetValue(symbolPath, out var s) ? s : TwinCATStatusMapper.Good); + } + /// Raw bytes of bit-write parent words, by parent path (#739 H18). The byte count is the parent's width. public Dictionary ParentWords { get; } = new(StringComparer.OrdinalIgnoreCase); /// Optional parent shapes; absent = an unsigned integer word as wide as its entry. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATArrayStringWriteTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATArrayStringWriteTests.cs new file mode 100644 index 00000000..b42c3f0a --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATArrayStringWriteTests.cs @@ -0,0 +1,199 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// #739 U9: array writes and STRING writes go through the same codecs the read side decodes with. +/// Arrays were advertised writable but every write threw InvalidCastException (reported BadTypeMismatch); +/// strings bypassed the codec (host-ANSI encoding, no length check against STRING(n)). +/// +[Trait("Category", "Unit")] +public sealed class TwinCATArrayStringWriteTests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + + // ---- string codec: encode ---- + + /// A STRING(n) encodes NUL-terminated and NUL-padded to the PLC width, in Windows-1252. + [Fact] + public void EncodeString_is_NUL_padded_Windows1252_at_the_PLC_width() + { + TwinCATStringCodec.EncodeString("AB", 4, utf8: false).ShouldBe(new byte[] { 0x41, 0x42, 0, 0 }); + TwinCATStringCodec.EncodeString("ä€", 3, utf8: false).ShouldBe(new byte[] { 0xE4, 0x80, 0 }); + TwinCATStringCodec.EncodeString("ä", 3, utf8: true).ShouldBe(new byte[] { 0xC3, 0xA4, 0 }); + TwinCATStringCodec.EncodeString("", 2, utf8: false).ShouldBe(new byte[] { 0, 0 }); + } + + /// The encode is the decode's inverse (a STRING(n) of exactly n characters fits). + [Fact] + public void EncodeString_round_trips_through_DecodeString() + { + foreach (var utf8 in new[] { false, true }) + { + var bytes = TwinCATStringCodec.EncodeString("Grüße 40°", 81, utf8); + TwinCATStringCodec.DecodeString(bytes, utf8).ShouldBe("Grüße 40°"); + } + TwinCATStringCodec.EncodeString("ABC", 4, utf8: false).Length.ShouldBe(4); + } + + /// A value longer than STRING(n), or a character the target encoding cannot hold, is out of range. + [Fact] + public void EncodeString_refuses_a_value_that_does_not_fit() + { + Should.Throw(() => TwinCATStringCodec.EncodeString("ABC", 3, utf8: false)); + Should.Throw(() => TwinCATStringCodec.EncodeString("中", 10, utf8: false)); + Should.Throw(() => TwinCATStringCodec.EncodeString("ää", 4, utf8: true)); // 4 bytes + NUL + } + + /// A WSTRING(n) encodes UTF-16LE with a 2-byte terminator and round-trips. + [Fact] + public void EncodeWString_is_UTF16LE_with_a_terminator() + { + TwinCATStringCodec.EncodeWString("Hi", 6).ShouldBe(new byte[] { 0x48, 0, 0x69, 0, 0, 0 }); + TwinCATStringCodec.DecodeWString(TwinCATStringCodec.EncodeWString("中文", 22)).ShouldBe("中文"); + Should.Throw(() => TwinCATStringCodec.EncodeWString("Hi", 4)); + } + + // ---- array codec: elements + encode ---- + + /// + /// A CLR list converts element-wise to the declared type (ScadaBridge writes an int[] to an + /// Int16 node); the element type is converted, not exact-matched. + /// + [Fact] + public void ToWriteElements_converts_each_element_to_the_declared_type() + { + var elements = TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, new[] { 1, -2, 300 }, 3); + elements.ShouldBe(new object[] { (short)1, (short)-2, (short)300 }); + TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Real, new List { 1, 2.5 }, 2) + .ShouldBe(new object[] { 1f, 2.5f }); + TwinCATArrayCodec.ToWriteElements(TwinCATDataType.String, new[] { "a", "b" }, 2) + .ShouldBe(new object[] { "a", "b" }); + } + + /// A length mismatch and an out-of-range element are out of range; a scalar is a type mismatch. + [Fact] + public void ToWriteElements_refuses_a_wrong_shape() + { + Should.Throw(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, new[] { 1, 2 }, 3)); + Should.Throw(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, new[] { 70_000 }, 1)); + Should.Throw(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, 5, 1)); + Should.Throw(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.String, "abc", 3)); + Should.Throw(() => TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Int, null, 1)); + } + + /// Encode lays elements out little-endian at the PLC stride — the exact inverse of Decode. + [Fact] + public void Encode_is_the_inverse_of_Decode() + { + var intLayout = new TwinCATArrayLayout(2, 2); + var intBytes = TwinCATArrayCodec.Encode(TwinCATDataType.Int, [(short)1, (short)-2], intLayout, utf8: false); + intBytes.ShouldBe(new byte[] { 0x01, 0x00, 0xFE, 0xFF }); + + RoundTrip(TwinCATDataType.Bool, [true, false, true], new TwinCATArrayLayout(3, 1)); + RoundTrip(TwinCATDataType.DInt, [int.MinValue, 7], new TwinCATArrayLayout(2, 4)); + RoundTrip(TwinCATDataType.ULInt, [ulong.MaxValue], new TwinCATArrayLayout(1, 8)); + RoundTrip(TwinCATDataType.LReal, [1.5, -2.25], new TwinCATArrayLayout(2, 8)); + RoundTrip(TwinCATDataType.String, ["ab", "Grüße"], new TwinCATArrayLayout(2, 11)); + RoundTrip(TwinCATDataType.WString, ["中", "x"], new TwinCATArrayLayout(2, 8)); + + // TIME elements arrive as the encoded uint ms (ConvertForWrite) and decode back to double ms. + var time = TwinCATArrayCodec.Encode(TwinCATDataType.Time, + TwinCATArrayCodec.ToWriteElements(TwinCATDataType.Time, new[] { 1500.0, 20.0 }, 2), new TwinCATArrayLayout(2, 4), false); + TwinCATArrayCodec.Decode(TwinCATDataType.Time, time, new TwinCATArrayLayout(2, 4), false) + .ShouldBe(new[] { 1500.0, 20.0 }); + } + + /// A string element longer than the element's STRING(n) is out of range. + [Fact] + public void Encode_refuses_a_string_element_wider_than_its_stride() + { + Should.Throw(() => + TwinCATArrayCodec.Encode(TwinCATDataType.String, ["toolong"], new TwinCATArrayLayout(1, 4), utf8: false)); + Should.Throw(() => + TwinCATArrayCodec.Encode(TwinCATDataType.Int, [(short)1], new TwinCATArrayLayout(2, 2), utf8: false)); + } + + private static void RoundTrip(TwinCATDataType type, object[] elements, TwinCATArrayLayout layout) + { + var bytes = TwinCATArrayCodec.Encode(type, elements, layout, utf8: false); + bytes.Length.ShouldBe(layout.ByteLength); + var decoded = TwinCATArrayCodec.Decode(type, bytes, layout, utf8: false); + decoded.Cast().ShouldBe(elements); + } + + // ---- driver ---- + + private static (TwinCATDriver drv, FakeTwinCATClient fake) NewDriver(params TwinCATTagDefinition[] tags) + { + var fake = new FakeTwinCATClient(); + var drv = new TwinCATDriver(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = TwinCATRawTags.Entries(tags), + Probe = new TwinCATProbeOptions { Enabled = false }, + }, "drv-1", new FakeTwinCATClientFactory { Customise = () => fake }); + return (drv, fake); + } + + /// An array tag's write sends ArrayLength converted elements through WriteArrayAsync. + [Fact] + public async Task An_array_write_sends_converted_elements() + { + var (drv, fake) = NewDriver(new TwinCATTagDefinition("Buf", Host, "MAIN.Buf", TwinCATDataType.Int, ArrayLength: 3)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var result = await drv.WriteAsync([new WriteRequest("Buf", new[] { 1, 2, 3 })], TestContext.Current.CancellationToken); + + result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.Good); + var write = fake.ArrayWriteLog.ShouldHaveSingleItem(); + write.symbol.ShouldBe("MAIN.Buf"); + write.elements.ShouldBe(new object[] { (short)1, (short)2, (short)3 }); + fake.WriteLog.ShouldBeEmpty(); // never the scalar path (that threw InvalidCastException) + } + + /// + /// A length mismatch is BadOutOfRange and a scalar value BadTypeMismatch — nothing reaches the wire + /// and health is untouched (a refusal is not a comms fault). Positive control: the good write above. + /// + [Fact] + public async Task A_wrong_shape_array_write_is_refused_before_the_wire() + { + var (drv, fake) = NewDriver(new TwinCATTagDefinition("Buf", Host, "MAIN.Buf", TwinCATDataType.Int, ArrayLength: 3)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var results = await drv.WriteAsync( + [ + new WriteRequest("Buf", new[] { 1, 2 }), + new WriteRequest("Buf", 5), + new WriteRequest("Buf", new[] { 1, 2, 70_000 }), + ], TestContext.Current.CancellationToken); + + results.Select(r => r.StatusCode).ShouldBe( + [ + TwinCATStatusMapper.BadOutOfRange, + TwinCATStatusMapper.BadTypeMismatch, + TwinCATStatusMapper.BadOutOfRange, + ]); + fake.ArrayWriteLog.ShouldBeEmpty(); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + /// An over-long string surfaced by the client as OverflowException is BadOutOfRange, not a comms fault. + [Fact] + public async Task An_over_long_string_write_is_BadOutOfRange() + { + var (drv, fake) = NewDriver(new TwinCATTagDefinition("Name", Host, "MAIN.Name", TwinCATDataType.String)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + fake.ThrowOnWrite = true; + fake.Exception = new OverflowException("6 bytes do not fit STRING(4)"); + + var result = await drv.WriteAsync([new WriteRequest("Name", "toolong")], TestContext.Current.CancellationToken); + + result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } +} From cc140593a02ee2e05c20633dd28234a6d12446c7 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:15:35 -0400 Subject: [PATCH 37/51] fix(focas): refuse String on PMC/PARAM/MACRO in the factory, editor and CLI (R07:U8) FocasDataType.String was authorable on any address, but no area read returns text: PMC fell through to a byte read, PARAM to Int32, MACRO to a double, and the String node received a boxed number at Good. One rule, FocasTagDefinitionFactory.DescribeDataTypeMismatch, now backs every path: - FromTagConfig rejects the pair, so the driver skips + reports it (BadNodeIdUnknown, never a Good number) and Inspect names the reason at deploy; - FocasTagConfigModel.Validate refuses it on the typed fields; - the AdminUI editor only offers String where it is readable and shows the rule's text inline (kept visible for a loaded blob that already carries it); - the CLI read/subscribe/probe/write commands refuse it at the boundary. String stays valid on FIXED: paths (Identity/Program name/mode text) and is ignored on ALARM: tags; an unparseable address keeps its own diagnosis. Razor change (FocasTagConfigEditor.razor) needs the Wave 4 live check. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Commands/ProbeCommand.cs | 1 + .../Commands/ReadCommand.cs | 3 +- .../Commands/SubscribeCommand.cs | 3 +- .../Commands/WriteCommand.cs | 3 +- .../FocasCommandBase.cs | 13 +++ .../FocasTagDefinitionFactory.cs | 27 +++++ .../Uns/TagEditors/FocasTagConfigEditor.razor | 18 ++- .../Uns/TagEditors/FocasTagConfigModel.cs | 4 + .../FocasCommandBaseValidationTests.cs | 21 ++++ .../FocasStringDataTypeTests.cs | 104 ++++++++++++++++++ .../Uns/FocasTagConfigModelTests.cs | 22 ++++ 11 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasStringDataTypeTests.cs diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/ProbeCommand.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/ProbeCommand.cs index d2d46458..9c31bffe 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/ProbeCommand.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/ProbeCommand.cs @@ -28,6 +28,7 @@ public sealed class ProbeCommand : FocasCommandBase { ConfigureLogging(); ValidateOptions(); + ValidateDataType(Address, DataType); var ct = console.RegisterCancellationHandler(); var probeTag = new FocasTagDefinition( diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/ReadCommand.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/ReadCommand.cs index df042344..9218a2c9 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/ReadCommand.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/ReadCommand.cs @@ -19,7 +19,7 @@ public sealed class ReadCommand : FocasCommandBase /// Gets the data type to interpret the address as. [CommandOption("type", 't', Description = - "Bit / Byte / Int16 / Int32 / Float32 / Float64 / String (default Int16).")] + "Bit / Byte / Int16 / Int32 / Float32 / Float64 / String (default Int16). String only on FIXED: paths.")] public FocasDataType DataType { get; init; } = FocasDataType.Int16; /// @@ -28,6 +28,7 @@ public sealed class ReadCommand : FocasCommandBase ConfigureLogging(); // Validate numeric option ranges before any driver work. ValidateOptions(); + ValidateDataType(Address, DataType); var ct = console.RegisterCancellationHandler(); var tagName = SynthesiseTagName(Address, DataType); diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/SubscribeCommand.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/SubscribeCommand.cs index 1a3cb5f4..ee126fc0 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/SubscribeCommand.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/SubscribeCommand.cs @@ -18,7 +18,7 @@ public sealed class SubscribeCommand : FocasCommandBase /// Gets the data type of the address. [CommandOption("type", 't', Description = - "Bit / Byte / Int16 / Int32 / Float32 / Float64 / String (default Int16).")] + "Bit / Byte / Int16 / Int32 / Float32 / Float64 / String (default Int16). String only on FIXED: paths.")] public FocasDataType DataType { get; init; } = FocasDataType.Int16; /// Gets the polling interval in milliseconds. @@ -33,6 +33,7 @@ public sealed class SubscribeCommand : FocasCommandBase // --interval-ms) before any driver work so a zero/negative interval surfaces as a // clean CommandException rather than a tight-spinning poll loop. ValidateOptions(IntervalMs); + ValidateDataType(Address, DataType); var ct = console.RegisterCancellationHandler(); var tagName = ReadCommand.SynthesiseTagName(Address, DataType); diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/WriteCommand.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/WriteCommand.cs index bd383041..22cfb431 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/WriteCommand.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/Commands/WriteCommand.cs @@ -20,7 +20,7 @@ public sealed class WriteCommand : FocasCommandBase /// Gets the data type of the value to write. [CommandOption("type", 't', Description = - "Bit / Byte / Int16 / Int32 / Float32 / Float64 / String (default Int16).")] + "Bit / Byte / Int16 / Int32 / Float32 / Float64 / String (default Int16). String only on FIXED: paths.")] public FocasDataType DataType { get; init; } = FocasDataType.Int16; /// Gets the value to write. @@ -37,6 +37,7 @@ public sealed class WriteCommand : FocasCommandBase // port/timeout surfaces as a clean CommandException rather than an opaque // downstream exception. ValidateOptions(); + ValidateDataType(Address, DataType); var ct = console.RegisterCancellationHandler(); var tagName = ReadCommand.SynthesiseTagName(Address, DataType); diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/FocasCommandBase.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/FocasCommandBase.cs index 62965d52..e57ce8bf 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/FocasCommandBase.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli/FocasCommandBase.cs @@ -89,4 +89,17 @@ public abstract class FocasCommandBase : DriverCommandBase throw new CliFx.Exceptions.CommandException( $"--interval-ms must be positive (got {iv})."); } + + /// + /// Refuses an --address / --type pair the driver cannot serve (String on a PMC / PARAM / MACRO + /// address — the driver skips such a tag, so the command would only ever print BadNodeIdUnknown). Same + /// rule as the driver and the AdminUI editor: . + /// + /// The FOCAS address. + /// The data type. + protected static void ValidateDataType(string address, FocasDataType type) + { + if (FocasTagDefinitionFactory.DescribeDataTypeMismatch(type, address) is { } error) + throw new CliFx.Exceptions.CommandException(error); + } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Contracts/FocasTagDefinitionFactory.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Contracts/FocasTagDefinitionFactory.cs index d495dad1..b65150f6 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Contracts/FocasTagDefinitionFactory.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Contracts/FocasTagDefinitionFactory.cs @@ -53,6 +53,9 @@ public static class FocasTagDefinitionFactory // Strict enum read: a typo'd dataType rejects the tag (→ BadNodeIdUnknown) // instead of silently defaulting to a wrong-width Good. if (!TagConfigJson.TryReadEnumStrict(root, "dataType", FocasDataType.Int32, out var dataType)) return false; + // A data type no read of this address can produce (String on PMC / PARAM / MACRO) rejects the tag: + // served, it published a boxed number at Good into a String node. + if (DescribeDataTypeMismatch(dataType, address) is not null) return false; // FOCAS authored tags carry no `writable` key (writability lives on the Tag entity / node ACL): // absent ⇒ read-only. An explicit writable:true is honoured for the driver's internal write gate // (default-false, so a missing key stays read-only — the note is preserved). @@ -73,6 +76,26 @@ public static class FocasTagDefinitionFactory catch (InvalidOperationException) { return false; } } + /// + /// The ONE address/data-type compatibility rule, shared by the mapper (and so the driver's init and the + /// deploy inspector), the AdminUI tag editor and the CLI. is only + /// servable from the fixed tree (Identity / Program name / mode text …): a PMC, parameter or macro read + /// returns a number, which a String node would receive at Good. An ALARM: tag has no value, so its + /// data type is irrelevant; an unparseable address is left to the address check, which names it better. + /// + /// The authored data type. + /// The authored address. + /// The reason the pair cannot be served, or when it can. + public static string? DescribeDataTypeMismatch(FocasDataType dataType, string? address) + { + if (dataType != FocasDataType.String) return null; + var parsed = FocasAddress.TryParse(address); + if (parsed is null || parsed.Kind is FocasAreaKind.Fixed or FocasAreaKind.Alarm) return null; + return $"FOCAS data type 'String' is not readable at '{address!.Trim()}' — PMC, PARAM: and MACRO: reads " + + "return numbers (String is only valid on FIXED: paths such as FIXED:Identity/SeriesNumber). " + + "Choose Bit / Byte / Int16 / Int32 / Float32 / Float64."; + } + /// /// The tag's cross-driver alarm object, or when it carries none. /// Lenient like the host's TagConfigIntent: a non-integer severity is simply absent. @@ -236,6 +259,10 @@ public static class FocasTagDefinitionFactory } if (TagConfigJson.DescribeInvalidEnum(root, "dataType") is { } w) rejections.Add(w); + else if (addr.ValueKind == JsonValueKind.String + && TagConfigJson.TryReadEnumStrict(root, "dataType", FocasDataType.Int32, out var dataType) + && DescribeDataTypeMismatch(dataType, addr.GetString()) is { } mismatch) + rejections.Add(mismatch + " The driver will skip this tag and it will read BadNodeIdUnknown."); } /// diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/FocasTagConfigEditor.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/FocasTagConfigEditor.razor index 8911b438..03fd6a15 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/FocasTagConfigEditor.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/FocasTagConfigEditor.razor @@ -15,8 +15,22 @@
+ @foreach (var v in Enum.GetValues()) + { + @* String is only readable from the fixed tree (FocasTagDefinitionFactory.DescribeDataTypeMismatch): + offered there, and kept visible when a loaded blob already carries it so the select never lies. *@ + if (v != FocasDataType.String || _m.DataType == FocasDataType.String + || FocasTagDefinitionFactory.DescribeDataTypeMismatch(v, _m.Address) is null) + { + + } + } + + @if (FocasTagDefinitionFactory.DescribeDataTypeMismatch(_m.DataType, _m.Address) is { } typeError) + { +
@typeError
+ } +
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/FocasTagConfigModel.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/FocasTagConfigModel.cs index 91851b8e..68904dff 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/FocasTagConfigModel.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/FocasTagConfigModel.cs @@ -80,6 +80,10 @@ public sealed class FocasTagConfigModel // ahead of the parse consult below and is never replaced by it. if (FocasAddress.TryParse(Address) is null) return DescribeUnparseableAddress(Address); + // The driver's own address/data-type rule (String only on FIXED: paths) — checked on the typed fields so a + // model built in the editor is judged too, not only a loaded blob the parse gate below can see. + if (FocasTagDefinitionFactory.DescribeDataTypeMismatch(DataType, Address) is { } typeError) return typeError; + // Parse-gate backstop. ⚠️ It gates on the PARSE VERDICT, never on Inspect being non-empty: FOCAS's // inspector emits advisories for blobs the reader ACCEPTS (an unparseable address, an authored // 'writable':true on a read-only wire), and promoting those to authoring errors would make the modal diff --git a/tests/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli.Tests/FocasCommandBaseValidationTests.cs b/tests/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli.Tests/FocasCommandBaseValidationTests.cs index 2efff255..66dcabde 100644 --- a/tests/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli.Tests/FocasCommandBaseValidationTests.cs +++ b/tests/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Cli.Tests/FocasCommandBaseValidationTests.cs @@ -33,6 +33,27 @@ public sealed class FocasCommandBaseValidationTests /// Invokes option validation without interval. public void InvokeValidateNoInterval() => ValidateOptions(intervalMs: null); + + /// Invokes the address/data-type check. + public void InvokeValidateDataType(string address, FocasDataType type) => ValidateDataType(address, type); + } + + /// + /// #740 / R07:U8 — --type String on a PMC / PARAM / MACRO address is refused at the CLI boundary + /// (the driver would skip the tag and the read would print BadNodeIdUnknown); a fixed-tree path keeps it. + /// + [Theory] + [InlineData("R100", FocasDataType.String, false)] + [InlineData("MACRO:500", FocasDataType.String, false)] + [InlineData("PARAM:6711", FocasDataType.String, false)] + [InlineData("R100", FocasDataType.Byte, true)] + [InlineData("FIXED:Identity/SeriesNumber", FocasDataType.String, true)] + public void Validate_data_type_against_address(string address, FocasDataType type, bool ok) + { + var sut = new Probe { CncHost = "host", IntervalMs = 1000 }; + if (ok) Should.NotThrow(() => sut.InvokeValidateDataType(address, type)); + else Should.Throw(() => sut.InvokeValidateDataType(address, type)) + .Message.ShouldContain("String"); } /// Verifies that default options are accepted. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasStringDataTypeTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasStringDataTypeTests.cs new file mode 100644 index 00000000..600a205a --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasStringDataTypeTests.cs @@ -0,0 +1,104 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; + +/// +/// #740 / R07:U8 — was authorable on any address, but no area read +/// produces a string: PMC fell through to a byte read, PARAM to an Int32, MACRO to a double, and a +/// String-typed node received a boxed number at Good. String is now refused on PMC / PARAM / MACRO by the +/// one shared check, everywhere a tag is authored or loaded; the fixed tree (Identity / Program name, …), +/// whose values really are strings, keeps it. +/// +[Trait("Category", "Unit")] +public sealed class FocasStringDataTypeTests +{ + private const string Host = "focas://10.0.0.5:8193"; + + private static string Config(string address, string dataType) => + $$"""{"address":"{{address}}","dataType":"{{dataType}}"}"""; + + /// String on an area read is refused by the mapper. + [Theory] + [InlineData("R100")] + [InlineData("X0.0")] + [InlineData("PARAM:6711")] + [InlineData("MACRO:500")] + public void Mapper_refuses_string_on_an_area_read(string address) + { + FocasTagDefinitionFactory.FromTagConfig(Config(address, "String"), "raw/T", out _).ShouldBeFalse(); + FocasTagDefinitionFactory.DescribeDataTypeMismatch(FocasDataType.String, address).ShouldNotBeNull(); + } + + /// Positive control: numeric types on the same addresses, and String on the fixed tree, still map. + [Theory] + [InlineData("R100", "Byte")] + [InlineData("PARAM:6711", "Int32")] + [InlineData("MACRO:500", "Float64")] + [InlineData("FIXED:Identity/SeriesNumber", "String")] + [InlineData("FIXED:Program/Name", "String")] + public void Mapper_accepts_numeric_area_reads_and_string_fixed_tree_paths(string address, string dataType) + { + FocasTagDefinitionFactory.FromTagConfig(Config(address, dataType), "raw/T", out var def).ShouldBeTrue(); + def.Address.ShouldBe(address); + } + + /// An ALARM: tag has no value, so its data type is irrelevant and String is not refused. + [Fact] + public void Mapper_accepts_string_on_an_alarm_tag() + { + FocasTagDefinitionFactory.FromTagConfig( + """{"address":"ALARM:*","dataType":"String","alarm":{"severity":500}}""", "raw/A", out _).ShouldBeTrue(); + } + + /// An unparseable address is left to the address check — it still draws THAT diagnosis. + [Fact] + public void Unparseable_address_keeps_its_own_diagnosis() + { + FocasTagDefinitionFactory.DescribeDataTypeMismatch(FocasDataType.String, "not-an-address").ShouldBeNull(); + FocasTagDefinitionFactory.FromTagConfig(Config("not-an-address", "String"), "raw/T", out _).ShouldBeTrue(); + } + + /// The deploy-time inspector names the reason rather than falling to the generic backstop. + [Fact] + public void Inspect_names_the_string_refusal() + { + var warnings = FocasTagDefinitionFactory.Inspect(Config("MACRO:500", "String")); + warnings.ShouldContain(w => w.Contains("String", StringComparison.Ordinal) && w.Contains("MACRO:500", StringComparison.Ordinal)); + FocasTagDefinitionFactory.Inspect(Config("MACRO:500", "Float64")).ShouldBeEmpty(); + } + + /// + /// At init a String area tag is skipped and reported (never served as a Good number); the numeric tag + /// beside it reads normally (positive control). + /// + [Fact] + public async Task Driver_skips_and_reports_a_string_area_tag() + { + var factory = new FakeFocasClientFactory + { + Customise = () => new FakeFocasClient { Values = { ["R100"] = (sbyte)7 } }, + }; + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(Host)], + RawTags = + [ + new RawTagEntry("raw/Str", Config("R100", "String"), false, Host), + new RawTagEntry("raw/Num", Config("R100", "Byte"), false, Host), + ], + Probe = new FocasProbeOptions { Enabled = false }, + }, "focas-u8", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + var snaps = await drv.ReadAsync(["raw/Str", "raw/Num"], CancellationToken.None); + snaps[0].StatusCode.ShouldBe(FocasStatusMapper.BadNodeIdUnknown); + snaps[0].Value.ShouldBeNull(); + snaps[1].StatusCode.ShouldBe(FocasStatusMapper.Good); + snaps[1].Value.ShouldBe((sbyte)7); + drv.GetSkippedTags()!.Sample.ShouldContain(s => s.RawPath == "raw/Str"); + await drv.ShutdownAsync(CancellationToken.None); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/FocasTagConfigModelTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/FocasTagConfigModelTests.cs index 334637cd..b3f16e8b 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/FocasTagConfigModelTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/FocasTagConfigModelTests.cs @@ -169,4 +169,26 @@ public sealed class FocasTagConfigModelTests [InlineData("""{"address":"R100","dataType":2}""")] public void Validate_accepts_every_dataType_the_driver_accepts(string json) => FocasTagConfigModel.FromJson(json).Validate().ShouldBeNull(); + + /// + /// #740 / R07:U8 — String on a PMC / PARAM / MACRO address is refused by the driver (no area read + /// returns text), so the modal refuses it too, for a model built in the editor and for a loaded blob. + /// + [Theory] + [InlineData("R100")] + [InlineData("PARAM:6711")] + [InlineData("MACRO:500")] + public void Validate_refuses_string_on_an_area_address(string address) + { + new FocasTagConfigModel { Address = address, DataType = FocasDataType.String } + .Validate().ShouldNotBeNull().ShouldContain("String"); + FocasTagConfigModel.FromJson($$"""{"address":"{{address}}","dataType":"String"}""") + .Validate().ShouldNotBeNull().ShouldContain("String"); + } + + /// Positive control: the fixed tree's string fields keep String. + [Fact] + public void Validate_accepts_string_on_a_fixed_tree_path() + => new FocasTagConfigModel { Address = "FIXED:Identity/SeriesNumber", DataType = FocasDataType.String } + .Validate().ShouldBeNull(); } From 256915b7e45a7bd757c24ad573431a2246212c8d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:18:16 -0400 Subject: [PATCH 38/51] perf(focas): read devices concurrently within a batch (R07:P9) ReadAsync walked the whole batch sequentially, so one SLOW CNC (TCP-accept-then-stall, the live 31i-B case) cost Timeout per tag per poll while no other device was read. ReadAsync now resolves every reference against one table snapshot, buckets the rest by device and reads the devices with Task.WhenAll; each device's tags still go in order on its own serialised socket (the PMC coalescing pre-pass runs per device, first). Results land at their own indices. Also precomputes each device's fixed-tree key prefix (it was built per reference, twice per poll). Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../FocasDriver.cs | 223 ++++++++++-------- .../FocasConcurrentDeviceReadTests.cs | 115 +++++++++ 2 files changed, 245 insertions(+), 93 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs index c8dad722..5c4953d3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs @@ -637,23 +637,23 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // ---- IReadable ---- /// + // Devices are read CONCURRENTLY, each one's tags in order on its own (serialised) socket: a slow CNC — a + // TCP-accept-then-stall control costs its Timeout per tag — used to hold every other device's read in the + // batch hostage, because the whole batch was one sequential walk. public async Task> ReadAsync( IReadOnlyList fullReferences, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(fullReferences); var now = DateTime.UtcNow; var results = new DataValueSnapshot[fullReferences.Count]; + // One snapshot of each swapped table for the whole call (see _devices). + var devices = Devices; + var alarmTags = Volatile.Read(ref _alarmTags); + var fixedTreeActive = _fixedTreeActive; - // Merge adjacent / near PMC reads of one device into fewer pmc_rdpmcrng calls. Indices it - // serves are marked here and skipped by the per-tag loop below; everything else (including every - // single-tag range and any range the CNC rejected) takes the unchanged per-tag path. - var served = _options.PmcCoalesce.Enabled - ? await CoalescePmcReadsAsync(fullReferences, results, now, cancellationToken).ConfigureAwait(false) - : null; - + Dictionary>? byDevice = null; for (var i = 0; i < fullReferences.Count; i++) { - if (served is not null && served[i]) continue; var reference = fullReferences[i]; // Fixed-tree T1 — fixed-tree references are synthesized from the cached @@ -663,25 +663,66 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // This branch matches a reference that IS a "{host}/{path}" fixed-tree key. Authored // tags do not take it — their reference is a RawPath — and are served further down by // their FIXED: address instead. - if (_fixedTreeActive && TryReadFixedTree(reference, now) is { } fx) + if (fixedTreeActive && TryReadFixedTree(devices, reference, now) is { } fx) { results[i] = fx; continue; } // An ALARM: tag is a Part 9 condition, not a value — there is nothing to read. - if (Volatile.Read(ref _alarmTags).ContainsKey(reference)) + if (alarmTags.ContainsKey(reference)) { results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadNotSupported, null, now); continue; } - if (!_resolver.TryResolve(reference, out var def)) + if (!_resolver.TryResolve(reference, out var def) + || !devices.TryGetValue(def.DeviceHostAddress, out var device)) { results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadNodeIdUnknown, null, now); continue; } - if (!Devices.TryGetValue(def.DeviceHostAddress, out var device)) + + byDevice ??= []; + if (!byDevice.TryGetValue(device, out var indices)) byDevice[device] = indices = []; + indices.Add(i); + } + + if (byDevice is null) return results; + if (byDevice.Count == 1) + { + foreach (var (device, indices) in byDevice) + await ReadDeviceAsync(device, indices, fullReferences, results, now, cancellationToken).ConfigureAwait(false); + } + else + { + await Task.WhenAll(byDevice.Select(kv => + ReadDeviceAsync(kv.Key, kv.Value, fullReferences, results, now, cancellationToken))).ConfigureAwait(false); + } + return results; + } + + /// + /// Reads one device's share of a batch, in request order: the PMC coalescing pre-pass first, then every + /// index it did not serve through the per-tag path. Each result lands at its own index, so concurrent + /// device reads never touch the same slot. + /// + private async Task ReadDeviceAsync( + DeviceState device, List indices, IReadOnlyList fullReferences, + DataValueSnapshot[] results, DateTime now, CancellationToken cancellationToken) + { + // Merge adjacent / near PMC reads of the device into fewer pmc_rdpmcrng calls. Indices it serves are + // marked here and skipped by the per-tag loop below; everything else (including every single-tag range + // and any range the CNC rejected) takes the unchanged per-tag path. + var served = _options.PmcCoalesce.Enabled + ? await CoalescePmcReadsAsync(device, indices, fullReferences, results, now, cancellationToken).ConfigureAwait(false) + : null; + + foreach (var i in indices) + { + if (served is not null && served.Contains(i)) continue; + var reference = fullReferences[i]; + if (!_resolver.TryResolve(reference, out var def)) { results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadNodeIdUnknown, null, now); continue; @@ -738,12 +779,10 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, Volatile.Read(ref _health).LastSuccessfulRead, ex.Message)); } } - - return results; } /// - /// PMC coalescing pre-pass. Groups the batch's PMC tags by device, plans merged ranges + /// PMC coalescing pre-pass for ONE device's share of a batch. Plans merged ranges /// () and serves every multi-member range with ONE /// pmc_rdpmcrng, slicing each tag's value out of the returned slots. /// @@ -756,106 +795,98 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// leaves the members UNSERVED, so the per-tag loop reads them individually and each gets its own /// true status — and the range is remembered so later polls skip the doomed merged read. /// - /// Returns null when nothing was served. + /// Returns the served indices, or null when nothing was served. /// - private async Task CoalescePmcReadsAsync( - IReadOnlyList references, DataValueSnapshot[] results, DateTime now, CancellationToken ct) + private async Task?> CoalescePmcReadsAsync( + DeviceState device, List indices, IReadOnlyList references, DataValueSnapshot[] results, + DateTime now, CancellationToken ct) { - Dictionary>? byDevice = null; - for (var i = 0; i < references.Count; i++) + List? items = null; + foreach (var i in indices) { try { - var reference = references[i]; - if (_fixedTreeActive && TryReadFixedTree(reference, now) is not null) continue; - if (!_resolver.TryResolve(reference, out var def)) continue; - if (!Devices.TryGetValue(def.DeviceHostAddress, out var device)) continue; + if (!_resolver.TryResolve(references[i], out var def)) continue; var parsed = ResolveParsedAddress(def); if (parsed.Kind != FocasAreaKind.Pmc || parsed.PmcLetter is null) continue; - (byDevice ??= []).TryAdd(device, []); - byDevice[device].Add(new FocasPmcReadItem(i, parsed.PmcLetter, parsed.Number, def.DataType, parsed.BitIndex)); + (items ??= []).Add(new FocasPmcReadItem(i, parsed.PmcLetter, parsed.Number, def.DataType, parsed.BitIndex)); } catch (Exception ex) when (ex is not OperationCanceledException) { /* malformed / unresolvable — the per-tag loop reports it with its usual status */ } } - if (byDevice is null) return null; + if (items is null || items.Count < 2) return null; - bool[]? served = null; - foreach (var (device, items) in byDevice) + var plans = FocasPmcCoalescer.Plan(items, _options.PmcCoalesce.MaxSpanBytes, _options.PmcCoalesce.MaxGapBytes); + if (!plans.Any(p => p.Members.Count >= 2 && !device.CoalesceRejected.ContainsKey(p.Key))) return null; + + HashSet? served = null; + // One connect attempt per device per batch: if it fails, every PMC tag of the device fails with + // that status here instead of paying a connect attempt per plan and again per tag. + IFocasClient client; + try { client = await EnsureConnectedAsync(device, ct).ConfigureAwait(false); } + catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } + catch (Exception) { - if (items.Count < 2) continue; - var plans = FocasPmcCoalescer.Plan(items, _options.PmcCoalesce.MaxSpanBytes, _options.PmcCoalesce.MaxGapBytes); - if (!plans.Any(p => p.Members.Count >= 2 && !device.CoalesceRejected.ContainsKey(p.Key))) continue; + FailMembers(items, FocasStatusMapper.BadCommunicationError, + $"FOCAS connect failed reading PMC tags of {device.Options.HostAddress}"); + return served; + } - // One connect attempt per device per batch: if it fails, every PMC tag of the device fails with - // that status here instead of paying a connect attempt per plan and again per tag. - IFocasClient client; - try { client = await EnsureConnectedAsync(device, ct).ConfigureAwait(false); } + foreach (var plan in plans) + { + if (plan.Members.Count < 2) continue; + if (device.CoalesceRejected.ContainsKey(plan.Key)) continue; + + var members = plan.Members; + try + { + var (slots, status) = await client + .ReadPmcSlotsAsync(plan.Letter, plan.SlotType, plan.Start, plan.End, ct) + .ConfigureAwait(false); + + if (status == FocasStatusMapper.Good && slots is not null) + { + foreach (var m in members) + { + var slot = plan.SlotOf(m); + results[m.Index] = slot < slots.Count + ? new DataValueSnapshot(FocasPmcCoalescer.MapSlot(slots[slot], m.Type, m.BitIndex), status, now, now) + : new DataValueSnapshot(null, FocasStatusMapper.BadOutOfRange, now, now); + (served ??= []).Add(m.Index); + } + Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, now, null)); + } + else + { + // The CNC REFUSED the merged range (any return code, mapped or not). Don't guess which + // member is at fault: let the per-tag loop give each its own answer. Remember the range + // so later polls skip the doomed read — except for a transient refusal (EW_BUSY maps to + // BadDeviceFailure), which says nothing about the range. + if (status != FocasStatusMapper.BadDeviceFailure) + device.CoalesceRejected[plan.Key] = 0; + _logger.LogDebug( + "FOCAS PMC range {Key} on {Host} rejected (0x{Status:X8}) — reading its {Count} tags individually", + plan.Key, device.Options.HostAddress, status, members.Count); + } + } catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } catch (Exception) { - FailMembers(items, FocasStatusMapper.BadCommunicationError, references, results, now, ref served, - $"FOCAS connect failed reading PMC tags of {device.Options.HostAddress}"); - continue; - } - - foreach (var plan in plans) - { - if (plan.Members.Count < 2) continue; - if (device.CoalesceRejected.ContainsKey(plan.Key)) continue; - - var members = plan.Members; - try - { - var (slots, status) = await client - .ReadPmcSlotsAsync(plan.Letter, plan.SlotType, plan.Start, plan.End, ct) - .ConfigureAwait(false); - - if (status == FocasStatusMapper.Good && slots is not null) - { - foreach (var m in members) - { - var slot = plan.SlotOf(m); - results[m.Index] = slot < slots.Count - ? new DataValueSnapshot(FocasPmcCoalescer.MapSlot(slots[slot], m.Type, m.BitIndex), status, now, now) - : new DataValueSnapshot(null, FocasStatusMapper.BadOutOfRange, now, now); - (served ??= new bool[references.Count])[m.Index] = true; - } - Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, now, null)); - } - else - { - // The CNC REFUSED the merged range (any return code, mapped or not). Don't guess which - // member is at fault: let the per-tag loop give each its own answer. Remember the range - // so later polls skip the doomed read — except for a transient refusal (EW_BUSY maps to - // BadDeviceFailure), which says nothing about the range. - if (status != FocasStatusMapper.BadDeviceFailure) - device.CoalesceRejected[plan.Key] = 0; - _logger.LogDebug( - "FOCAS PMC range {Key} on {Host} rejected (0x{Status:X8}) — reading its {Count} tags individually", - plan.Key, device.Options.HostAddress, status, members.Count); - } - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } - catch (Exception) - { - // Transport failure / timeout: the link is suspect, so fail the members without per-tag retries. - FailMembers(members, FocasStatusMapper.BadCommunicationError, references, results, now, ref served, - $"FOCAS transport failure reading PMC range {plan.Key}"); - } + // Transport failure / timeout: the link is suspect, so fail the members without per-tag retries. + FailMembers(members, FocasStatusMapper.BadCommunicationError, + $"FOCAS transport failure reading PMC range {plan.Key}"); } } return served; - void FailMembers(IEnumerable failed, uint status, IReadOnlyList refs, - DataValueSnapshot[] res, DateTime stamp, ref bool[]? servedRef, string reason) + void FailMembers(IEnumerable failed, uint status, string reason) { foreach (var m in failed) { - res[m.Index] = new DataValueSnapshot(null, status, null, stamp); - (servedRef ??= new bool[refs.Count])[m.Index] = true; + results[m.Index] = new DataValueSnapshot(null, status, null, now); + (served ??= []).Add(m.Index); } Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, Volatile.Read(ref _health).LastSuccessfulRead, @@ -1604,12 +1635,13 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// the reference looks like a fixed-tree FullName; null when it doesn't /// (callers fall through to the user-authored tag path). ///
- private DataValueSnapshot? TryReadFixedTree(string reference, DateTime now) + private DataValueSnapshot? TryReadFixedTree( + IReadOnlyDictionary devices, string reference, DateTime now) { - foreach (var state in Devices.Values) + foreach (var state in devices.Values) { - if (!reference.StartsWith(state.Options.HostAddress + "/", StringComparison.OrdinalIgnoreCase)) continue; - if (TryReadFixedTreeOnDevice(state, reference[(state.Options.HostAddress.Length + 1)..], now) is { } hit) + if (!reference.StartsWith(state.FixedTreePrefix, StringComparison.OrdinalIgnoreCase)) continue; + if (TryReadFixedTreeOnDevice(state, reference[state.FixedTreePrefix.Length..], now) is { } hit) return hit; } return null; @@ -2088,6 +2120,11 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, public FocasHostAddress ParsedAddress { get; } = parsedAddress; /// Gets the device configuration options. public FocasDeviceOptions Options { get; } = options; + /// + /// "{HostAddress}/" — the prefix of this device's fixed-tree cache keys, built once rather than + /// per reference per poll (twice per poll before #740). + /// + public string FixedTreePrefix { get; } = options.HostAddress + "/"; /// Gets or sets the FOCAS client instance. public IFocasClient? Client { get; set; } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs new file mode 100644 index 00000000..9ca35cf7 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs @@ -0,0 +1,115 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; + +/// +/// #740 / R07:P9 — walked the whole batch sequentially, so a SLOW CNC +/// (TCP-accept-then-stall, the live 31i-B case) cost its timeout per tag per poll during which no other +/// device was read. Devices are now read concurrently, each still serialised on its own socket. +/// +[Trait("Category", "Unit")] +public sealed class FocasConcurrentDeviceReadTests +{ + private const string HostA = "focas://10.0.0.5:8193"; + private const string HostB = "focas://10.0.0.6:8193"; + + /// + /// A client that records which device it dialled. Device A's read only completes once device B's read has + /// STARTED — so a sequential walk (A first) can never finish A, while a concurrent one does. Also counts + /// the peak number of reads in flight on any one client. + /// + private sealed class OrderingClient(TaskCompletionSource bStarted) : FakeFocasClient + { + private string _host = ""; + private int _inFlight; + public int PeakInFlight; + + public override Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken ct) + { + _host = address.ToString(); + return base.ConnectAsync(address, timeout, ct); + } + + public override async Task<(object? value, uint status)> ReadAsync( + FocasAddress address, FocasDataType type, CancellationToken ct) + { + var now = Interlocked.Increment(ref _inFlight); + InterlockedMax(ref PeakInFlight, now); + try + { + if (_host.Contains("10.0.0.6", StringComparison.Ordinal)) bStarted.TrySetResult(); + else await bStarted.Task.WaitAsync(TimeSpan.FromSeconds(2), ct).ConfigureAwait(false); + await Task.Delay(20, ct).ConfigureAwait(false); + return ((sbyte)1, FocasStatusMapper.Good); + } + finally { Interlocked.Decrement(ref _inFlight); } + } + + private static void InterlockedMax(ref int target, int value) + { + int seen; + while ((seen = Volatile.Read(ref target)) < value + && Interlocked.CompareExchange(ref target, value, seen) != seen) { } + } + } + + /// A stalled device no longer holds every other device's read hostage — and each device stays serialised. + [Fact] + public async Task Devices_are_read_concurrently_and_each_stays_serialised() + { + var bStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var factory = new FakeFocasClientFactory { Customise = () => new OrderingClient(bStarted) }; + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(HostA, DeviceName: "A"), new FocasDeviceOptions(HostB, DeviceName: "B")], + RawTags = + [ + FocasRawTags.Entry(new FocasTagDefinition("raw/A1", "A", "R100", FocasDataType.Byte)), + FocasRawTags.Entry(new FocasTagDefinition("raw/A2", "A", "R101", FocasDataType.Byte)), + FocasRawTags.Entry(new FocasTagDefinition("raw/B1", "B", "R100", FocasDataType.Byte)), + ], + Probe = new FocasProbeOptions { Enabled = false }, + // Off so every tag takes the per-tag path — this pins device-level concurrency, not coalescing. + PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false }, + }, "focas-p9", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + var snaps = await drv.ReadAsync(["raw/A1", "raw/A2", "raw/B1"], CancellationToken.None); + + snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.Good, + "device A's read can only finish once device B's has started — a sequential walk times A out"); + factory.Clients.Count.ShouldBe(2); + factory.Clients.Cast().ShouldAllBe(c => c.PeakInFlight == 1, + "one device's socket is still request→response: never two reads in flight on it"); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// Results land at their own indices whatever order the devices finish in. + [Fact] + public async Task Results_keep_their_request_order_across_devices() + { + var factory = new FakeFocasClientFactory(); + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(HostA, DeviceName: "A"), new FocasDeviceOptions(HostB, DeviceName: "B")], + RawTags = + [ + FocasRawTags.Entry(new FocasTagDefinition("raw/A1", "A", "R100", FocasDataType.Byte)), + FocasRawTags.Entry(new FocasTagDefinition("raw/B1", "B", "R200", FocasDataType.Byte)), + ], + Probe = new FocasProbeOptions { Enabled = false }, + }, "focas-p9-order", factory) + ; + factory.Customise = () => new FakeFocasClient { Values = { ["R100"] = (sbyte)1, ["R200"] = (sbyte)2 } }; + await drv.InitializeAsync("{}", CancellationToken.None); + + var snaps = await drv.ReadAsync(["raw/B1", "nope", "raw/A1"], CancellationToken.None); + + snaps[0].Value.ShouldBe((sbyte)2); + snaps[1].StatusCode.ShouldBe(FocasStatusMapper.BadNodeIdUnknown); + snaps[2].Value.ShouldBe((sbyte)1); + await drv.ShutdownAsync(CancellationToken.None); + } +} From 4c4dfaadcc1721f79f8bf6bfb2bab21f325e3ec6 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:21:02 -0400 Subject: [PATCH 39/51] fix(focas): recycle the device handle under ConnectGate and drain it first (R07:L6) The handle-recycle loop disposed the device client with no gate. It could null a client a concurrent EnsureConnectedAsync had just installed (orphaning a live two-socket session), or dispose one a poll was mid-read on (ObjectDisposedException -> BadCommunicationError + Degraded on every recycle interval). The recycle now takes the device's ConnectGate, detaches the client, and disposes it via SynchronizedFocasClient.DisposeWhenIdleAsync, which waits (bounded by the call timeout plus slack) for ops that already hold the client. The gate stays held through the drain, so no fresh session opens beside the old one. SynchronizedFocasClient no longer disposes its call gate. Releasing a disposed semaphore threw out of reads that had otherwise completed. Ops that reach a disposed client now get ObjectDisposedException and are never run on the dead socket. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../FocasDriver.cs | 30 ++++++- .../SynchronizedFocasClient.cs | 79 +++++++++++++++---- .../FocasHandleRecycleTests.cs | 67 ++++++++++++++++ 3 files changed, 158 insertions(+), 18 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs index 5c4953d3..f65bb2f4 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs @@ -1779,6 +1779,13 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, return null; } + /// + /// How long a handle recycle waits for in-flight ops on the detached client: one call timeout (each op is + /// bounded by it) plus slack, so a recycle never kills a read that would have completed. + /// + private TimeSpan RecycleDrainBudget => + (_options.Timeout > TimeSpan.Zero ? _options.Timeout : TimeSpan.FromSeconds(2)) + TimeSpan.FromMilliseconds(500); + private async Task RecycleLoopAsync(DeviceState state, CancellationToken ct) { while (!ct.IsCancellationRequested) @@ -1788,14 +1795,31 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // Close the current handle — the next Read / Write / Probe call triggers // EnsureConnectedAsync, which reopens a fresh one. We don't block here on - // reconnect because the goal is just to release the FWLIB handle slot; a - // readable tick one probe cycle later is an acceptable cost. - try { state.DisposeClient(); } + // reconnect because the goal is just to release the FWLIB handle slot. + // + // Under the device's ConnectGate, exactly like EnsureConnectedAsync's install: unserialised, the + // recycle could null a client a concurrent connect had just installed (orphaning a live two-socket + // session) or dispose one a poll was mid-read on (BadCommunicationError + Degraded every interval). + // The gate is held through the drain so no fresh session opens beside the old one — the CNC's + // handle pool is small — and the detached client is disposed only once the ops already holding it + // have finished (bounded by the call timeout). + try { await state.ConnectGate.WaitAsync(ct).ConfigureAwait(false); } + catch (OperationCanceledException) { break; } + try + { + var old = state.Client; + state.Client = null; + if (old is SynchronizedFocasClient sync) + await sync.DisposeWhenIdleAsync(RecycleDrainBudget).ConfigureAwait(false); + else + old?.Dispose(); + } catch (Exception ex) { /* already disposed or race — next EnsureConnected recovers */ _logger.LogDebug(ex, "FOCAS handle-recycle dispose failed for {Host}", state.Options.HostAddress); } + finally { state.ConnectGate.Release(); } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/SynchronizedFocasClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/SynchronizedFocasClient.cs index 9dc81fca..a0bade40 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/SynchronizedFocasClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/SynchronizedFocasClient.cs @@ -32,6 +32,9 @@ public sealed class SynchronizedFocasClient : IFocasClient private readonly IFocasClient _inner; private readonly TimeSpan _callTimeout; private readonly SemaphoreSlim _gate = new(1, 1); + /// Ops holding or queued for — what drains. + private int _pending; + private int _disposed; /// Wraps with per-device serialization + a per-call timeout. /// The underlying FOCAS client to serialize access to. @@ -115,26 +118,66 @@ public sealed class SynchronizedFocasClient : IFocasClient public Task> GetSpindleMotorSpeedsAsync(CancellationToken cancellationToken) => RunBoundedAsync(ct => _inner.GetSpindleMotorSpeedsAsync(ct), cancellationToken); - /// Disposes the inner client and the call gate. + /// + /// Disposes the inner client. Ops still queued on the call gate then fail with + /// instead of running on a disposed socket. The gate itself is NOT + /// disposed: an op that already holds it releases it on the way out, and releasing a disposed + /// threw out of a read that had otherwise completed. + /// public void Dispose() { + if (Interlocked.Exchange(ref _disposed, 1) != 0) return; _inner.Dispose(); - _gate.Dispose(); + } + + /// + /// Disposes once every op that already holds (or is queued for) this client has finished, waiting at most + /// . Used by the handle-recycle loop, which detaches the client under the device's + /// connect gate first — so nothing new can reach it — and then must not kill a read that is mid-flight on + /// it (that read used to surface BadCommunicationError + Degraded on every recycle tick). + /// + /// Upper bound on the wait; the client is disposed regardless once it passes. + /// A task that completes when the client is disposed. + public async Task DisposeWhenIdleAsync(TimeSpan maxWait) + { + var deadline = DateTime.UtcNow + maxWait; + while (Volatile.Read(ref _pending) > 0 && DateTime.UtcNow < deadline) + await Task.Delay(5, CancellationToken.None).ConfigureAwait(false); + Dispose(); + } + + /// Gets the number of ops currently holding or queued for the call gate. Test seam. + internal int PendingOperations => Volatile.Read(ref _pending); + + private void ThrowIfDisposed() + { + if (Volatile.Read(ref _disposed) != 0) + throw new ObjectDisposedException(nameof(SynchronizedFocasClient), "The FOCAS client was recycled or disposed."); } // Gate only — the caller already governs the budget (connect timeout arg / probe linked token). private async Task RunGatedAsync(Func> op, CancellationToken ct) { - await _gate.WaitAsync(ct).ConfigureAwait(false); - try { return await op(ct).ConfigureAwait(false); } - finally { _gate.Release(); } + Interlocked.Increment(ref _pending); + try + { + await _gate.WaitAsync(ct).ConfigureAwait(false); + try { ThrowIfDisposed(); return await op(ct).ConfigureAwait(false); } + finally { _gate.Release(); } + } + finally { Interlocked.Decrement(ref _pending); } } private async Task RunGatedAsync(Func op, CancellationToken ct) { - await _gate.WaitAsync(ct).ConfigureAwait(false); - try { await op(ct).ConfigureAwait(false); } - finally { _gate.Release(); } + Interlocked.Increment(ref _pending); + try + { + await _gate.WaitAsync(ct).ConfigureAwait(false); + try { ThrowIfDisposed(); await op(ct).ConfigureAwait(false); } + finally { _gate.Release(); } + } + finally { Interlocked.Decrement(ref _pending); } } // Gate + per-call timeout. A fired timeout surfaces as OperationCanceledException whose token is @@ -142,16 +185,22 @@ public sealed class SynchronizedFocasClient : IFocasClient // their own token's IsCancellationRequested. private async Task RunBoundedAsync(Func> op, CancellationToken ct) { - await _gate.WaitAsync(ct).ConfigureAwait(false); + Interlocked.Increment(ref _pending); try { - if (_callTimeout <= TimeSpan.Zero) - return await op(ct).ConfigureAwait(false); + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + ThrowIfDisposed(); + if (_callTimeout <= TimeSpan.Zero) + return await op(ct).ConfigureAwait(false); - using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct); - linked.CancelAfter(_callTimeout); - return await op(linked.Token).ConfigureAwait(false); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct); + linked.CancelAfter(_callTimeout); + return await op(linked.Token).ConfigureAwait(false); + } + finally { _gate.Release(); } } - finally { _gate.Release(); } + finally { Interlocked.Decrement(ref _pending); } } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasHandleRecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasHandleRecycleTests.cs index 97042231..4c287b4f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasHandleRecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasHandleRecycleTests.cs @@ -62,6 +62,73 @@ public sealed class FocasHandleRecycleTests await drv.ShutdownAsync(CancellationToken.None); } + /// A client whose reads take a while — long enough for a recycle tick to land mid-read. + private sealed class SlowReadClient : FakeFocasClient + { + public override async Task<(object? value, uint status)> ReadAsync( + FocasAddress address, FocasDataType type, CancellationToken ct) + { + await Task.Delay(200, ct).ConfigureAwait(false); + if (!IsConnected) throw new ObjectDisposedException(nameof(SlowReadClient), "disposed mid-read"); + return ((sbyte)5, FocasStatusMapper.Good); + } + } + + /// + /// #740 / R07:L6 — the recycle loop disposed the device client with no gate, so a recycle tick landing + /// mid-read killed that read (BadCommunicationError + Degraded, every interval). The recycle now takes the + /// device's connect gate and lets the in-flight read finish before disposing the client it detached. + /// + [Fact] + public async Task Recycle_tick_landing_mid_read_lets_the_read_finish() + { + var factory = new FakeFocasClientFactory { Customise = () => new SlowReadClient() }; + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions("focas://10.0.0.5:8193")], + RawTags = [FocasRawTags.Entry(new FocasTagDefinition("R", "focas://10.0.0.5:8193", "R100", FocasDataType.Byte))], + Probe = new FocasProbeOptions { Enabled = false }, + HandleRecycle = new FocasHandleRecycleOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(50) }, + }, "drv-1", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + // Each 200 ms read spans at least three recycle ticks. + for (var i = 0; i < 3; i++) + { + var snap = (await drv.ReadAsync(["R"], CancellationToken.None))[0]; + snap.StatusCode.ShouldBe(FocasStatusMapper.Good, $"read {i} was killed by a recycle tick"); + snap.Value.ShouldBe((sbyte)5); + } + drv.GetHealth().State.ShouldBe(Core.Abstractions.DriverState.Healthy); + + // Positive control: the recycle really ran — the first client was disposed and a fresh one opened. + await WaitFor(() => factory.Clients[0].DisposeCount > 0, TimeSpan.FromSeconds(3)); + factory.Clients[0].DisposeCount.ShouldBeGreaterThan(0); + factory.Clients.Count.ShouldBeGreaterThan(1); + await drv.ShutdownAsync(CancellationToken.None); + factory.Clients.ShouldAllBe(c => !c.IsConnected, "no client opened around a recycle may be orphaned"); + } + + /// DisposeWhenIdleAsync drains the in-flight op, then disposes; a later op is refused, not run on a dead socket. + [Fact] + public async Task DisposeWhenIdle_drains_the_in_flight_op_then_refuses_new_ones() + { + var inner = new SlowReadClient(); + await inner.ConnectAsync(new FocasHostAddress("10.0.0.5", 8193), TimeSpan.FromSeconds(1), CancellationToken.None); + var sync = new SynchronizedFocasClient(inner, TimeSpan.FromSeconds(5)); + var addr = FocasAddress.TryParse("R100")!; + + var read = sync.ReadAsync(addr, FocasDataType.Byte, CancellationToken.None); + await Task.Delay(20); + sync.PendingOperations.ShouldBe(1); + var dispose = sync.DisposeWhenIdleAsync(TimeSpan.FromSeconds(5)); + + (await read).status.ShouldBe(FocasStatusMapper.Good); + await dispose; + inner.DisposeCount.ShouldBe(1); + await Should.ThrowAsync(() => sync.ReadAsync(addr, FocasDataType.Byte, CancellationToken.None)); + } + private static async Task WaitFor(Func pred, TimeSpan timeout) { var deadline = DateTime.UtcNow + timeout; From 29b566d51b435d383a47566839706f8a0c85abe4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:24:05 -0400 Subject: [PATCH 40/51] fix(focas): disambiguate the cnc_rdalmmsg2 record layout at 1,520-byte multiples (R07:H20) ParseAlarms took the 76-byte vendor layout whenever Length % 76 == 0, so a legacy 80-byte reply of exactly 19 records (1,520 B = 20 x 76) decoded as 20 garbage vendor records. The response block carries no record count (its 16-byte header is length/class/path/ command/rc/payload length), so the layout is chosen from what is known: the record count the request asked for (a reply never holds more), then each layout's field invariants on every record. For legacy records, the int32 type/axis/length must fit a short and the length must be at most 64. For vendor records, the i16 length must be at most 64. If both still fit, the layout whose declared lengths match the decoded text wins. Vendor stays the default when both readings are equally plausible, and unambiguous lengths are unchanged. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Wire/FocasWireClient.cs | 77 ++++++++++++++- .../FocasAlarmLayoutTests.cs | 96 +++++++++++++++++++ 2 files changed, 171 insertions(+), 2 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmLayoutTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireClient.cs index 53a23d2c..fd76bf66 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/Wire/FocasWireClient.cs @@ -1137,15 +1137,88 @@ public sealed class FocasWireClient : IAsyncDisposable, IDisposable /// The requested record count. /// The decoded alarms. /// The payload length matches neither record layout. + /// + /// Choosing the layout. The response block carries no record count, so a length that is a + /// multiple of BOTH strides (1,520 B = 20 × 76 = 19 × 80, and its multiples) is ambiguous; picking the + /// vendor stride on Length % 76 == 0 alone decoded 19 legacy records as 20 garbage ones. The choice + /// now uses what IS known: (1) the record count the REQUEST asked for — a reply never holds more, so a + /// layout implying more records than is ruled out; (2) each layout's own field + /// invariants, checked on every record ( / ). + /// Only when both readings remain plausible does the vendor layout win, as before. + /// internal static IReadOnlyList ParseAlarms(byte[] payload, short count) { - if (payload.Length % 76 == 0) return ParseVendorAlarms(payload, count); - if (payload.Length % 80 == 0) return ParseLegacyWireAlarms(payload, count); + if (payload.Length == 0) return []; + var vendor = payload.Length % VendorAlarmStride == 0 && payload.Length / VendorAlarmStride <= Math.Max((int)count, 1); + var legacy = payload.Length % LegacyAlarmStride == 0 && payload.Length / LegacyAlarmStride <= Math.Max((int)count, 1); + if (vendor && legacy) + { + // Ambiguous length: a legacy record's int32 type/axis/length fields must each fit the short they are + // read into and its length must fit the 64-byte message — a vendor payload read at the legacy stride + // fails that on its first non-empty message. Prefer the reading whose every record is self-consistent. + var legacyOk = LegacyRecordsValid(payload); + var vendorOk = VendorRecordsValid(payload); + if (legacyOk && !vendorOk) vendor = false; + else if (legacyOk && vendorOk && LengthsMatchText(payload, LegacyAlarmStride, 12, 4, 16) + && !LengthsMatchText(payload, VendorAlarmStride, 10, 2, 12)) vendor = false; + } + if (vendor) return ParseVendorAlarms(payload, count); + if (legacy) return ParseLegacyWireAlarms(payload, count); + if (payload.Length % VendorAlarmStride == 0 || payload.Length % LegacyAlarmStride == 0) + { + // Fits a stride but holds more records than were requested: fall back to the stride it fits, capped + // at the count, exactly as before this check existed. + return payload.Length % VendorAlarmStride == 0 + ? ParseVendorAlarms(payload, count) + : ParseLegacyWireAlarms(payload, count); + } throw new FocasWireException( $"cnc_rdalmmsg2 returned a {payload.Length}-byte payload, which fits neither the 76- nor the 80-byte " + "record layout; refusing to read it as an empty alarm list.", rc: null); } + private const int VendorAlarmStride = 76; + private const int LegacyAlarmStride = 80; + private const int AlarmMessageBytes = 64; + + /// Every 80-byte record's int32 type / axis / length fit a short, and the length fits the message. + private static bool LegacyRecordsValid(byte[] payload) + { + for (var o = 0; o + LegacyAlarmStride <= payload.Length; o += LegacyAlarmStride) + { + var type = ReadInt32(payload, o + 4); + var axis = ReadInt32(payload, o + 8); + var length = ReadInt32(payload, o + 12); + if (type is < short.MinValue or > short.MaxValue) return false; + if (axis is < short.MinValue or > short.MaxValue) return false; + if (length is < 0 or > AlarmMessageBytes) return false; + } + return true; + } + + /// Every 76-byte record's i16 message length fits the 64-byte message. + private static bool VendorRecordsValid(byte[] payload) + { + for (var o = 0; o + VendorAlarmStride <= payload.Length; o += VendorAlarmStride) + { + var length = ReadInt16(payload, o + 10); + if (length is < 0 or > AlarmMessageBytes) return false; + } + return true; + } + + /// Every record's declared message length equals the length of the text actually in its message field. + private static bool LengthsMatchText(byte[] payload, int stride, int lengthOffset, int lengthWidth, int messageOffset) + { + for (var o = 0; o + stride <= payload.Length; o += stride) + { + var length = lengthWidth == 4 ? ReadInt32(payload, o + lengthOffset) : ReadInt16(payload, o + lengthOffset); + var text = FocasWireProtocol.ReadAscii(payload.AsSpan(o + messageOffset, AlarmMessageBytes)); + if (length != text.Length) return false; + } + return true; + } + private static IReadOnlyList ParseVendorAlarms(byte[] payload, short count) { var alarms = new List(); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmLayoutTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmLayoutTests.cs new file mode 100644 index 00000000..bc1bc1b2 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmLayoutTests.cs @@ -0,0 +1,96 @@ +using System.Buffers.Binary; +using System.Text; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; + +/// +/// #740 / R07:H20 — ParseAlarms picked the 76-byte vendor layout whenever Length % 76 == 0, so a +/// legacy 80-byte reply of exactly 19 records (1,520 B = 20 × 76) decoded as 20 garbage vendor records. The +/// response block carries no record count, so the layout is now chosen from the two things that are known: the +/// count the REQUEST asked for (a reply never holds more) and each layout's own field invariants. +/// +[Trait("Category", "Unit")] +public sealed class FocasAlarmLayoutTests +{ + private static byte[] Legacy(params (int no, int type, int axis, string msg)[] records) + { + var buf = new byte[records.Length * 80]; + for (var i = 0; i < records.Length; i++) + { + var o = i * 80; + var (no, type, axis, msg) = records[i]; + BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o, 4), no); + BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o + 4, 4), type); + BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o + 8, 4), axis); + BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o + 12, 4), msg.Length); + Encoding.ASCII.GetBytes(msg).CopyTo(buf.AsSpan(o + 16)); + } + return buf; + } + + private static byte[] Vendor(params (int no, short type, short axis, string msg)[] records) + { + var buf = new byte[records.Length * 76]; + for (var i = 0; i < records.Length; i++) + { + var o = i * 76; + var (no, type, axis, msg) = records[i]; + BinaryPrimitives.WriteInt32BigEndian(buf.AsSpan(o, 4), no); + BinaryPrimitives.WriteInt16BigEndian(buf.AsSpan(o + 4, 2), type); + BinaryPrimitives.WriteInt16BigEndian(buf.AsSpan(o + 6, 2), axis); + BinaryPrimitives.WriteInt16BigEndian(buf.AsSpan(o + 10, 2), (short)msg.Length); + Encoding.ASCII.GetBytes(msg).CopyTo(buf.AsSpan(o + 12)); + } + return buf; + } + + private static (int, int, int, string)[] LegacySet(int n) => + [.. Enumerable.Range(0, n).Select(i => (400 + i, 2 + (i % 3), 1 + (i % 4), $"SERVO ALARM {400 + i}"))]; + + private static (int, short, short, string)[] VendorSet(int n) => + [.. Enumerable.Range(0, n).Select(i => (1000 + i, (short)(4 + (i % 3)), (short)(i % 2), $"OVERTRAVEL +{i}"))]; + + /// The ambiguous length: 19 legacy records decode as 19 legacy records, not 20 vendor ones. + [Fact] + public void Nineteen_legacy_records_decode_as_legacy() + { + var payload = Legacy(LegacySet(19)); + payload.Length.ShouldBe(1520); + (payload.Length % 76).ShouldBe(0, "the length really is ambiguous"); + + var alarms = FocasWireClient.ParseAlarms(payload, 32); + + alarms.Count.ShouldBe(19); + alarms[0].AlarmNumber.ShouldBe(400); + alarms[0].Message.ShouldBe("SERVO ALARM 400"); + alarms[18].AlarmNumber.ShouldBe(418); + alarms[18].Axis.ShouldBe((short)(1 + (18 % 4))); + } + + /// Positive control at the same length: 20 vendor records still decode as vendor. + [Fact] + public void Twenty_vendor_records_decode_as_vendor() + { + var payload = Vendor(VendorSet(20)); + payload.Length.ShouldBe(1520); + + var alarms = FocasWireClient.ParseAlarms(payload, 32); + + alarms.Count.ShouldBe(20); + alarms[0].AlarmNumber.ShouldBe(1000); + alarms[19].Message.ShouldBe("OVERTRAVEL +19"); + } + + /// The unambiguous lengths are unchanged. + [Fact] + public void Unambiguous_lengths_are_unchanged() + { + FocasWireClient.ParseAlarms(Vendor(VendorSet(3)), 32).Count.ShouldBe(3); + FocasWireClient.ParseAlarms(Legacy(LegacySet(3)), 32).Count.ShouldBe(3); + FocasWireClient.ParseAlarms([], 32).ShouldBeEmpty(); + Should.Throw(() => FocasWireClient.ParseAlarms(new byte[77], 32)); + } +} From cbacaee4ee3289e3b830997570cf4b4d374417eb Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:24:10 -0400 Subject: [PATCH 41/51] fix(twincat): bounded, race-safe lifecycle; drop counter; post-unsubscribe purge (L8) - Device and tag tables are build-and-swap: InitializeAsync builds a fresh generation and publishes it by one swap under a lifecycle lock, then releases the replaced generation; teardown swaps in empty tables. A retained poll loop never sees a table mid-Clear (it used to read _tagsByRawPath while a re-init only ever added to it), and each device set is released by exactly one caller. - A lifecycle generation makes an InitializeAsync overtaken by a shutdown (or a newer init) release what it built instead of installing it - the Lane A abandoned-init-overlaps-shutdown case. - ShutdownAsync joins all probe loops together within 2 s (it awaited each unbounded - a probe wedged in the blocking ADS Connect hung the actor's teardown) in parallel with the poll-engine dispose, the whole bounded at 4.5 s; DisposeAsync may run concurrently and returns at once. A connect that completes on a released device disposes its client. - Releasing a generation disposes its native registrations, so a direct re-init no longer leaves live notifications bound to dead DeviceStates. - Connect gates and RMW semaphores are no longer disposed under in-flight users (no kernel handle is ever allocated); a fault on a released device is not recorded as driver health. - The notification channel counts DropWrite overflows (ITwinCATClient.DroppedNotificationCount); the probe tick warns with the delta. A queued value whose registration was disposed is purged in the client dispatch loop and dropped driver-side for any client. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdsTwinCATClient.cs | 17 +- .../ITwinCATClient.cs | 6 + .../TwinCATDriver.cs | 359 +++++++++++++----- .../FakeTwinCATClient.cs | 12 + .../TwinCATLifecycleTests.cs | 271 +++++++++++++ 5 files changed, 561 insertions(+), 104 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs index 1781bedf..a779b09b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs @@ -721,15 +721,24 @@ internal sealed class AdsTwinCATClient : ITwinCATClient value = DecodeString(reg.Type, raw, reg.StringsUtf8); else if (TwinCATTemporalCodec.IsTemporal(reg.Type) && value is uint rawTemporal) value = TwinCATTemporalCodec.Decode(reg.Type, rawTemporal); - _notificationQueue.Writer.TryWrite(new PendingNotification(reg, value)); + if (!_notificationQueue.Writer.TryWrite(new PendingNotification(reg, value))) + Interlocked.Increment(ref _droppedNotifications); // queue full (DropWrite): count, never block } + private long _droppedNotifications; + + /// + public long DroppedNotificationCount => Interlocked.Read(ref _droppedNotifications); + private async Task DispatchLoopAsync(CancellationToken ct) { try { await foreach (var pending in _notificationQueue.Reader.ReadAllAsync(ct).ConfigureAwait(false)) { + // A value queued before its registration was deleted is purged here rather than delivered + // after the unsubscribe. + if (pending.Registration.IsDeleted) continue; try { pending.Registration.OnChange(pending.Registration.SymbolPath, pending.Value); } catch { /* consumer-side errors stay on this managed task, not the ADS thread */ } } @@ -980,9 +989,15 @@ internal sealed class AdsTwinCATClient : ITwinCATClient /// public void Dispose() { + IsDeleted = true; // Fire-and-forget AMS call — caller has already committed to the tear-down. _ = owner.DeleteNotificationAsync(handle, CancellationToken.None); } + + /// Gets a value indicating whether the registration was disposed; queued values for it are dropped. + public bool IsDeleted { get => Volatile.Read(ref _deleted); private set => Volatile.Write(ref _deleted, value); } + + private bool _deleted; } private static Type MapToClrType(TwinCATDataType type) => type switch diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs index b44f3038..cb552f0c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/ITwinCATClient.cs @@ -102,6 +102,12 @@ public interface ITwinCATClient : IDisposable /// The overall ADS code of the most recent such fallback (0 = a malformed response). uint LastSumFallbackCode => 0; + /// + /// Native notifications this client dropped because its bounded dispatch queue was full (the router + /// thread never blocks on a slow consumer). The driver warns when it grows. Default 0. + /// + long DroppedNotificationCount => 0; + /// /// Write a symbolic value. Returns the mapped OPC UA status for the operation /// (0 = Good, non-zero = error mapped via ). diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index 30d9f488..8542bcad 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -22,30 +22,60 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery public SkippedTagReport? GetSkippedTags() => _skippedTags.Snapshot(); // Mutable so ReinitializeAsync can apply a new config generation. The constructor seeds - // it; InitializeAsync re-parses driverConfigJson over the top of it. - private TwinCATDriverOptions _options; + // it; InitializeAsync re-parses driverConfigJson over the top of it (committed under _lifecycleLock). + private volatile TwinCATDriverOptions _options; private readonly string _driverInstanceId; private readonly ITwinCATClientFactory _clientFactory; private readonly ILogger _logger; private readonly PollGroupEngine _poll; - // ConcurrentDictionary so ShutdownAsync (Clear) and ReadAsync/WriteAsync/SubscribeAsync - // (TryGetValue) don't race — plain Dictionary is not safe for concurrent read+write. - private readonly ConcurrentDictionary _devices = - new(StringComparer.OrdinalIgnoreCase); + + // Device + tag tables are BUILD-AND-SWAP (#739 L8): InitializeAsync builds fresh tables and publishes + // them by reference under _lifecycleLock; ShutdownAsync / Dispose swap in empty ones and release what + // they took. A retained poll loop, the probe tick or an in-flight write therefore always sees one whole + // generation — never a table mid-Clear — and two overlapping teardowns can never release the same + // device twice (each device set is taken by exactly one swap). + private volatile ConcurrentDictionary _devices = NewDeviceTable(); // v3: the authored RawPath → definition table. Keyed case-sensitive ordinal (the RawPath is // the exact wire reference). Built in InitializeAsync from _options.RawTags via the pure mapper. - private readonly ConcurrentDictionary _tagsByRawPath = - new(StringComparer.Ordinal); + private volatile IReadOnlyDictionary _tagsByRawPath = EmptyTags; + + private static readonly IReadOnlyDictionary EmptyTags = + new Dictionary(StringComparer.Ordinal); + + private static ConcurrentDictionary NewDeviceTable() => new(StringComparer.OrdinalIgnoreCase); + + // Serialises the table swaps against each other. _lifecycleGeneration is bumped by every + // InitializeAsync and every teardown, so an InitializeAsync that was overtaken while it awaited + // (an abandoned init overlapping the runtime's next ShutdownAsync, or a newer init) sees a + // different generation at its commit and releases what it built instead of installing it. + private readonly object _lifecycleLock = new(); + private long _lifecycleGeneration; + + /// Test seam: awaited after the new generation is built and before it is committed, so a test can + /// interleave a teardown deterministically. Never set in production. + internal Func? BeforeCommitForTest { get; set; } + + /// Upper bound on joining released devices' probe loops (one wait for all of them). + internal static readonly TimeSpan ProbeJoinTimeout = TimeSpan.FromSeconds(2); + + /// + /// Upper bound on the whole of . The runtime gives a final stop 5 s, so + /// the driver stops waiting a little earlier; anything still running past it (a poll loop wedged in + /// a read) is cancelled already and finishes in the background. + /// + internal static readonly TimeSpan ShutdownBudget = TimeSpan.FromSeconds(4.5); + + // References that already logged their bad-read Warning in the current bad episode (#739 P10). + // Removed when the tag reads Good; cleared on (re-)initialise and shutdown. + private readonly ConcurrentDictionary _warnedRefs = new(StringComparer.Ordinal); // Per-parent-word RMW gate for BOOL-within-word writes: a single-bit write is a // read-modify-write of the parent word (TwinCAT's symbol table doesn't expose "Word.N"), // so concurrent bit-writers to the same word must serialise or they lose each other's // updates. Keyed by device + parent symbol. (Cannot guard against the PLC program writing - // the word between our read and write — inherent to RMW.) - // References that already logged their bad-read Warning in the current bad episode (#739 P10). - // Removed when the tag reads Good; cleared on (re-)initialise and shutdown. - private readonly ConcurrentDictionary _warnedRefs = new(StringComparer.Ordinal); - + // the word between our read and write — inherent to RMW.) Teardown only CLEARS it: a + // SemaphoreSlim owns no kernel handle unless AvailableWaitHandle is read, and disposing one an + // in-flight write still holds turns its Release into an ObjectDisposedException. private readonly ConcurrentDictionary _bitRmwLocks = new(StringComparer.OrdinalIgnoreCase); @@ -118,29 +148,40 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { // InitializeAsync is reachable on a live driver with no teardown first (the IDriver contract - // allows it, and a reconnect retry can overlap a running attempt). Release the previous device states' probe loops + clients before - // the overwrite below. Deliberately NOT ShutdownAsync: that also drops _poll and the subscriptions. - await ReleaseDeviceStatesAsync().ConfigureAwait(false); + // allows it, and a reconnect retry can overlap a running attempt). The new generation is BUILT + // first and published by one swap; the previous device states (probe loops, clients, native + // registrations) are released right after the swap — never orphaned, and a retained poll loop + // never sees an empty table in between. Deliberately NOT ShutdownAsync: that also drops _poll and + // the subscriptions. + long generation; + lock (_lifecycleLock) + { + generation = ++_lifecycleGeneration; + _health = new DriverHealth(DriverState.Initializing, null, null); + } _warnedRefs.Clear(); - _health = new DriverHealth(DriverState.Initializing, null, null); + + var devices = NewDeviceTable(); + var committed = false; try { // Apply the supplied config generation. A blank or content-free document keeps the // constructor-seeded options — that path covers callers that have already // materialised options up front (the factory passes both, in agreement). + var options = _options; if (!string.IsNullOrWhiteSpace(driverConfigJson)) { var parsed = TwinCATDriverFactoryExtensions.ParseOptions(driverConfigJson, _driverInstanceId); if (parsed.Devices.Count > 0 || parsed.RawTags.Count > 0) - _options = parsed; + options = parsed; } - foreach (var device in _options.Devices) + foreach (var device in options.Devices) { var addr = TwinCATAmsAddress.TryParse(device.HostAddress) ?? throw new InvalidOperationException( $"TwinCAT device has invalid HostAddress '{device.HostAddress}' — expected 'ads://{{netId}}:{{port}}'."); - _devices[device.HostAddress] = new DeviceState(addr, device); + devices[device.HostAddress] = new DeviceState(addr, device); } // Build the RawPath → definition table from the authored raw tags. Each entry's TagConfig @@ -149,14 +190,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // DeviceName is resolved to the target's live host (def.DeviceHostAddress) via ResolveDeviceHost // so the existing device routing (_devices keyed by HostAddress) is unchanged. A mapper miss is // skipped (logged), never thrown — a bad tag must not fail the whole driver init. - _skippedTags.BeginRebuild(_options.RawTags.Count); - foreach (var entry in _options.RawTags) + var tags = new Dictionary(StringComparer.Ordinal); + _skippedTags.BeginRebuild(options.RawTags.Count); + foreach (var entry in options.RawTags) { if (TwinCATTagDefinitionFactory.FromTagConfig(entry.TagConfig, entry.RawPath, out var def)) - _tagsByRawPath[entry.RawPath] = def with + tags[entry.RawPath] = def with { WriteIdempotent = entry.WriteIdempotent, - DeviceHostAddress = ResolveDeviceHost(entry.DeviceName), + DeviceHostAddress = ResolveDeviceHost(options, entry.DeviceName), }; else { @@ -168,47 +210,113 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } _skippedTags.Commit(); - if (_options.Probe.Enabled) + if (BeforeCommitForTest is { } pause) await pause().ConfigureAwait(false); + + ConcurrentDictionary previous; + lock (_lifecycleLock) { - foreach (var state in _devices.Values) + if (generation != _lifecycleGeneration) + throw new OperationCanceledException( + "TwinCAT InitializeAsync was superseded by a shutdown or a newer initialise while it ran."); + + _options = options; + _tagsByRawPath = tags; + previous = _devices; + _devices = devices; + committed = true; + // Probe loops start INSIDE the commit so a teardown that swaps this generation out right + // after always finds (and cancels) them. + if (options.Probe.Enabled) { - state.ProbeCts = new CancellationTokenSource(); - var ct = state.ProbeCts.Token; - // Store the task so ShutdownAsync can await it. - state.ProbeTask = Task.Run(() => ProbeLoopAsync(state, ct), ct); + foreach (var state in devices.Values) + { + state.ProbeCts = new CancellationTokenSource(); + var ct = state.ProbeCts.Token; + // Store the task so the teardown can join it. + state.ProbeTask = Task.Run(() => ProbeLoopAsync(state, ct), ct); + } + } + _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null); + } + + // The replaced generation is now unreachable from the tables: release it (bounded). + await ReleaseDevicesAsync(previous.Values).ConfigureAwait(false); + } + catch (Exception ex) when (!committed) + { + // Nothing of this generation was published: release what was built. A superseded init leaves + // health to whoever overtook it. + await ReleaseDevicesAsync(devices.Values).ConfigureAwait(false); + if (ex is not OperationCanceledException) + { + lock (_lifecycleLock) + { + if (generation == _lifecycleGeneration) + _health = new DriverHealth(DriverState.Faulted, null, ex.Message); } } - _health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null); - } - catch (Exception ex) - { - _health = new DriverHealth(DriverState.Faulted, null, ex.Message); throw; } } /// - /// Cancels, awaits and disposes every existing device state's probe loop, client and gate, then - /// clears the device table. Device part of only. + /// Releases a set of device states that have already been swapped out of : + /// marks them released (so a probe wedged in the blocking ADS connect disposes the client it + /// eventually gets), disposes their native registrations (a re-initialise must not leave live ADS + /// notifications bound to a dead generation), cancels every probe loop and joins them all together + /// within , then disposes their clients. The connect gates are NOT + /// disposed — an in-flight read/write may still hold one. /// - private async Task ReleaseDeviceStatesAsync() + /// The released device states (each owned by exactly one caller). + private async Task ReleaseDevicesAsync(ICollection states) { - if (_devices.IsEmpty) return; - foreach (var state in _devices.Values) + if (states.Count == 0) return; + var probes = BeginRelease(states); + if (probes.Count > 0) { - try { state.ProbeCts?.Cancel(); } catch { } - if (state.ProbeTask is Task pt) - { - try { await pt.WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false); } - catch { /* cancelled, faulted or wedged — teardown continues */ } - } - state.ProbeCts?.Dispose(); + try { await Task.WhenAll(probes).WaitAsync(ProbeJoinTimeout).ConfigureAwait(false); } + catch { /* cancelled, faulted or wedged in the synchronous ADS connect — teardown continues */ } + } + EndRelease(states); + } + + /// The non-blocking first half of a release: mark, retract registrations, cancel probes. + private static List BeginRelease(ICollection states) + { + var probes = new List(states.Count); + foreach (var state in states) + { + state.Released = true; + foreach (var reg in state.NativeRegistrations.Values) { try { reg.Dispose(); } catch { } } + try { state.ProbeCts?.Cancel(); } catch { /* already disposed */ } + if (state.ProbeTask is Task pt) probes.Add(pt); + } + return probes; + } + + /// The second half of a release, after the bounded probe join: dispose CTSs and clients. + private static void EndRelease(ICollection states) + { + foreach (var state in states) + { + try { state.ProbeCts?.Dispose(); } catch { } state.ProbeCts = null; state.ProbeTask = null; - state.DisposeClient(); - state.DisposeGate(); + try { state.DisposeClient(); } catch { /* best-effort */ } + } + } + + /// Swaps out every table under the lifecycle lock and returns the device set the caller now owns. + private ICollection TakeAllForTeardown() + { + lock (_lifecycleLock) + { + _lifecycleGeneration++; + var taken = _devices; + _devices = NewDeviceTable(); + _tagsByRawPath = EmptyTags; + return taken.Values; } - _devices.Clear(); } /// @@ -219,6 +327,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } /// + /// + /// Safe to run concurrently with itself, with / and with + /// an abandoned : the tables are taken by one atomic swap, so each device + /// is released exactly once and an overtaken init releases its own generation. Bounded by + /// . + /// public async Task ShutdownAsync(CancellationToken cancellationToken) { // Native subs first — disposing the handles is cheap + lets the client close its @@ -227,30 +341,22 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery foreach (var r in sub.Registrations) { try { r.Dispose(); } catch { } } _nativeSubs.Clear(); - await _poll.DisposeAsync().ConfigureAwait(false); + var taken = TakeAllForTeardown(); - // Cancel every probe loop and await its task before disposing the client + gate so the - // loop can never touch a disposed object. - foreach (var state in _devices.Values) + // The poll loops and the device release wind down in parallel (releasing a client can unblock a + // poll loop stuck in a read on it); the whole teardown is bounded. + var teardown = Task.WhenAll(_poll.DisposeAsync().AsTask(), ReleaseDevicesAsync(taken)); + try { await teardown.WaitAsync(ShutdownBudget).ConfigureAwait(false); } + catch (TimeoutException) { - try { state.ProbeCts?.Cancel(); } catch { } - if (state.ProbeTask is Task pt) - { - try { await pt.ConfigureAwait(false); } - catch (OperationCanceledException) { /* expected — probe loop exits on cancel */ } - catch { /* other probe errors are not fatal to shutdown */ } - } - state.ProbeCts?.Dispose(); - state.ProbeCts = null; - state.DisposeClient(); - state.DisposeGate(); + _logger.LogWarning( + "TwinCAT driver '{DriverInstanceId}' shutdown exceeded {Budget}; the remaining teardown continues in the background", + _driverInstanceId, ShutdownBudget); } - _devices.Clear(); - _tagsByRawPath.Clear(); + catch { /* best-effort teardown */ } + + _warnedRefs.Clear(); _resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry - // Dispose + clear the per-parent-word RMW gates so ReinitializeAsync cycles don't orphan - // their SemaphoreSlim instances (each leaks a wait handle once contended). - foreach (var sem in _bitRmwLocks.Values) sem.Dispose(); _bitRmwLocks.Clear(); _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null); } @@ -340,7 +446,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery catch (OperationCanceledException) { throw; } catch (Exception ex) { - MarkCommsFault(ex.Message); + MarkCommsFault(ex.Message, device); return new DataValueSnapshot(null, TwinCATStatusMapper.BadCommunicationError, null, now); } } @@ -463,7 +569,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { foreach (var e in chunk) results[e.Index] = new DataValueSnapshot(null, TwinCATStatusMapper.BadCommunicationError, null, now); - MarkCommsFault(ex.Message); + MarkCommsFault(ex.Message, device); // Once per device: the health message is overwritten by the next good read in the same poll, so // without this a sum-read path failing on every poll left no trace (hw2 live gate, 2026-10-08). if (connected && !device.SumChunkFailureWarned) @@ -575,7 +681,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery catch (Exception ex) { results[i] = new WriteResult(TwinCATStatusMapper.BadCommunicationError); - MarkCommsFault(ex.Message); + MarkCommsFault(ex.Message, device); } } @@ -771,7 +877,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // The device could not be connected (offline, router down, backoff window open). // Every tag of this device goes Bad + dead for the retry; other devices carry on. (unreachable ??= []).Add(device); - MarkCommsFault(unavailable.InnerException?.Message ?? unavailable.Message); + MarkCommsFault(unavailable.InnerException?.Message ?? unavailable.Message, device); RaiseDataChange(handle, reference, new DataValueSnapshot( null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)); } @@ -783,7 +889,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery "TwinCAT driver '{DriverInstanceId}' native-notification registration failed for " + "'{Reference}'; it will be retried on the next healthy probe tick", _driverInstanceId, reference); - MarkCommsFault(ex.Message); + MarkCommsFault(ex.Message, device); RaiseDataChange(handle, reference, new DataValueSnapshot( null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)); } @@ -801,6 +907,14 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery return handle; } + /// + /// The device state was released (re-initialise / teardown) while a connect on it ran. Not an + /// : the caller's token was not cancelled, so the data path maps + /// it like any failed request (Bad for this call) and ignores it. + /// + private sealed class DeviceReleasedException(string host) + : Exception($"TwinCAT device {host} was released during connect."); + /// The connect step of a native registration failed — the whole device is unreachable. private sealed class DeviceUnavailableException(Exception inner) : Exception("TwinCAT device unreachable: " + inner.Message, inner); @@ -809,8 +923,13 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// Records a comms fault: preserving LastSuccessfulRead. /// Never downgrades . /// - private void MarkCommsFault(string message) + /// The fault description. + /// The device the fault was observed on; a fault on a device already swapped out of + /// the table (a read/write racing a re-initialise or teardown) belongs to a dead generation and is + /// not recorded. + private void MarkCommsFault(string message, DeviceState? device = null) { + if (device?.Released == true) return; var current = _health; if (current.State != DriverState.Faulted) _health = new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, message); @@ -1006,8 +1125,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery BitIndex = bitIndex; ArrayLength = arrayLength; Interval = interval; - OnChange = onChange; - OnUnavailableStatus = onUnavailable; + // A notification already queued in the client's dispatch channel can be delivered after this + // registration is disposed (unsubscribe / teardown): drop it here, for every client. + OnChange = (symbol, value) => { if (!IsDisposed) onChange(symbol, value); }; + OnUnavailableStatus = status => { if (!IsDisposed) onUnavailable(status); }; } /// Installs a freshly-registered handle and returns the previous one (null on first set). @@ -1025,9 +1146,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery public ITwinCATNotificationHandle? MarkHandleDead() => Interlocked.Exchange(ref _handle, null); + private volatile bool _disposed; + + /// Gets a value indicating whether this registration has been disposed (its callbacks are dropped). + public bool IsDisposed => _disposed; + /// Removes this registration from its device and disposes its live handle. public void Dispose() { + _disposed = true; Device.NativeRegistrations.TryRemove(Id, out _); var h = Interlocked.Exchange(ref _handle, null); try { h?.Dispose(); } catch { /* best-effort */ } @@ -1085,6 +1212,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { state.ConsecutiveProbeFailures = 0; RecordDeviceAnswered(DateTime.UtcNow, force: true); + if (probed is not null) ReportNotificationDrops(state, probed); } else if (++state.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded) { @@ -1107,6 +1235,24 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } } + /// + /// Logs a Warning when the client's native-notification queue dropped values since the last probe + /// tick (the queue is bounded and drops on overflow — silently, before #739 L8). At most one + /// Warning per probe interval, carrying the delta and the total. + /// + private void ReportNotificationDrops(DeviceState state, ITwinCATClient client) + { + var total = client.DroppedNotificationCount; + var delta = total - state.ReportedNotificationDrops; + if (delta <= 0) return; + state.ReportedNotificationDrops = total; + _logger.LogWarning( + "TwinCAT driver '{DriverInstanceId}' device '{HostAddress}' dropped {Dropped} native notification(s) " + + "since the last probe ({Total} on this connection): the notification queue was full — the consumer " + + "is not keeping up with the PLC's change rate", + _driverInstanceId, state.Options.HostAddress, delta, total); + } + // ---- IDriverDiagnostics ---- // Well-known system symbols, exported by TC3 (and most TC2) PLC runtimes under TwinCAT_SystemInfoVarList. @@ -1296,11 +1442,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// CLI, pass the host string as the DeviceName). An empty DeviceName routes to the sole device /// on a single-device instance; an unmatched name leaves the def unrouted (→ BadNodeIdUnknown). /// + /// The options generation being initialised. /// The owning device's name (or host) from the raw-tag entry. /// The resolved device host address, or empty when unresolved. - private string ResolveDeviceHost(string deviceName) + private static string ResolveDeviceHost(TwinCATDriverOptions options, string deviceName) { - var devices = _options.Devices; + var devices = options.Devices; if (string.IsNullOrEmpty(deviceName)) return devices.Count == 1 ? devices[0].HostAddress : ""; foreach (var d in devices) @@ -1372,6 +1519,8 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { // Re-check under the gate: another caller may have connected while we waited. if (device.Client is { IsConnected: true } c) return c; + // A released device (swapped out by a re-initialise / teardown) never connects again. + if (device.Released) throw new DeviceReleasedException(device.Options.HostAddress); // Inside an open backoff window a data-path caller fails fast — no client // create, no wire connect. The probe loop bypasses this (its interval is the recovery @@ -1412,7 +1561,16 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery device.Backoff.RecordFailure(DateTime.UtcNow); // open the backoff window throw; } + if (device.Released) + { + // This device was torn down while the (blocking) connect ran: never install a client on a + // released generation, where nothing would ever dispose it. + client.OnSymbolVersionChanged -= HandleSymbolVersionChanged; + try { client.Dispose(); } catch { /* best-effort */ } + throw new DeviceReleasedException(device.Options.HostAddress); + } device.Client = client; + device.ReportedNotificationDrops = 0; device.UnavailableDiagSymbols.Clear(); // a rebuilt client may face a PLC that now exports them device.Backoff.RecordSuccess(); // recovery resets the window immediately @@ -1608,29 +1766,17 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery foreach (var r in sub.Registrations) { try { r.Dispose(); } catch { } } _nativeSubs.Clear(); - // PollGroupEngine.DisposeAsync awaits loop tasks; we drive that synchronously here - // (bounded wait — same 5s ceiling DisposeAsync uses internally) using Wait() on the - // returned ValueTask so no sync-context capture happens. - try { _poll.DisposeAsync().AsTask().Wait(TimeSpan.FromSeconds(5)); } catch { } + var taken = TakeAllForTeardown(); + var probes = BeginRelease(taken); - foreach (var state in _devices.Values) - { - try { state.ProbeCts?.Cancel(); } catch { } - if (state.ProbeTask is Task pt) - { - try { pt.Wait(TimeSpan.FromSeconds(2)); } catch { /* probe-cancel races are expected */ } - } - state.ProbeCts?.Dispose(); - state.ProbeCts = null; - state.DisposeClient(); - state.DisposeGate(); - } - _devices.Clear(); - _tagsByRawPath.Clear(); + // PollGroupEngine.DisposeAsync awaits loop tasks; drive it synchronously with a bounded Wait() on + // the returned task (no sync-context capture), in parallel with the probe join. + var poll = _poll.DisposeAsync().AsTask(); + try { Task.WhenAll([poll, .. probes]).Wait(ProbeJoinTimeout); } catch { /* cancel races are expected */ } + EndRelease(taken); + + _warnedRefs.Clear(); _resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry - // Dispose + clear the per-parent-word RMW gates (mirrors ShutdownAsync) so the - // SemaphoreSlim instances aren't orphaned on disposal. - foreach (var sem in _bitRmwLocks.Values) sem.Dispose(); _bitRmwLocks.Clear(); _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null); } @@ -1711,7 +1857,14 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery UnavailableDiagSymbols.Clear(); } - /// Disposes the connection gate semaphore. - public void DisposeGate() => ConnectGate.Dispose(); + /// + /// Set once this device state has been swapped out of the driver's table. A connect that + /// completes afterwards (a probe wedged in the blocking ADS Connect past the bounded join) + /// disposes its client instead of installing it on a dead generation. + /// + public volatile bool Released; + + /// Total notification drops last reported for this device's current client. + public long ReportedNotificationDrops; } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs index 39f795be..909fc31f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs @@ -56,10 +56,22 @@ internal class FakeTwinCATClient : ITwinCATClient { ConnectCount++; if (ThrowOnConnect) throw Exception ?? new InvalidOperationException(); + if (BlockConnect is { } block) + { + // Models the real client's blocking SDK Connect: ignores the token until released. + return block.Task.ContinueWith(_ => { IsConnected = true; }, TaskScheduler.Default); + } IsConnected = true; return Task.CompletedTask; } + /// When set, does not complete (and ignores cancellation) until this + /// is completed — a probe wedged in the synchronous ADS Connect. + public TaskCompletionSource? BlockConnect { get; set; } + + /// Gets or sets the simulated notification-queue drop count. + public long DroppedNotificationCount { get; set; } + /// Simulates reading a value from a symbol. /// The path to the symbol to read. /// The data type of the symbol. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs new file mode 100644 index 00000000..c419289b --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs @@ -0,0 +1,271 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// #739 L8 + the Lane A runtime contract (final stop = ShutdownAsync then DisposeAsync, +/// bounded 5 s + 250 ms): a probe wedged in the blocking ADS connect cannot hang teardown; dispose may run +/// while a shutdown is still running; an abandoned initialise overlapping the next shutdown never installs +/// its generation; a value queued before unsubscribe is not delivered after it; re-init swaps its tables +/// and retracts the previous generation's notifications; queue drops are reported. +/// +[Trait("Category", "Unit")] +public sealed class TwinCATLifecycleTests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + + private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, bool probe = true, bool native = true, + TwinCATCapturingLogger? log = null, params TwinCATTagDefinition[] tags) => + new(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = TwinCATRawTags.Entries(tags.Length > 0 + ? tags + : [new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)]), + Probe = probe + ? new TwinCATProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(30), Timeout = TimeSpan.FromMilliseconds(100) } + : new TwinCATProbeOptions { Enabled = false }, + UseNativeNotifications = native, + }, "drv-1", factory, log); + + private static async Task WaitForAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(10); + } + + /// + /// A probe wedged in the (uncancellable) connect no longer hangs ShutdownAsync — the join is bounded — + /// and the client the wedged connect eventually produces is disposed, not installed on a dead device. + /// + [Fact] + public async Task Shutdown_is_bounded_when_a_probe_is_wedged_in_connect() + { + var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { BlockConnect = wedge } }; + var drv = NewDriver(factory); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); + factory.Clients.Count.ShouldBe(1); // the probe is inside the wedged connect + + var sw = Stopwatch.StartNew(); + await drv.ShutdownAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ShutdownBudget + TimeSpan.FromMilliseconds(500)); + + wedge.SetResult(); + await WaitForAsync(() => factory.Clients[0].DisposeCount == 1, TimeSpan.FromSeconds(2)); + factory.Clients[0].DisposeCount.ShouldBe(1); + drv.DeviceCount.ShouldBe(0); + } + + /// + /// DisposeAsync while a ShutdownAsync is still running (joining a wedged probe) returns promptly and + /// both complete without throwing; the device is released exactly once. + /// + [Fact] + public async Task Dispose_concurrent_with_a_running_shutdown_completes_promptly() + { + var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { BlockConnect = wedge } }; + var drv = NewDriver(factory); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); + + var shutdown = drv.ShutdownAsync(CancellationToken.None); + var sw = Stopwatch.StartNew(); + await drv.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromMilliseconds(250)); + shutdown.IsCompleted.ShouldBeFalse("the positive control: the first shutdown really was still joining the probe"); + + wedge.SetResult(); + await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + drv.Dispose(); // and a late sync Dispose is a no-op + drv.GetHealth().State.ShouldBe(DriverState.Unknown); + } + + /// + /// An InitializeAsync overtaken by a ShutdownAsync before it commits must not install its devices or + /// start probe loops when it resumes. Positive control: a fresh initialise afterwards installs normally. + /// + [Fact] + public async Task An_initialise_overtaken_by_shutdown_before_commit_installs_nothing() + { + var factory = new FakeTwinCATClientFactory(); + var drv = NewDriver(factory); + var paused = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var resume = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + drv.BeforeCommitForTest = async () => { paused.TrySetResult(); await resume.Task; }; + + var abandoned = drv.InitializeAsync("{}", CancellationToken.None); + await paused.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + resume.SetResult(); + + await Should.ThrowAsync(() => abandoned.WaitAsync(TimeSpan.FromSeconds(5))); + drv.DeviceCount.ShouldBe(0); + await Task.Delay(150, TestContext.Current.CancellationToken); // several probe intervals + factory.Clients.ShouldBeEmpty("no probe loop of the overtaken generation may be running"); + drv.GetHealth().State.ShouldBe(DriverState.Unknown); + + drv.BeforeCommitForTest = null; + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + drv.DeviceCount.ShouldBe(1); + await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); + factory.Clients.Count.ShouldBe(1); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + /// + /// The Lane A shape: an InitializeAsync the runtime abandoned while it is still releasing the previous + /// generation (joining a probe wedged in connect) overlaps the next ShutdownAsync. Shutdown stays + /// bounded and, once everything settles, no device, probe loop or live client survives. + /// + [Fact] + public async Task An_abandoned_initialise_overlapping_shutdown_leaves_nothing_running() + { + var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var first = true; + var factory = new FakeTwinCATClientFactory + { + Customise = () => + { + var c = new FakeTwinCATClient { BlockConnect = first ? wedge : null }; + first = false; + return c; + }, + }; + var drv = NewDriver(factory); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); + + var abandoned = drv.InitializeAsync("{}", CancellationToken.None); // joins the wedged generation-1 probe + await Task.Delay(50, TestContext.Current.CancellationToken); + var sw = Stopwatch.StartNew(); + await drv.ShutdownAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ShutdownBudget + TimeSpan.FromMilliseconds(500)); + + wedge.SetResult(); + try { await abandoned.WaitAsync(TimeSpan.FromSeconds(5)); } catch (OperationCanceledException) { } + + drv.DeviceCount.ShouldBe(0); + await WaitForAsync(() => factory.Clients.All(c => c.DisposeCount > 0 || !c.IsConnected), TimeSpan.FromSeconds(2)); + var settled = factory.Clients.Count; + await Task.Delay(150, TestContext.Current.CancellationToken); + factory.Clients.Count.ShouldBe(settled, "no probe loop may still be creating clients"); + factory.Clients.ShouldAllBe(c => c.DisposeCount > 0 || !c.IsConnected); + } + + /// + /// A value the client queued before the unsubscribe is dropped if dispatched after it. + /// Positive control: the same callback delivers while subscribed. + /// + [Fact] + public async Task A_notification_dispatched_after_unsubscribe_is_dropped() + { + var factory = new FakeTwinCATClientFactory(); + var drv = NewDriver(factory, probe: false); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + var handle = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + var callback = factory.Clients[0].Notifications.Single().OnChange; // what the client's queue holds + + callback("MAIN.Speed", 1); + events.Count.ShouldBe(1); + + await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken); + callback("MAIN.Speed", 2); + events.Count.ShouldBe(1); + } + + /// + /// A direct re-initialise with a changed config swaps the tag table (the removed tag is unknown, the + /// added one resolves) and retracts the previous generation's live notifications. + /// + [Fact] + public async Task Reinitialise_swaps_tables_and_retracts_the_previous_generations_notifications() + { + var factory = new FakeTwinCATClientFactory(); + var drv = NewDriver(factory, probe: false); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.SubscribeAsync(["Speed"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + factory.Clients[0].Notifications.Count.ShouldBe(1); + + var next = System.Text.Json.JsonSerializer.Serialize(new + { + devices = new[] { new { hostAddress = Host } }, + rawTags = new[] + { + new + { + rawPath = "Torque", + tagConfig = TwinCATTagDefinitionFactory.ToTagConfig( + new TwinCATTagDefinition("Torque", Host, "MAIN.Torque", TwinCATDataType.DInt)), + writeIdempotent = false, + deviceName = Host, + }, + }, + probe = new { enabled = false }, + }); + await drv.InitializeAsync(next, TestContext.Current.CancellationToken); + + factory.Clients[0].Notifications.ShouldBeEmpty(); + factory.Clients[0].DisposeCount.ShouldBe(1); + (await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken)).Single().StatusCode + .ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + (await drv.ReadAsync(["Torque"], TestContext.Current.CancellationToken)).Single().StatusCode + .ShouldBe(TwinCATStatusMapper.Good); + } + + /// Reads racing a stream of re-initialises never throw (no table is ever mid-Clear). + [Fact] + public async Task Reads_racing_reinitialise_never_throw() + { + var factory = new FakeTwinCATClientFactory(); + var drv = NewDriver(factory, probe: false); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + using var stop = new CancellationTokenSource(TimeSpan.FromMilliseconds(400)); + var good = 0; + var reader = Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + var r = await drv.ReadAsync(["Speed"], CancellationToken.None); + if (r[0].StatusCode == TwinCATStatusMapper.Good) Interlocked.Increment(ref good); + } + }, TestContext.Current.CancellationToken); + while (!stop.IsCancellationRequested) + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + await reader; // throws if any read threw + good.ShouldBeGreaterThan(0); + } + + /// Notification-queue drops are reported once per probe tick that sees new drops. + [Fact] + public async Task Notification_queue_drops_are_reported() + { + var fake = new FakeTwinCATClient(); + var log = new TwinCATCapturingLogger(); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, log: log); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + await WaitForAsync(() => fake.IsConnected, TimeSpan.FromSeconds(2)); + await Task.Delay(100, TestContext.Current.CancellationToken); + log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(0); // no drops, no warning + + fake.DroppedNotificationCount = 5; + await WaitForAsync(() => log.Warnings.Any(w => w.Contains("dropped 5")), TimeSpan.FromSeconds(2)); + await Task.Delay(100, TestContext.Current.CancellationToken); + log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(1); + + fake.DroppedNotificationCount = 8; + await WaitForAsync(() => log.Warnings.Any(w => w.Contains("dropped 3")), TimeSpan.FromSeconds(2)); + log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(2); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } +} From 9beeed7a7649956b793e9015623348bac0c1ba35 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:25:20 -0400 Subject: [PATCH 42/51] feat(twincat): implement IBatchSubscribable (R02:P1) The poll engine already builds a per-cycle batch; TwinCAT now forwards it as OnDataChangeBatch, so DriverInstanceActor gets one message per poll cycle instead of one per changed tag. Because the actor subscribes ONLY the batch event of a batch-capable driver, every native-path data change (pushes, the BadNodeIdUnknown / refusal / unavailable snapshots) is raised through a single RaiseDataChange that fires both the per-tag event and a fresh one-item batch. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../TwinCATDriver.cs | 19 +++- .../TwinCATBatchSubscribeTests.cs | 89 +++++++++++++++++++ 2 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATBatchSubscribeTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index 8542bcad..fc0469ae 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -11,7 +11,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; /// the skeleton; read / write / discover / subscribe / probe / host- /// resolver land in PRs 2 and 3. /// -public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery, ISubscribable, +public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery, IBatchSubscribable, IHostConnectivityProbe, IPerCallHostResolver, IRediscoverable, ISkippedTagReporter, IDriverDiagnostics, IDisposable, IAsyncDisposable { @@ -88,6 +88,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// Occurs when a subscribed tag value changes. public event EventHandler? OnDataChange; + /// + /// Batched data changes (R02:P1): one raise per poll cycle on the poll path, one per notification on + /// the native path (each ADS notification is its own burst). DriverInstanceActor subscribes only this + /// event of a batch-capable driver, so EVERY data change the driver publishes goes through both. + /// + public event EventHandler? OnDataChangeBatch; /// Occurs when a device host connectivity status changes. public event EventHandler? OnHostStatusChanged; /// Occurs when the Galaxy object hierarchy or TwinCAT symbol table is rediscovered. @@ -115,6 +121,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, tagRef, snapshot)), onError: HandlePollError, backoffCap: PollBackoffCap, + onBatch: (handle, items) => OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)), // The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this // closure IS the production consumer of its overrun signal. onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1, @@ -972,9 +979,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery : $"TwinCAT device {unreachable.Options.HostAddress} unreachable ({unreachable.ConsecutiveProbeFailures} consecutive failed probes)"); } - /// Raises one data change for a native subscription. - private void RaiseDataChange(ISubscriptionHandle handle, string reference, DataValueSnapshot snapshot) => + /// + /// Raises one data change for a native subscription on BOTH events: the per-tag one (legacy consumers) + /// and a one-item batch (a fresh list per raise — the consumer may retain it). + /// + private void RaiseDataChange(ISubscriptionHandle handle, string reference, DataValueSnapshot snapshot) + { OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, reference, snapshot)); + OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, [new DataChangeItem(reference, snapshot)])); + } /// /// Registers one native notification and records its replayable intent on the device — both diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATBatchSubscribeTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATBatchSubscribeTests.cs new file mode 100644 index 00000000..29148186 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATBatchSubscribeTests.cs @@ -0,0 +1,89 @@ +using System.Collections.Concurrent; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// #739 / R02:P1 — TwinCAT is an . DriverInstanceActor subscribes ONLY the +/// batch event of a batch-capable driver, so every data change — poll cycle AND native push, including the +/// Bad snapshots the native path publishes itself — must reach OnDataChangeBatch; the per-tag event +/// keeps firing for legacy consumers (the CLI). +/// +[Trait("Category", "Unit")] +public sealed class TwinCATBatchSubscribeTests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + + private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, bool native) => + new(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = TwinCATRawTags.Entries( + [ + new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt), + new TwinCATTagDefinition("B", Host, "MAIN.B", TwinCATDataType.DInt), + ]), + Probe = new TwinCATProbeOptions { Enabled = false }, + UseNativeNotifications = native, + }, "drv-1", factory); + + /// The driver advertises the batch capability. + [Fact] + public void Driver_is_batch_subscribable() => + NewDriver(new FakeTwinCATClientFactory(), native: false).ShouldBeAssignableTo(); + + /// A poll cycle that changes two tags raises ONE batch carrying both, and the per-tag event twice. + [Fact] + public async Task A_poll_cycle_raises_one_batch_with_every_changed_tag() + { + var factory = new FakeTwinCATClientFactory + { + Customise = () => new FakeTwinCATClient { Values = { ["MAIN.A"] = 1, ["MAIN.B"] = 2 } }, + }; + var drv = NewDriver(factory, native: false); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var batches = new ConcurrentQueue(); + var single = new ConcurrentQueue(); + drv.OnDataChangeBatch += (_, e) => batches.Enqueue(e); + drv.OnDataChange += (_, e) => single.Enqueue(e); + + var handle = await drv.SubscribeAsync(["A", "B"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + await WaitForAsync(() => !batches.IsEmpty, TimeSpan.FromSeconds(3)); + await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken); + + var first = batches.First(); + first.SubscriptionHandle.ShouldBe(handle); + first.Items.Select(i => i.FullReference).ShouldBe(["A", "B"], ignoreOrder: true); + single.Count(e => e.FullReference is "A" or "B").ShouldBeGreaterThanOrEqualTo(2); + } + + /// A native push and the native path's own Bad snapshot both reach the batch event. + [Fact] + public async Task Native_pushes_and_native_bad_snapshots_reach_the_batch_event() + { + var fake = new FakeTwinCATClient(); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, native: true); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var batches = new ConcurrentQueue(); + drv.OnDataChangeBatch += (_, e) => batches.Enqueue(e); + + _ = await drv.SubscribeAsync(["A", "Ghost"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + fake.FireNotification("MAIN.A", 42); + + var items = batches.SelectMany(b => b.Items).ToList(); + items.Single(i => i.FullReference == "Ghost").Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + var a = items.Single(i => i.FullReference == "A"); + a.Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.Good); + a.Snapshot.Value.ShouldBe(42); + } + + private static async Task WaitForAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(20); + } +} From 2724d7dae84d5e5ddaeef727ca9f9456203a7968 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:26:28 -0400 Subject: [PATCH 43/51] fix(focas): a CNC refusal is per-tag Bad, not a driver Degraded (lane health checklist) Applies the driver-lane health contract to the value path. Every non-Good read status used to write Degraded on every poll, so one vacant macro (BadNoData), one EW_NUMBER address (BadOutOfRange) or one fractional macro on an integer tag made a reachable CNC look sick forever. Now: - a comms-class status (BadCommunicationError / BadInternalError (EW_HANDLE) / BadTimeout) still degrades, with LastSuccessfulRead preserved; - any other non-Good status is a refusal: the tag reads Bad, one Warning is logged per tag (re-armed when it next reads Good, cleared at init), driver state is untouched, and LastSuccessfulRead advances because the CNC answered (CAS, since devices now read concurrently); - a write refused with NotSupportedException no longer degrades the driver. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../FocasDriver.cs | 54 ++++++- .../FocasAlarmProjectionTests.cs | 3 +- .../FocasReadHealthContractTests.cs | 140 ++++++++++++++++++ 3 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs index f65bb2f4..c5bb0dda 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs @@ -235,6 +235,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, var generation = Interlocked.Increment(ref _lifecycleGeneration); await ReleaseDeviceStatesAsync().ConfigureAwait(false); ResetAlarmHealth(); + _refusalWarned.Clear(); WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Initializing, null, null)); var devices = new Dictionary(StringComparer.OrdinalIgnoreCase); try @@ -756,11 +757,20 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, results[i] = new DataValueSnapshot(value, status, now, now); if (status == FocasStatusMapper.Good) + { + _refusalWarned.TryRemove(reference, out _); Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, now, null)); - else + } + else if (IsCommsStatus(status)) + { Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, Volatile.Read(ref _health).LastSuccessfulRead, $"FOCAS status 0x{status:X8} reading {reference}")); + } + else + { + NoteRefusal(reference, def.Address, status, now); + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } catch (OperationCanceledException) @@ -781,6 +791,43 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, } } + /// + /// RawPaths whose CNC refusal has already been logged — a refusal warns ONCE per tag, not once per poll, + /// and re-arms when the tag next reads Good. Cleared at initialise. + /// + private readonly ConcurrentDictionary _refusalWarned = new(StringComparer.Ordinal); + + /// + /// True for a status that says the LINK failed (or the read never got an answer), as opposed to the CNC + /// answering with a refusal. maps EW_SOCKET / EW_UNEXP / + /// EW_PARITY / unknown codes to BadCommunicationError and EW_HANDLE to BadInternalError; the wire client + /// reports a dead session as BadCommunicationError. + /// + private static bool IsCommsStatus(uint status) => status is FocasStatusMapper.BadCommunicationError + or FocasStatusMapper.BadInternalError or FocasStatusMapper.BadTimeout; + + /// + /// A deterministic CNC refusal (drivers.md health contract): per-tag Bad only — one Warning per tag, no + /// state change — but the CNC DID answer, so LastSuccessfulRead advances (else a reachable control whose + /// only tags are refused would look silent to the comms-loss watchdog). + /// + private void NoteRefusal(string reference, string address, uint status, DateTime now) + { + // CAS, not a plain write: devices are read concurrently, and a read-modify-write could otherwise put back + // the state another device's comms fault had just replaced. + while (true) + { + var current = Volatile.Read(ref _health); + if (current.LastSuccessfulRead is { } last && last >= now) break; + if (ReferenceEquals(Interlocked.CompareExchange(ref _health, current with { LastSuccessfulRead = now }, current), current)) + break; + } + if (_refusalWarned.TryAdd(reference, 0)) + _logger.LogWarning( + "FOCAS CNC refused {RawPath} ({Address}) with 0x{Status:X8}; the tag reads Bad until it is fixed (warned once). Driver={DriverInstanceId}", + reference, address, status, _driverInstanceId); + } + /// /// PMC coalescing pre-pass for ONE device's share of a batch. Plans merged ranges /// () and serves every multi-member range with ONE @@ -956,9 +1003,10 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, } catch (NotSupportedException nse) { + // A refusal, not a comms fault (drivers.md, #611): nothing was wrong with the link, so health is + // left alone — degrading here handed the comms-loss watchdog a reason to reconnect a healthy CNC. results[i] = new WriteResult(FocasStatusMapper.BadNotSupported); - Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, - Volatile.Read(ref _health).LastSuccessfulRead, nse.Message)); + _logger.LogDebug(nse, "FOCAS write to {RawPath} refused as unsupported", w.FullReference); } catch (Exception ex) when (ex is FormatException or InvalidCastException) { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmProjectionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmProjectionTests.cs index 9006936d..0749aafc 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmProjectionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmProjectionTests.cs @@ -645,7 +645,8 @@ public sealed class FocasAlarmProjectionTests drv.GetHealth().State.ShouldBe(DriverState.Healthy); // Value path Degraded, alarm path good ⇒ Degraded: an alarm success cannot mask it. - factory.Clients.Single().ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange; + // A COMMS-class status: a CNC refusal (e.g. BadOutOfRange) is not a value-path Degraded since #740. + factory.Clients.Single().ReadStatuses["R100"] = FocasStatusMapper.BadCommunicationError; await drv.ReadAsync([valuePath], CancellationToken.None); drv.NoteAlarmReadSucceeded(Host); drv.GetHealth().State.ShouldBe(DriverState.Degraded); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs new file mode 100644 index 00000000..88e7f206 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs @@ -0,0 +1,140 @@ +using Microsoft.Extensions.Logging; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests; + +/// +/// #740 driver-lane health checklist (drivers.md, #527/#611): a deterministic CNC refusal — the control +/// answered with a non-OK return code, or with a value the tag's type cannot hold (vacant macro, fractional +/// macro on an integer tag) — is per-tag Bad with ONE Warning per tag and NO health write, and it still +/// proves the device answered (LastSuccessfulRead advances). Only a comms-class status degrades the driver. +/// Before this, every non-Good status latched Degraded every poll, so one vacant macro made a reachable CNC +/// look sick forever. +/// +[Trait("Category", "Unit")] +public sealed class FocasReadHealthContractTests +{ + private const string Host = "focas://10.0.0.5:8193"; + + private sealed class WarningCounter : ILogger + { + private int _warnings; + public int Warnings => Volatile.Read(ref _warnings); + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, + Func formatter) + { + if (logLevel == LogLevel.Warning && formatter(state, exception).Contains("refused", StringComparison.Ordinal)) + Interlocked.Increment(ref _warnings); + } + } + + private static (FocasDriver drv, FakeFocasClient client, WarningCounter log) NewDriver(uint r100Status) + { + var client = new FakeFocasClient { Values = { ["R101"] = (sbyte)1 } }; + client.ReadStatuses["R100"] = r100Status; + var log = new WarningCounter(); + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(Host)], + RawTags = + [ + FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte)), + FocasRawTags.Entry(new FocasTagDefinition("raw/R101", Host, "R101", FocasDataType.Byte)), + ], + Probe = new FocasProbeOptions { Enabled = false }, + PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false }, + }, "focas-contract", new FakeFocasClientFactory { Customise = () => client }, log); + return (drv, client, log); + } + + /// A refusal leaves health alone, warns once, and still credits the answer. + [Theory] + [InlineData(FocasStatusMapper.BadOutOfRange)] // EW_NUMBER + [InlineData(FocasStatusMapper.BadNotSupported)] // EW_FUNC / EW_NOOPT + [InlineData(FocasStatusMapper.BadNoData)] // vacant macro + [InlineData(FocasStatusMapper.BadTypeMismatch)] // fractional macro on an integer tag + [InlineData(FocasStatusMapper.BadDeviceFailure)] // EW_BUSY + public async Task Cnc_refusal_is_per_tag_bad_with_one_warning_and_no_health_write(uint refusal) + { + var (drv, _, log) = NewDriver(refusal); + await drv.InitializeAsync("{}", CancellationToken.None); + var atInit = drv.GetHealth().LastSuccessfulRead; + + for (var i = 0; i < 3; i++) + { + await Task.Delay(5); + var snap = (await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0]; + snap.StatusCode.ShouldBe(refusal); + } + + var health = drv.GetHealth(); + health.State.ShouldBe(DriverState.Healthy, "a CNC refusal is not a comms fault"); + health.LastSuccessfulRead!.Value.ShouldBeGreaterThan(atInit!.Value, "the CNC answered — that is evidence of life"); + log.Warnings.ShouldBe(1, "one Warning per refused tag, not one per poll"); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// A refused tag that recovers and is refused again warns again (the dedup clears on Good). + [Fact] + public async Task Refusal_warning_rearms_after_the_tag_reads_good() + { + var (drv, client, log) = NewDriver(FocasStatusMapper.BadOutOfRange); + await drv.InitializeAsync("{}", CancellationToken.None); + await drv.ReadAsync(["raw/R100"], CancellationToken.None); + client.ReadStatuses.Remove("R100"); + client.Values["R100"] = (sbyte)3; + (await drv.ReadAsync(["raw/R100"], CancellationToken.None))[0].StatusCode.ShouldBe(FocasStatusMapper.Good); + client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange; + await drv.ReadAsync(["raw/R100"], CancellationToken.None); + log.Warnings.ShouldBe(2); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// Positive control: a comms-class status still degrades, preserving LastSuccessfulRead. + [Fact] + public async Task Comms_status_still_degrades_with_last_successful_read_preserved() + { + var (drv, _, _) = NewDriver(FocasStatusMapper.BadCommunicationError); + await drv.InitializeAsync("{}", CancellationToken.None); + await drv.ReadAsync(["raw/R101"], CancellationToken.None); + var lastGood = drv.GetHealth().LastSuccessfulRead; + + await drv.ReadAsync(["raw/R100"], CancellationToken.None); + + var health = drv.GetHealth(); + health.State.ShouldBe(DriverState.Degraded); + health.LastSuccessfulRead.ShouldBe(lastGood); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// A write the driver refuses (NotSupportedException) is not a comms fault either. + [Fact] + public async Task Write_refusal_does_not_degrade() + { + var client = new ThrowingWriteClient(); + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(Host)], + RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/W", Host, "R100", FocasDataType.Byte, Writable: true))], + Probe = new FocasProbeOptions { Enabled = false }, + }, "focas-write-refusal", new FakeFocasClientFactory { Customise = () => client }); + await drv.InitializeAsync("{}", CancellationToken.None); + + var result = await drv.WriteAsync([new WriteRequest("raw/W", (sbyte)1)], CancellationToken.None); + + result[0].StatusCode.ShouldBe(FocasStatusMapper.BadNotSupported); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + await drv.ShutdownAsync(CancellationToken.None); + } + + private sealed class ThrowingWriteClient : FakeFocasClient + { + public override Task WriteAsync(FocasAddress address, FocasDataType type, object? value, CancellationToken ct) => + throw new NotSupportedException("bit writes are not supported on this area"); + } +} From 8ffa6178b577b7030e03a1e5892e4b2b68aa917e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:27:38 -0400 Subject: [PATCH 44/51] test(focas): integration case for re-initialise under live wire reads (R07:H13) Against the focas-mock fixture: the fixed-tree loop is running and reads are in flight while the driver re-initialises five times. No reader throws and reads keep returning Good. After shutdown, a fresh driver still connects and reads. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Series/WireBackendCoverageTests.cs | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendCoverageTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendCoverageTests.cs index 4cbf460e..ae0055b3 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendCoverageTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendCoverageTests.cs @@ -141,6 +141,86 @@ public sealed class WireBackendCoverageTests } } + /// + /// #740 / R07:H13 — re-initialising a live driver (the fixed-tree loop running, reads in flight over the real + /// wire) swaps the device table instead of clearing it under its readers: no reader throws, reads keep + /// returning Good, and shutdown leaves no session behind (a fresh driver can still connect afterwards). + /// + [Fact] + public async Task Reinitialise_under_live_reads_swaps_the_device_table() + { + if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason); + var ct = TestContext.Current.CancellationToken; + + await _fx.LoadProfileAsync("FWLIB64", ct); + await _fx.PatchStateAsync(new + { + macros = new Dictionary { ["500"] = new { value = 42000, @decimal = 3 } }, + }, ct); + + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(DeviceHost)], + RawTags = FocasRawTags.Entries( + new FocasTagDefinition("Macro500", DeviceHost, "MACRO:500", FocasDataType.Float64, Writable: false)), + Probe = new FocasProbeOptions { Enabled = false }, + FixedTree = new FocasFixedTreeOptions + { + Enabled = true, + PollInterval = TimeSpan.FromMilliseconds(50), + ProgramPollInterval = TimeSpan.Zero, + TimerPollInterval = TimeSpan.Zero, + }, + }, driverInstanceId: "wire-reinit-swap", clientFactory: new WireFocasClientFactory()); + + await using (drv) + { + await drv.InitializeAsync("{}", ct); + using var stop = CancellationTokenSource.CreateLinkedTokenSource(ct); + var errors = new ConcurrentQueue(); + var good = 0; + var reader = Task.Run(async () => + { + while (!stop.IsCancellationRequested) + { + try + { + _ = drv.GetHostStatuses(); + var s = await drv.ReadAsync(["Macro500", $"{DeviceHost}/Identity/SeriesNumber"], CancellationToken.None); + if (s[0].StatusCode == FocasStatusMapper.Good) Interlocked.Increment(ref good); + } + catch (Exception ex) { errors.Enqueue(ex); } + } + }, CancellationToken.None); + + for (var i = 0; i < 5; i++) + { + await Task.Delay(150, ct); + await drv.InitializeAsync("{}", ct); + } + await Task.Delay(300, ct); + stop.Cancel(); + await reader; + + errors.ShouldBeEmpty(); + good.ShouldBeGreaterThan(0, "positive control: reads went over the wire between re-initialises"); + } + + // A fresh driver still connects and reads Good after the re-initialise churn and the shutdown. + var after = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(DeviceHost)], + RawTags = FocasRawTags.Entries( + new FocasTagDefinition("Macro500", DeviceHost, "MACRO:500", FocasDataType.Float64, Writable: false)), + Probe = new FocasProbeOptions { Enabled = false }, + }, driverInstanceId: "wire-reinit-after", clientFactory: new WireFocasClientFactory()); + await using (after) + { + await after.InitializeAsync("{}", ct); + (await after.ReadAsync(["Macro500"], ct))[0].StatusCode.ShouldBe(FocasStatusMapper.Good); + } + } + /// Verifies that discover emits device folder and tag variables. [Fact] public async Task Discover_emits_device_folder_and_tag_variables() From 6201bafea4714d92f2926719de3e6871421d280c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:28:53 -0400 Subject: [PATCH 45/51] test(twincat): pass the test token to the abandoned-init wait Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../TwinCATLifecycleTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs index c419289b..222495fd 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs @@ -151,7 +151,8 @@ public sealed class TwinCATLifecycleTests sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ShutdownBudget + TimeSpan.FromMilliseconds(500)); wedge.SetResult(); - try { await abandoned.WaitAsync(TimeSpan.FromSeconds(5)); } catch (OperationCanceledException) { } + try { await abandoned.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } + catch (OperationCanceledException) when (!TestContext.Current.CancellationToken.IsCancellationRequested) { } drv.DeviceCount.ShouldBe(0); await WaitForAsync(() => factory.Clients.All(c => c.DisposeCount > 0 || !c.IsConnected), TimeSpan.FromSeconds(2)); From 8de5947e2d48d42799ac6b610bf9eb47817225c9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:30:05 -0400 Subject: [PATCH 46/51] test(twincat): XAR-gated wire cases for H11/H12/H18/U9 Native-subscribe isolation of an unknown symbol, LastSuccessfulRead advance from pushes alone, a WORD bit write flipping exactly one bit (bit 16 refused), a whole-array REAL write round-trip (length mismatch refused) and a STRING(80) write round-trip (overflow refused). Every writing case restores the seed it touched. Skipped without the TwinCAT XAR fixture. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../TwinCATRemediationGateTests.cs | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.IntegrationTests/TwinCATRemediationGateTests.cs diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.IntegrationTests/TwinCATRemediationGateTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.IntegrationTests/TwinCATRemediationGateTests.cs new file mode 100644 index 00000000..c592495b --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.IntegrationTests/TwinCATRemediationGateTests.cs @@ -0,0 +1,146 @@ +using System.Collections.Concurrent; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.IntegrationTests; + +/// +/// Wire-level cases for the #739 High/Medium findings, against the live TwinCAT 3 XAR fixture +/// (skipped without it, like ). Every case that writes restores what it +/// changed: GVL_Primitives.vWord and vString are seeds the primitives theory reads. +/// +[Collection("TwinCATXar")] +[Trait("Category", "Integration")] +[Trait("Simulator", "TwinCAT-XAR")] +public sealed class TwinCATRemediationGateTests(TwinCATXarFixture sim) +{ + private string Host => $"ads://{sim.TargetNetId}:{sim.AmsPort}"; + + private TwinCATDriver NewDriver(string id, bool native, params TwinCATTagDefinition[] tags) => new(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host, "XAR-VM")], + RawTags = TwinCATRawTags.Entries(tags), + UseNativeNotifications = native, + Timeout = TimeSpan.FromSeconds(5), + Probe = new TwinCATProbeOptions { Enabled = false }, + }, id); + + /// H11: a typo'd symbol in a native subscribe is isolated; the real counter keeps delivering. + [TwinCATFact] + public async Task Native_subscribe_isolates_an_unknown_symbol() + { + if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason); + await using var drv = NewDriver("tc3-gate-h11", native: true, + new TwinCATTagDefinition("Counter", Host, "GVL_Fixture.nCounter", TwinCATDataType.DInt), + new TwinCATTagDefinition("Typo", Host, "GVL_Fixture.nCounterTypo", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + + var handle = await drv.SubscribeAsync(["Counter", "Typo"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5); + while (events.Count(e => e.FullReference == "Counter") < 2 && DateTime.UtcNow < deadline) + await Task.Delay(50, TestContext.Current.CancellationToken); + await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken); + + events.Count(e => e.FullReference == "Counter" && e.Snapshot.StatusCode == 0u).ShouldBeGreaterThanOrEqualTo(2); + events.First(e => e.FullReference == "Typo").Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a refused symbol is not a comms fault"); + } + + /// H12: native pushes alone advance LastSuccessfulRead. + [TwinCATFact] + public async Task Native_pushes_advance_LastSuccessfulRead() + { + if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason); + await using var drv = NewDriver("tc3-gate-h12", native: true, + new TwinCATTagDefinition("Counter", Host, "GVL_Fixture.nCounter", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var initStamp = drv.GetHealth().LastSuccessfulRead!.Value; + + var handle = await drv.SubscribeAsync(["Counter"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + await Task.Delay(1500, TestContext.Current.CancellationToken); + await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken); + + drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp.AddMilliseconds(500)); + } + + /// + /// H18: a bit write on the 16-bit vWord (0xBEEF) flips exactly that bit at the WORD's own width + /// (the old 32-bit UDInt RMW could not even address a WORD), and a bit beyond it is refused. + /// + [TwinCATFact] + public async Task Bit_write_on_a_WORD_flips_one_bit_and_refuses_bit_16() + { + if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason); + await using var drv = NewDriver("tc3-gate-h18", native: false, + new TwinCATTagDefinition("Word", Host, "GVL_Primitives.vWord", TwinCATDataType.UInt, Writable: false), + new TwinCATTagDefinition("Bit4", Host, "GVL_Primitives.vWord.4", TwinCATDataType.Bool), + new TwinCATTagDefinition("Bit16", Host, "GVL_Primitives.vWord.16", TwinCATDataType.Bool)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var before = Convert.ToUInt16((await drv.ReadAsync(["Word"], TestContext.Current.CancellationToken))[0].Value); + var bit4WasSet = (before & 0x10) != 0; + try + { + (await drv.WriteAsync([new WriteRequest("Bit4", !bit4WasSet)], TestContext.Current.CancellationToken))[0] + .StatusCode.ShouldBe(0u); + var after = Convert.ToUInt16((await drv.ReadAsync(["Word"], TestContext.Current.CancellationToken))[0].Value); + after.ShouldBe((ushort)(before ^ 0x10), "only bit 4 may change"); + + (await drv.WriteAsync([new WriteRequest("Bit16", true)], TestContext.Current.CancellationToken))[0] + .StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange); + } + finally + { + await drv.WriteAsync([new WriteRequest("Bit4", bit4WasSet)], CancellationToken.None); + } + Convert.ToUInt16((await drv.ReadAsync(["Word"], TestContext.Current.CancellationToken))[0].Value).ShouldBe(before); + } + + /// U9: a whole-array write round-trips through the codec (CLR double[] converts to REAL elements). + [TwinCATFact] + public async Task Whole_array_write_round_trips() + { + if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason); + await using var drv = NewDriver("tc3-gate-u9-array", native: false, + new TwinCATTagDefinition("Arr", Host, "GVL_Arrays.aReal1D", TwinCATDataType.Real, ArrayLength: 32)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var values = Enumerable.Range(0, 32).Select(i => i * 1.5 + 0.25).ToArray(); + + (await drv.WriteAsync([new WriteRequest("Arr", values)], TestContext.Current.CancellationToken))[0] + .StatusCode.ShouldBe(0u); + var read = (await drv.ReadAsync(["Arr"], TestContext.Current.CancellationToken))[0]; + read.StatusCode.ShouldBe(0u); + ((float[])read.Value!).ShouldBe(values.Select(v => (float)v).ToArray()); + + (await drv.WriteAsync([new WriteRequest("Arr", values[..31])], TestContext.Current.CancellationToken))[0] + .StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange); + } + + /// U9: a STRING write is NUL-padded at the PLC width (no stale tail) and an over-long one is refused. + [TwinCATFact] + public async Task String_write_round_trips_and_refuses_overflow() + { + if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason); + await using var drv = NewDriver("tc3-gate-u9-string", native: false, + new TwinCATTagDefinition("S", Host, "GVL_Primitives.vString", TwinCATDataType.String)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var original = (string)(await drv.ReadAsync(["S"], TestContext.Current.CancellationToken))[0].Value!; + try + { + (await drv.WriteAsync([new WriteRequest("S", "Grüße 40°")], TestContext.Current.CancellationToken))[0] + .StatusCode.ShouldBe(0u); + (await drv.ReadAsync(["S"], TestContext.Current.CancellationToken))[0].Value.ShouldBe("Grüße 40°"); + + (await drv.WriteAsync([new WriteRequest("S", new string('x', 81))], TestContext.Current.CancellationToken))[0] + .StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange, "vString is STRING(80)"); + } + finally + { + await drv.WriteAsync([new WriteRequest("S", original)], CancellationToken.None); + } + (await drv.ReadAsync(["S"], TestContext.Current.CancellationToken))[0].Value.ShouldBe(original); + } +} From fd64fed21784253cd3de0c1851e6905894a373be Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:44:42 -0400 Subject: [PATCH 47/51] fix(focas): a stalled device fails fast for the rest of its batch; generation-stamp health (#740 review round 1) I1: Task.WhenAll waits for the slowest device, so concurrency alone still let a device that accepts TCP and then stalls cost (its tag count x call timeout) per batch. After the first timeout, transport failure or comms-class status on a device, its remaining tags in the batch now fail BadCommunicationError immediately, the same as the coalescer's FailMembers, which now also stops after a failed range. FIXED: tags are still served from the cache. A malformed address is BadNodeIdUnknown and does not count as a link failure. Test: a 10-tag stalled device beside a healthy one completes in under 3 call timeouts with the healthy value Good. The P9 test now also asserts per-client read counts and that two reads overlapped across devices. Minors: - 1: a superseded init releases its table only if it was never published (flag). - 2: a fixed-tree refusal credits LastSuccessfulRead; a value-path refusal after a comms fault restores Healthy. - 5: alarm/fixed/refusal health is reset inside the publish lock, after the generation check. - 6: FocasFixedTreeHealthTests wait on served-failure/refusal counters, not wall-clock. - 8: data-path health writes (reads, coalescer, writes, poll errors, refusals) carry the generation they started under and are dropped once a lifecycle call has moved on. Each write is a check-and-set under the lifecycle lock. Also adds a unit case where 20 vendor records with type 0 and empty messages stay vendor. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../FocasDriver.cs | 203 ++++++++++++------ .../FocasAlarmLayoutTests.cs | 18 ++ .../FocasConcurrentDeviceReadTests.cs | 75 ++++++- .../FocasFixedTreeHealthTests.cs | 53 +++-- .../FocasReadHealthContractTests.cs | 72 +++++++ 5 files changed, 335 insertions(+), 86 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs index c5bb0dda..5177119b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs @@ -200,10 +200,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, internal void HandlePollError(Exception ex) { _logger.LogWarning(ex, "FOCAS poll reader failed. Driver={DriverInstanceId}", _driverInstanceId); - var current = Volatile.Read(ref _health); - if (current.State != DriverState.Faulted) - Volatile.Write(ref _health, - new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, ex.Message)); + DegradeHealth(CurrentGeneration, ex.Message); } /// @@ -234,10 +231,11 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // shutdown) is superseded by that shutdown's bump, and checks for it before publishing anything. var generation = Interlocked.Increment(ref _lifecycleGeneration); await ReleaseDeviceStatesAsync().ConfigureAwait(false); - ResetAlarmHealth(); - _refusalWarned.Clear(); WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Initializing, null, null)); var devices = new Dictionary(StringComparer.OrdinalIgnoreCase); + // Set once this attempt's table is live: from then on it is the NEXT lifecycle call's to release, never + // this attempt's (review minor 1 — a reference compare could release a table twice). + var published = false; try { // adopt a changed config. Options and the backend client factory are re-derived by ONE @@ -361,6 +359,11 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // leaves the previous snapshot standing rather than a truncated one that would read as // "we now skip fewer tags". _skippedTags.Commit(); + // The per-path health of the previous table, reset only now that this attempt is known to be the + // live one (review minor 5): a superseded attempt must not wipe the health of the call that + // superseded it. + ResetAlarmHealth(); + _refusalWarned.Clear(); Volatile.Write(ref _parsedAddressesByTagName, parsedAddresses); Volatile.Write(ref _tagsByRawPath, tagsByRawPath); Volatile.Write(ref _alarmTags, alarmTags); @@ -370,6 +373,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, _fixedTreeActive = _options.FixedTree.Enabled || parsedAddresses.Values.Any(a => a.Kind == FocasAreaKind.Fixed); Volatile.Write(ref _devices, devices); + published = true; StartDeviceLoops(devices.Values); } @@ -390,7 +394,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, { // Superseded: whatever this attempt built but never published is released here (a published table // is the superseding call's to release); health belongs to that call too. - if (!ReferenceEquals(Devices, devices)) await ReleaseTableAsync(devices.Values).ConfigureAwait(false); + if (!published) await ReleaseTableAsync(devices.Values).ConfigureAwait(false); throw; } catch (Exception ex) @@ -647,7 +651,9 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, ArgumentNullException.ThrowIfNull(fullReferences); var now = DateTime.UtcNow; var results = new DataValueSnapshot[fullReferences.Count]; - // One snapshot of each swapped table for the whole call (see _devices). + // The generation FIRST, then one snapshot of each swapped table for the whole call (see _devices): health + // written by this batch is dropped if a lifecycle call has moved on meanwhile. + var generation = CurrentGeneration; var devices = Devices; var alarmTags = Volatile.Read(ref _alarmTags); var fixedTreeActive = _fixedTreeActive; @@ -693,12 +699,12 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, if (byDevice.Count == 1) { foreach (var (device, indices) in byDevice) - await ReadDeviceAsync(device, indices, fullReferences, results, now, cancellationToken).ConfigureAwait(false); + await ReadDeviceAsync(device, indices, fullReferences, results, now, generation, cancellationToken).ConfigureAwait(false); } else { await Task.WhenAll(byDevice.Select(kv => - ReadDeviceAsync(kv.Key, kv.Value, fullReferences, results, now, cancellationToken))).ConfigureAwait(false); + ReadDeviceAsync(kv.Key, kv.Value, fullReferences, results, now, generation, cancellationToken))).ConfigureAwait(false); } return results; } @@ -708,16 +714,25 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// index it did not serve through the per-tag path. Each result lands at its own index, so concurrent /// device reads never touch the same slot. /// + /// + /// One link failure ends the device's batch (#740 review I1). Concurrency alone does not bound a + /// batch: Task.WhenAll waits for the slowest device, and a CNC that accepts TCP and then stalls costs + /// the full call timeout on EVERY tag — so a 10-tag stalled device held the healthy devices' values back by + /// ten timeouts. After the first timeout / transport failure (or a comms-class status) on this device, its + /// remaining tags in the batch fail fast with BadCommunicationError, exactly as the coalescer's + /// FailMembers does for a merged range. Fixed-tree tags are still served from the cache: they need no + /// link. A CNC REFUSAL does not end the batch — the link answered. + /// private async Task ReadDeviceAsync( DeviceState device, List indices, IReadOnlyList fullReferences, - DataValueSnapshot[] results, DateTime now, CancellationToken cancellationToken) + DataValueSnapshot[] results, DateTime now, int generation, CancellationToken cancellationToken) { // Merge adjacent / near PMC reads of the device into fewer pmc_rdpmcrng calls. Indices it serves are // marked here and skipped by the per-tag loop below; everything else (including every single-tag range // and any range the CNC rejected) takes the unchanged per-tag path. - var served = _options.PmcCoalesce.Enabled - ? await CoalescePmcReadsAsync(device, indices, fullReferences, results, now, cancellationToken).ConfigureAwait(false) - : null; + var (served, linkDown) = _options.PmcCoalesce.Enabled + ? await CoalescePmcReadsAsync(device, indices, fullReferences, results, now, generation, cancellationToken).ConfigureAwait(false) + : (null, false); foreach (var i in indices) { @@ -729,29 +744,43 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, continue; } + FocasAddress parsed; + try { parsed = ResolveParsedAddress(def); } + catch (InvalidOperationException) + { + // A malformed address is a config fault of this tag, not of the link. + results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadNodeIdUnknown, null, now); + continue; + } + + // A FIXED: tag names a field of this device's fixed-tree snapshot, which the + // fixed-tree loop already maintains — there is no area read to issue, and no + // connection to force. A null here means the path resolves to nothing on THIS + // device (an unsupported capability, a renamed axis, a typo), which is a genuine + // BadNodeIdUnknown; a device that has not answered yet reports a comms fault + // instead, because the address may well be fine. + if (parsed.Kind == FocasAreaKind.Fixed) + { + results[i] = TryReadFixedTreeOnDevice(device, parsed.FixedPath!, now) + ?? new DataValueSnapshot( + null, + device.LastFixedPollUtc is null + ? FocasStatusMapper.BadCommunicationError + : FocasStatusMapper.BadNodeIdUnknown, + null, + now); + continue; + } + + if (linkDown) + { + // Already failed (and health already degraded) this batch — don't pay another timeout. + results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadCommunicationError, null, now); + continue; + } + try { - var parsed = ResolveParsedAddress(def); - - // A FIXED: tag names a field of this device's fixed-tree snapshot, which the - // fixed-tree loop already maintains — there is no area read to issue, and no - // connection to force. A null here means the path resolves to nothing on THIS - // device (an unsupported capability, a renamed axis, a typo), which is a genuine - // BadNodeIdUnknown; a device that has not answered yet reports a comms fault - // instead, because the address may well be fine. - if (parsed.Kind == FocasAreaKind.Fixed) - { - results[i] = TryReadFixedTreeOnDevice(device, parsed.FixedPath!, now) - ?? new DataValueSnapshot( - null, - device.LastFixedPollUtc is null - ? FocasStatusMapper.BadCommunicationError - : FocasStatusMapper.BadNodeIdUnknown, - null, - now); - continue; - } - var client = await EnsureConnectedAsync(device, cancellationToken).ConfigureAwait(false); var (value, status) = await client.ReadAsync(parsed, def.DataType, cancellationToken).ConfigureAwait(false); @@ -759,17 +788,16 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, if (status == FocasStatusMapper.Good) { _refusalWarned.TryRemove(reference, out _); - Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, now, null)); + CreditHealth(generation, now); } else if (IsCommsStatus(status)) { - Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, - Volatile.Read(ref _health).LastSuccessfulRead, - $"FOCAS status 0x{status:X8} reading {reference}")); + DegradeHealth(generation, $"FOCAS status 0x{status:X8} reading {reference}"); + linkDown = true; } else { - NoteRefusal(reference, def.Address, status, now); + NoteRefusal(reference, def.Address, status, now, generation); } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } @@ -779,18 +807,48 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // Timeout budget. Surface a recoverable comm error so the BadWaitingForInitialData // seed is overwritten and health degrades, instead of the read hanging forever. results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadCommunicationError, null, now); - Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, - Volatile.Read(ref _health).LastSuccessfulRead, $"FOCAS read timed out for {reference}")); + DegradeHealth(generation, $"FOCAS read timed out for {reference}"); + linkDown = true; } catch (Exception ex) { results[i] = new DataValueSnapshot(null, FocasStatusMapper.BadCommunicationError, null, now); - Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, - Volatile.Read(ref _health).LastSuccessfulRead, ex.Message)); + DegradeHealth(generation, ex.Message); + linkDown = true; } } } + /// + /// Applies a data-path health transition — only while is still the live + /// lifecycle generation (#740 review minor 8): a read that started against the previous table must not + /// write its Degraded onto the next initialise's health. Taken under the lifecycle lock, so the check and + /// the write cannot straddle a publish; the lock is uncontended outside a lifecycle transition. + /// + /// The generation the caller's operation started under. + /// The transition; returns its input unchanged to leave health alone. + private void UpdateHealth(int generation, Func next) + { + lock (_lifecycleLock) + { + if (generation != Volatile.Read(ref _lifecycleGeneration)) return; + Volatile.Write(ref _health, next(Volatile.Read(ref _health))); + } + } + + /// A comms fault: Degraded with LastSuccessfulRead preserved; never downgrades Faulted. + private void DegradeHealth(int generation, string reason) => + UpdateHealth(generation, h => h.State == DriverState.Faulted + ? h + : new DriverHealth(DriverState.Degraded, h.LastSuccessfulRead, reason)); + + /// An answered read: Healthy with LastSuccessfulRead advanced. + private void CreditHealth(int generation, DateTime now) => + UpdateHealth(generation, _ => new DriverHealth(DriverState.Healthy, now, null)); + + /// The live lifecycle generation — captured by a data-path operation when it starts. + private int CurrentGeneration => Volatile.Read(ref _lifecycleGeneration); + /// /// RawPaths whose CNC refusal has already been logged — a refusal warns ONCE per tag, not once per poll, /// and re-arms when the tag next reads Good. Cleared at initialise. @@ -811,17 +869,14 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// state change — but the CNC DID answer, so LastSuccessfulRead advances (else a reachable control whose /// only tags are refused would look silent to the comms-loss watchdog). /// - private void NoteRefusal(string reference, string address, uint status, DateTime now) + private void NoteRefusal(string reference, string address, uint status, DateTime now, int generation) { - // CAS, not a plain write: devices are read concurrently, and a read-modify-write could otherwise put back - // the state another device's comms fault had just replaced. - while (true) - { - var current = Volatile.Read(ref _health); - if (current.LastSuccessfulRead is { } last && last >= now) break; - if (ReferenceEquals(Interlocked.CompareExchange(ref _health, current with { LastSuccessfulRead = now }, current), current)) - break; - } + // The link answered, so a value-path Degraded left by an earlier comms fault is over (review minor 2): a + // device that recovers and then only refuses must not stay Degraded forever. Under the lifecycle lock + // (UpdateHealth), so concurrent device reads cannot interleave a read-modify-write. + UpdateHealth(generation, h => h.State is DriverState.Healthy or DriverState.Degraded + ? new DriverHealth(DriverState.Healthy, h.LastSuccessfulRead is { } l && l > now ? l : now, null) + : h); if (_refusalWarned.TryAdd(reference, 0)) _logger.LogWarning( "FOCAS CNC refused {RawPath} ({Address}) with 0x{Status:X8}; the tag reads Bad until it is fixed (warned once). Driver={DriverInstanceId}", @@ -844,9 +899,9 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// /// Returns the served indices, or null when nothing was served. /// - private async Task?> CoalescePmcReadsAsync( + private async Task<(HashSet? Served, bool LinkDown)> CoalescePmcReadsAsync( DeviceState device, List indices, IReadOnlyList references, DataValueSnapshot[] results, - DateTime now, CancellationToken ct) + DateTime now, int generation, CancellationToken ct) { List? items = null; foreach (var i in indices) @@ -863,12 +918,13 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /* malformed / unresolvable — the per-tag loop reports it with its usual status */ } } - if (items is null || items.Count < 2) return null; + if (items is null || items.Count < 2) return (null, false); var plans = FocasPmcCoalescer.Plan(items, _options.PmcCoalesce.MaxSpanBytes, _options.PmcCoalesce.MaxGapBytes); - if (!plans.Any(p => p.Members.Count >= 2 && !device.CoalesceRejected.ContainsKey(p.Key))) return null; + if (!plans.Any(p => p.Members.Count >= 2 && !device.CoalesceRejected.ContainsKey(p.Key))) return (null, false); HashSet? served = null; + var linkDown = false; // One connect attempt per device per batch: if it fails, every PMC tag of the device fails with // that status here instead of paying a connect attempt per plan and again per tag. IFocasClient client; @@ -878,13 +934,20 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, { FailMembers(items, FocasStatusMapper.BadCommunicationError, $"FOCAS connect failed reading PMC tags of {device.Options.HostAddress}"); - return served; + return (served, true); } foreach (var plan in plans) { if (plan.Members.Count < 2) continue; if (device.CoalesceRejected.ContainsKey(plan.Key)) continue; + if (linkDown) + { + // The link failed on an earlier range this batch: fail the rest without another timeout. + FailMembers(plan.Members, FocasStatusMapper.BadCommunicationError, + $"FOCAS transport failure on {device.Options.HostAddress} earlier in this batch"); + continue; + } var members = plan.Members; try @@ -903,7 +966,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, : new DataValueSnapshot(null, FocasStatusMapper.BadOutOfRange, now, now); (served ??= []).Add(m.Index); } - Volatile.Write(ref _health, new DriverHealth(DriverState.Healthy, now, null)); + CreditHealth(generation, now); } else { @@ -924,9 +987,10 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // Transport failure / timeout: the link is suspect, so fail the members without per-tag retries. FailMembers(members, FocasStatusMapper.BadCommunicationError, $"FOCAS transport failure reading PMC range {plan.Key}"); + linkDown = true; } } - return served; + return (served, linkDown); void FailMembers(IEnumerable failed, uint status, string reason) { @@ -935,9 +999,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, results[m.Index] = new DataValueSnapshot(null, status, null, now); (served ??= []).Add(m.Index); } - Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, - Volatile.Read(ref _health).LastSuccessfulRead, - $"{reason} (0x{status:X8})")); + DegradeHealth(generation, $"{reason} (0x{status:X8})"); } } @@ -948,6 +1010,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, IReadOnlyList writes, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(writes); + var generation = CurrentGeneration; var results = new WriteResult[writes.Count]; for (var i = 0; i < writes.Count; i++) @@ -998,8 +1061,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, // Per-call timeout (not external cancellation) — the write stalled past the device // Timeout budget. Surface a recoverable comm error rather than aborting the batch. results[i] = new WriteResult(FocasStatusMapper.BadCommunicationError); - Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, - Volatile.Read(ref _health).LastSuccessfulRead, $"FOCAS write timed out for {w.FullReference}")); + DegradeHealth(generation, $"FOCAS write timed out for {w.FullReference}"); } catch (NotSupportedException nse) { @@ -1019,8 +1081,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, catch (Exception ex) { results[i] = new WriteResult(FocasStatusMapper.BadCommunicationError); - Volatile.Write(ref _health, new DriverHealth(DriverState.Degraded, - Volatile.Read(ref _health).LastSuccessfulRead, ex.Message)); + DegradeHealth(generation, ex.Message); } } @@ -1992,6 +2053,7 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// An answered fixed-tree pass of : clears its failure, advances LastSuccessfulRead. private void NoteFixedTreeAnswered(DeviceState state) { + if (state.IsRetired) return; _fixedTreeFailures.TryRemove(state.Options.HostAddress, out _); Interlocked.Exchange(ref _fixedTreeLastSuccessTicks, DateTime.UtcNow.Ticks); } @@ -2004,9 +2066,12 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// private void NoteFixedTreeFailed(DeviceState state, Exception error) { + if (state.IsRetired) return; // a released device's late tick says nothing about the live table if (FocasAlarmProjection.IsCncRefusal(error)) { - _fixedTreeFailures.TryRemove(state.Options.HostAddress, out _); + // The CNC answered — evidence of life (review minor 2): clear the failure AND credit the answer, so a + // control that only refuses after an outage does not look silent to the comms-loss watchdog. + NoteFixedTreeAnswered(state); return; } var since = state.LastFixedPollUtc ?? state.FixedTreeStartedUtc; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmLayoutTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmLayoutTests.cs index bc1bc1b2..467e72c3 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmLayoutTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasAlarmLayoutTests.cs @@ -84,6 +84,24 @@ public sealed class FocasAlarmLayoutTests alarms[19].Message.ShouldBe("OVERTRAVEL +19"); } + /// + /// Review I2 edge: vendor records with type 0 and EMPTY messages also satisfy the legacy invariants (the + /// int32 read of type<<16 | axis is just the axis, and the "length" int32 is the empty message's + /// zero bytes) — both readings are plausible, so the vendor default must hold and the numbers decode. + /// + [Fact] + public void Twenty_vendor_records_with_type_zero_and_empty_messages_stay_vendor() + { + var payload = Vendor([.. Enumerable.Range(0, 20).Select(i => (2000 + i, (short)0, (short)(i % 3), ""))]); + payload.Length.ShouldBe(1520); + + var alarms = FocasWireClient.ParseAlarms(payload, 32); + + alarms.Count.ShouldBe(20); + alarms.Select(a => a.AlarmNumber).ShouldBe(Enumerable.Range(2000, 20)); + alarms.ShouldAllBe(a => a.Type == 0 && a.Message == ""); + } + /// The unambiguous lengths are unchanged. [Fact] public void Unambiguous_lengths_are_unchanged() diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs index 9ca35cf7..23e0e4e3 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs @@ -20,11 +20,12 @@ public sealed class FocasConcurrentDeviceReadTests /// STARTED — so a sequential walk (A first) can never finish A, while a concurrent one does. Also counts /// the peak number of reads in flight on any one client. /// - private sealed class OrderingClient(TaskCompletionSource bStarted) : FakeFocasClient + private sealed class OrderingClient(TaskCompletionSource bStarted, int[] global) : FakeFocasClient { private string _host = ""; private int _inFlight; public int PeakInFlight; + public int Reads; public override Task ConnectAsync(FocasHostAddress address, TimeSpan timeout, CancellationToken ct) { @@ -37,6 +38,9 @@ public sealed class FocasConcurrentDeviceReadTests { var now = Interlocked.Increment(ref _inFlight); InterlockedMax(ref PeakInFlight, now); + Interlocked.Increment(ref Reads); + // global[0] = reads in flight across EVERY client, global[1] = its peak. + InterlockedMax(ref global[1], Interlocked.Increment(ref global[0])); try { if (_host.Contains("10.0.0.6", StringComparison.Ordinal)) bStarted.TrySetResult(); @@ -44,7 +48,11 @@ public sealed class FocasConcurrentDeviceReadTests await Task.Delay(20, ct).ConfigureAwait(false); return ((sbyte)1, FocasStatusMapper.Good); } - finally { Interlocked.Decrement(ref _inFlight); } + finally + { + Interlocked.Decrement(ref _inFlight); + Interlocked.Decrement(ref global[0]); + } } private static void InterlockedMax(ref int target, int value) @@ -60,7 +68,8 @@ public sealed class FocasConcurrentDeviceReadTests public async Task Devices_are_read_concurrently_and_each_stays_serialised() { var bStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var factory = new FakeFocasClientFactory { Customise = () => new OrderingClient(bStarted) }; + var global = new int[2]; + var factory = new FakeFocasClientFactory { Customise = () => new OrderingClient(bStarted, global) }; var drv = new FocasDriver(new FocasDriverOptions { Devices = [new FocasDeviceOptions(HostA, DeviceName: "A"), new FocasDeviceOptions(HostB, DeviceName: "B")], @@ -81,8 +90,66 @@ public sealed class FocasConcurrentDeviceReadTests snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.Good, "device A's read can only finish once device B's has started — a sequential walk times A out"); factory.Clients.Count.ShouldBe(2); - factory.Clients.Cast().ShouldAllBe(c => c.PeakInFlight == 1, + var clients = factory.Clients.Cast().ToList(); + // Every client really served reads (2 on A, 1 on B), so "peak 1" is measured, not vacuous … + clients.Select(c => c.Reads).OrderBy(n => n).ShouldBe([1, 2]); + clients.ShouldAllBe(c => c.PeakInFlight == 1, "one device's socket is still request→response: never two reads in flight on it"); + // … and the two devices overlapped: two reads were in flight at once across the clients. + global[1].ShouldBe(2, "the devices must be read at the same time, not one after the other"); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// A device that accepted TCP and then stopped answering: every read hangs until the call timeout. + private sealed class StalledClient : FakeFocasClient + { + public int Reads; + + public override async Task<(object? value, uint status)> ReadAsync( + FocasAddress address, FocasDataType type, CancellationToken ct) + { + Interlocked.Increment(ref Reads); + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + return (null, FocasStatusMapper.Good); + } + } + + /// + /// #740 review I1 — Task.WhenAll waits for the slowest device, so concurrency alone still made a + /// stalled device cost (its tag count × the call timeout) per batch, and the healthy device's values waited + /// that long. After the first timeout on a device its remaining tags in the batch fail fast: the batch is + /// bounded by about ONE call timeout and the healthy device's values are Good. + /// + [Fact] + public async Task A_stalled_device_costs_one_call_timeout_per_batch_not_one_per_tag() + { + var callTimeout = TimeSpan.FromMilliseconds(200); + var factory = new FakeFocasClientFactory(); + factory.Customise = () => factory.Clients.Count == 0 ? new StalledClient() : new FakeFocasClient { Values = { ["R100"] = (sbyte)9 } }; + var stalledTags = Enumerable.Range(0, 10) + .Select(i => FocasRawTags.Entry(new FocasTagDefinition($"raw/A{i}", "A", $"R{100 + i}", FocasDataType.Byte))); + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(HostA, DeviceName: "A"), new FocasDeviceOptions(HostB, DeviceName: "B")], + RawTags = [.. stalledTags, FocasRawTags.Entry(new FocasTagDefinition("raw/B1", "B", "R100", FocasDataType.Byte))], + Probe = new FocasProbeOptions { Enabled = false }, + PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = false }, + Timeout = callTimeout, + }, "focas-stall", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + // Connect device A first so it gets the stalled client (connect order = client order). + await drv.ReadAsync(["raw/A0"], CancellationToken.None); + + var refs = Enumerable.Range(0, 10).Select(i => $"raw/A{i}").Append("raw/B1").ToArray(); + var sw = System.Diagnostics.Stopwatch.StartNew(); + var snaps = await drv.ReadAsync(refs, CancellationToken.None); + sw.Stop(); + + sw.Elapsed.ShouldBeLessThan(callTimeout * 3, "one stalled device must cost ~one call timeout, not ten"); + snaps[10].StatusCode.ShouldBe(FocasStatusMapper.Good); + snaps[10].Value.ShouldBe((sbyte)9); + snaps.Take(10).ShouldAllBe(s => s.StatusCode == FocasStatusMapper.BadCommunicationError); + ((StalledClient)factory.Clients[0]).Reads.ShouldBe(2, "one read in the warm-up batch, one in the measured batch"); await drv.ShutdownAsync(CancellationToken.None); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs index b83898e8..fae11d58 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs @@ -24,6 +24,10 @@ public sealed class FocasFixedTreeHealthTests { public volatile bool Down; public volatile bool Refuse; + /// Transport failures served so far — what the tests wait on instead of wall-clock delays. + public int Failures; + /// Refusals served so far. + public int Refusals; } private sealed class SwitchableCnc(CncSwitch sw) : FakeFocasClient @@ -31,13 +35,25 @@ public sealed class FocasFixedTreeHealthTests private bool Down => sw.Down; private bool Refuse => sw.Refuse; - public override Task GetSysInfoAsync(CancellationToken ct) => - Down ? throw new IOException("socket closed") : base.GetSysInfoAsync(ct); + public override Task GetSysInfoAsync(CancellationToken ct) + { + if (!Down) return base.GetSysInfoAsync(ct); + Interlocked.Increment(ref sw.Failures); + throw new IOException("socket closed"); + } public override Task ReadDynamicAsync(int axisIndex, CancellationToken ct) { - if (Down) throw new IOException("socket closed"); - if (Refuse) throw new FocasWireException("cnc_rddynamic2 refused", rc: 6); + if (Down) + { + Interlocked.Increment(ref sw.Failures); + throw new IOException("socket closed"); + } + if (Refuse) + { + Interlocked.Increment(ref sw.Refusals); + throw new FocasWireException("cnc_rddynamic2 refused", rc: 6); + } return base.ReadDynamicAsync(axisIndex, ct); } } @@ -71,18 +87,22 @@ public sealed class FocasFixedTreeHealthTests await drv.InitializeAsync("{}", CancellationToken.None); await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null); - var first = drv.GetHealth().LastSuccessfulRead; - await Task.Delay(100); - var healthy = drv.GetHealth(); - healthy.State.ShouldBe(DriverState.Healthy); - healthy.LastSuccessfulRead!.Value.ShouldBeGreaterThan(first!.Value, "an answered axis pass is a successful read"); + var first = drv.GetHealth().LastSuccessfulRead!.Value; + // An answered axis pass is a successful read: wait for it to move rather than sleeping. + await WaitUntil(() => drv.GetHealth().LastSuccessfulRead > first); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); sw.Down = true; - await Task.Delay(100); - drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade"); + var lastAnswer = drv.GetDeviceState(Host)!.LastFixedPollUtc!.Value; + // Two failed ticks have demonstrably happened (positive evidence for the absence check below) … + await WaitUntil(() => Volatile.Read(ref sw.Failures) >= 2); + // … and, if they landed inside the budget, the driver is still Healthy. + if (DateTime.UtcNow - lastAnswer < Stale) + drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade"); await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded); var frozen = drv.GetHealth().LastSuccessfulRead; - await Task.Delay(150); + var failuresAtDegrade = Volatile.Read(ref sw.Failures); + await WaitUntil(() => Volatile.Read(ref sw.Failures) >= failuresAtDegrade + 5); drv.GetHealth().State.ShouldBe(DriverState.Degraded); drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "LastSuccessfulRead must stay frozen while the CNC is silent"); @@ -116,8 +136,15 @@ public sealed class FocasFixedTreeHealthTests await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null); sw.Refuse = true; - await Task.Delay(Stale * 3); + var started = DateTime.UtcNow; + await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 1); + var atFirstRefusal = drv.GetHealth().LastSuccessfulRead!.Value; + // Refused ticks really happened, for longer than the staleness budget … + await WaitUntil(() => Volatile.Read(ref sw.Refusals) >= 5 && DateTime.UtcNow - started > Stale * 2); + // … and the driver stayed Healthy, with each refusal crediting the answer (review minor 2). drv.GetHealth().State.ShouldBe(DriverState.Healthy); + drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(atFirstRefusal, + "a refusal is an answer from the CNC — LastSuccessfulRead advances"); sw.Refuse = false; } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs index 88e7f206..923ec354 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasReadHealthContractTests.cs @@ -112,6 +112,78 @@ public sealed class FocasReadHealthContractTests await drv.ShutdownAsync(CancellationToken.None); } + /// + /// Review minor 2 — a device that recovers from a comms fault and then only REFUSES (every tag it serves is + /// refused) is answering, so it is restored to Healthy instead of staying Degraded forever. + /// + [Fact] + public async Task Refusal_after_a_comms_fault_restores_healthy() + { + var (drv, client, _) = NewDriver(FocasStatusMapper.BadCommunicationError); + await drv.InitializeAsync("{}", CancellationToken.None); + await drv.ReadAsync(["raw/R100"], CancellationToken.None); + drv.GetHealth().State.ShouldBe(DriverState.Degraded, "positive control: the comms fault degraded it"); + + client.ReadStatuses["R100"] = FocasStatusMapper.BadOutOfRange; + await drv.ReadAsync(["raw/R100"], CancellationToken.None); + + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// A read whose CNC answer is held until released, then fails at the transport. + private sealed class HeldReadClient(TaskCompletionSource entered, Task release) : FakeFocasClient + { + public volatile bool Hold = true; + + public override async Task<(object? value, uint status)> ReadAsync( + FocasAddress address, FocasDataType type, CancellationToken ct) + { + if (!Hold) return await base.ReadAsync(address, type, ct).ConfigureAwait(false); + entered.TrySetResult(); + await release.ConfigureAwait(false); + throw new IOException("socket reset"); + } + } + + /// + /// Review minor 8 — a read that started against the PREVIOUS table and fails after a re-initialise must not + /// write its Degraded onto the new generation's health. Positive control: the same failure under the live + /// generation does degrade. + /// + [Fact] + public async Task A_read_from_the_previous_generation_cannot_degrade_the_new_one() + { + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var clients = new List(); + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(Host)], + RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/R100", Host, "R100", FocasDataType.Byte))], + Probe = new FocasProbeOptions { Enabled = false }, + Timeout = TimeSpan.FromSeconds(10), + }, "focas-gen", new FakeFocasClientFactory + { + Customise = () => { var c = new HeldReadClient(entered, release.Task); lock (clients) clients.Add(c); return c; }, + }); + await drv.InitializeAsync("{}", CancellationToken.None); + + var stale = drv.ReadAsync(["raw/R100"], CancellationToken.None); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await drv.InitializeAsync("{}", CancellationToken.None); // a new generation, Healthy + release.SetResult(); + (await stale.WaitAsync(TimeSpan.FromSeconds(5)))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError); + + drv.GetHealth().State.ShouldBe(DriverState.Healthy, "the old generation's failure must not land on the new one"); + + // Positive control: a failure under the live generation degrades. + var live = drv.ReadAsync(["raw/R100"], CancellationToken.None); + (await live.WaitAsync(TimeSpan.FromSeconds(5)))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + await drv.ShutdownAsync(CancellationToken.None); + } + /// A write the driver refuses (NotSupportedException) is not a comms fault either. [Fact] public async Task Write_refusal_does_not_degrade() From de54b23b1a71a6301b35588ecf299ac64d56eb41 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:44:42 -0400 Subject: [PATCH 48/51] test(focas): wire-level cases for fixed-tree health and the legacy alarm layout (#740 review I2) Against focas-mock: - H15: a local TCP relay in front of the shared mock takes "the CNC" away and brings it back without stopping the fixture. A FIXED-only driver goes Degraded with LastSuccessfulRead frozen and its value Bad, then returns to Healthy with Good values (the positive control). - H20: the mock answers cnc_rdalmmsg2 in the 80-byte legacy layout. Nineteen alarms (1,520 B = 20 x 76) now decode as the 19 seeded alarms; the old length test gives 20, verified red. A 3-alarm reply is the control. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Series/WireBackendHealthAndLayoutTests.cs | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendHealthAndLayoutTests.cs diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendHealthAndLayoutTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendHealthAndLayoutTests.cs new file mode 100644 index 00000000..30d39394 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendHealthAndLayoutTests.cs @@ -0,0 +1,206 @@ +using System.Net; +using System.Net.Sockets; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Wire; + +namespace ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests.Series; + +/// +/// #740 fix round 1 (review I2) — wire-level cases for H15 (fixed-tree health) and H20 (alarm record layout) +/// against the focas-mock fixture. +/// +[Collection(FocasSimCollection.Name)] +public sealed class WireBackendHealthAndLayoutTests +{ + private readonly FocasSimFixture _fx; + + /// Initializes a new instance of the test class with the FOCAS simulation fixture. + /// The FOCAS simulation fixture. + public WireBackendHealthAndLayoutTests(FocasSimFixture fx) => _fx = fx; + + /// + /// A local TCP relay in front of the shared mock, so ONE test can take "the CNC" away and bring it back + /// without stopping a fixture other suites use: closes every relayed session and stops + /// listening (new connects are refused); listens again on the same port. + /// + private sealed class CncRelay : IAsyncDisposable + { + private readonly string _targetHost; + private readonly int _targetPort; + private readonly List _live = []; + private TcpListener? _listener; + private CancellationTokenSource? _cts; + + public CncRelay(string targetHost, int targetPort) + { + _targetHost = targetHost; + _targetPort = targetPort; + Port = Listen(0); + } + + public int Port { get; private set; } + + private int Listen(int port) + { + _cts = new CancellationTokenSource(); + _listener = new TcpListener(IPAddress.Loopback, port); + _listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + _listener.Start(); + var ct = _cts.Token; + var listener = _listener; + _ = Task.Run(() => AcceptLoopAsync(listener, ct), CancellationToken.None); + return ((IPEndPoint)_listener.LocalEndpoint).Port; + } + + private async Task AcceptLoopAsync(TcpListener listener, CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + TcpClient inbound; + try { inbound = await listener.AcceptTcpClientAsync(ct); } + catch { return; } + var outbound = new TcpClient(); + try { await outbound.ConnectAsync(_targetHost, _targetPort, ct); } + catch { inbound.Dispose(); outbound.Dispose(); continue; } + lock (_live) { _live.Add(inbound); _live.Add(outbound); } + _ = Pump(inbound, outbound, ct); + _ = Pump(outbound, inbound, ct); + } + } + + private static async Task Pump(TcpClient from, TcpClient to, CancellationToken ct) + { + try { await from.GetStream().CopyToAsync(to.GetStream(), ct); } + catch { /* the relay was taken down, or a side closed */ } + finally { from.Dispose(); to.Dispose(); } + } + + public void Down() + { + _cts?.Cancel(); + _listener?.Stop(); + lock (_live) + { + foreach (var c in _live) c.Dispose(); + _live.Clear(); + } + } + + public void Up() => Listen(Port); + + public ValueTask DisposeAsync() + { + Down(); + return ValueTask.CompletedTask; + } + } + + /// + /// H15 over the wire: a driver serving only FIXED: tags is Healthy with a moving LastSuccessfulRead + /// while its CNC answers; when the CNC disappears it degrades with LastSuccessfulRead frozen (the #527 + /// watchdog's conjunction); and it recovers to Healthy when the CNC returns (positive control). + /// + [Fact] + public async Task Fixed_only_driver_degrades_on_cnc_loss_and_recovers_over_the_wire() + { + if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason); + var ct = TestContext.Current.CancellationToken; + await _fx.LoadProfileAsync("FWLIB64", ct); + + await using var relay = new CncRelay(_fx.Host, _fx.Port); + var host = $"focas://127.0.0.1:{relay.Port}"; + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(host)], + RawTags = FocasRawTags.Entries( + new FocasTagDefinition("Series", host, "FIXED:Identity/SeriesNumber", FocasDataType.String, Writable: false)), + Probe = new FocasProbeOptions { Enabled = false }, + Timeout = TimeSpan.FromSeconds(1), + FixedTree = new FocasFixedTreeOptions + { + PollInterval = TimeSpan.FromMilliseconds(50), + ProgramPollInterval = TimeSpan.Zero, + TimerPollInterval = TimeSpan.Zero, + }, + }, driverInstanceId: "wire-fixed-health", clientFactory: new WireFocasClientFactory()) + { + FixedTreeStaleBudgetOverride = TimeSpan.FromMilliseconds(500), + }; + + await using (drv) + { + await drv.InitializeAsync("{}", ct); + await WaitFor(() => drv.GetDeviceState(host)?.LastFixedPollUtc is not null, TimeSpan.FromSeconds(10)); + var first = drv.GetHealth().LastSuccessfulRead!.Value; + await WaitFor(() => drv.GetHealth().LastSuccessfulRead > first, TimeSpan.FromSeconds(5)); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + + relay.Down(); + await WaitFor(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(15)); + var frozen = drv.GetHealth().LastSuccessfulRead; + await Task.Delay(TimeSpan.FromSeconds(1), ct); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + drv.GetHealth().LastSuccessfulRead.ShouldBe(frozen, "nothing answered, so LastSuccessfulRead stays frozen"); + (await drv.ReadAsync(["Series"], ct))[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError, + "the stale fixed-tree value reads Bad on the same budget that degrades the driver"); + + // Positive control: the CNC returns ⇒ Healthy again, LastSuccessfulRead moves. The connect backoff + // (1 s doubling) bounds how soon the loop re-dials. + relay.Up(); + await WaitFor(() => drv.GetHealth().State == DriverState.Healthy, TimeSpan.FromSeconds(40)); + drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(frozen!.Value); + (await drv.ReadAsync(["Series"], ct))[0].StatusCode.ShouldBe(FocasStatusMapper.Good); + } + } + + /// + /// H20 over the wire: the mock answers cnc_rdalmmsg2 in the 80-byte LEGACY layout. Nineteen active + /// alarms make a 1,520-byte payload — also exactly 20 × 76 — which the old length test decoded as twenty + /// garbage vendor records. Positive control: the unambiguous 3-alarm reply decodes the same alarms. + /// + [Fact] + public async Task Nineteen_legacy_alarms_decode_as_nineteen_over_the_wire() + { + if (_fx.SkipReason is not null) Assert.Skip(_fx.SkipReason); + var ct = TestContext.Current.CancellationToken; + await _fx.LoadProfileAsync("FWLIB64", ct); + + var client = new WireFocasClientFactory().Create(); + try + { + await client.ConnectAsync(FocasHostAddress.TryParse(_fx.DeviceUri)!, TimeSpan.FromSeconds(2), ct); + + await _fx.PatchStateAsync(new { alarms = Alarms(3) }, ct); + var three = await client.ReadAlarmsAsync(ct); + three.Count.ShouldBe(3); + three[2].AlarmNumber.ShouldBe(402); + + await _fx.PatchStateAsync(new { alarms = Alarms(19) }, ct); + var nineteen = await client.ReadAlarmsAsync(ct); + nineteen.Count.ShouldBe(19, "a 1,520-byte legacy reply is 19 records, not 20 vendor ones"); + nineteen.Select(a => a.AlarmNumber).ShouldBe(Enumerable.Range(400, 19)); + nineteen[18].Message.ShouldBe("SERVO ALARM 418"); + nineteen[18].Axis.ShouldBe((short)(1 + (18 % 4))); + } + finally + { + client.Dispose(); + await _fx.PatchStateAsync(new { alarms = Array.Empty() }, ct); + } + + static object[] Alarms(int n) => [.. Enumerable.Range(0, n) + .Select(i => (object)new { alm_no = 400 + i, type = 2 + (i % 3), axis = 1 + (i % 4), msg = $"SERVO ALARM {400 + i}" })]; + } + + private static async Task WaitFor(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (!condition()) + { + if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached"); + await Task.Delay(25); + } + } +} From f179829ca6ccf3a99428fd9eb59df32c7b32f331 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:47:24 -0400 Subject: [PATCH 49/51] fix(twincat): review round 1 - PLC-not-in-RUN degrades, per-device health, retry backoff - IMPORTANT: a successful probe records the PLC's ADS state; a PLC not in Run keeps the driver Degraded with 'PLC in {state}' while LastSuccessfulRead still advances (the device answered). RecordDeviceAnswered never restores Healthy while any device is out of Run. - Health is per device, rolled up worst-of: a comms fault is remembered on the device it was seen on and only that device answering clears it (device A answering no longer clears device B's fault). - Refused (non-comms) dead registrations back off 30 s doubling to 10 min and are re-armed by a symbol-version change or retried by the reconnect replay; comms-class failures keep the every-tick retry. A retry whose failure class changes republishes the tag's status, once. - Bit-RMW gates are kept across generations and teardowns (never cleared). - ShutdownAsync honours its CancellationToken: it stops waiting, finishes the bookkeeping and throws OperationCanceledException. - Remarks on why a superseded InitializeAsync throws OperationCanceledException. - REAL / LREAL values (scalar or array element) outside the target range are OverflowException -> BadOutOfRange instead of being written as Infinity. - String / array write encoding factored into AdsTwinCATClient.PlanStringWrite / PlanArrayWrite; the fake runs them so a driver-level test covers the real encode path against a fake resolved symbol. - Timing-fragile tests use condition waits / measured probe-tick windows; the dispose-during-shutdown test asserts ordering with a 1 s bound. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdsTwinCATClient.cs | 83 ++++- .../TwinCATDriver.cs | 184 +++++++++-- .../FakeTwinCATClient.cs | 40 ++- .../TwinCATHealthContractTests.cs | 4 +- .../TwinCATLifecycleTests.cs | 56 +++- .../TwinCATReadWarningTests.cs | 3 +- .../TwinCATReviewFixRound1Tests.cs | 294 ++++++++++++++++++ 7 files changed, 608 insertions(+), 56 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound1Tests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs index a779b09b..9eecbd34 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs @@ -478,11 +478,8 @@ internal sealed class AdsTwinCATClient : ITwinCATClient var (shape, _, symbol, symbolStatus) = await DescribeSymbolAsync(symbolPath, cancellationToken) .ConfigureAwait(false); if (symbolStatus != TwinCATStatusMapper.Good) return symbolStatus; - if (TwinCATTypeGuard.CheckRead(type, shape) is not null) return TwinCATStatusMapper.BadTypeMismatch; - var text = Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; - var encoded = type == TwinCATDataType.WString - ? TwinCATStringCodec.EncodeWString(text, shape.ByteSize) - : TwinCATStringCodec.EncodeString(text, shape.ByteSize, StringsAreUtf8()); + var (planStatus, encoded) = PlanStringWrite(type, shape, value, StringsAreUtf8()); + if (planStatus != TwinCATStatusMapper.Good) return planStatus; return await WriteRawAsync(symbolPath, symbol!, encoded, cancellationToken).ConfigureAwait(false); } @@ -499,6 +496,47 @@ internal sealed class AdsTwinCATClient : ITwinCATClient } } + /// + /// The pure encode plan of a STRING / WSTRING write against the RESOLVED symbol: the declared kind must + /// match the PLC's (, else BadTypeMismatch), then the value is + /// encoded at the PLC's byte width by . + /// + /// The declared String / WString type. + /// The resolved PLC symbol shape. + /// The written value. + /// True when the target reports UTF-8 STRING data. + /// Good with the bytes to write, or the refusal status with no bytes. + /// The value does not fit the PLC string. + internal static (uint status, byte[] bytes) PlanStringWrite( + TwinCATDataType type, TwinCATPlcSymbolShape shape, object? value, bool utf8) + { + if (TwinCATTypeGuard.CheckRead(type, shape) is not null) return (TwinCATStatusMapper.BadTypeMismatch, []); + var text = Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture) ?? string.Empty; + return (TwinCATStatusMapper.Good, type == TwinCATDataType.WString + ? TwinCATStringCodec.EncodeWString(text, shape.ByteSize) + : TwinCATStringCodec.EncodeString(text, shape.ByteSize, utf8)); + } + + /// + /// The pure encode plan of an array write against the RESOLVED PLC array: the layout comes from + /// (the array read's rule, else BadTypeMismatch) and the + /// bytes from . + /// + /// The declared element type. + /// The converted elements. + /// The resolved PLC symbol shape. + /// The resolved PLC array shape, or null for a non-array symbol. + /// True when the target reports UTF-8 STRING data. + /// Good with the bytes to write, or the refusal status with no bytes. + /// A string element does not fit its stride. + internal static (uint status, byte[] bytes) PlanArrayWrite(TwinCATDataType type, IReadOnlyList elements, + TwinCATPlcSymbolShape shape, TwinCATPlcArrayShape? array, bool utf8) + { + if (TwinCATTypeGuard.CheckArray(type, elements.Count, shape, array, out var layout) is not null) + return (TwinCATStatusMapper.BadTypeMismatch, []); + return (TwinCATStatusMapper.Good, TwinCATArrayCodec.Encode(type, elements, layout, utf8)); + } + /// public async Task WriteArrayAsync(string symbolPath, TwinCATDataType type, IReadOnlyList elements, CancellationToken cancellationToken) @@ -510,9 +548,8 @@ internal sealed class AdsTwinCATClient : ITwinCATClient var (shape, array, symbol, symbolStatus) = await DescribeSymbolAsync(symbolPath, cancellationToken) .ConfigureAwait(false); if (symbolStatus != TwinCATStatusMapper.Good) return symbolStatus; - if (TwinCATTypeGuard.CheckArray(type, elements.Count, shape, array, out var layout) is not null) - return TwinCATStatusMapper.BadTypeMismatch; - var bytes = TwinCATArrayCodec.Encode(type, elements, layout, StringsAreUtf8()); + var (planStatus, bytes) = PlanArrayWrite(type, elements, shape, array, StringsAreUtf8()); + if (planStatus != TwinCATStatusMapper.Good) return planStatus; return await WriteRawAsync(symbolPath, symbol!, bytes, cancellationToken).ConfigureAwait(false); } catch (AdsErrorException ex) @@ -1034,14 +1071,40 @@ internal sealed class AdsTwinCATClient : ITwinCATClient TwinCATDataType.UDInt => Convert.ToUInt32(value), TwinCATDataType.LInt => Convert.ToInt64(value), TwinCATDataType.ULInt => Convert.ToUInt64(value), - TwinCATDataType.Real => Convert.ToSingle(value), - TwinCATDataType.LReal => Convert.ToDouble(value), + TwinCATDataType.Real => ToSingleInRange(value), + TwinCATDataType.LReal => ToDoubleInRange(value), TwinCATDataType.String or TwinCATDataType.WString => Convert.ToString(value) ?? string.Empty, TwinCATDataType.Time or TwinCATDataType.Date or TwinCATDataType.DateTime or TwinCATDataType.TimeOfDay => TwinCATTemporalCodec.Encode(type, value), _ => throw new NotSupportedException($"TwinCATDataType {type} not writable."), }; + // Convert.ToSingle(1e300) is +Infinity, and Convert.ToDouble("1e400") too: writing that to the PLC + // would store a value nobody wrote. A finite source that overflows the target is out of range; an + // explicit infinity (or NaN) passes through as the caller's own value. + private static float ToSingleInRange(object? value) + { + var f = Convert.ToSingle(value, System.Globalization.CultureInfo.InvariantCulture); + if (float.IsInfinity(f) && !IsExplicitInfinity(value)) + throw new OverflowException($"{value} is outside the REAL range (±{float.MaxValue:R})"); + return f; + } + + private static double ToDoubleInRange(object? value) + { + var d = Convert.ToDouble(value, System.Globalization.CultureInfo.InvariantCulture); + if (double.IsInfinity(d) && !IsExplicitInfinity(value)) + throw new OverflowException($"{value} is outside the LREAL range"); + return d; + } + + private static bool IsExplicitInfinity(object? value) => value switch + { + float f => float.IsInfinity(f), + double d => double.IsInfinity(d), + _ => false, + }; + private static bool ExtractBit(object? rawWord, int bit) => rawWord switch { short s => (s & (1 << bit)) != 0, diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index fc0469ae..4d9be07b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -51,6 +51,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery private readonly object _lifecycleLock = new(); private long _lifecycleGeneration; + /// Test accessor: the RMW gate for a device + parent word, or null. + internal SemaphoreSlim? BitRmwGateForTest(string host, string parentPath) => + _bitRmwLocks.TryGetValue($"{host}|{parentPath}", out var g) ? g : null; + /// Test seam: awaited after the new generation is built and before it is committed, so a test can /// interleave a teardown deterministically. Never set in production. internal Func? BeforeCommitForTest { get; set; } @@ -73,9 +77,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // read-modify-write of the parent word (TwinCAT's symbol table doesn't expose "Word.N"), // so concurrent bit-writers to the same word must serialise or they lose each other's // updates. Keyed by device + parent symbol. (Cannot guard against the PLC program writing - // the word between our read and write — inherent to RMW.) Teardown only CLEARS it: a - // SemaphoreSlim owns no kernel handle unless AvailableWaitHandle is read, and disposing one an - // in-flight write still holds turns its Release into an ObjectDisposedException. + // the word between our read and write — inherent to RMW.) The gates are KEPT across generations + // and teardowns (never cleared or disposed): a write still in flight on the old generation and the + // first write of the next one must serialise on the SAME gate. A SemaphoreSlim owns no kernel handle + // unless AvailableWaitHandle is read, and the key set is bounded by the authored bit tags. private readonly ConcurrentDictionary _bitRmwLocks = new(StringComparer.OrdinalIgnoreCase); @@ -152,6 +157,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery public string DriverType => "TwinCAT"; /// + /// + /// An initialise that is overtaken before it commits — a / + /// or a newer bumped the lifecycle generation while it was building — releases + /// what it built and throws . Cancellation is the honest outcome: + /// the work was abandoned by a later lifecycle call, not failed, so it must not write Faulted over + /// the newer state or let a caller believe this generation is live; returning normally would claim a + /// driver that was never installed. The runtime has usually abandoned the task by then, and an + /// overlapping retry treats it like any other cancelled attempt. + /// public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { // InitializeAsync is reachable on a live driver with no teardown first (the IDriver contract @@ -353,7 +367,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // The poll loops and the device release wind down in parallel (releasing a client can unblock a // poll loop stuck in a read on it); the whole teardown is bounded. var teardown = Task.WhenAll(_poll.DisposeAsync().AsTask(), ReleaseDevicesAsync(taken)); - try { await teardown.WaitAsync(ShutdownBudget).ConfigureAwait(false); } + var cancelled = false; + try { await teardown.WaitAsync(ShutdownBudget, cancellationToken).ConfigureAwait(false); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // The caller stopped waiting. The tables are already swapped out and every loop is cancelled, so + // the rest of the teardown finishes in the background; report the cancellation after the cheap + // bookkeeping below. + cancelled = true; + } catch (TimeoutException) { _logger.LogWarning( @@ -364,8 +386,8 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery _warnedRefs.Clear(); _resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry - _bitRmwLocks.Clear(); _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null); + if (cancelled) throw new OperationCanceledException(cancellationToken); } /// @@ -448,7 +470,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery var (value, status) = await client.ReadValueAsync( symbolName, def.DataType, parsed?.BitIndex, def.ArrayLength, cancellationToken) .ConfigureAwait(false); - return CompleteRead(reference, def, symbolName, value, status, now); + return CompleteRead(device, reference, def, symbolName, value, status, now); } catch (OperationCanceledException) { throw; } catch (Exception ex) @@ -459,13 +481,13 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } /// Maps one read outcome to a snapshot and applies the health / log side effects. - private DataValueSnapshot CompleteRead( + private DataValueSnapshot CompleteRead(DeviceState device, string reference, TwinCATTagDefinition def, string symbolName, object? value, uint status, DateTime now) { if (status == TwinCATStatusMapper.Good) { _warnedRefs.TryRemove(reference, out _); // a later failure is a new episode and warns again - RecordDeviceAnswered(now, force: true); + RecordDeviceAnswered(now, device, force: true); return new DataValueSnapshot(value, status, now, now); } @@ -487,7 +509,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } if (IsCommsStatus(status)) - MarkCommsFault($"ADS status {status:X8} reading {reference}"); + MarkCommsFault($"ADS status {status:X8} reading {reference}", device); else RecordDeviceAnsweredKeepState(now); // a refusal: the device answered — no state change return new DataValueSnapshot(value, status, now, now); @@ -558,7 +580,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { var e = chunk[k]; var (value, status) = k < bulk.Count ? bulk[k] : (null, TwinCATStatusMapper.BadCommunicationError); - results[e.Index] = CompleteRead(e.Reference, e.Def, e.SymbolName, value, status, now); + results[e.Index] = CompleteRead(device, e.Reference, e.Def, e.SymbolName, value, status, now); } } catch (OperationCanceledException) { throw; } @@ -842,7 +864,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // A push is the device answering: stamp LastSuccessfulRead so a notification-only // driver does not look silent to the comms-loss watchdog. var now = DateTime.UtcNow; - RecordDeviceAnswered(now); + RecordDeviceAnswered(now, device); RaiseDataChange(handle, reference, new DataValueSnapshot(value, TwinCATStatusMapper.Good, now, now)); }, // When the handle goes dead (a failed replay) push a Bad snapshot so @@ -856,6 +878,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // Same device already failed to connect in this call: keep the intent dead for the // probe-tick retry / reconnect replay and surface Bad now. device.NativeRegistrations[reg.Id] = reg; + NoteDeadFailure(reg, TwinCATStatusMapper.BadCommunicationError, refused: false, DateTime.UtcNow); RaiseDataChange(handle, reference, new DataValueSnapshot( null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)); continue; @@ -876,6 +899,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery _logger.LogWarning( "TwinCAT driver '{DriverInstanceId}' refused native notification for '{Reference}': {Reason}", _driverInstanceId, reference, refused.Message); + NoteDeadFailure(reg, refused.StatusCode, refused: true, DateTime.UtcNow); RaiseDataChange(handle, reference, new DataValueSnapshot( null, refused.StatusCode, null, DateTime.UtcNow)); } @@ -885,6 +909,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // Every tag of this device goes Bad + dead for the retry; other devices carry on. (unreachable ??= []).Add(device); MarkCommsFault(unavailable.InnerException?.Message ?? unavailable.Message, device); + NoteDeadFailure(reg, TwinCATStatusMapper.BadCommunicationError, refused: false, DateTime.UtcNow); RaiseDataChange(handle, reference, new DataValueSnapshot( null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)); } @@ -897,6 +922,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery "'{Reference}'; it will be retried on the next healthy probe tick", _driverInstanceId, reference); MarkCommsFault(ex.Message, device); + NoteDeadFailure(reg, TwinCATStatusMapper.BadCommunicationError, refused: false, DateTime.UtcNow); RaiseDataChange(handle, reference, new DataValueSnapshot( null, TwinCATStatusMapper.BadCommunicationError, null, DateTime.UtcNow)); } @@ -928,7 +954,9 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// /// Records a comms fault: preserving LastSuccessfulRead. - /// Never downgrades . + /// Never downgrades . A fault observed on a device is remembered ON + /// that device () and only that device answering clears it — the + /// driver state is the worst of every device (#739 fix round 1). /// /// The fault description. /// The device the fault was observed on; a fault on a device already swapped out of @@ -937,6 +965,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery private void MarkCommsFault(string message, DeviceState? device = null) { if (device?.Released == true) return; + if (device is not null) device.CommsFault = message; var current = _health; if (current.State != DriverState.Faulted) _health = new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, message); @@ -954,29 +983,90 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery private static readonly TimeSpan AnsweredStampInterval = TimeSpan.FromMilliseconds(250); /// - /// Records that a device answered (a native push, a successful probe, a Good read): advances - /// LastSuccessfulRead and restores unless a device is still - /// probe-unreachable (then the state stays Degraded, the stamp still advances). Throttled so a + /// Why one device keeps the driver Degraded, or null when it does not: probe-unreachable, a PLC whose + /// ADS state is not Run (coordinator ruling: a stopped / config-mode PLC is Degraded with a + /// reason even though it answers), or an uncleared comms fault seen on it. + /// + private static string? DeviceDegradedReason(DeviceState d) + { + if (d.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded) + return $"TwinCAT device {d.Options.HostAddress} unreachable ({d.ConsecutiveProbeFailures} consecutive failed probes)"; + if (d.PlcState is { } st && !string.Equals(st, "Run", StringComparison.OrdinalIgnoreCase)) + return $"TwinCAT device {d.Options.HostAddress}: PLC in {st}"; + return d.CommsFault; + } + + /// + /// Records that a device answered (a native push, a successful probe, a Good read): clears THAT + /// device's comms fault, advances LastSuccessfulRead, and sets the state to the worst of every + /// device — Healthy only when no device is unreachable, out of RUN or still faulted (then Degraded + /// with that device's reason; the stamp still advances, since a device did answer). Throttled so a /// notification burst does not allocate a health record per value; never touches Faulted. /// /// The time the device answered. + /// The device that answered; null when not attributable. /// Write even inside the throttle window (an explicit read / probe outcome). - private void RecordDeviceAnswered(DateTime now, bool force = false) + private void RecordDeviceAnswered(DateTime now, DeviceState? device = null, bool force = false) { + if (device?.Released == true) return; + if (device?.CommsFault is not null) + { + device.CommsFault = null; + force = true; + } var current = _health; if (current.State == DriverState.Faulted) return; // Hot path (a notification on a healthy driver inside the window): no device scan, no allocation. if (!force && current.State == DriverState.Healthy && current.LastSuccessfulRead is { } recent && now - recent < AnsweredStampInterval) return; - var unreachable = _devices.Values.FirstOrDefault(d => d.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded); - var state = unreachable is null ? DriverState.Healthy : DriverState.Degraded; + string? reason = null; + foreach (var d in _devices.Values) + if ((reason = DeviceDegradedReason(d)) is not null) break; + var state = reason is null ? DriverState.Healthy : DriverState.Degraded; if (!force && current.State == state && current.LastSuccessfulRead is { } last && now - last < AnsweredStampInterval) return; - _health = new DriverHealth(state, now, unreachable is null - ? null - : $"TwinCAT device {unreachable.Options.HostAddress} unreachable ({unreachable.ConsecutiveProbeFailures} consecutive failed probes)"); + _health = new DriverHealth(state, now, reason); + } + + /// First probe-tick retry delay of a REFUSED dead registration (the PLC answered and refused it). + internal static readonly TimeSpan RefusedRetryInitial = TimeSpan.FromSeconds(30); + + /// Cap of the refused-registration retry backoff. + internal static readonly TimeSpan RefusedRetryMax = TimeSpan.FromMinutes(10); + + /// + /// Books a failed registration attempt. A REFUSAL is deterministic until the PLC program changes, so it + /// backs off exponentially (30 s doubling to 10 min) and is re-armed at once by a symbol-version change + /// or retried by the reconnect replay; a comms-class failure stays on the every-tick retry. + /// + /// True when the published status changes (the caller republishes the tag). + private static bool NoteDeadFailure(NativeRegistration reg, uint status, bool refused, DateTime now) + { + if (refused) + { + reg.RefusalBackoff = reg.RefusalBackoff == TimeSpan.Zero + ? RefusedRetryInitial + : TimeSpan.FromTicks(Math.Min(reg.RefusalBackoff.Ticks * 2, RefusedRetryMax.Ticks)); + reg.NextRetryUtc = now + reg.RefusalBackoff; + } + else + { + reg.RefusalBackoff = TimeSpan.Zero; + reg.NextRetryUtc = DateTime.MinValue; + } + var changed = reg.DeadStatus != status; + reg.DeadStatus = status; + return changed; + } + + /// Clears the dead-registration bookkeeping after a successful (re-)registration. + private static void NoteLive(NativeRegistration reg) + { + reg.DeadStatus = null; + reg.RefusalBackoff = TimeSpan.Zero; + reg.NextRetryUtc = DateTime.MinValue; } /// @@ -1159,6 +1249,15 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery public ITwinCATNotificationHandle? MarkHandleDead() => Interlocked.Exchange(ref _handle, null); + /// The Bad status last published for this registration while dead; null while live. + public uint? DeadStatus { get; set; } + + /// Earliest time the probe-tick retry may re-attempt this dead registration. + public DateTime NextRetryUtc { get; set; } + + /// Current refusal backoff (zero after a comms-class failure or a success). + public TimeSpan RefusalBackoff { get; set; } + private volatile bool _disposed; /// Gets a value indicating whether this registration has been disposed (its callbacks are dropped). @@ -1195,6 +1294,9 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery var client = await EnsureConnectedAsync(state, ct, _options.Probe.Timeout, bypassBackoff: true) .ConfigureAwait(false); success = await client.ProbeAsync(ct).ConfigureAwait(false); + // The probe's ReadState also says whether the PLC is in RUN; a stopped PLC answers but + // serves no live data, so it keeps the driver Degraded (RecordDeviceAnswered below). + if (success) state.PlcState = client.LastAdsState; // A locally-connected client whose wire probe FAILS is wire-dead but port-open: // force a recycle so the next tick rebuilds + replays native notifications. Without @@ -1224,13 +1326,13 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery if (success) { state.ConsecutiveProbeFailures = 0; - RecordDeviceAnswered(DateTime.UtcNow, force: true); + RecordDeviceAnswered(DateTime.UtcNow, state, force: true); if (probed is not null) ReportNotificationDrops(state, probed); } else if (++state.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded) { MarkCommsFault($"TwinCAT device {state.Options.HostAddress} unreachable " + - $"({state.ConsecutiveProbeFailures} consecutive failed probes)"); + $"({state.ConsecutiveProbeFailures} consecutive failed probes)", state); } TransitionDeviceState(state, success ? HostState.Running : HostState.Stopped); @@ -1625,6 +1727,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery try { orphan?.Dispose(); } catch { /* best-effort */ } continue; } + NoteLive(reg); replayed++; } catch (TwinCATNotificationRefusedException refused) @@ -1638,7 +1741,8 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery _driverInstanceId, reg.SymbolName, device.Options.HostAddress, refused.Message); var dead = reg.MarkHandleDead(); try { dead?.Dispose(); } catch { /* already-dead handle from the disposed client */ } - try { reg.OnRefused(refused.StatusCode); } catch { /* never abort recovery of the rest */ } + if (NoteDeadFailure(reg, refused.StatusCode, refused: true, DateTime.UtcNow)) + try { reg.OnRefused(refused.StatusCode); } catch { /* never abort recovery of the rest */ } } catch (Exception ex) { @@ -1652,8 +1756,9 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // no quality channel), and degrade health preserving the last successful read. var dead = reg.MarkHandleDead(); try { dead?.Dispose(); } catch { /* already-dead handle from the disposed client */ } - try { reg.OnUnavailable(); } catch { /* never let a subscriber callback abort recovery of the rest */ } - MarkCommsFault($"native-notification replay failed for '{reg.SymbolName}' on {device.Options.HostAddress}"); + if (NoteDeadFailure(reg, TwinCATStatusMapper.BadCommunicationError, refused: false, DateTime.UtcNow)) + try { reg.OnUnavailable(); } catch { /* never let a subscriber callback abort recovery of the rest */ } + MarkCommsFault($"native-notification replay failed for '{reg.SymbolName}' on {device.Options.HostAddress}", device); } } @@ -1682,9 +1787,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery if (device.Client is not { IsConnected: true } client) return; var recovered = 0; + var now = DateTime.UtcNow; foreach (var reg in device.NativeRegistrations.Values) { if (reg.HasLiveHandle) continue; + // A refused registration waits out its backoff (a symbol-version change re-arms it). + if (now < reg.NextRetryUtc) continue; try { var newHandle = await client.AddNotificationAsync( @@ -1698,11 +1806,18 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery try { orphan?.Dispose(); } catch { /* best-effort */ } continue; } + NoteLive(reg); recovered++; } catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } catch (Exception ex) { + // Republish only when the failure CLASS changes (a comms blip that turned into a refusal, + // or the reverse); the same failure again publishes nothing. + var refusal = ex as TwinCATNotificationRefusedException; + var status = refusal?.StatusCode ?? TwinCATStatusMapper.BadCommunicationError; + if (NoteDeadFailure(reg, status, refusal is not null, now)) + try { reg.OnRefused(status); } catch { /* never abort the rest */ } _logger.LogDebug(ex, "TwinCAT driver '{DriverInstanceId}' probe-tick retry of dead notification " + "'{Symbol}' on device '{HostAddress}' failed; will retry next tick", @@ -1758,10 +1873,18 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// space must be rebuilt — this is the documented TwinCAT failure mode, not a transient /// connection error. /// - private void HandleSymbolVersionChanged(object? sender, EventArgs e) => + private void HandleSymbolVersionChanged(object? sender, EventArgs e) + { + // A program change can make a refused registration valid: re-arm every backed-off dead intent of the + // device whose client saw it, so the next probe tick re-checks them at once. + foreach (var d in _devices.Values) + if (ReferenceEquals(d.Client, sender)) + foreach (var reg in d.NativeRegistrations.Values) + reg.NextRetryUtc = DateTime.MinValue; OnRediscoveryNeeded?.Invoke(this, new RediscoveryEventArgs( "TwinCAT symbol-version-changed (DeviceSymbolVersionInvalid 0x0711) — PLC program re-downloaded", ScopeHint: "TwinCAT")); + } /// /// Synchronous teardown — no await, no captured sync context. The OPC UA stack @@ -1790,7 +1913,6 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery _warnedRefs.Clear(); _resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry - _bitRmwLocks.Clear(); _health = new DriverHealth(DriverState.Unknown, _health.LastSuccessfulRead, null); } @@ -1852,6 +1974,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// True once an unexpected sum-read chunk failure was logged for this device. public volatile bool SumChunkFailureWarned; + /// The last comms fault seen on this device; cleared when this device answers. Null = none. + public volatile string? CommsFault; + + /// The ADS state (Run, Stop, Config, …) the last successful probe read; null = unknown. + public volatile string? PlcState; + /// Consecutive failed probe ticks; reset by a successful probe. Written only by this device's probe loop. public volatile int ConsecutiveProbeFailures; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs index 909fc31f..49a6deea 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Runtime.CompilerServices; using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; @@ -142,6 +143,14 @@ internal class FakeTwinCATClient : ITwinCATClient string symbolPath, TwinCATDataType type, int? bitIndex, object? value, CancellationToken ct) { if (ThrowOnWrite) throw Exception ?? new InvalidOperationException(); + if (type is TwinCATDataType.String or TwinCATDataType.WString + && SymbolShapes.TryGetValue(symbolPath, out var shape)) + { + // The real client's encode plan (resolve → CheckRead → codec at the PLC width). + var (planStatus, bytes) = AdsTwinCATClient.PlanStringWrite(type, shape, value, utf8: false); + if (planStatus != TwinCATStatusMapper.Good) return Task.FromResult(planStatus); + RawWrites[symbolPath] = bytes; + } WriteLog.Add((symbolPath, type, bitIndex, value)); Values[symbolPath] = value; var status = WriteStatuses.TryGetValue(symbolPath, out var s) ? s : TwinCATStatusMapper.Good; @@ -161,11 +170,26 @@ internal class FakeTwinCATClient : ITwinCATClient CancellationToken ct) { if (ThrowOnWrite) throw Exception ?? new InvalidOperationException(); + if (ArrayShapes.TryGetValue(symbolPath, out var array) && SymbolShapes.TryGetValue(symbolPath, out var shape)) + { + // The real client's encode plan (CheckArray layout → TwinCATArrayCodec.Encode). + var (planStatus, bytes) = AdsTwinCATClient.PlanArrayWrite(type, elements, shape, array, utf8: false); + if (planStatus != TwinCATStatusMapper.Good) return Task.FromResult(planStatus); + RawWrites[symbolPath] = bytes; + } ArrayWriteLog.Add((symbolPath, type, elements)); Values[symbolPath] = elements; return Task.FromResult(WriteStatuses.TryGetValue(symbolPath, out var s) ? s : TwinCATStatusMapper.Good); } + /// Resolved PLC symbol shapes; a string / array write to a symbol listed here runs the real client's + /// encode plan and lands its bytes in (#739 fix round 1). + public Dictionary SymbolShapes { get; } = new(StringComparer.OrdinalIgnoreCase); + /// Resolved PLC array shapes for entries that are arrays. + public Dictionary ArrayShapes { get; } = new(StringComparer.OrdinalIgnoreCase); + /// The raw bytes the encode plan produced, by symbol. + public ConcurrentDictionary RawWrites { get; } = new(StringComparer.OrdinalIgnoreCase); + /// Raw bytes of bit-write parent words, by parent path (#739 H18). The byte count is the parent's width. public Dictionary ParentWords { get; } = new(StringComparer.OrdinalIgnoreCase); /// Optional parent shapes; absent = an unsigned integer word as wide as its entry. @@ -205,10 +229,16 @@ internal class FakeTwinCATClient : ITwinCATClient /// A task that returns the probe result. public virtual Task ProbeAsync(CancellationToken ct) { + Interlocked.Increment(ref _probeCount); if (ThrowOnProbe) return Task.FromResult(false); return Task.FromResult(ProbeResult); } + private int _probeCount; + + /// Gets how many times ran — a measured window for absence assertions. + public int ProbeCount => Volatile.Read(ref _probeCount); + /// Releases unmanaged resources. public virtual void Dispose() { @@ -224,7 +254,14 @@ internal class FakeTwinCATClient : ITwinCATClient public bool ThrowOnAddNotification { get; set; } /// Per-symbol exceptions throws (e.g. a /// for one refused tag). - public Dictionary AddNotificationErrors { get; } = new(StringComparer.OrdinalIgnoreCase); + public ConcurrentDictionary AddNotificationErrors { get; } = new(StringComparer.OrdinalIgnoreCase); + + private readonly ConcurrentDictionary _addAttempts = new(StringComparer.OrdinalIgnoreCase); + + /// How many times was called for . + /// The symbol path. + /// The attempt count. + public int AttemptsFor(string symbolPath) => _addAttempts.TryGetValue(symbolPath, out var n) ? n : 0; /// Records the most recently-supplied maxDelayMs for Driver.TwinCAT-014 tests. public int LastMaxDelayMs { get; private set; } /// When set, awaits this gate before creating the handle — @@ -245,6 +282,7 @@ internal class FakeTwinCATClient : ITwinCATClient string symbolPath, TwinCATDataType type, int? bitIndex, int? arrayCount, TimeSpan cycleTime, int maxDelayMs, Action onChange, CancellationToken cancellationToken) { + _addAttempts.AddOrUpdate(symbolPath, 1, (_, n) => n + 1); if (ThrowOnAddNotification) throw Exception ?? new InvalidOperationException("fake AddNotification failure"); if (AddNotificationErrors.TryGetValue(symbolPath, out var perSymbol)) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATHealthContractTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATHealthContractTests.cs index f1c06bfd..19beaeff 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATHealthContractTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATHealthContractTests.cs @@ -50,7 +50,9 @@ public sealed class TwinCATHealthContractTests drv.GetHealth().State.ShouldBe(DriverState.Degraded); drv.GetHealth().LastSuccessfulRead.ShouldBe(initStamp); - await Task.Delay(300, TestContext.Current.CancellationToken); + // Condition, not a fixed delay: the clock only has to move past the stamp (the Degraded state means + // the notification is not inside the Healthy throttle window). + await WaitForAsync(() => DateTime.UtcNow > initStamp!.Value, TimeSpan.FromSeconds(2)); fake.FireNotification("MAIN.Speed", 7); var after = drv.GetHealth(); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs index 222495fd..915316f8 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATLifecycleTests.cs @@ -80,8 +80,10 @@ public sealed class TwinCATLifecycleTests var shutdown = drv.ShutdownAsync(CancellationToken.None); var sw = Stopwatch.StartNew(); await drv.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); - sw.Elapsed.ShouldBeLessThan(TimeSpan.FromMilliseconds(250)); + // Ordering is the real assertion: dispose finished while the first shutdown was still joining the + // wedged probe. The elapsed bound is generous (the runtime allows 250 ms; a loaded CI box gets 1 s). shutdown.IsCompleted.ShouldBeFalse("the positive control: the first shutdown really was still joining the probe"); + sw.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(1)); wedge.SetResult(); await shutdown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); @@ -109,15 +111,18 @@ public sealed class TwinCATLifecycleTests await Should.ThrowAsync(() => abandoned.WaitAsync(TimeSpan.FromSeconds(5))); drv.DeviceCount.ShouldBe(0); - await Task.Delay(150, TestContext.Current.CancellationToken); // several probe intervals - factory.Clients.ShouldBeEmpty("no probe loop of the overtaken generation may be running"); + factory.Clients.ShouldBeEmpty(); drv.GetHealth().State.ShouldBe(DriverState.Unknown); + // Positive control + measured window: a fresh generation's probe creates exactly one client and + // ticks several times; an overtaken-generation probe loop (same interval, started earlier) would have + // created a second client within that window. drv.BeforeCommitForTest = null; await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); drv.DeviceCount.ShouldBe(1); - await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); - factory.Clients.Count.ShouldBe(1); + await WaitForAsync(() => factory.Clients.Count >= 1 && factory.Clients[0].ProbeCount >= 4, TimeSpan.FromSeconds(3)); + factory.Clients[0].ProbeCount.ShouldBeGreaterThanOrEqualTo(4); + factory.Clients.Count.ShouldBe(1, "no probe loop of the overtaken generation may be running"); await drv.ShutdownAsync(TestContext.Current.CancellationToken); } @@ -145,7 +150,7 @@ public sealed class TwinCATLifecycleTests await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); var abandoned = drv.InitializeAsync("{}", CancellationToken.None); // joins the wedged generation-1 probe - await Task.Delay(50, TestContext.Current.CancellationToken); + await WaitForAsync(() => drv.DeviceCount == 1 && factory.Clients.Count >= 2, TimeSpan.FromSeconds(3)); // gen 2 committed + probing var sw = Stopwatch.StartNew(); await drv.ShutdownAsync(TestContext.Current.CancellationToken).WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ShutdownBudget + TimeSpan.FromMilliseconds(500)); @@ -157,7 +162,15 @@ public sealed class TwinCATLifecycleTests drv.DeviceCount.ShouldBe(0); await WaitForAsync(() => factory.Clients.All(c => c.DisposeCount > 0 || !c.IsConnected), TimeSpan.FromSeconds(2)); var settled = factory.Clients.Count; - await Task.Delay(150, TestContext.Current.CancellationToken); + + // Measured window: a sibling driver on the same probe interval ticks 4 times; any surviving probe + // loop of the torn-down driver would have built a new client in that time. + var witnessFactory = new FakeTwinCATClientFactory(); + var witness = NewDriver(witnessFactory); + await witness.InitializeAsync("{}", TestContext.Current.CancellationToken); + await WaitForAsync(() => witnessFactory.Clients.Count >= 1 && witnessFactory.Clients[0].ProbeCount >= 4, TimeSpan.FromSeconds(3)); + witnessFactory.Clients[0].ProbeCount.ShouldBeGreaterThanOrEqualTo(4); + await witness.ShutdownAsync(TestContext.Current.CancellationToken); factory.Clients.Count.ShouldBe(settled, "no probe loop may still be creating clients"); factory.Clients.ShouldAllBe(c => c.DisposeCount > 0 || !c.IsConnected); } @@ -231,20 +244,33 @@ public sealed class TwinCATLifecycleTests var factory = new FakeTwinCATClientFactory(); var drv = NewDriver(factory, probe: false); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); - using var stop = new CancellationTokenSource(TimeSpan.FromMilliseconds(400)); + // Run for at least 300 ms of overlap AND until the reader has seen a Good read mid-churn (positive + // control), bounded at 10 s — a condition, not a fixed window, so a loaded machine cannot starve it. var good = 0; + var reads = 0; + var inits = 0; + var minUntil = DateTime.UtcNow.AddMilliseconds(300); + var deadline = DateTime.UtcNow.AddSeconds(10); + bool Done() => (DateTime.UtcNow > minUntil && Volatile.Read(ref good) > 0 && Volatile.Read(ref inits) > 10) + || DateTime.UtcNow > deadline; var reader = Task.Run(async () => { - while (!stop.IsCancellationRequested) + while (!Done()) { var r = await drv.ReadAsync(["Speed"], CancellationToken.None); + Interlocked.Increment(ref reads); if (r[0].StatusCode == TwinCATStatusMapper.Good) Interlocked.Increment(ref good); } }, TestContext.Current.CancellationToken); - while (!stop.IsCancellationRequested) + while (!Done()) + { await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + Interlocked.Increment(ref inits); + await Task.Yield(); // let the reader in between generations + } await reader; // throws if any read threw good.ShouldBeGreaterThan(0); + inits.ShouldBeGreaterThan(10); } /// Notification-queue drops are reported once per probe tick that sees new drops. @@ -255,14 +281,14 @@ public sealed class TwinCATLifecycleTests var log = new TwinCATCapturingLogger(); var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, log: log); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); - await WaitForAsync(() => fake.IsConnected, TimeSpan.FromSeconds(2)); - await Task.Delay(100, TestContext.Current.CancellationToken); - log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(0); // no drops, no warning + await WaitForAsync(() => fake.ProbeCount >= 3, TimeSpan.FromSeconds(3)); + log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(0); // 3 ticks, no drops, no warning fake.DroppedNotificationCount = 5; await WaitForAsync(() => log.Warnings.Any(w => w.Contains("dropped 5")), TimeSpan.FromSeconds(2)); - await Task.Delay(100, TestContext.Current.CancellationToken); - log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(1); + var ticks = fake.ProbeCount; + await WaitForAsync(() => fake.ProbeCount >= ticks + 3, TimeSpan.FromSeconds(3)); + log.Warnings.Count(w => w.Contains("dropped")).ShouldBe(1); // 3 more ticks, no new drops, no repeat fake.DroppedNotificationCount = 8; await WaitForAsync(() => log.Warnings.Any(w => w.Contains("dropped 3")), TimeSpan.FromSeconds(2)); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWarningTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWarningTests.cs index dd669164..eeb78937 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWarningTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReadWarningTests.cs @@ -87,7 +87,8 @@ public sealed class TwinCATReadWarningTests await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); var initStamp = drv.GetHealth().LastSuccessfulRead!.Value; fake.ReadStatuses["MAIN.Typo"] = TwinCATStatusMapper.BadNodeIdUnknown; - await Task.Delay(300, TestContext.Current.CancellationToken); + var deadline = DateTime.UtcNow.AddSeconds(2); + while (DateTime.UtcNow <= initStamp && DateTime.UtcNow < deadline) await Task.Delay(5, TestContext.Current.CancellationToken); _ = await drv.ReadAsync(["Typo"], TestContext.Current.CancellationToken); var afterRefusal = drv.GetHealth(); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound1Tests.cs new file mode 100644 index 00000000..35ff945b --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound1Tests.cs @@ -0,0 +1,294 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using Shouldly; +using TwinCAT.TypeSystem; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// #739 review fix round 1: the PLC-not-in-RUN health ruling, refused-registration retry backoff, +/// per-device worst-of health, RMW gates surviving teardown, a cancellable shutdown, the real +/// STRING / array encode path through the client planning helpers, and REAL range checks. +/// +[Trait("Category", "Unit")] +public sealed class TwinCATReviewFixRound1Tests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + private const string HostB = "ads://5.23.91.24.1.1:851"; + + private static TwinCATProbeOptions FastProbe => new() + { + Enabled = true, + Interval = TimeSpan.FromMilliseconds(30), + Timeout = TimeSpan.FromMilliseconds(100), + }; + + private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, TwinCATProbeOptions probe, bool native, + params TwinCATTagDefinition[] tags) => + new(new TwinCATDriverOptions + { + Devices = [.. tags.Select(t => t.DeviceHostAddress).Append(Host).Distinct().Select(h => new TwinCATDeviceOptions(h))], + RawTags = TwinCATRawTags.Entries(tags), + Probe = probe, + UseNativeNotifications = native, + }, "drv-1", factory); + + private static async Task WaitForAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(10); + } + + // ---- IMPORTANT: a PLC not in RUN leaves the driver Degraded ---- + + /// + /// A probe that answers with ADS state Stop writes Degraded "PLC in Stop" and still advances + /// LastSuccessfulRead (the device answered); back in Run the driver is Healthy (positive control). + /// + [Fact] + public async Task A_PLC_not_in_Run_is_Degraded_and_Run_restores_Healthy() + { + var fake = new FakeTwinCATClient { AdsStateText = "Stop" }; + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: false, + new TwinCATTagDefinition("Speed", Host, "MAIN.Speed", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var initStamp = drv.GetHealth().LastSuccessfulRead!.Value; + + await WaitForAsync(() => drv.GetHealth().State == DriverState.Degraded, TimeSpan.FromSeconds(3)); + var stopped = drv.GetHealth(); + stopped.State.ShouldBe(DriverState.Degraded); + stopped.LastError.ShouldNotBeNull().ShouldContain("PLC in Stop"); + await WaitForAsync(() => drv.GetHealth().LastSuccessfulRead > initStamp, TimeSpan.FromSeconds(3)); + drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp); + + // A Good read while the PLC is stopped must not paper over it. + _ = await drv.ReadAsync(["Speed"], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + + fake.AdsStateText = "Run"; + await WaitForAsync(() => drv.GetHealth().State == DriverState.Healthy, TimeSpan.FromSeconds(3)); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + // ---- MINOR 1: refused dead registrations back off; a class change republishes ---- + + /// + /// A REFUSED dead registration is not re-attempted every probe tick; a comms-failed one is (positive + /// control: it recovers). A symbol-version-changed signal re-arms the refused one at once. + /// + [Fact] + public async Task Refused_registrations_back_off_while_comms_failures_retry_each_tick() + { + var fake = new FakeTwinCATClient(); + fake.AddNotificationErrors["MAIN.T"] = new TwinCATTypeMismatchException("MAIN.T", "declared DInt, PLC STRING"); + fake.AddNotificationErrors["MAIN.C"] = new InvalidOperationException("transient"); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: true, + new TwinCATTagDefinition("T", Host, "MAIN.T", TwinCATDataType.DInt), + new TwinCATTagDefinition("C", Host, "MAIN.C", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.SubscribeAsync(["T", "C"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + + await WaitForAsync(() => fake.AttemptsFor("MAIN.C") >= 3, TimeSpan.FromSeconds(3)); + fake.AttemptsFor("MAIN.C").ShouldBeGreaterThanOrEqualTo(3); // comms class: every tick + fake.AttemptsFor("MAIN.T").ShouldBe(1); // refusal: backed off + + fake.AddNotificationErrors.TryRemove("MAIN.C", out _); + await WaitForAsync(() => fake.Notifications.Any(n => n.SymbolPath == "MAIN.C"), TimeSpan.FromSeconds(3)); + fake.Notifications.ShouldContain(n => n.SymbolPath == "MAIN.C"); + + fake.FireSymbolVersionChanged(); + await WaitForAsync(() => fake.AttemptsFor("MAIN.T") >= 2, TimeSpan.FromSeconds(3)); + fake.AttemptsFor("MAIN.T").ShouldBe(2); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + /// When a retry fails with a different failure class the tag's status is republished, once. + [Fact] + public async Task A_retry_whose_failure_class_changes_republishes_the_status() + { + var fake = new FakeTwinCATClient(); + fake.AddNotificationErrors["MAIN.X"] = new InvalidOperationException("router hiccup"); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: true, + new TwinCATTagDefinition("X", Host, "MAIN.X", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + _ = await drv.SubscribeAsync(["X"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + events.ShouldHaveSingleItem().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadCommunicationError); + await WaitForAsync(() => fake.AttemptsFor("MAIN.X") >= 3, TimeSpan.FromSeconds(3)); + events.Count.ShouldBe(1, "a repeat of the same failure class publishes nothing"); + + fake.AddNotificationErrors["MAIN.X"] = new TwinCATNotificationRefusedException( + "MAIN.X", "symbol gone", TwinCATStatusMapper.BadNodeIdUnknown); + await WaitForAsync(() => events.Count >= 2, TimeSpan.FromSeconds(3)); + events.Last().Snapshot.StatusCode.ShouldBe(TwinCATStatusMapper.BadNodeIdUnknown); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + // ---- MINOR 2: per-device worst-of health ---- + + /// + /// A comms fault on device B is not cleared by device A answering; B answering clears it + /// (positive control). + /// + [Fact] + public async Task One_devices_answer_does_not_clear_another_devices_comms_fault() + { + var clients = new ConcurrentDictionary(); + var build = 0; + var factory = new FakeTwinCATClientFactory + { + Customise = () => + { + var c = new FakeTwinCATClient(); + clients[Interlocked.Increment(ref build)] = c; + return c; + }, + }; + var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }, native: false, + new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt), + new TwinCATTagDefinition("B", HostB, "MAIN.B", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); // client 1 = device A + _ = await drv.ReadAsync(["B"], TestContext.Current.CancellationToken); // client 2 = device B + clients[2].ReadStatuses["MAIN.B"] = TwinCATStatusMapper.BadTimeout; + + _ = await drv.ReadAsync(["B"], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + + _ = await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded, "device A answering says nothing about device B"); + + clients[2].ReadStatuses.Remove("MAIN.B"); + _ = await drv.ReadAsync(["B"], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + // ---- MINOR 3: RMW gates are kept across generations ---- + + /// The per-parent RMW gate survives a teardown (a write still in flight keeps the same gate). + [Fact] + public async Task Bit_RMW_gates_survive_a_shutdown() + { + var fake = new FakeTwinCATClient { ParentWords = { ["MAIN.Flags"] = [0, 0] } }; + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, + new TwinCATProbeOptions { Enabled = false }, native: false, + new TwinCATTagDefinition("Bit", Host, "MAIN.Flags.1", TwinCATDataType.Bool)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.WriteAsync([new WriteRequest("Bit", true)], TestContext.Current.CancellationToken); + var gate = drv.BitRmwGateForTest(Host, "MAIN.Flags").ShouldNotBeNull(); + + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.WriteAsync([new WriteRequest("Bit", false)], TestContext.Current.CancellationToken); + + drv.BitRmwGateForTest(Host, "MAIN.Flags").ShouldBeSameAs(gate); + } + + // ---- MINOR 4: ShutdownAsync honours its token ---- + + /// + /// A cancelled ShutdownAsync stops waiting on a probe wedged in connect and throws + /// OperationCanceledException; the teardown itself still happened (tables emptied). + /// + [Fact] + public async Task Shutdown_honours_its_cancellation_token() + { + var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var factory = new FakeTwinCATClientFactory { Customise = () => new FakeTwinCATClient { BlockConnect = wedge } }; + var drv = NewDriver(factory, FastProbe, native: false); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + await WaitForAsync(() => factory.Clients.Count == 1, TimeSpan.FromSeconds(2)); + + using var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100)); + var sw = Stopwatch.StartNew(); + await Should.ThrowAsync(() => drv.ShutdownAsync(cts.Token)); + sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ProbeJoinTimeout, "the wait ended on the token, not the join bound"); + drv.DeviceCount.ShouldBe(0); + wedge.SetResult(); + } + + // ---- MINOR 7: the real STRING / array encode path through the client planning helpers ---- + + /// + /// A STRING write through the driver lands as NUL-padded Windows-1252 bytes at the PLC symbol's width; + /// an over-long value is BadOutOfRange and a WSTRING symbol under a String tag BadTypeMismatch. + /// + [Fact] + public async Task String_writes_go_through_the_client_encode_plan() + { + var fake = new FakeTwinCATClient(); + fake.SymbolShapes["MAIN.Name"] = TwinCATPlcSymbolShape.Classify("STRING(4)", DataTypeCategory.String, 5); + fake.SymbolShapes["MAIN.Wide"] = TwinCATPlcSymbolShape.Classify("WSTRING(4)", DataTypeCategory.String, 10); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, + new TwinCATProbeOptions { Enabled = false }, native: false, + new TwinCATTagDefinition("Name", Host, "MAIN.Name", TwinCATDataType.String), + new TwinCATTagDefinition("Wide", Host, "MAIN.Wide", TwinCATDataType.String)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var results = await drv.WriteAsync( + [ + new WriteRequest("Name", "äb"), + new WriteRequest("Name", "toolong"), + new WriteRequest("Wide", "x"), + ], TestContext.Current.CancellationToken); + + results.Select(r => r.StatusCode).ShouldBe( + [TwinCATStatusMapper.Good, TwinCATStatusMapper.BadOutOfRange, TwinCATStatusMapper.BadTypeMismatch]); + fake.RawWrites["MAIN.Name"].ShouldBe(new byte[] { 0xE4, (byte)'b', 0, 0, 0 }); + fake.RawWrites.ShouldNotContainKey("MAIN.Wide"); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + /// An array write through the driver lands as the codec's bytes at the PLC element stride. + [Fact] + public async Task Array_writes_go_through_the_client_encode_plan() + { + var fake = new FakeTwinCATClient(); + var element = TwinCATPlcSymbolShape.Classify("INT", DataTypeCategory.Primitive, 2); + fake.SymbolShapes["MAIN.Buf"] = TwinCATPlcSymbolShape.Classify("ARRAY [0..3] OF INT", DataTypeCategory.Array, 8); + fake.ArrayShapes["MAIN.Buf"] = new TwinCATPlcArrayShape([4], 8, element); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, + new TwinCATProbeOptions { Enabled = false }, native: false, + new TwinCATTagDefinition("Buf", Host, "MAIN.Buf", TwinCATDataType.Int, ArrayLength: 2), + new TwinCATTagDefinition("BufAsReal", Host, "MAIN.Buf", TwinCATDataType.Real, ArrayLength: 2)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var results = await drv.WriteAsync( + [ + new WriteRequest("Buf", new[] { 1, -2 }), + new WriteRequest("BufAsReal", new[] { 1.0, 2.0 }), + ], TestContext.Current.CancellationToken); + + results[0].StatusCode.ShouldBe(TwinCATStatusMapper.Good); + fake.RawWrites["MAIN.Buf"].ShouldBe(new byte[] { 0x01, 0x00, 0xFE, 0xFF }); // leading sub-range only + results[1].StatusCode.ShouldBe(TwinCATStatusMapper.BadTypeMismatch); + } + + // ---- MINOR 8: REAL / LREAL range ---- + + /// A REAL (scalar or array element) beyond float range is BadOutOfRange, never written as Infinity. + [Fact] + public async Task Out_of_range_REAL_values_are_BadOutOfRange() + { + Should.Throw(() => AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, 1e300)); + Should.Throw(() => AdsTwinCATClient.ConvertForWrite(TwinCATDataType.LReal, "1e400")); + AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, 3.5).ShouldBe(3.5f); + AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, float.PositiveInfinity).ShouldBe(float.PositiveInfinity); + float.IsNaN((float)AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, double.NaN)).ShouldBeTrue(); + + var fake = new FakeTwinCATClient(); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, + new TwinCATProbeOptions { Enabled = false }, native: false, + new TwinCATTagDefinition("Arr", Host, "MAIN.Arr", TwinCATDataType.Real, ArrayLength: 2)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var result = await drv.WriteAsync([new WriteRequest("Arr", new[] { 1.0, 1e300 })], TestContext.Current.CancellationToken); + result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange); + fake.ArrayWriteLog.ShouldBeEmpty(); + } +} From 09e0fe18418e2cfb8c7909e66d04fce11e4214be Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 14:55:37 -0400 Subject: [PATCH 50/51] fix(focas): a returned status never ends a device's batch; round-2 minors (#740 review round 2) N1: linkDown is set only when NO reply arrived, i.e. a timeout, a transport exception or a connect failure. A status the CNC returned proves it answered. EW_REJECT and every unlisted rc map to BadCommunicationError, so a deterministic rc on one tag used to fail every later tag of a reachable control on every poll. The comms-classed status still degrades health. The coalescer already treated a returned refusal this way; it is unchanged. Tests: a returned EW_REJECT on tag 1 leaves tags 2 and 3 read and Good (red before). A timeout still fails the rest without reading them (positive control). Minors: - m1: the _lifecycleLock doc now covers the health transitions it also guards. - m2: the NoteFixedTreeFailed summary says a refusal credits the answer. - m3: ReadAsync stamps a failed batch's exception with the generation the batch started under, and HandlePollError degrades that generation only. - m4: the fixed-tree "blip" absence check runs unconditionally under a wide budget, and asserts that both ticks landed inside it. - m5: direct unit cases for the coalescer link-down paths. A transport fault on the first range fails the second range and the per-tag tag without reading them. A connect failure costs one connect per batch. CncRelay.Up() retries the re-bind. Refs #740 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../FocasDriver.cs | 53 ++++++--- .../Series/WireBackendHealthAndLayoutTests.cs | 14 ++- .../FocasConcurrentDeviceReadTests.cs | 102 ++++++++++++++++++ .../FocasFixedTreeHealthTests.cs | 15 +-- .../FocasPollErrorHealthTests.cs | 28 +++++ 5 files changed, 190 insertions(+), 22 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs index 5177119b..bebe023a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS/FocasDriver.cs @@ -59,8 +59,9 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, private IReadOnlyDictionary Devices => Volatile.Read(ref _devices); /// - /// Guards the publish/retire of the device table against an overlapping lifecycle call. Held only for - /// synchronous swaps — never across an await, and nothing a loop runs ever takes it. + /// Guards the publish/retire of the device table against an overlapping lifecycle call, and makes each + /// generation-checked health transition atomic (). Held only for synchronous swaps + /// and single health transitions — never across an await, and nothing is called under it. /// private readonly object _lifecycleLock = new(); @@ -200,7 +201,9 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, internal void HandlePollError(Exception ex) { _logger.LogWarning(ex, "FOCAS poll reader failed. Driver={DriverInstanceId}", _driverInstanceId); - DegradeHealth(CurrentGeneration, ex.Message); + // The generation the failed batch started under, when ReadAsync stamped it; otherwise the live one. + var generation = ex.Data.Contains(BatchGenerationKey) && ex.Data[BatchGenerationKey] is int g ? g : CurrentGeneration; + DegradeHealth(generation, ex.Message); } /// @@ -696,19 +699,35 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, } if (byDevice is null) return results; - if (byDevice.Count == 1) + try { - foreach (var (device, indices) in byDevice) - await ReadDeviceAsync(device, indices, fullReferences, results, now, generation, cancellationToken).ConfigureAwait(false); + if (byDevice.Count == 1) + { + foreach (var (device, indices) in byDevice) + await ReadDeviceAsync(device, indices, fullReferences, results, now, generation, cancellationToken).ConfigureAwait(false); + } + else + { + await Task.WhenAll(byDevice.Select(kv => + ReadDeviceAsync(kv.Key, kv.Value, fullReferences, results, now, generation, cancellationToken))).ConfigureAwait(false); + } } - else + catch (Exception ex) when (ex is not OperationCanceledException) { - await Task.WhenAll(byDevice.Select(kv => - ReadDeviceAsync(kv.Key, kv.Value, fullReferences, results, now, generation, cancellationToken))).ConfigureAwait(false); + // Stamp the generation this batch STARTED under (review round 2, m3), so the poll engine's onError + // (HandlePollError) cannot degrade a generation that re-initialised after the batch began. + ex.Data[BatchGenerationKey] = generation; + throw; } return results; } + /// key carrying the lifecycle generation a failed batch started under. + internal const string BatchGenerationKey = "FocasDriver.BatchGeneration"; + + /// Test seam — the live lifecycle generation. + internal int LifecycleGeneration => CurrentGeneration; + /// /// Reads one device's share of a batch, in request order: the PMC coalescing pre-pass first, then every /// index it did not serve through the per-tag path. Each result lands at its own index, so concurrent @@ -718,10 +737,11 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, /// One link failure ends the device's batch (#740 review I1). Concurrency alone does not bound a /// batch: Task.WhenAll waits for the slowest device, and a CNC that accepts TCP and then stalls costs /// the full call timeout on EVERY tag — so a 10-tag stalled device held the healthy devices' values back by - /// ten timeouts. After the first timeout / transport failure (or a comms-class status) on this device, its - /// remaining tags in the batch fail fast with BadCommunicationError, exactly as the coalescer's + /// ten timeouts. After the first timeout / transport failure / connect failure on this device — i.e. NO + /// reply arrived — its remaining tags in the batch fail fast with BadCommunicationError, exactly as the coalescer's /// FailMembers does for a merged range. Fixed-tree tags are still served from the cache: they need no - /// link. A CNC REFUSAL does not end the batch — the link answered. + /// link. Any status the CNC RETURNED — a refusal, or a comms-classed rc such as EW_REJECT — does not end the + /// batch: a reply proves the link answered. /// private async Task ReadDeviceAsync( DeviceState device, List indices, IReadOnlyList fullReferences, @@ -792,8 +812,11 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, } else if (IsCommsStatus(status)) { + // Degrade, but do NOT end the batch (#740 review round 2, N1): this status came back in a reply, + // so the CNC answered — EW_REJECT and every unlisted rc map to BadCommunicationError. Ending the + // batch here starved every later tag of a reachable control forever (authored order is fixed). + // A session that is really dead throws or times out on the next tag, which does end it. DegradeHealth(generation, $"FOCAS status 0x{status:X8} reading {reference}"); - linkDown = true; } else { @@ -2059,8 +2082,8 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery, } /// - /// A failed fixed-tree tick. A CNC REFUSAL (non-OK rc, transport fine) is not a comms loss and clears any - /// failure without crediting a read; a transport failure degrades the driver only once the device has been + /// A failed fixed-tree tick. A CNC REFUSAL (non-OK rc, transport fine) is not a comms loss: it clears any + /// failure AND credits the answer to LastSuccessfulRead (the CNC replied); a transport failure degrades the driver only once the device has been /// silent past (measured from its last answer, or from loop start when /// it never answered), so one dropped tick on a slow control does not flap /hosts. /// diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendHealthAndLayoutTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendHealthAndLayoutTests.cs index 30d39394..c704b21f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendHealthAndLayoutTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.IntegrationTests/Series/WireBackendHealthAndLayoutTests.cs @@ -88,7 +88,19 @@ public sealed class WireBackendHealthAndLayoutTests } } - public void Up() => Listen(Port); + /// + /// Listens again on the SAME port (the driver's host string names it). The re-bind is retried briefly: the + /// OS can hold the port for a moment after , or hand it to another process — the latter + /// surfaces as a clear failure after the retries rather than a random flake. + /// + public void Up() + { + for (var attempt = 1; ; attempt++) + { + try { Listen(Port); return; } + catch (SocketException) when (attempt < 20) { Thread.Sleep(100); } + } + } public ValueTask DisposeAsync() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs index 23e0e4e3..f4fa7d11 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasConcurrentDeviceReadTests.cs @@ -153,6 +153,108 @@ public sealed class FocasConcurrentDeviceReadTests await drv.ShutdownAsync(CancellationToken.None); } + /// Counts per-tag reads; one address can be made to time out (throw) or answer a status. + private sealed class CountingClient : FakeFocasClient + { + public int Reads; + public string? TimeoutAt; + + public override async Task<(object? value, uint status)> ReadAsync( + FocasAddress address, FocasDataType type, CancellationToken ct) + { + Interlocked.Increment(ref Reads); + if (address.Canonical == TimeoutAt) + { + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + } + return await base.ReadAsync(address, type, ct).ConfigureAwait(false); + } + } + + private static FocasDriver OneDevice(FakeFocasClientFactory factory, bool coalesce, params string[] addresses) => new( + new FocasDriverOptions + { + Devices = [new FocasDeviceOptions(HostA, DeviceName: "A")], + RawTags = [.. addresses.Select(a => FocasRawTags.Entry(new FocasTagDefinition($"raw/{a}", "A", a, FocasDataType.Byte)))], + Probe = new FocasProbeOptions { Enabled = false }, + PmcCoalesce = new FocasPmcCoalesceOptions { Enabled = coalesce }, + Timeout = TimeSpan.FromMilliseconds(200), + }, "focas-linkdown", factory); + + /// + /// #740 review round 2 N1 — a status the CNC RETURNED (EW_REJECT and every unlisted rc map to + /// BadCommunicationError) proves the device answered, so it must not end the device's batch: the tags after it + /// still read. Before the fix, a deterministic rc on the first tag starved every later tag forever. + /// + [Fact] + public async Task A_returned_comms_status_does_not_fail_the_rest_of_the_batch() + { + var client = new CountingClient { Values = { ["R101"] = (sbyte)1, ["R102"] = (sbyte)2 } }; + client.ReadStatuses["R100"] = FocasStatusMapper.MapFocasReturn(13); // EW_REJECT ⇒ BadCommunicationError + var drv = OneDevice(new FakeFocasClientFactory { Customise = () => client }, coalesce: false, "R100", "R101", "R102"); + await drv.InitializeAsync("{}", CancellationToken.None); + + var snaps = await drv.ReadAsync(["raw/R100", "raw/R101", "raw/R102"], CancellationToken.None); + + snaps[0].StatusCode.ShouldBe(FocasStatusMapper.BadCommunicationError); + snaps[1].StatusCode.ShouldBe(FocasStatusMapper.Good); + snaps[2].Value.ShouldBe((sbyte)2); + client.Reads.ShouldBe(3, "every tag was really read"); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// Positive control for N1: a TIMEOUT (no reply at all) still fails the rest of the batch without reading it. + [Fact] + public async Task A_timeout_still_fails_the_rest_of_the_batch_fast() + { + var client = new CountingClient { TimeoutAt = "R100", Values = { ["R101"] = (sbyte)1 } }; + var drv = OneDevice(new FakeFocasClientFactory { Customise = () => client }, coalesce: false, "R100", "R101", "R102"); + await drv.InitializeAsync("{}", CancellationToken.None); + + var snaps = await drv.ReadAsync(["raw/R100", "raw/R101", "raw/R102"], CancellationToken.None); + + snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.BadCommunicationError); + client.Reads.ShouldBe(1, "only the tag that timed out touched the wire"); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// + /// Review round 2 m5 — the coalescer's link-down paths: a transport fault on the first merged range fails the + /// second range WITHOUT a second range read, and the per-tag tags of the same device without a per-tag read. + /// + [Fact] + public async Task Coalescer_transport_fault_fails_later_ranges_and_per_tag_tags_without_reading_them() + { + var client = new CountingClient { ThrowOnPmcRange = true }; + // Two ranges far apart (gap > MaxGapBytes) + one lone tag that takes the per-tag path. + var drv = OneDevice(new FakeFocasClientFactory { Customise = () => client }, coalesce: true, + "R100", "R101", "R500", "R501", "D900"); + await drv.InitializeAsync("{}", CancellationToken.None); + + var snaps = await drv.ReadAsync(["raw/R100", "raw/R101", "raw/R500", "raw/R501", "raw/D900"], CancellationToken.None); + + snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.BadCommunicationError); + client.PmcRangeReads.Count.ShouldBe(1, "the second range must not pay another transport attempt"); + client.Reads.ShouldBe(0, "the per-tag tag must not pay one either"); + await drv.ShutdownAsync(CancellationToken.None); + } + + /// Review round 2 m5 — a coalescer CONNECT failure fails the per-tag tags of the device without a second connect. + [Fact] + public async Task Coalescer_connect_failure_fails_per_tag_tags_without_reconnecting() + { + var factory = new FakeFocasClientFactory { Customise = () => new CountingClient { ThrowOnConnect = true } }; + var drv = OneDevice(factory, coalesce: true, "R100", "R101", "D900"); + await drv.InitializeAsync("{}", CancellationToken.None); + + var snaps = await drv.ReadAsync(["raw/R100", "raw/R101", "raw/D900"], CancellationToken.None); + + snaps.ShouldAllBe(s => s.StatusCode == FocasStatusMapper.BadCommunicationError); + factory.Clients.Count.ShouldBe(1, "one connect attempt per device per batch"); + factory.Clients.Sum(c => c.ConnectCount).ShouldBe(1); + await drv.ShutdownAsync(CancellationToken.None); + } + /// Results land at their own indices whatever order the devices finish in. [Fact] public async Task Results_keep_their_request_order_across_devices() diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs index fae11d58..ff5aef11 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasFixedTreeHealthTests.cs @@ -18,6 +18,7 @@ public sealed class FocasFixedTreeHealthTests { private const string Host = "focas://10.0.0.5:8193"; private static readonly TimeSpan Stale = TimeSpan.FromMilliseconds(300); + private static readonly TimeSpan BlipBudget = TimeSpan.FromMilliseconds(1500); /// A CNC that can be switched off (transport failure) or made to refuse the dynamic read. private sealed class CncSwitch @@ -58,7 +59,7 @@ public sealed class FocasFixedTreeHealthTests } } - private static FocasDriver FixedOnlyDriver(CncSwitch sw) => new(new FocasDriverOptions + private static FocasDriver FixedOnlyDriver(CncSwitch sw, TimeSpan? budget = null) => new(new FocasDriverOptions { Devices = [new FocasDeviceOptions(Host)], RawTags = [FocasRawTags.Entry(new FocasTagDefinition("raw/X", Host, "FIXED:Axes/X/AbsolutePosition", FocasDataType.Float64))], @@ -71,7 +72,7 @@ public sealed class FocasFixedTreeHealthTests }, }, "focas-fixed-health", new FakeFocasClientFactory { Customise = () => new SwitchableCnc(sw) }) { - FixedTreeStaleBudgetOverride = Stale, + FixedTreeStaleBudgetOverride = budget ?? Stale, }; /// @@ -83,7 +84,9 @@ public sealed class FocasFixedTreeHealthTests public async Task Fixed_only_driver_degrades_when_its_cnc_disappears_and_recovers_when_it_returns() { var sw = new CncSwitch(); - await using var drv = FixedOnlyDriver(sw); + // A wide budget (review round 2 m4): two 20 ms ticks always land well inside it, so the "blip does not + // degrade" absence check below runs unconditionally instead of being skipped on a slow agent. + await using var drv = FixedOnlyDriver(sw, BlipBudget); await drv.InitializeAsync("{}", CancellationToken.None); await WaitUntil(() => drv.GetDeviceState(Host)?.LastFixedPollUtc is not null); @@ -96,9 +99,9 @@ public sealed class FocasFixedTreeHealthTests var lastAnswer = drv.GetDeviceState(Host)!.LastFixedPollUtc!.Value; // Two failed ticks have demonstrably happened (positive evidence for the absence check below) … await WaitUntil(() => Volatile.Read(ref sw.Failures) >= 2); - // … and, if they landed inside the budget, the driver is still Healthy. - if (DateTime.UtcNow - lastAnswer < Stale) - drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade"); + // … inside the budget (asserted, not assumed), and the driver is still Healthy. + (DateTime.UtcNow - lastAnswer).ShouldBeLessThan(BlipBudget, "the two failed ticks must land inside the budget"); + drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a blip inside the staleness budget does not degrade"); await WaitUntil(() => drv.GetHealth().State == DriverState.Degraded); var frozen = drv.GetHealth().LastSuccessfulRead; var failuresAtDegrade = Volatile.Read(ref sw.Failures); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasPollErrorHealthTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasPollErrorHealthTests.cs index f738c275..3e89a3ff 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasPollErrorHealthTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.FOCAS.Tests/FocasPollErrorHealthTests.cs @@ -57,4 +57,32 @@ public sealed class FocasPollErrorHealthTests drv.GetHealth().State.ShouldBe(DriverState.Faulted); } + + /// + /// #740 review round 2 m3 — a poll failure carries the generation its batch STARTED under, so a batch that + /// began before a re-initialise cannot degrade the new generation. Positive control: a failure stamped with + /// the live generation (or unstamped) does degrade. + /// + [Fact] + public async Task PollReaderFailure_from_a_previous_generation_does_not_degrade_the_new_one() + { + var drv = new FocasDriver(new FocasDriverOptions + { + Devices = [new FocasDeviceOptions("focas://10.0.0.5:8193")], + Probe = new FocasProbeOptions { Enabled = false }, + }, "focas-1", new FakeFocasClientFactory()); + await drv.InitializeAsync("{}", CancellationToken.None); + var oldGeneration = drv.LifecycleGeneration; + await drv.InitializeAsync("{}", CancellationToken.None); + + var stale = new InvalidOperationException("old batch boom"); + stale.Data[FocasDriver.BatchGenerationKey] = oldGeneration; + drv.HandlePollError(stale); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + + var live = new InvalidOperationException("live batch boom"); + live.Data[FocasDriver.BatchGenerationKey] = drv.LifecycleGeneration; + drv.HandlePollError(live); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + } } From a8d313ed5ade26d3041da863776d979507f25fcd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:00:23 -0400 Subject: [PATCH 51/51] fix(twincat): review round 2 - teardown confined to its generation; write health; re-arm epoch - I1: the poll engine belongs to a teardown generation. ShutdownAsync/Dispose retire the current engine and install a fresh one in the same swap as the device/tag tables, then dispose the retired one (possibly in the background after a cancelled or over-budget shutdown). A subscription made after the next InitializeAsync can no longer land in an engine whose late dispose tail would dispose its CTS without cancelling it (a loop no UnsubscribeAsync could find). Poll handles are generation-qualified (TwinCATPollHandle) because each engine numbers handles from 1; SubscribeAsync re-homes a subscription that raced a retirement. - M1: a write's status feeds per-device health - Good clears that device's comms fault and advances LastSuccessfulRead, a comms status degrades, a refusal keeps the state. - M2: a symbol-version re-arm bumps a per-registration epoch; an in-flight retry that fails afterwards no longer re-imposes the backoff. - M3: a re-arm restarts the refusal backoff at 30 s. - M4: documented that PLC-not-in-RUN detection needs the probe. - M5: WSTRING and multi-dimensional array encode-plan tests; the cancelled- shutdown test asserts Unknown health and awaits its background teardown. - M6: an infinity spelled as a string ('Infinity', '-inf') passes like a float/double infinity; a finite overflow stays BadOutOfRange. Refs #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AdsTwinCATClient.cs | 3 + .../TwinCATDriver.cs | 143 ++++++++-- .../FakeTwinCATClient.cs | 23 +- .../TwinCATReviewFixRound1Tests.cs | 8 + .../TwinCATReviewFixRound2Tests.cs | 262 ++++++++++++++++++ 5 files changed, 417 insertions(+), 22 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound2Tests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs index 9eecbd34..649998b0 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/AdsTwinCATClient.cs @@ -1098,10 +1098,13 @@ internal sealed class AdsTwinCATClient : ITwinCATClient return d; } + // An explicit infinity — a float/double infinity, or a string that SPELLS one ("Infinity", "-inf", "∞") + // — is the caller's own value and passes; a finite number that overflows ("1e400") does not. private static bool IsExplicitInfinity(object? value) => value switch { float f => float.IsInfinity(f), double d => double.IsInfinity(d), + string s => s.Contains("inf", StringComparison.OrdinalIgnoreCase) || s.Contains('∞'), _ => false, }; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs index 4d9be07b..8b6c0927 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT/TwinCATDriver.cs @@ -27,7 +27,28 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery private readonly string _driverInstanceId; private readonly ITwinCATClientFactory _clientFactory; private readonly ILogger _logger; - private readonly PollGroupEngine _poll; + // The poll engine belongs to a teardown GENERATION (#739 fix round 2): ShutdownAsync / Dispose retire the + // current engine and install a fresh one in the same swap, then dispose the retired one — possibly in the + // background after a cancelled or over-budget shutdown. PollGroupEngine.DisposeAsync clears whatever its + // own table holds when it finishes, so a subscription made after a later InitializeAsync must never land + // in a retiring engine, or the old teardown would dispose its CTS without cancelling it (a poll loop no + // UnsubscribeAsync can find, running forever). + private volatile PollGeneration _poll; + private long _nextPollGeneration; + + /// One generation's poll engine plus its serial (the serial is part of every handle it issues). + private sealed record PollGeneration(long Serial, PollGroupEngine Engine); + + /// + /// A poll subscription handle, qualified by the engine generation that issued it. Each engine numbers its + /// handles from 1, so without the generation a retired engine's late callback would carry a handle EQUAL + /// to a new subscription's and be taken for it. + /// + private sealed record TwinCATPollHandle(long Generation, ISubscriptionHandle Inner) : ISubscriptionHandle + { + /// + public string DiagnosticId => $"twincat-poll-g{Generation}-{Inner.DiagnosticId}"; + } // Device + tag tables are BUILD-AND-SWAP (#739 L8): InitializeAsync builds fresh tables and publishes // them by reference under _lifecycleLock; ShutdownAsync / Dispose swap in empty ones and release what @@ -120,18 +141,27 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery _logger = logger ?? NullLogger.Instance; _resolver = new EquipmentTagRefResolver( r => _tagsByRawPath.TryGetValue(r, out var t) ? t : null); - _poll = new PollGroupEngine( + _poll = CreatePollEngine(); + } + + /// Builds one generation's poll engine; its callbacks publish generation-qualified handles. + private PollGeneration CreatePollEngine() + { + var driverInstanceId = _driverInstanceId; + var serial = Interlocked.Increment(ref _nextPollGeneration); + return new PollGeneration(serial, new PollGroupEngine( reader: ReadAsync, onChange: (handle, tagRef, snapshot) => - OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, tagRef, snapshot)), + OnDataChange?.Invoke(this, new DataChangeEventArgs(new TwinCATPollHandle(serial, handle), tagRef, snapshot)), onError: HandlePollError, backoffCap: PollBackoffCap, - onBatch: (handle, items) => OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)), + onBatch: (handle, items) => OnDataChangeBatch?.Invoke(this, + new DataChangeBatchEventArgs(new TwinCATPollHandle(serial, handle), items)), // The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this // closure IS the production consumer of its overrun signal. onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1, new KeyValuePair(OtOpcUaTelemetry.PollOverrunDriverTypeTag, DriverTypeNames.TwinCAT), - new KeyValuePair(OtOpcUaTelemetry.PollOverrunDriverInstanceTag, driverInstanceId))); + new KeyValuePair(OtOpcUaTelemetry.PollOverrunDriverInstanceTag, driverInstanceId)))); } /// Upper bound on the poll-loop failure backoff — adopts the S7-proven 30 s cap fleet-wide. @@ -327,8 +357,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } } - /// Swaps out every table under the lifecycle lock and returns the device set the caller now owns. - private ICollection TakeAllForTeardown() + /// + /// Swaps out every table AND the poll engine under the lifecycle lock and returns what the caller now + /// owns. Everything returned belongs to the retired generation only; nothing a later + /// / creates is reachable from it. + /// + private (ICollection devices, PollGroupEngine poll) TakeAllForTeardown() { lock (_lifecycleLock) { @@ -336,10 +370,16 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery var taken = _devices; _devices = NewDeviceTable(); _tagsByRawPath = EmptyTags; - return taken.Values; + var poll = _poll; + _poll = CreatePollEngine(); + return (taken.Values, poll.Engine); } } + /// The most recent teardown's background work (poll-engine dispose + device release). Test + /// visibility only: a cancelled shutdown leaves it running. + internal Task? LastTeardownForTest { get; private set; } + /// public async Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { @@ -362,11 +402,13 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery foreach (var r in sub.Registrations) { try { r.Dispose(); } catch { } } _nativeSubs.Clear(); - var taken = TakeAllForTeardown(); + var (taken, retiredPoll) = TakeAllForTeardown(); // The poll loops and the device release wind down in parallel (releasing a client can unblock a - // poll loop stuck in a read on it); the whole teardown is bounded. - var teardown = Task.WhenAll(_poll.DisposeAsync().AsTask(), ReleaseDevicesAsync(taken)); + // poll loop stuck in a read on it); the whole teardown is bounded. Both halves own ONLY the retired + // generation, so finishing in the background after a cancelled wait is safe. + var teardown = Task.WhenAll(retiredPoll.DisposeAsync().AsTask(), ReleaseDevicesAsync(taken)); + LastTeardownForTest = teardown; var cancelled = false; try { await teardown.WaitAsync(ShutdownBudget, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) @@ -515,6 +557,19 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery return new DataValueSnapshot(value, status, now, now); } + /// + /// Applies the health contract to a write the client returned a status for: Good is the device + /// answering (clears that device's comms fault, advances LastSuccessfulRead); a comms-class + /// status is a comms fault; anything else is a refusal — the device answered, the state is unchanged. + /// + private void NoteWriteOutcome(DeviceState device, uint status, string reference) + { + var now = DateTime.UtcNow; + if (status == TwinCATStatusMapper.Good) RecordDeviceAnswered(now, device, force: true); + else if (IsCommsStatus(status)) MarkCommsFault($"ADS status {status:X8} writing {reference}", device); + else RecordDeviceAnsweredKeepState(now); + } + /// /// True for a status that means the request did not get a device answer (router / transport / /// timeout). Every other non-Good status is the device refusing this one tag (symbol not found, type @@ -675,6 +730,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery { gate.Release(); } + NoteWriteOutcome(device, results[i].StatusCode, w.FullReference); continue; } @@ -685,12 +741,14 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // array — the inverse of the array read. results[i] = new WriteResult(await client.WriteArrayAsync( symbolName, def.DataType, elements, cancellationToken).ConfigureAwait(false)); + NoteWriteOutcome(device, results[i].StatusCode, w.FullReference); continue; } var status = await client.WriteValueAsync( symbolName, def.DataType, parsed?.BitIndex, w.Value, cancellationToken).ConfigureAwait(false); results[i] = new WriteResult(status); + NoteWriteOutcome(device, status, w.FullReference); } catch (OperationCanceledException) { throw; } catch (NotSupportedException) @@ -826,7 +884,17 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery IReadOnlyList fullReferences, TimeSpan publishingInterval, CancellationToken cancellationToken) { if (!_options.UseNativeNotifications) - return _poll.Subscribe(fullReferences, publishingInterval); + { + // A teardown racing this call may retire the engine we read: re-home the subscription on the + // current one so it never lives in an engine that is being disposed. + while (true) + { + var generation = _poll; + var inner = generation.Engine.Subscribe(fullReferences, publishingInterval); + if (ReferenceEquals(generation, _poll)) return new TwinCATPollHandle(generation.Serial, inner); + generation.Engine.Unsubscribe(inner); + } + } var id = Interlocked.Increment(ref _nextNativeSubId); var handle = new NativeSubscriptionHandle(id); @@ -987,6 +1055,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// ADS state is not Run (coordinator ruling: a stopped / config-mode PLC is Degraded with a /// reason even though it answers), or an uncleared comms fault seen on it. /// + /// + /// The ADS state comes from the probe's ReadState, so PLC-not-in-RUN detection needs the probe + /// (Probe.Enabled, the default); with the probe off the state is unknown and never degrades. + /// private static string? DeviceDegradedReason(DeviceState d) { if (d.ConsecutiveProbeFailures >= ProbeFailuresBeforeDegraded) @@ -1042,9 +1114,20 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// or retried by the reconnect replay; a comms-class failure stays on the every-tick retry. /// /// True when the published status changes (the caller republishes the tag). - private static bool NoteDeadFailure(NativeRegistration reg, uint status, bool refused, DateTime now) + /// The registration. + /// The status the failure maps to. + /// True for a refusal (backs off), false for a comms-class failure. + /// The failure time. + /// The registration's when the attempt + /// started; when a re-arm happened while it was in flight the backoff is NOT re-imposed (the re-arm wins). + private static bool NoteDeadFailure(NativeRegistration reg, uint status, bool refused, DateTime now, + int? epochAtAttempt = null) { - if (refused) + if (epochAtAttempt is int epoch && epoch != reg.RearmEpoch) + { + // A symbol-version change re-armed this registration mid-attempt: keep it re-armed. + } + else if (refused) { reg.RefusalBackoff = reg.RefusalBackoff == TimeSpan.Zero ? RefusedRetryInitial @@ -1141,7 +1224,10 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery } // Async teardown: the engine drains the in-flight poll without blocking the caller's // thread — this runs inside a DriverInstanceActor mailbox turn. - await _poll.UnsubscribeAsync(handle).ConfigureAwait(false); + // A handle of a retired generation has nothing left to stop: its engine was disposed with it. + var current = _poll; + if (handle is TwinCATPollHandle poll && poll.Generation == current.Serial) + await current.Engine.UnsubscribeAsync(poll.Inner).ConfigureAwait(false); } private sealed record NativeSubscriptionHandle(long Id) : ISubscriptionHandle @@ -1255,9 +1341,25 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery /// Earliest time the probe-tick retry may re-attempt this dead registration. public DateTime NextRetryUtc { get; set; } - /// Current refusal backoff (zero after a comms-class failure or a success). + /// Current refusal backoff (zero after a comms-class failure, a success or a re-arm). public TimeSpan RefusalBackoff { get; set; } + private int _rearmEpoch; + + /// Bumped by every symbol-version re-arm; a retry that started before it must not re-impose a backoff. + public int RearmEpoch => Volatile.Read(ref _rearmEpoch); + + /// + /// Re-arms the retry after a PLC program change: retry on the next tick and restart the backoff + /// from (the refusal it backed off from may no longer apply). + /// + public void Rearm() + { + Interlocked.Increment(ref _rearmEpoch); + RefusalBackoff = TimeSpan.Zero; + NextRetryUtc = DateTime.MinValue; + } + private volatile bool _disposed; /// Gets a value indicating whether this registration has been disposed (its callbacks are dropped). @@ -1793,6 +1895,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery if (reg.HasLiveHandle) continue; // A refused registration waits out its backoff (a symbol-version change re-arms it). if (now < reg.NextRetryUtc) continue; + var epoch = reg.RearmEpoch; try { var newHandle = await client.AddNotificationAsync( @@ -1816,7 +1919,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery // or the reverse); the same failure again publishes nothing. var refusal = ex as TwinCATNotificationRefusedException; var status = refusal?.StatusCode ?? TwinCATStatusMapper.BadCommunicationError; - if (NoteDeadFailure(reg, status, refusal is not null, now)) + if (NoteDeadFailure(reg, status, refusal is not null, now, epoch)) try { reg.OnRefused(status); } catch { /* never abort the rest */ } _logger.LogDebug(ex, "TwinCAT driver '{DriverInstanceId}' probe-tick retry of dead notification " + @@ -1880,7 +1983,7 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery foreach (var d in _devices.Values) if (ReferenceEquals(d.Client, sender)) foreach (var reg in d.NativeRegistrations.Values) - reg.NextRetryUtc = DateTime.MinValue; + reg.Rearm(); OnRediscoveryNeeded?.Invoke(this, new RediscoveryEventArgs( "TwinCAT symbol-version-changed (DeviceSymbolVersionInvalid 0x0711) — PLC program re-downloaded", ScopeHint: "TwinCAT")); @@ -1902,12 +2005,12 @@ public sealed class TwinCATDriver : IDriver, IReadable, IWritable, ITagDiscovery foreach (var r in sub.Registrations) { try { r.Dispose(); } catch { } } _nativeSubs.Clear(); - var taken = TakeAllForTeardown(); + var (taken, retiredPoll) = TakeAllForTeardown(); var probes = BeginRelease(taken); // PollGroupEngine.DisposeAsync awaits loop tasks; drive it synchronously with a bounded Wait() on // the returned task (no sync-context capture), in parallel with the probe join. - var poll = _poll.DisposeAsync().AsTask(); + var poll = retiredPoll.DisposeAsync().AsTask(); try { Task.WhenAll([poll, .. probes]).Wait(ProbeJoinTimeout); } catch { /* cancel races are expected */ } EndRelease(taken); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs index 49a6deea..8559f9af 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/FakeTwinCATClient.cs @@ -83,16 +83,31 @@ internal class FakeTwinCATClient : ITwinCATClient public virtual Task<(object? value, uint status)> ReadValueAsync( string symbolPath, TwinCATDataType type, int? bitIndex, int? arrayCount, CancellationToken ct) { + if (BlockRead is { } block) + { + // A read wedged in the (uncancellable) wire call; completes with the normal answer once released. + return block.Task.ContinueWith(_ => + { + BlockRead = null; + return ReadValueAsync(symbolPath, type, bitIndex, arrayCount, CancellationToken.None).Result; + }, TaskScheduler.Default); + } if (ThrowOnRead) throw Exception ?? new InvalidOperationException(); - SingleReadCount++; + Interlocked.Increment(ref _singleReadCount); LastReadArrayCount = arrayCount; var status = ReadStatuses.TryGetValue(symbolPath, out var s) ? s : TwinCATStatusMapper.Good; var value = Values.TryGetValue(symbolPath, out var v) ? v : null; return Task.FromResult((value, status)); } + /// When set, does not complete (and ignores cancellation) until this is + /// completed — a poll read wedged in the wire call. + public TaskCompletionSource? BlockRead { get; set; } + + private int _singleReadCount; + /// Gets the number of individual calls served. - public int SingleReadCount { get; private set; } + public int SingleReadCount { get => Volatile.Read(ref _singleReadCount); private set => Volatile.Write(ref _singleReadCount, value); } /// Gets the size of each batch the driver issued, in order (#645). public List SumReadBatches { get; } = new(); /// Gets or sets an exception throws instead of answering — a @@ -264,6 +279,9 @@ internal class FakeTwinCATClient : ITwinCATClient public int AttemptsFor(string symbolPath) => _addAttempts.TryGetValue(symbolPath, out var n) ? n : 0; /// Records the most recently-supplied maxDelayMs for Driver.TwinCAT-014 tests. public int LastMaxDelayMs { get; private set; } + /// When set, awaits this gate BEFORE applying + /// — wedges an attempt in flight whose outcome is a failure. + public TaskCompletionSource? PreAttemptGate { get; set; } /// When set, awaits this gate before creating the handle — /// lets a test wedge a replay/register mid-flight (STAB-17 orphan race). public TaskCompletionSource? AddNotificationGate { get; set; } @@ -283,6 +301,7 @@ internal class FakeTwinCATClient : ITwinCATClient int maxDelayMs, Action onChange, CancellationToken cancellationToken) { _addAttempts.AddOrUpdate(symbolPath, 1, (_, n) => n + 1); + if (PreAttemptGate is { } pre) await pre.Task.ConfigureAwait(false); if (ThrowOnAddNotification) throw Exception ?? new InvalidOperationException("fake AddNotification failure"); if (AddNotificationErrors.TryGetValue(symbolPath, out var perSymbol)) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound1Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound1Tests.cs index 35ff945b..a25d5292 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound1Tests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound1Tests.cs @@ -210,7 +210,15 @@ public sealed class TwinCATReviewFixRound1Tests await Should.ThrowAsync(() => drv.ShutdownAsync(cts.Token)); sw.Elapsed.ShouldBeLessThan(TwinCATDriver.ProbeJoinTimeout, "the wait ended on the token, not the join bound"); drv.DeviceCount.ShouldBe(0); + drv.GetHealth().State.ShouldBe(DriverState.Unknown); + + // The background teardown finishes once the wedge releases, and disposes the client the wedged + // connect produced (no dangling teardown left behind the test). + var teardown = drv.LastTeardownForTest.ShouldNotBeNull(); wedge.SetResult(); + await teardown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + await WaitForAsync(() => factory.Clients[0].DisposeCount > 0, TimeSpan.FromSeconds(3)); + factory.Clients[0].DisposeCount.ShouldBeGreaterThan(0); } // ---- MINOR 7: the real STRING / array encode path through the client planning helpers ---- diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound2Tests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound2Tests.cs new file mode 100644 index 00000000..0acc860f --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATReviewFixRound2Tests.cs @@ -0,0 +1,262 @@ +using System.Collections.Concurrent; +using Shouldly; +using TwinCAT.TypeSystem; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.TwinCAT; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// #739 review fix round 2: a cancelled shutdown's background teardown is confined to its own generation +/// (I1); write outcomes feed per-device health (M1); a symbol-version re-arm survives an in-flight retry +/// and restarts the backoff (M2/M3); WSTRING / multi-dimensional encode plans (M5); infinity strings (M6). +/// +[Trait("Category", "Unit")] +public sealed class TwinCATReviewFixRound2Tests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + + private static TwinCATProbeOptions FastProbe => new() + { + Enabled = true, + Interval = TimeSpan.FromMilliseconds(30), + Timeout = TimeSpan.FromMilliseconds(100), + }; + + private static TwinCATDriver NewDriver(FakeTwinCATClientFactory factory, TwinCATProbeOptions probe, bool native, + params TwinCATTagDefinition[] tags) => + new(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = TwinCATRawTags.Entries(tags), + Probe = probe, + UseNativeNotifications = native, + }, "drv-1", factory); + + private static async Task WaitForAsync(Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (!condition() && DateTime.UtcNow < deadline) + await Task.Delay(10); + } + + // ---- I1 ---- + + /// + /// The exact reviewed sequence: a poll read is wedged; ShutdownAsync is cancelled (its teardown + /// continues in the background); the same instance is re-initialised and subscribed; the wedge is + /// released and the OLD teardown completes. The NEW subscription keeps polling and delivering (positive + /// control), UnsubscribeAsync finds and stops it, and a final shutdown leaves nothing polling. + /// + [Fact] + public async Task A_cancelled_shutdowns_background_teardown_never_touches_the_next_generations_subscription() + { + var wedge = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var counter = 0; + var clients = new ConcurrentQueue(); + var factory = new FakeTwinCATClientFactory + { + Customise = () => + { + // Client 1 (generation 1) wedges its first read; later clients answer an ever-changing value + // so every poll publishes. + var c = new ChangingValueFake(() => Interlocked.Increment(ref counter)) + { + BlockRead = clients.IsEmpty ? wedge : null, + }; + clients.Enqueue(c); + return c; + }, + }; + var tag = new TwinCATTagDefinition("A", Host, "MAIN.A", TwinCATDataType.DInt); + var drv = NewDriver(factory, new TwinCATProbeOptions { Enabled = false }, native: false, tag); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var events = new ConcurrentQueue(); + drv.OnDataChange += (_, e) => events.Enqueue(e); + + _ = await drv.SubscribeAsync(["A"], TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); + await WaitForAsync(() => !clients.IsEmpty, TimeSpan.FromSeconds(2)); // gen-1 poll read is wedged + + using (var cts = new CancellationTokenSource(TimeSpan.FromMilliseconds(100))) + await Should.ThrowAsync(() => drv.ShutdownAsync(cts.Token)); + var oldTeardown = drv.LastTeardownForTest.ShouldNotBeNull(); + oldTeardown.IsCompleted.ShouldBeFalse("the old teardown must still be running in the background"); + + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var handle = await drv.SubscribeAsync(["A"], TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); + + wedge.SetResult(); + await oldTeardown.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + // Positive control: the new subscription delivers AFTER the old teardown finished. + var afterTeardown = events.Count(e => e.SubscriptionHandle.Equals(handle)); + await WaitForAsync(() => events.Count(e => e.SubscriptionHandle.Equals(handle)) >= afterTeardown + 3, TimeSpan.FromSeconds(3)); + events.Count(e => e.SubscriptionHandle.Equals(handle)).ShouldBeGreaterThanOrEqualTo(afterTeardown + 3); + + // UnsubscribeAsync can still find and stop it. + await drv.UnsubscribeAsync(handle, TestContext.Current.CancellationToken); + var stoppedAt = events.Count(e => e.SubscriptionHandle.Equals(handle)); + // Measured window: a sibling driver's poll publishes 5 times; the stopped loop must publish nothing. + var witnessFactory = new FakeTwinCATClientFactory { Customise = () => new ChangingValueFake(() => Environment.TickCount) }; + var witnessDrv = NewDriver(witnessFactory, new TwinCATProbeOptions { Enabled = false }, native: false, tag); + await witnessDrv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var witnessEvents = 0; + witnessDrv.OnDataChange += (_, _) => Interlocked.Increment(ref witnessEvents); + _ = await witnessDrv.SubscribeAsync(["A"], TimeSpan.FromMilliseconds(50), TestContext.Current.CancellationToken); + await WaitForAsync(() => Volatile.Read(ref witnessEvents) >= 5, TimeSpan.FromSeconds(3)); + Volatile.Read(ref witnessEvents).ShouldBeGreaterThanOrEqualTo(5); + await witnessDrv.ShutdownAsync(TestContext.Current.CancellationToken); + + events.Count(e => e.SubscriptionHandle.Equals(handle)).ShouldBe(stoppedAt, "the unsubscribed loop must have stopped"); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + /// Answers each read with a new value so every poll cycle publishes. + private sealed class ChangingValueFake(Func next) : FakeTwinCATClient + { + public override Task<(object? value, uint status)> ReadValueAsync( + string symbolPath, TwinCATDataType type, int? bitIndex, int? arrayCount, CancellationToken ct) + { + if (BlockRead is not null) return base.ReadValueAsync(symbolPath, type, bitIndex, arrayCount, ct); + Values[symbolPath] = next(); + return base.ReadValueAsync(symbolPath, type, bitIndex, arrayCount, ct); + } + } + + // ---- M1 ---- + + /// + /// A write-only device with the probe off: a wire-failed write degrades; a later Good write is the device + /// answering — Healthy again with the stamp advanced. A refused write (BadNotWritable) changes nothing. + /// + [Fact] + public async Task A_good_write_clears_that_devices_comms_fault() + { + var fake = new FakeTwinCATClient(); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, + new TwinCATProbeOptions { Enabled = false }, native: false, + new TwinCATTagDefinition("W", Host, "MAIN.W", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var initStamp = drv.GetHealth().LastSuccessfulRead!.Value; + + fake.ThrowOnWrite = true; + _ = await drv.WriteAsync([new WriteRequest("W", 1)], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + + fake.ThrowOnWrite = false; + fake.WriteStatuses["MAIN.W"] = TwinCATStatusMapper.BadNotWritable; + _ = await drv.WriteAsync([new WriteRequest("W", 1)], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded, "a refusal says nothing about the comms fault"); + + await WaitForAsync(() => DateTime.UtcNow > initStamp, TimeSpan.FromSeconds(1)); + fake.WriteStatuses.Remove("MAIN.W"); + _ = await drv.WriteAsync([new WriteRequest("W", 1)], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + drv.GetHealth().LastSuccessfulRead!.Value.ShouldBeGreaterThan(initStamp); + } + + // ---- M2 + M3 ---- + + /// + /// A symbol-version re-arm that lands while a retry of the refused registration is in flight is not + /// overwritten by that retry's failure: the next tick re-attempts at once. (Without the epoch the + /// failure re-imposed a 30 s+ backoff.) + /// + [Fact] + public async Task A_rearm_during_an_in_flight_retry_is_not_lost() + { + var fake = new FakeTwinCATClient(); + fake.AddNotificationErrors["MAIN.T"] = new TwinCATTypeMismatchException("MAIN.T", "PLC STRING"); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: true, + new TwinCATTagDefinition("T", Host, "MAIN.T", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.SubscribeAsync(["T"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + fake.AttemptsFor("MAIN.T").ShouldBe(1); + + var inFlight = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fake.PreAttemptGate = inFlight; + fake.FireSymbolVersionChanged(); // re-arm: the next tick starts a retry, which wedges on the gate + await WaitForAsync(() => fake.AttemptsFor("MAIN.T") >= 2, TimeSpan.FromSeconds(3)); + fake.FireSymbolVersionChanged(); // a second program change while that retry is in flight + fake.PreAttemptGate = null; + inFlight.SetResult(); // the in-flight retry now fails (still refused) + + await WaitForAsync(() => fake.AttemptsFor("MAIN.T") >= 3, TimeSpan.FromSeconds(3)); + fake.AttemptsFor("MAIN.T").ShouldBeGreaterThanOrEqualTo(3); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + /// A re-arm restarts the refusal backoff from the initial delay instead of doubling the old one. + [Fact] + public async Task A_rearm_restarts_the_refusal_backoff() + { + var fake = new FakeTwinCATClient(); + fake.AddNotificationErrors["MAIN.T"] = new TwinCATTypeMismatchException("MAIN.T", "PLC STRING"); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, FastProbe, native: true, + new TwinCATTagDefinition("T", Host, "MAIN.T", TwinCATDataType.DInt)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + _ = await drv.SubscribeAsync(["T"], TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + var reg = drv.GetDeviceState(Host)!.NativeRegistrations.Values.Single(); + reg.RefusalBackoff.ShouldBe(TwinCATDriver.RefusedRetryInitial); + + fake.FireSymbolVersionChanged(); + await WaitForAsync(() => fake.AttemptsFor("MAIN.T") >= 2, TimeSpan.FromSeconds(3)); + await WaitForAsync(() => reg.RefusalBackoff != TimeSpan.Zero, TimeSpan.FromSeconds(1)); + reg.RefusalBackoff.ShouldBe(TwinCATDriver.RefusedRetryInitial); + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + } + + // ---- M5 ---- + + /// A WSTRING tag over a WSTRING symbol writes UTF-16LE with an aligned 2-byte terminator, zero-padded. + [Fact] + public async Task A_WSTRING_write_goes_through_the_client_encode_plan() + { + var fake = new FakeTwinCATClient(); + fake.SymbolShapes["MAIN.W"] = TwinCATPlcSymbolShape.Classify("WSTRING(3)", DataTypeCategory.String, 8); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, + new TwinCATProbeOptions { Enabled = false }, native: false, + new TwinCATTagDefinition("W", Host, "MAIN.W", TwinCATDataType.WString)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var results = await drv.WriteAsync( + [new WriteRequest("W", "中x"), new WriteRequest("W", "abcd")], TestContext.Current.CancellationToken); + + results[0].StatusCode.ShouldBe(TwinCATStatusMapper.Good); + results[1].StatusCode.ShouldBe(TwinCATStatusMapper.BadOutOfRange); // 4 chars do not fit WSTRING(3) + fake.RawWrites["MAIN.W"].ShouldBe(new byte[] { 0x2D, 0x4E, (byte)'x', 0, 0, 0, 0, 0 }); + } + + /// A multi-dimensional array is written flattened in PLC memory order (last index fastest). + [Fact] + public async Task A_multi_dimensional_array_write_is_flattened_row_major() + { + var fake = new FakeTwinCATClient(); + var element = TwinCATPlcSymbolShape.Classify("INT", DataTypeCategory.Primitive, 2); + fake.SymbolShapes["MAIN.M"] = TwinCATPlcSymbolShape.Classify("ARRAY [0..1,0..2] OF INT", DataTypeCategory.Array, 12); + fake.ArrayShapes["MAIN.M"] = new TwinCATPlcArrayShape([2, 3], 12, element); + var drv = NewDriver(new FakeTwinCATClientFactory { Customise = () => fake }, + new TwinCATProbeOptions { Enabled = false }, native: false, + new TwinCATTagDefinition("M", Host, "MAIN.M", TwinCATDataType.Int, ArrayLength: 6)); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var result = await drv.WriteAsync([new WriteRequest("M", new[] { 1, 2, 3, 4, 5, 6 })], TestContext.Current.CancellationToken); + + result.Single().StatusCode.ShouldBe(TwinCATStatusMapper.Good); + fake.RawWrites["MAIN.M"].ShouldBe(new byte[] { 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0 }); + TwinCATArrayCodec.Decode(TwinCATDataType.Int, fake.RawWrites["MAIN.M"], new TwinCATArrayLayout(6, 2), false) + .ShouldBe(new short[] { 1, 2, 3, 4, 5, 6 }); + } + + // ---- M6 ---- + + /// An infinity spelled as a string passes like a float/double infinity; a finite overflow does not. + [Fact] + public void Infinity_strings_are_explicit_infinities() + { + AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, "Infinity").ShouldBe(float.PositiveInfinity); + AdsTwinCATClient.ConvertForWrite(TwinCATDataType.LReal, "-Infinity").ShouldBe(double.NegativeInfinity); + Should.Throw(() => AdsTwinCATClient.ConvertForWrite(TwinCATDataType.Real, "1e300")); + } +}